From arigo at codespeak.net Sat Dec 1 11:05:52 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 1 Dec 2007 11:05:52 +0100 (CET) Subject: [pypy-svn] r49237 - in pypy/dist/pypy: lang/prolog/interpreter translator/backendopt/test translator/goal Message-ID: <20071201100552.7600780C6@code0.codespeak.net> Author: arigo Date: Sat Dec 1 11:05:51 2007 New Revision: 49237 Modified: pypy/dist/pypy/lang/prolog/interpreter/term.py pypy/dist/pypy/translator/backendopt/test/test_removenoops.py pypy/dist/pypy/translator/goal/targetrpystonex.py Log: Some remaining usages of id(). Modified: pypy/dist/pypy/lang/prolog/interpreter/term.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/term.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/term.py Sat Dec 1 11:05:51 2007 @@ -1,5 +1,6 @@ import math from pypy.rlib.objectmodel import we_are_translated, UnboxedValue +from pypy.rlib.objectmodel import compute_unique_id from pypy.rlib.rarithmetic import intmask from pypy.lang.prolog.interpreter.error import UnificationFailed, UncatchableError from pypy.lang.prolog.interpreter import error @@ -380,7 +381,7 @@ raise UnificationFailed def get_unify_hash(self, heap): - return intmask(id(self) << TAGBITS | self.TAG) + return intmask(hash(self) << TAGBITS | self.TAG) @@ -572,7 +573,7 @@ return c if isinstance(obj1, Var): assert isinstance(obj2, Var) - return rcmp(id(obj1), id(obj2)) + return rcmp(compute_unique_id(obj1), compute_unique_id(obj2)) if isinstance(obj1, Atom): assert isinstance(obj2, Atom) return rcmp(obj1.name, obj2.name) Modified: pypy/dist/pypy/translator/backendopt/test/test_removenoops.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_removenoops.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_removenoops.py Sat Dec 1 11:05:51 2007 @@ -92,7 +92,7 @@ s2 = lltype.malloc(S) llop.keepalive(lltype.Void, s1) llop.keepalive(lltype.Void, s2) - return id(s1) + id(s2) + return lltype.cast_ptr_to_int(s1) + lltype.cast_ptr_to_int(s2) graph, t = get_graph(f, []) remove_superfluous_keep_alive(graph) ops = getops(graph) Modified: pypy/dist/pypy/translator/goal/targetrpystonex.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetrpystonex.py (original) +++ pypy/dist/pypy/translator/goal/targetrpystonex.py Sat Dec 1 11:05:51 2007 @@ -1,4 +1,5 @@ from pypy.translator.test import rpystone +from pypy.rlib.objectmodel import current_object_addr_as_int def make_target_definition(LOOPS): @@ -15,7 +16,7 @@ g.Array2Glob[i][j] = 0 g.PtrGlb = None g.PtrGlbNext = None - return rpystone.pystones(loops), id(g) + return rpystone.pystones(loops), current_object_addr_as_int(g) def target(*args): return entry_point, [int] From arigo at codespeak.net Sat Dec 1 11:20:54 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 1 Dec 2007 11:20:54 +0100 (CET) Subject: [pypy-svn] r49238 - in pypy/dist/pypy/rpython: lltypesystem test Message-ID: <20071201102054.1480A80C8@code0.codespeak.net> Author: arigo Date: Sat Dec 1 11:20:53 2007 New Revision: 49238 Modified: pypy/dist/pypy/rpython/lltypesystem/rclass.py pypy/dist/pypy/rpython/test/test_rclass.py Log: Use again the invert of the address of the object as its default hash, for Boehm. Modified: pypy/dist/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rclass.py Sat Dec 1 11:20:53 2007 @@ -701,7 +701,11 @@ return 0 # for None cached = ins.hash_cache if cached == 0: - cached = ins.hash_cache = cast_ptr_to_int(ins) + # XXX this should ideally be done in a GC-dependent way: we only + # need a hash_cache for moving GCs, and we only need the '~' to + # avoid Boehm keeping the object alive if the value is passed + # around + cached = ins.hash_cache = ~cast_ptr_to_int(ins) return cached def ll_inst_type(obj): Modified: pypy/dist/pypy/rpython/test/test_rclass.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rclass.py (original) +++ pypy/dist/pypy/rpython/test/test_rclass.py Sat Dec 1 11:20:53 2007 @@ -399,16 +399,19 @@ d = D() def f(): d2 = D() - # xxx check for this CPython peculiarity for now: - # (this is true on top of the llinterp too) - x = ((hash(d2) & sys.maxint) == - (current_object_addr_as_int(d2) & sys.maxint)) - return x, hash(c)+hash(d) + return hash(d2), current_object_addr_as_int(d2), hash(c), hash(d) res = self.interpret(f, []) - - assert res.item0 == True - assert res.item1 == intmask(hash(c)+hash(d)) + # xxx this is too precise, checking the exact implementation + if isinstance(self, OORtypeMixin): + assert res.item0 == res.item1 + else: + assert res.item0 == ~res.item1 + # the following property is essential on top of the lltypesystem + # otherwise prebuilt dictionaries are broken. It's not that + # relevant on top of the ootypesystem though. + assert res.item2 == hash(c) + assert res.item3 == hash(d) def test_type(self): class A: From arigo at codespeak.net Sat Dec 1 11:25:57 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 1 Dec 2007 11:25:57 +0100 (CET) Subject: [pypy-svn] r49239 - pypy/dist/pypy/translator/c/test Message-ID: <20071201102557.E256E820D@code0.codespeak.net> Author: arigo Date: Sat Dec 1 11:25:57 2007 New Revision: 49239 Modified: pypy/dist/pypy/translator/c/test/test_typed.py Log: Port of r49238. Modified: pypy/dist/pypy/translator/c/test/test_typed.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_typed.py (original) +++ pypy/dist/pypy/translator/c/test/test_typed.py Sat Dec 1 11:25:57 2007 @@ -573,6 +573,7 @@ assert f(255) == 255 def test_hash_preservation(self): + from pypy.rlib.objectmodel import current_object_addr_as_int class C: pass class D(C): @@ -581,16 +582,16 @@ d = D() def fn(): d2 = D() - # xxx check for this CPython peculiarity for now: - x = (hash(d2) & sys.maxint) == (id(d2) & sys.maxint) - return x, hash(c)+hash(d) + return hash(d2), current_object_addr_as_int(d2), hash(c), hash(d) f = self.getcompiled(fn) res = f() - assert res[0] == True - assert res[1] == intmask(hash(c)+hash(d)) + # xxx this is too precise, checking the exact implementation + assert res[0] == ~res[1] + assert res[2] == hash(c) + assert res[3] == hash(d) def test_list_basic_ops(self): def list_basic_ops(i, j): From arigo at codespeak.net Sat Dec 1 11:38:37 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 1 Dec 2007 11:38:37 +0100 (CET) Subject: [pypy-svn] r49240 - pypy/dist/pypy/rpython/test Message-ID: <20071201103837.196038210@code0.codespeak.net> Author: arigo Date: Sat Dec 1 11:38:36 2007 New Revision: 49240 Modified: pypy/dist/pypy/rpython/test/test_rbuiltin.py Log: These checks are too precise, e.g. when llvm reuses the test. Modified: pypy/dist/pypy/rpython/test/test_rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rbuiltin.py (original) +++ pypy/dist/pypy/rpython/test/test_rbuiltin.py Sat Dec 1 11:38:36 2007 @@ -445,9 +445,12 @@ assert isinstance(x1, int) assert isinstance(x2, (int, r_longlong)) assert isinstance(x3, int) - assert x1 == intmask(x0) # at least on top of llinterp - assert x3 == intmask(x2) # at least on top of llinterp assert x0 != x2 + # the following checks are probably too precise, but work at + # least on top of llinterp + if type(self) is TestLLtype: + assert x1 == intmask(x0) + assert x3 == intmask(x2) class TestLLtype(BaseTestRbuiltin, LLRtypeMixin): From arigo at codespeak.net Sat Dec 1 11:42:42 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 1 Dec 2007 11:42:42 +0100 (CET) Subject: [pypy-svn] r49241 - pypy/dist/pypy/translator/tool/test Message-ID: <20071201104242.C317F8210@code0.codespeak.net> Author: arigo Date: Sat Dec 1 11:42:42 2007 New Revision: 49241 Modified: pypy/dist/pypy/translator/tool/test/test_cbuild.py Log: This doesn't make much sense any more. Modified: pypy/dist/pypy/translator/tool/test/test_cbuild.py ============================================================================== --- pypy/dist/pypy/translator/tool/test/test_cbuild.py (original) +++ pypy/dist/pypy/translator/tool/test/test_cbuild.py Sat Dec 1 11:42:42 2007 @@ -20,26 +20,6 @@ testexec = build_executable([t], eci) out = py.process.cmdexec(testexec) assert out.startswith('hello world') - -def test_compile_threads(): - if sys.platform == 'nt': - py.test.skip("Linux-specific test") - try: - import ctypes - except ImportError: - py.test.skip("Need ctypes for that test") - from pypy.tool.autopath import pypydir - pypydir = py.path.local(pypydir) - csourcedir = pypydir.join('translator', 'c', 'src') - include_dirs = [str(csourcedir.dirpath())] - files = [csourcedir.join('thread.c')] - eci = ExternalCompilationInfo( - include_dirs=include_dirs, - libraries=['pthread'] - ) - mod = compile_c_module(files, '_thread', eci) - cdll = ctypes.CDLL(mod) - assert hasattr(cdll, 'RPyThreadLockInit') class TestEci: def setup_class(cls): From cfbolz at codespeak.net Sat Dec 1 12:49:11 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Dec 2007 12:49:11 +0100 (CET) Subject: [pypy-svn] r49242 - pypy/dist/pypy/rlib/test Message-ID: <20071201114911.55D038249@code0.codespeak.net> Author: cfbolz Date: Sat Dec 1 12:49:09 2007 New Revision: 49242 Modified: pypy/dist/pypy/rlib/test/test_rope.py Log: the test hit a bug under 2.4 and 2.3 Modified: pypy/dist/pypy/rlib/test/test_rope.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rope.py (original) +++ pypy/dist/pypy/rlib/test/test_rope.py Sat Dec 1 12:49:09 2007 @@ -479,6 +479,8 @@ def test_find_iterator_unicode(): + if sys.version_info > (2, 5): + py.test.skip("bug in unicode.find that was fixed in 2.5") for searchstring in [ u"\uAAAA\uBBBB\uCCCC", u"\uAAAA", u"", u"\u6666", u"\u6666\u7777\u8888", From cfbolz at codespeak.net Sat Dec 1 12:58:30 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 1 Dec 2007 12:58:30 +0100 (CET) Subject: [pypy-svn] r49243 - pypy/dist/pypy/rlib/test Message-ID: <20071201115830.27A948249@code0.codespeak.net> Author: cfbolz Date: Sat Dec 1 12:58:29 2007 New Revision: 49243 Modified: pypy/dist/pypy/rlib/test/test_rope.py Log: wrong comparison order - thanks alexander Modified: pypy/dist/pypy/rlib/test/test_rope.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rope.py (original) +++ pypy/dist/pypy/rlib/test/test_rope.py Sat Dec 1 12:58:29 2007 @@ -479,7 +479,7 @@ def test_find_iterator_unicode(): - if sys.version_info > (2, 5): + if sys.version_info < (2, 5): py.test.skip("bug in unicode.find that was fixed in 2.5") for searchstring in [ u"\uAAAA\uBBBB\uCCCC", u"\uAAAA", u"", u"\u6666", From exarkun at codespeak.net Sat Dec 1 18:05:47 2007 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Sat, 1 Dec 2007 18:05:47 +0100 (CET) Subject: [pypy-svn] r49252 - in pypy/dist/pypy: module/posix module/posix/test rpython/module rpython/module/test Message-ID: <20071201170547.B074280C4@code0.codespeak.net> Author: exarkun Date: Sat Dec 1 18:05:46 2007 New Revision: 49252 Added: pypy/dist/pypy/rpython/module/test/execve_tests.py Modified: pypy/dist/pypy/module/posix/__init__.py pypy/dist/pypy/module/posix/test/test_posix2.py pypy/dist/pypy/rpython/module/ll_os.py pypy/dist/pypy/rpython/module/test/test_ll_os.py Log: llimpl for os.execve Modified: pypy/dist/pypy/module/posix/__init__.py ============================================================================== --- pypy/dist/pypy/module/posix/__init__.py (original) +++ pypy/dist/pypy/module/posix/__init__.py Sat Dec 1 18:05:46 2007 @@ -75,7 +75,7 @@ interpleveldefs['waitpid'] = 'interp_posix.waitpid' if hasattr(os, 'execv'): interpleveldefs['execv'] = 'interp_posix.execv' - if hasattr(os, 'execve') and 0: # XXX XXX in-progress + if hasattr(os, 'execve'): interpleveldefs['execve'] = 'interp_posix.execve' if False and hasattr(os, 'uname'): interpleveldefs['uname'] = 'interp_posix.uname' Modified: pypy/dist/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/dist/pypy/module/posix/test/test_posix2.py (original) +++ pypy/dist/pypy/module/posix/test/test_posix2.py Sat Dec 1 18:05:46 2007 @@ -206,8 +206,9 @@ t([3, "a"]) def test_execve(self): - skip("Not implemented") os = self.posix + if not hasattr(os, "fork"): + skip("Need fork() to test execve()") pid = os.fork() if pid == 0: os.execve("/usr/bin/env", ["env", "python", "-c", "import os; open('onefile', 'w').write(os.environ['ddd'])"], {'ddd':'xxx'}) Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Sat Dec 1 18:05:46 2007 @@ -137,6 +137,36 @@ return extdef([str, [str]], s_ImpossibleValue, llimpl=execv_llimpl, export_name="ll_os.ll_os_execv") + + @registering_if(os, 'execve') + def register_os_execve(self): + os_execve = self.llexternal( + 'execve', [rffi.CCHARP, rffi.CCHARPP, rffi.CCHARPP], rffi.INT) + + def execve_llimpl(path, args, env): + # XXX Check path, args, env for \0 and raise TypeErrors as + # appropriate + envstrs = [] + for item in env.iteritems(): + envstrs.append("%s=%s" % item) + + l_args = rffi.liststr2charpp(args) + l_env = rffi.liststr2charpp(envstrs) + os_execve(path, l_args, l_env) + + # XXX untested + rffi.free_charpp(l_env) + rffi.free_charpp(l_args) + + raise OSError(rposix.get_errno(), "execve failed") + + return extdef( + [str, [str], {str: str}], + s_ImpossibleValue, + llimpl=execve_llimpl, + export_name="ll_os.ll_os_execve") + + @registering_if(posix, 'spawnv') def register_os_spawnv(self): os_spawnv = self.llexternal('spawnv', Added: pypy/dist/pypy/rpython/module/test/execve_tests.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rpython/module/test/execve_tests.py Sat Dec 1 18:05:46 2007 @@ -0,0 +1,25 @@ + +""" +Functions which call the llimpl execve function with various arguments. Since +execve replaces the process with a new one in the successful case, these are +here to be run as a child process of the test process. +""" + +import os, sys + +from pypy.rpython.module.test.test_ll_os import EXECVE_ENV, getllimpl + +execve = getllimpl(os.execve) + + +def execve_true(): + execve("/bin/true", ["/bin/true"], {}) + +def execve_false(): + execve("/bin/false", ["/bin/false"], {}) + +def execve_env(): + execve("/usr/bin/env", ["/usr/bin/env"], EXECVE_ENV) + +if __name__ == '__main__': + globals()[sys.argv[1]]() Modified: pypy/dist/pypy/rpython/module/test/test_ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/test/test_ll_os.py (original) +++ pypy/dist/pypy/rpython/module/test/test_ll_os.py Sat Dec 1 18:05:46 2007 @@ -1,8 +1,11 @@ import os +from py.path import local + from pypy.tool.udir import udir from pypy.translator.c.test.test_genc import compile from pypy.rpython import extregistry +import errno import sys import py @@ -58,6 +61,43 @@ assert file(filename).read().strip() == '2' os.unlink(filename) + +EXECVE_ENV = {"foo": "bar", "baz": "quux"} +execve_tests = str(local(__file__).dirpath().join('execve_tests.py')) + +def test_execve(): + if os.name != 'posix': + py.test.skip('posix specific function') + base = sys.executable + " " + execve_tests + " " + + # Test exit status and code + result = os.system(base + "execve_true") + assert os.WIFEXITED(result) + assert os.WEXITSTATUS(result) == 0 + result = os.system(base + "execve_false") + assert os.WIFEXITED(result) + assert os.WEXITSTATUS(result) == 1 + + # Test environment + result = os.popen(base + "execve_env").read() + assert dict([line.split('=') for line in result.splitlines()]) == EXECVE_ENV + + # These won't actually execute anything, so they don't need a child process + # helper. + execve = getllimpl(os.execve) + + # If the target does not exist, an OSError should result + info = py.test.raises( + OSError, execve, execve_tests + "-non-existent", [], {}) + assert info.value.errno == errno.ENOENT + + # If the target is not executable, an OSError should result + info = py.test.raises( + OSError, execve, execve_tests, [], {}) + assert info.value.errno == errno.EACCES + + + class ExpectTestOs: def setup_class(cls): if not hasattr(os, 'ttyname'): From niko at codespeak.net Sat Dec 1 18:16:58 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Sat, 1 Dec 2007 18:16:58 +0100 (CET) Subject: [pypy-svn] r49253 - pypy/dist/pypy/translator/jvm Message-ID: <20071201171658.76FC88166@code0.codespeak.net> Author: niko Date: Sat Dec 1 18:16:55 2007 New Revision: 49253 Modified: pypy/dist/pypy/translator/jvm/generator.py Log: update the debug info we print out: now the line number corresponds to the line in the file, and we emit local variable debug information with the name of the variable from the rpython source. this helps when viewing code in java debuggers Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Sat Dec 1 18:16:55 2007 @@ -497,8 +497,10 @@ def __init__(self, classty, superclassty): self.class_type = classty self.superclass_type = superclassty + self.line_number = 1 def out(self, arg): self.file.write(arg) + self.line_number += arg.count("\n") class FunctionState(object): """ When you invoked begin_function(), one of these objects is allocated @@ -508,20 +510,27 @@ self.local_vars = {} self.function_arguments = [] self.instr_counter = 0 - def add_var(self, jvar, jtype): + def add_var(self, jvar, jtype, is_param): """ Adds new entry for variable 'jvar', of java type 'jtype' """ idx = self.next_offset self.next_offset += jtype.descriptor.type_width() if jvar: assert jvar.name not in self.local_vars # never been added before - self.local_vars[jvar.name] = idx - self.function_arguments.append((jtype, idx)) + self.local_vars[jvar.name] = (idx, jtype) + if is_param: + self.function_arguments.append((jtype, idx)) return idx def var_offset(self, jvar, jtype): """ Returns offset for variable 'jvar', of java type 'jtype' """ if jvar.name in self.local_vars: - return self.local_vars[jvar.name] - return self.add_var(jvar, jtype) + return self.local_vars[jvar.name][0] + return self.add_var(jvar, jtype, False) + def var_info_list(self): + var_info_list = [None] * self.next_offset + for name, (idx, jtype) in self.local_vars.items(): + var_info_list[idx] = (name, jtype) + return var_info_list + # ___________________________________________________________________________ @@ -654,7 +663,7 @@ for idx, ty in enumerate(argtypes): if idx < len(argvars): var = argvars[idx] else: var = None - self.curfunc.add_var(var, ty) + self.curfunc.add_var(var, ty, True) # Prepare a map for the local variable indices we will add # Let the subclass do the rest of the work; note that it does # not need to know the argvars parameter, so don't pass it @@ -1371,12 +1380,34 @@ funcname, "".join([a.descriptor for a in argtypes]), rettype.descriptor)) - self._abstract_method = abstract + self.abstract_method = abstract + + if not self.abstract_method: + self.function_start_label = self.unique_label( + 'function_start', True) def _end_function(self): - if not self._abstract_method: + + if not self.abstract_method: + function_end_label = self.unique_label('function_end', True) + self.curclass.out('.limit stack 100\n') # HACK, track max offset self.curclass.out('.limit locals %d\n' % self.curfunc.next_offset) + + # Declare debug information for each variable: + var_info_list = self.curfunc.var_info_list() + for idx, data in enumerate(var_info_list): + if data: + name, jtype = data + if jtype is not jVoid: + self.curclass.out( + '.var %d is %s %s from %s to %s\n' % ( + idx, + name, + jtype.descriptor, + self.function_start_label.label, + function_end_label.label)) + self.curclass.out('.end method\n') def mark(self, lbl): @@ -1394,7 +1425,7 @@ return str(arg) strargs = [jasmin_syntax(arg) for arg in args] instr_text = '%s %s' % (jvmstr, " ".join(strargs)) - self.curclass.out(' .line %d\n' % self.curfunc.instr_counter) + self.curclass.out(' .line %d\n' % self.curclass.line_number) self.curclass.out(' %-60s\n' % (instr_text,)) self.curfunc.instr_counter+=1 From niko at codespeak.net Sat Dec 1 18:59:01 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Sat, 1 Dec 2007 18:59:01 +0100 (CET) Subject: [pypy-svn] r49254 - in pypy/dist/pypy/translator: jvm jvm/src/pypy oosupport/test_template Message-ID: <20071201175901.9BCD28128@code0.codespeak.net> Author: niko Date: Sat Dec 1 18:59:01 2007 New Revision: 49254 Modified: pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/node.py pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java pypy/dist/pypy/translator/jvm/typesystem.py pypy/dist/pypy/translator/oosupport/test_template/builtin.py Log: fix math.modf in the jvm: 1. the code in PyPy.java was slightly wrong 2. the static signature cannot reference generated objects, so when they are returned we must use jObject instead and insert a downcast 3. add a test to oosupport/builtin for modf Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Sat Dec 1 18:59:01 2007 @@ -1047,15 +1047,32 @@ def call_primitive(self, op, module, name): callee = op.args[0].value - argtypes, rettype = self.db.types_for_signature( + jargtypes, jrettype = self.db.types_for_signature( callee._TYPE.ARGS, callee._TYPE.RESULT) + + # Determine what class the primitive is implemented in: if module == 'll_os': jcls = jll_os else: jcls = jPyPy - mthd = Method.v(jcls, name, argtypes, rettype) + + # Determine the method signature: + # n.b.: if the method returns a generated type, then + # it's static type will be Object. This is because + # the method cannot directly refer to the Java type in + # .java source, as its name is not yet known. + if jrettype.is_generated(): + mthd = Method.v(jcls, name, jargtypes, jObject) + else: + mthd = Method.v(jcls, name, jargtypes, jrettype) + + # Invoke the method self.emit(mthd) + # Cast the result, if needed + if jrettype.is_generated(): + self.downcast_jtype(jrettype) + def prepare_call_oostring(self, OOTYPE): # Load the PyPy object pointer onto the stack: self.push_pypy() Modified: pypy/dist/pypy/translator/jvm/node.py ============================================================================== --- pypy/dist/pypy/translator/jvm/node.py (original) +++ pypy/dist/pypy/translator/jvm/node.py Sat Dec 1 18:59:01 2007 @@ -20,9 +20,9 @@ from pypy.rpython.ootypesystem import \ ootype, rclass from pypy.translator.jvm.typesystem import \ - JvmClassType, jString, jStringArray, jVoid, jThrowable, jInt, \ + JvmGeneratedClassType, jString, jStringArray, jVoid, jThrowable, jInt, \ jObject, JvmType, jStringBuilder, jPyPyInterlink, jCallbackInterfaces, \ - JvmInterfaceType, jPyPy + JvmGeneratedInterfaceType, jPyPy from pypy.translator.jvm.opcodes import \ opcodes from pypy.translator.jvm.option import \ @@ -445,7 +445,7 @@ OOFunction._render_op(self, op) -class StaticMethodInterface(Node, JvmClassType): +class StaticMethodInterface(Node, JvmGeneratedClassType): """ We generate an abstract base class when we need function pointers, which correspond to constants of StaticMethod ootype. We need a @@ -467,7 +467,7 @@ argtypes: list of JvmTypes rettype: JvmType """ - JvmClassType.__init__(self, name) + JvmGeneratedClassType.__init__(self, name) assert isinstance(jrettype, JvmType) self.java_argument_types = [self] + list(jargtypes) self.java_return_type = jrettype @@ -539,7 +539,7 @@ gen.end_class() -class StaticMethodImplementation(Node, JvmClassType): +class StaticMethodImplementation(Node, JvmGeneratedClassType): """ In addition to the StaticMethodInterface, we must generate an implementation for each specific method that is called. These @@ -567,7 +567,7 @@ } """ def __init__(self, name, super_class, bound_to_jty, impl_method): - JvmClassType.__init__(self, name) + JvmGeneratedClassType.__init__(self, name) self.super_class = super_class self.impl_method = impl_method self.dump_method = ConstantStringDumpMethod( @@ -633,13 +633,13 @@ gen.end_function() gen.end_class() -class Interface(Node, JvmInterfaceType): +class Interface(Node, JvmGeneratedInterfaceType): """ Represents an interface to be generated. The only class that we currently generate into an interface is ootype.ROOT. """ def __init__(self, name): - JvmClassType.__init__(self, name) + JvmGeneratedInterfaceType.__init__(self, name) self.super_class = jObject self.rendered = False self.properties = {} @@ -675,7 +675,7 @@ gen.end_class() -class Class(Node, JvmClassType): +class Class(Node, JvmGeneratedClassType): """ Represents a class to be emitted. Note that currently, classes are emitted all in one shot, not piecemeal. """ @@ -685,7 +685,7 @@ 'name' should be a fully qualified Java class name like "java.lang.String", supercls is a Class object """ - JvmClassType.__init__(self, name) + JvmGeneratedClassType.__init__(self, name) self.rendered = False # has rendering occurred? self.abstract = False # is this an abstract class? self.fields = {} # maps field name to jvmgen.Field object Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Sat Dec 1 18:59:01 2007 @@ -988,13 +988,8 @@ } public Object ll_math_modf(double x) { - if (x >= 0) { - double floor_x = Math.floor(x); - return interlink.recordFloatFloat(floor_x, x - floor_x); - } - - double ceil_x = Math.ceil(x); - return interlink.recordFloatFloat(ceil_x, x + ceil_x); + double integer_x = (x >= 0 ? Math.floor(x) : Math.ceil(x)); + return interlink.recordFloatFloat(x - integer_x, integer_x); } public double ll_math_exp(double x) { Modified: pypy/dist/pypy/translator/jvm/typesystem.py ============================================================================== --- pypy/dist/pypy/translator/jvm/typesystem.py (original) +++ pypy/dist/pypy/translator/jvm/typesystem.py Sat Dec 1 18:59:01 2007 @@ -127,6 +127,11 @@ exist on this type. """ raise NotImplementedException + def is_generated(self): + """ Indicates whether the source for this type is generated by + pypy. """ + return False + def __repr__(self): return "%s<%s>" % (self.__class__.__name__, self.descriptor) @@ -146,9 +151,21 @@ def lookup_method(self, methodnm): raise KeyError(fieldnm) # we treat as opaque type +class JvmGeneratedClassType(JvmClassType): + """ Abstract class extended by the classes in node.py that represent + generated classes """ + def is_generated(self): + return True + class JvmInterfaceType(JvmClassType): pass +class JvmGeneratedInterfaceType(JvmInterfaceType): + """ Abstract class extended by the classes in node.py that represent + generated interfaces """ + def is_generated(self): + return True + jIntegerClass = JvmClassType('java.lang.Integer') jLongClass = JvmClassType('java.lang.Long') jDoubleClass = JvmClassType('java.lang.Double') Modified: pypy/dist/pypy/translator/oosupport/test_template/builtin.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/test_template/builtin.py (original) +++ pypy/dist/pypy/translator/oosupport/test_template/builtin.py Sat Dec 1 18:59:01 2007 @@ -1,4 +1,4 @@ -import os +import os, math import errno import stat from py.builtin import sorted @@ -138,6 +138,16 @@ # XXX: remember to test ll_os_readlink and ll_os_pipe as soon as # they are implemented + def test_math_modf(self): + def fn(x): + return math.modf(x) + for x in (.5, 1, 1.5): + for y in (1, -1): + act_res = self.interpret(fn, [x*y]) + exp_res = math.modf(x*y) + assert act_res.item0 == exp_res[0] + assert act_res.item1 == exp_res[1] + class BaseTestTime(llBaseTestTime): From niko at codespeak.net Sat Dec 1 19:02:44 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Sat, 1 Dec 2007 19:02:44 +0100 (CET) Subject: [pypy-svn] r49255 - pypy/dist/pypy/translator/jvm Message-ID: <20071201180244.7A49E812A@code0.codespeak.net> Author: niko Date: Sat Dec 1 19:02:44 2007 New Revision: 49255 Modified: pypy/dist/pypy/translator/jvm/node.py Log: use the new is_generated() test in InterlinkFunction to determine when the rewrite the signature to use Object, since it applies there as well -- before we just checked if the type was a reference type Modified: pypy/dist/pypy/translator/jvm/node.py ============================================================================== --- pypy/dist/pypy/translator/jvm/node.py (original) +++ pypy/dist/pypy/translator/jvm/node.py Sat Dec 1 19:02:44 2007 @@ -795,11 +795,10 @@ self.name = name self.helper = helper - # The functions in Interlink.java either return Object, - # because they are returning an instance of a class generated - # by us which the JVM doesn't know about, or they return a - # scalar. - if self.helper.return_type.descriptor.is_reference(): + # Since the names of classes we generate are not statically + # known, the functions in Interlink.java simply return + # Object when they create an instance of one of those types. + if self.helper.return_type.is_generated(): self.return_type = jObject else: self.return_type = self.helper.return_type From xoraxax at codespeak.net Sat Dec 1 19:14:39 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 1 Dec 2007 19:14:39 +0100 (CET) Subject: [pypy-svn] r49256 - pypy/dist/pypy/translator/goal Message-ID: <20071201181439.C10218193@code0.codespeak.net> Author: xoraxax Date: Sat Dec 1 19:14:39 2007 New Revision: 49256 Modified: pypy/dist/pypy/translator/goal/app_main.py pypy/dist/pypy/translator/goal/nanos.py Log: app_main does not use the normal os module now anymore. Nanos can expose its module even in scenarios without a pypy interpreter - this makes app_main testable again. This should fix the broken -u option. Modified: pypy/dist/pypy/translator/goal/app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/app_main.py (original) +++ pypy/dist/pypy/translator/goal/app_main.py Sat Dec 1 19:14:39 2007 @@ -401,8 +401,8 @@ if __name__ == '__main__': - import os import autopath + import nanos # obscure! try removing the following line, see how it crashes, and # guess why... ImStillAroundDontForgetMe = sys.modules['__main__'] @@ -420,5 +420,6 @@ from pypy.module.sys.version import PYPY_VERSION sys.pypy_version_info = PYPY_VERSION sys.pypy_initial_path = pypy_initial_path + os = nanos.os_module_for_testing sys.exit(entry_point(sys.argv[0], sys.argv[1:], os)) #sys.exit(entry_point('app_main.py', sys.argv[1:])) Modified: pypy/dist/pypy/translator/goal/nanos.py ============================================================================== --- pypy/dist/pypy/translator/goal/nanos.py (original) +++ pypy/dist/pypy/translator/goal/nanos.py Sat Dec 1 19:14:39 2007 @@ -35,7 +35,7 @@ app_os = applevel(r''' # NOT_RPYTHON - from os import sep, pathsep + from os import sep, pathsep, getenv, name, fdopen try: from os import readlink except ImportError: @@ -53,3 +53,14 @@ space.setattr(w_os, space.wrap('path'), w_os_path) space.setattr(w_os, space.wrap('getenv'), space.wrap(getenv_w)) return w_os + + +# in order to be able to test app_main without the pypy interpreter +# we create the nanos module with the same names here like it would +# be created while translation +path_module_for_testing = type(os)("os.path") +os_module_for_testing = type(os)("os") +os_module_for_testing.path = path_module_for_testing +eval(app_os_path.code, path_module_for_testing.__dict__) +eval(app_os.code, os_module_for_testing.__dict__) + From xoraxax at codespeak.net Sat Dec 1 19:41:52 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 1 Dec 2007 19:41:52 +0100 (CET) Subject: [pypy-svn] r49257 - pypy/dist/pypy/translator/goal Message-ID: <20071201184152.4A36981BC@code0.codespeak.net> Author: xoraxax Date: Sat Dec 1 19:41:51 2007 New Revision: 49257 Removed: pypy/dist/pypy/translator/goal/Ratthing-b246-benchs.txt pypy/dist/pypy/translator/goal/module-list.example pypy/dist/pypy/translator/goal/module-list.pedronis Log: I think these files are not needed anymore. From xoraxax at codespeak.net Sat Dec 1 19:49:32 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 1 Dec 2007 19:49:32 +0100 (CET) Subject: [pypy-svn] r49258 - pypy/dist/pypy/translator Message-ID: <20071201184932.A836A8190@code0.codespeak.net> Author: xoraxax Date: Sat Dec 1 19:49:32 2007 New Revision: 49258 Modified: pypy/dist/pypy/translator/driver.py Log: Remove two status messages in the regular code path. Modified: pypy/dist/pypy/translator/driver.py ============================================================================== --- pypy/dist/pypy/translator/driver.py (original) +++ pypy/dist/pypy/translator/driver.py Sat Dec 1 19:49:32 2007 @@ -322,12 +322,11 @@ def sanity_check_annotation(self): translator = self.translator irreg = query.qoutput(query.check_exceptblocks_qgen(translator)) - if not irreg: - self.log.info("All exceptblocks seem sane") + if irreg: + self.log.info("Some exceptblocks seem insane") lost = query.qoutput(query.check_methods_qgen(translator)) assert not lost, "lost methods, something gone wrong with the annotation of method defs" - self.log.info("No lost method defs") so = query.qoutput(query.polluted_qgen(translator)) tot = len(translator.graphs) From arigo at codespeak.net Sun Dec 2 12:01:27 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 12:01:27 +0100 (CET) Subject: [pypy-svn] r49265 - pypy/dist/pypy/translator/c/test Message-ID: <20071202110127.CD2378166@code0.codespeak.net> Author: arigo Date: Sun Dec 2 12:01:27 2007 New Revision: 49265 Modified: pypy/dist/pypy/translator/c/test/test_newgc.py Log: Forgot to check this in. Another forgotten id() usage. Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Sun Dec 2 12:01:27 2007 @@ -841,6 +841,7 @@ should_be_moving = True def test_many_ids(self): + from pypy.rlib.objectmodel import compute_unique_id class A(object): pass def f(): @@ -852,7 +853,7 @@ # remember the ids, it will trigger some collections itself i = 0 while i < len(alist): - idarray[i] = id(alist[i]) + idarray[i] = compute_unique_id(alist[i]) i += 1 j = 0 while j < 2: @@ -860,7 +861,7 @@ [A() for i in range(20000)] i = 0 while i < len(alist): - if idarray[i] != id(alist[i]): + if idarray[i] != compute_unique_id(alist[i]): return j * 1000000 + i i += 1 j += 1 From arigo at codespeak.net Sun Dec 2 12:07:43 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 12:07:43 +0100 (CET) Subject: [pypy-svn] r49266 - pypy/dist/pypy/module/posix Message-ID: <20071202110743.A303E814A@code0.codespeak.net> Author: arigo Date: Sun Dec 2 12:07:43 2007 New Revision: 49266 Modified: pypy/dist/pypy/module/posix/interp_posix.py Log: * Style fixes. * What was that ValueError case here for? Modified: pypy/dist/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/dist/pypy/module/posix/interp_posix.py (original) +++ pypy/dist/pypy/module/posix/interp_posix.py Sun Dec 2 12:07:43 2007 @@ -476,16 +476,14 @@ args: iterable of arguments env: dictionary of strings mapping to strings """ + args = [space.str_w(w_arg) for w_arg in space.unpackiterable(w_args)] + env = {} + w_keys = space.call_method(w_env, 'keys') + for w_key in space.unpackiterable(w_keys): + w_value = space.getitem(w_env, w_key) + env[space.str_w(w_key)] = space.str_w(w_value) try: - args = [space.str_w(i) for i in space.unpackiterable(w_args)] - env = {} - keys = space.call_function(space.getattr(w_env, space.wrap('keys'))) - for key in space.unpackiterable(keys): - value = space.getitem(w_env, key) - env[space.str_w(key)] = space.str_w(value) os.execve(command, args, env) - except ValueError, e: - raise OperationError(space.w_ValueError, space.wrap(str(e))) except OSError, e: raise wrap_oserror(space, e) execve.unwrap_spec = [ObjSpace, str, W_Root, W_Root] From arigo at codespeak.net Sun Dec 2 12:36:27 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 12:36:27 +0100 (CET) Subject: [pypy-svn] r49267 - pypy/dist/pypy/tool/pytest Message-ID: <20071202113627.F38868170@code0.codespeak.net> Author: arigo Date: Sun Dec 2 12:36:26 2007 New Revision: 49267 Modified: pypy/dist/pypy/tool/pytest/appsupport.py Log: We can get a real None here, e.g. if the signature of the app_test function itself is bogus. Modified: pypy/dist/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/dist/pypy/tool/pytest/appsupport.py (original) +++ pypy/dist/pypy/tool/pytest/appsupport.py Sun Dec 2 12:36:26 2007 @@ -112,7 +112,7 @@ def __init__(self, space, apptb): l = [] - while apptb is not space.w_None: + while apptb is not space.w_None and apptb is not None: l.append(self.Entry(space, apptb)) apptb = space.getattr(apptb, space.wrap('tb_next')) list.__init__(self, l) From cfbolz at codespeak.net Sun Dec 2 12:39:32 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 2 Dec 2007 12:39:32 +0100 (CET) Subject: [pypy-svn] r49268 - in pypy/dist/pypy/rlib: . test Message-ID: <20071202113932.DE29B8135@code0.codespeak.net> Author: cfbolz Date: Sun Dec 2 12:39:32 2007 New Revision: 49268 Modified: pypy/dist/pypy/rlib/rope.py pypy/dist/pypy/rlib/test/test_rope.py Log: fix a rope multiplication: make the resulting ropes as balanced as possible by default. Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Sun Dec 2 12:39:32 2007 @@ -8,7 +8,7 @@ NBITS = int(math.log(sys.maxint) / LOG2) + 2 # XXX should optimize the numbers -NEW_NODE_WHEN_LENGTH = 16 +NEW_NODE_WHEN_LENGTH = 32 CONVERT_WHEN_SMALLER = 8 MAX_DEPTH = 32 # maybe should be smaller CONCATENATE_WHEN_MULTIPLYING = 128 @@ -476,37 +476,33 @@ return node.getslice(0, stop) - def multiply(node, times): if times <= 0: return LiteralStringNode.EMPTY if times == 1: return node - end_length = node.length() * times - num_bits = 2 - mask = times >> 2 - while mask: - num_bits += 1 - mask >>= 1 - result = node - mask = 1 << (num_bits - 2) - #import pdb; pdb.set_trace() - for i in range(num_bits - 1): - if mask & times: - if result.length() < CONCATENATE_WHEN_MULTIPLYING: - result = concatenate(result, result) - result = concatenate(result, node) + twopower = node + number = 1 + result = None + while number < times: + if number & times: + if result is None: + result = twopower + elif result.length() < CONCATENATE_WHEN_MULTIPLYING: + result = concatenate(result, twopower) else: - result = BinaryConcatNode(result, result) - result = BinaryConcatNode(result, node) + result = BinaryConcatNode(result, twopower) + try: + number = ovfcheck(number * 2) + except OverflowError: + break + if twopower.length() < CONCATENATE_WHEN_MULTIPLYING: + twopower = concatenate(twopower, twopower) else: - if result.length() < CONCATENATE_WHEN_MULTIPLYING: - result = concatenate(result, result) - else: - result = BinaryConcatNode(result, result) - mask >>= 1 + twopower = BinaryConcatNode(twopower, twopower) return result + def join(node, l): if node.length() == 0: return rebalance(l) @@ -538,7 +534,7 @@ first_node = None while stack: curr = stack.pop() - while isinstance(curr, BinaryConcatNode) and not curr.balanced: + while isinstance(curr, BinaryConcatNode) and not curr.check_balanced(): stack.append(curr.right) curr = curr.left Modified: pypy/dist/pypy/rlib/test/test_rope.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rope.py (original) +++ pypy/dist/pypy/rlib/test/test_rope.py Sun Dec 2 12:39:32 2007 @@ -838,3 +838,7 @@ res = str_decode_utf8(node) assert res is None + +def test_multiply_result_needs_no_rebalancing(): + r1 = multiply(LiteralStringNode("s"), 2**31 - 2) + assert r1.rebalance() is r1 From arigo at codespeak.net Sun Dec 2 12:44:50 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 12:44:50 +0100 (CET) Subject: [pypy-svn] r49269 - in pypy/dist/pypy: module/_socket/test rlib rlib/test Message-ID: <20071202114450.B681D8134@code0.codespeak.net> Author: arigo Date: Sun Dec 2 12:44:50 2007 New Revision: 49269 Modified: pypy/dist/pypy/module/_socket/test/test_sock_app.py pypy/dist/pypy/rlib/rsocket.py pypy/dist/pypy/rlib/test/test_rsocket.py Log: issue318 resolved Fix accept() on AF_UNIX sockets. Add tests. Modified: pypy/dist/pypy/module/_socket/test/test_sock_app.py ============================================================================== --- pypy/dist/pypy/module/_socket/test/test_sock_app.py (original) +++ pypy/dist/pypy/module/_socket/test/test_sock_app.py Sun Dec 2 12:44:50 2007 @@ -1,13 +1,13 @@ from pypy.conftest import gettestobjspace import sys import py +from pypy.tool.udir import udir def setup_module(mod): mod.space = gettestobjspace(usemodules=['_socket']) global socket import socket mod.w_socket = space.appexec([], "(): import _socket as m; return m") - from pypy.tool.udir import udir mod.path = udir.join('fd') mod.path.write('fo') mod.raises = py.test.raises # make raises available from app-level tests @@ -233,6 +233,7 @@ class AppTestSocket: def setup_class(cls): cls.space = space + cls.w_udir = space.wrap(str(udir)) def test_ntoa_exception(self): import _socket @@ -407,7 +408,32 @@ s = _socket.socket(_socket.AF_INET, _socket.SOCK_DGRAM, 0) s.sendto(buffer(''), ('localhost', 9)) # Send to discard port. s.close() - + + def test_unix_socket_connect(self): + import _socket, os + if not hasattr(_socket, 'AF_UNIX'): + skip('AF_UNIX not supported.') + sockpath = os.path.join(self.udir, 'app_test_unix_socket_connect') + + serversock = _socket.socket(_socket.AF_UNIX) + serversock.bind(sockpath) + serversock.listen(1) + + clientsock = _socket.socket(_socket.AF_UNIX) + clientsock.connect(sockpath) + s, addr = serversock.accept() + assert not addr + + s.send('X') + data = clientsock.recv(100) + assert data == 'X' + clientsock.send('Y') + data = s.recv(100) + assert data == 'Y' + + clientsock.close() + s.close() + class AppTestSocketTCP: def setup_class(cls): Modified: pypy/dist/pypy/rlib/rsocket.py ============================================================================== --- pypy/dist/pypy/rlib/rsocket.py (original) +++ pypy/dist/pypy/rlib/rsocket.py Sun Dec 2 12:44:50 2007 @@ -377,7 +377,7 @@ class UNIXAddress(Address): family = AF_UNIX struct = _c.sockaddr_un - minlen = offsetof(_c.sockaddr_un, 'c_sun_path') + 1 + minlen = offsetof(_c.sockaddr_un, 'c_sun_path') maxlen = sizeof(struct) def __init__(self, path): @@ -406,7 +406,7 @@ def get_path(self): a = self.lock(_c.sockaddr_un) maxlength = self.addrlen - offsetof(_c.sockaddr_un, 'c_sun_path') - if _c.linux and a.c_sun_path[0] == '\x00': + if _c.linux and maxlength > 0 and a.c_sun_path[0] == '\x00': # Linux abstract namespace length = maxlength else: Modified: pypy/dist/pypy/rlib/test/test_rsocket.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rsocket.py (original) +++ pypy/dist/pypy/rlib/test/test_rsocket.py Sun Dec 2 12:44:50 2007 @@ -345,6 +345,32 @@ py.test.skip("no inet_ntop()") assert inet_ntop(AF_INET, '\x01\x02\x03\x05') == '1.2.3.5' +def test_unix_socket_connect(): + if getattr(rsocket, 'AF_UNIX', None) is None: + py.test.skip('AF_UNIX not supported.') + from pypy.tool.udir import udir + sockpath = str(udir.join('test_unix_socket_connect')) + a = UNIXAddress(sockpath) + + serversock = RSocket(AF_UNIX) + serversock.bind(a) + serversock.listen(1) + + clientsock = RSocket(AF_UNIX) + clientsock.connect(a) + s, addr = serversock.accept() + + s.send('X') + data = clientsock.recv(100) + assert data == 'X' + clientsock.send('Y') + data = s.recv(100) + assert data == 'Y' + + clientsock.close() + s.close() + + class TestTCP: PORT = 50007 HOST = 'localhost' From arigo at codespeak.net Sun Dec 2 12:59:57 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 12:59:57 +0100 (CET) Subject: [pypy-svn] r49271 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071202115957.F049B8167@code0.codespeak.net> Author: arigo Date: Sun Dec 2 12:59:57 2007 New Revision: 49271 Modified: pypy/dist/pypy/rpython/lltypesystem/rlist.py Log: issue326 in-progress Reduce memory allocations when creating RPython lists. For now there is a choice of either using a prebuilt empty initial array, or always pre-allocating for N elements. Modified: pypy/dist/pypy/rpython/lltypesystem/rlist.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rlist.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rlist.py Sun Dec 2 12:59:57 2007 @@ -71,6 +71,7 @@ ITEMARRAY = GcArray(ITEM, adtmeths = ADTIFixedList({ "ll_newlist": ll_fixed_newlist, + "ll_newemptylist": ll_fixed_newemptylist, "ll_length": ll_fixed_length, "ll_items": ll_fixed_items, ##"list_builder": self.list_builder, @@ -184,6 +185,7 @@ ("items", Ptr(ITEMARRAY)), adtmeths = ADTIList({ "ll_newlist": ll_newlist, + "ll_newemptylist": ll_newemptylist, "ll_length": ll_length, "ll_items": ll_items, ##"list_builder": self.list_builder, @@ -362,6 +364,26 @@ ll_newlist = typeMethod(ll_newlist) ll_newlist.oopspec = 'newlist(length)' +# should empty lists start with no allocated memory, or with a preallocated +# minimal number of entries? XXX compare memory usage versus speed, and +# check how many always-empty lists there are in a typical pypy-c run... +INITIAL_EMPTY_LIST_ALLOCATION = 0 + +def _ll_prebuilt_empty_array(LISTITEM): + return malloc(LISTITEM, 0) +_ll_prebuilt_empty_array._annspecialcase_ = 'specialize:memo' + +def ll_newemptylist(LIST): + l = malloc(LIST) + l.length = 0 + if INITIAL_EMPTY_LIST_ALLOCATION > 0: + l.items = malloc(LIST.items.TO, INITIAL_EMPTY_LIST_ALLOCATION) + else: + l.items = _ll_prebuilt_empty_array(LIST.items.TO) + return l +ll_newemptylist = typeMethod(ll_newemptylist) +ll_newemptylist.oopspec = 'newlist(0)' + def ll_length(l): return l.length @@ -385,6 +407,10 @@ ll_fixed_newlist = typeMethod(ll_fixed_newlist) ll_fixed_newlist.oopspec = 'newlist(length)' +def ll_fixed_newemptylist(LIST): + return ll_fixed_newlist(LIST, 0) +ll_fixed_newemptylist = typeMethod(ll_fixed_newemptylist) + def ll_fixed_length(l): return len(l) @@ -401,8 +427,11 @@ def newlist(llops, r_list, items_v): LIST = r_list.LIST - cno = inputconst(Signed, len(items_v)) - v_result = llops.gendirectcall(LIST.ll_newlist, cno) + if len(items_v) == 0: + v_result = llops.gendirectcall(LIST.ll_newemptylist) + else: + cno = inputconst(Signed, len(items_v)) + v_result = llops.gendirectcall(LIST.ll_newlist, cno) v_func = inputconst(Void, dum_nocheck) for i, v_item in enumerate(items_v): ci = inputconst(Signed, i) From arigo at codespeak.net Sun Dec 2 13:29:06 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 13:29:06 +0100 (CET) Subject: [pypy-svn] r49272 - in pypy/dist: lib-python/modified-2.4.1 pypy/interpreter/test Message-ID: <20071202122906.5526B80C9@code0.codespeak.net> Author: arigo Date: Sun Dec 2 13:29:04 2007 New Revision: 49272 Added: pypy/dist/lib-python/modified-2.4.1/inspect.py - copied, changed from r49241, pypy/dist/lib-python/2.4.1/inspect.py Modified: pypy/dist/pypy/interpreter/test/test_code.py Log: issue306 in-progress Hacking at inspect.py to let getargs() decode the signature of our internal-code' objects too. Modified: pypy/dist/pypy/interpreter/test/test_code.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_code.py (original) +++ pypy/dist/pypy/interpreter/test/test_code.py Sun Dec 2 13:29:04 2007 @@ -126,3 +126,12 @@ exec "def f(): pass" in d2 assert d1['f'].func_code == d2['f'].func_code assert hash(d1['f'].func_code) == hash(d2['f'].func_code) + + def test_inspect(self): + if not hasattr(len, 'func_code'): + skip("CPython: no func_code attribute on built-in functions") + import inspect + args, varargs, varkw = inspect.getargs(len.func_code) + assert args == ['obj'] + assert varargs is None + assert varkw is None From arigo at codespeak.net Sun Dec 2 13:39:08 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 13:39:08 +0100 (CET) Subject: [pypy-svn] r49274 - pypy/dist/lib-python/modified-2.4.1 Message-ID: <20071202123908.104CF815C@code0.codespeak.net> Author: arigo Date: Sun Dec 2 13:39:07 2007 New Revision: 49274 Added: pypy/dist/lib-python/modified-2.4.1/pydoc.py - copied, changed from r49241, pypy/dist/lib-python/2.4.1/pydoc.py Modified: pypy/dist/lib-python/modified-2.4.1/inspect.py Log: Fix help(builtin-function) to show the signature on top of PyPy. Modified: pypy/dist/lib-python/modified-2.4.1/inspect.py ============================================================================== --- pypy/dist/lib-python/modified-2.4.1/inspect.py (original) +++ pypy/dist/lib-python/modified-2.4.1/inspect.py Sun Dec 2 13:39:07 2007 @@ -679,7 +679,9 @@ if ismethod(func): func = func.im_func - if not isfunction(func): + if not (isfunction(func) or + isbuiltin(func) and hasattr(func, 'func_code')): + # PyPy extension: this works for built-in functions too raise TypeError('arg is not a Python function') args, varargs, varkw = getargs(func.func_code) return args, varargs, varkw, func.func_defaults From arigo at codespeak.net Sun Dec 2 13:44:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 13:44:24 +0100 (CET) Subject: [pypy-svn] r49276 - pypy/dist/pypy/interpreter Message-ID: <20071202124424.5695F815C@code0.codespeak.net> Author: arigo Date: Sun Dec 2 13:44:24 2007 New Revision: 49276 Modified: pypy/dist/pypy/interpreter/typedef.py Log: Tweak. Modified: pypy/dist/pypy/interpreter/typedef.py ============================================================================== --- pypy/dist/pypy/interpreter/typedef.py (original) +++ pypy/dist/pypy/interpreter/typedef.py Sun Dec 2 13:44:24 2007 @@ -287,7 +287,8 @@ return miniglobals['descr_typecheck_%s' % func.__name__] def unknown_objclass_getter(space): - raise OperationError(space.w_TypeError, + # NB. this is an AttributeError to make inspect.py happy + raise OperationError(space.w_AttributeError, space.wrap("generic property has no __objclass__")) def make_objclass_getter(func, cls, cache={}): From arigo at codespeak.net Sun Dec 2 15:51:34 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 15:51:34 +0100 (CET) Subject: [pypy-svn] r49280 - pypy/dist/pypy/interpreter Message-ID: <20071202145134.0742B80F9@code0.codespeak.net> Author: arigo Date: Sun Dec 2 15:51:34 2007 New Revision: 49280 Modified: pypy/dist/pypy/interpreter/executioncontext.py Log: ExecutionContext instances should not be seen during translation, but they occasionally do (there is a hard-to-reproduce order dependency). This change might help us locate where they come from. Modified: pypy/dist/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/dist/pypy/interpreter/executioncontext.py (original) +++ pypy/dist/pypy/interpreter/executioncontext.py Sun Dec 2 15:51:34 2007 @@ -258,3 +258,8 @@ def add_pending_action(self, action): self.pending_actions.append(action) self.ticker = 0 + + def _freeze_(self): + raise Exception("ExecutionContext instances should not be seen during" + " translation. Now is a good time to inspect the" + " traceback and see where this one comes from :-)") From arigo at codespeak.net Sun Dec 2 16:53:06 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 16:53:06 +0100 (CET) Subject: [pypy-svn] r49286 - in pypy/dist/pypy/module: _file marshal marshal/test Message-ID: <20071202155306.F28568138@code0.codespeak.net> Author: arigo Date: Sun Dec 2 16:53:05 2007 New Revision: 49286 Modified: pypy/dist/pypy/module/_file/interp_file.py pypy/dist/pypy/module/marshal/interp_marshal.py pypy/dist/pypy/module/marshal/test/test_marshal.py Log: A performance hack for interp-level code that manipulates app-level file objects. It gives direct access to the underlying stream, shortcutting calls that need to go through app-level (which performs various checks and locking). For marshal, for example, a single lock/unlock pair around the whole operation is enough. Modified: pypy/dist/pypy/module/_file/interp_file.py ============================================================================== --- pypy/dist/pypy/module/_file/interp_file.py (original) +++ pypy/dist/pypy/module/_file/interp_file.py Sun Dec 2 16:53:05 2007 @@ -87,6 +87,36 @@ assert self.slockowner is None return False + def do_read(self, n): + """ + An interface for direct interp-level usage of W_Stream, + e.g. from interp_marshal.py. + NOTE: this assumes that the stream lock is already acquired. + Like os.read(), this can return less than n bytes. + """ + try: + return self.stream.read(n) + except streamio.StreamError, e: + raise OperationError(space.w_ValueError, + space.wrap(e.message)) + except OSError, e: + raise wrap_oserror_as_ioerror(space, e) + + def do_write(self, data): + """ + An interface for direct interp-level usage of W_Stream, + e.g. from interp_marshal.py. + NOTE: this assumes that the stream lock is already acquired. + """ + try: + self.stream.write(data) + except streamio.StreamError, e: + raise OperationError(space.w_ValueError, + space.wrap(e.message)) + except OSError, e: + raise wrap_oserror_as_ioerror(space, e) + + for name, argtypes in streamio.STREAM_METHODS.iteritems(): numargs = len(argtypes) args = ", ".join(["v%s" % i for i in range(numargs)]) @@ -136,3 +166,20 @@ space, streamio.fdopen_as_stream(fd, mode, buffering))) fdopen_as_stream.unwrap_spec = [ObjSpace, int, str, int] + +def file2stream(space, w_f): + """A hack for direct interp-level access to W_Stream objects, + for better performance e.g. when marshalling directly from/to a + real file object. This peels off the app-level layers of the file class + defined in app_file.py. It complains if the file is already closed. + """ + w_stream = space.findattr(w_f, space.wrap('stream')) + if w_stream is None: + return None + w_stream = space.interpclass_w(w_stream) + if not isinstance(w_stream, W_Stream): + return None + if space.is_true(space.getattr(w_f, space.wrap('_closed'))): + raise OperationError(space.w_ValueError, + space.wrap('I/O operation on closed file')) + return w_stream Modified: pypy/dist/pypy/module/marshal/interp_marshal.py ============================================================================== --- pypy/dist/pypy/module/marshal/interp_marshal.py (original) +++ pypy/dist/pypy/module/marshal/interp_marshal.py Sun Dec 2 16:53:05 2007 @@ -1,6 +1,7 @@ from pypy.interpreter.baseobjspace import ObjSpace from pypy.interpreter.error import OperationError from pypy.rlib.rarithmetic import intmask +from pypy.module._file.interp_file import file2stream import sys # Py_MARSHAL_VERSION = 2 @@ -14,12 +15,19 @@ def dump(space, w_data, w_f, w_version=Py_MARSHAL_VERSION): """Write the 'data' object into the open file 'f'.""" - writer = FileWriter(space, w_f) - # note: bound methods are currently not supported, - # so we have to pass the instance in, instead. - ##m = Marshaller(space, writer.write, space.int_w(w_version)) - m = Marshaller(space, writer, space.int_w(w_version)) - m.put_w_obj(w_data) + w_stream = file2stream(space, w_f) + if w_stream is not None: + writer = StreamWriter(space, w_stream) + else: + writer = FileWriter(space, w_f) + try: + # note: bound methods are currently not supported, + # so we have to pass the instance in, instead. + ##m = Marshaller(space, writer.write, space.int_w(w_version)) + m = Marshaller(space, writer, space.int_w(w_version)) + m.put_w_obj(w_data) + finally: + writer.finished() def dumps(space, w_data, w_version=Py_MARSHAL_VERSION): """Return the string that would have been written to a file @@ -30,9 +38,17 @@ def load(space, w_f): """Read one value from the file 'f' and return it.""" - reader = FileReader(space, w_f) - u = Unmarshaller(space, reader) - return u.get_w_obj(False) + # special case real files for performance + w_stream = file2stream(space, w_f) + if w_stream is not None: + reader = StreamReader(space, w_stream) + else: + reader = FileReader(space, w_f) + try: + u = Unmarshaller(space, reader) + return u.get_w_obj(False) + finally: + reader.finished() def loads(space, w_str): """Convert a string back to a value. Extra characters in the string are @@ -41,9 +57,22 @@ return u.get_w_obj(False) -class FileWriter(object): - def __init__(self, space, w_f): +class AbstractReaderWriter(object): + def __init__(self, space): self.space = space + + def raise_eof(self): + space = self.space + raise OperationError(space.w_EOFError, space.wrap( + 'EOF read where object expected')) + + def finished(self): + pass + + +class FileWriter(AbstractReaderWriter): + def __init__(self, space, w_f): + AbstractReaderWriter.__init__(self, space) try: self.func = space.getattr(w_f, space.wrap('write')) # XXX how to check if it is callable? @@ -53,19 +82,14 @@ raise OperationError(space.w_TypeError, space.wrap( 'marshal.dump() 2nd arg must be file-like object')) - def raise_eof(self): - space = self.space - raise OperationError(space.w_EOFError, space.wrap( - 'EOF read where object expected')) - def write(self, data): space = self.space space.call_function(self.func, space.wrap(data)) -class FileReader(object): +class FileReader(AbstractReaderWriter): def __init__(self, space, w_f): - self.space = space + AbstractReaderWriter.__init__(self, space) try: self.func = space.getattr(w_f, space.wrap('read')) # XXX how to check if it is callable? @@ -83,10 +107,29 @@ self.raise_eof() return ret - def raise_eof(self): - space = self.space - raise OperationError(space.w_EOFError, space.wrap( - 'EOF read where object expected')) + +class StreamReaderWriter(AbstractReaderWriter): + def __init__(self, space, w_stream): + AbstractReaderWriter.__init__(self, space) + self.w_stream = w_stream + w_stream.descr_lock() + + def finished(self): + self.w_stream.descr_unlock() + +class StreamWriter(StreamReaderWriter): + def write(self, data): + self.w_stream.do_write(data) + +class StreamReader(StreamReaderWriter): + def read(self, n): + result = data = self.w_stream.do_read(n) + while len(result) < n: + if len(data) == 0: + self.raise_eof() + data = self.w_stream.do_read(n) + result += data + return result MAX_MARSHAL_DEPTH = 5000 Modified: pypy/dist/pypy/module/marshal/test/test_marshal.py ============================================================================== --- pypy/dist/pypy/module/marshal/test/test_marshal.py (original) +++ pypy/dist/pypy/module/marshal/test/test_marshal.py Sun Dec 2 16:53:05 2007 @@ -1,5 +1,11 @@ +from pypy.tool.udir import udir + class AppTestMarshal: + def setup_class(cls): + tmpfile = udir.join('AppTestMarshal.tmp') + cls.w_tmpfile = cls.space.wrap(str(tmpfile)) + def test_None(self): import sys hello = "he" @@ -589,6 +595,26 @@ x = marshal.load(f) assert x == case and type(x) is type(case) + def test_stream_reader_writer(self): + # for performance, we have a special case when reading/writing real + # file objects + import marshal + obj1 = [4, ("hello", 7.5)] + obj2 = "foobar" + f = open(self.tmpfile, 'wb') + marshal.dump(obj1, f) + marshal.dump(obj2, f) + f.write('END') + f.close() + f = open(self.tmpfile, 'rb') + obj1b = marshal.load(f) + obj2b = marshal.load(f) + tail = f.read() + f.close() + assert obj1b == obj1 + assert obj2b == obj2 + assert tail == 'END' + class AppTestMultiDict(object): def setup_class(cls): @@ -602,3 +628,4 @@ def setup_class(cls): from pypy.conftest import gettestobjspace cls.space = gettestobjspace(**{"objspace.std.withrope": True}) + AppTestMarshal.setup_class.im_func(cls) From arigo at codespeak.net Sun Dec 2 17:18:15 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 2 Dec 2007 17:18:15 +0100 (CET) Subject: [pypy-svn] r49287 - pypy/dist/pypy/module/_file Message-ID: <20071202161815.B86BE8171@code0.codespeak.net> Author: arigo Date: Sun Dec 2 17:18:13 2007 New Revision: 49287 Modified: pypy/dist/pypy/module/_file/interp_file.py Log: Typos. Modified: pypy/dist/pypy/module/_file/interp_file.py ============================================================================== --- pypy/dist/pypy/module/_file/interp_file.py (original) +++ pypy/dist/pypy/module/_file/interp_file.py Sun Dec 2 17:18:13 2007 @@ -97,10 +97,10 @@ try: return self.stream.read(n) except streamio.StreamError, e: - raise OperationError(space.w_ValueError, - space.wrap(e.message)) + raise OperationError(self.space.w_ValueError, + self.space.wrap(e.message)) except OSError, e: - raise wrap_oserror_as_ioerror(space, e) + raise wrap_oserror_as_ioerror(self.space, e) def do_write(self, data): """ @@ -111,10 +111,10 @@ try: self.stream.write(data) except streamio.StreamError, e: - raise OperationError(space.w_ValueError, - space.wrap(e.message)) + raise OperationError(self.space.w_ValueError, + self.space.wrap(e.message)) except OSError, e: - raise wrap_oserror_as_ioerror(space, e) + raise wrap_oserror_as_ioerror(self.space, e) for name, argtypes in streamio.STREAM_METHODS.iteritems(): From cfbolz at codespeak.net Sun Dec 2 19:20:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 2 Dec 2007 19:20:23 +0100 (CET) Subject: [pypy-svn] r49292 - pypy/dist/pypy/rlib/test Message-ID: <20071202182023.71BD88140@code0.codespeak.net> Author: cfbolz Date: Sun Dec 2 19:20:21 2007 New Revision: 49292 Modified: pypy/dist/pypy/rlib/test/test_rope.py Log: this should really be indented one more level Modified: pypy/dist/pypy/rlib/test/test_rope.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rope.py (original) +++ pypy/dist/pypy/rlib/test/test_rope.py Sun Dec 2 19:20:21 2007 @@ -400,7 +400,7 @@ for c in result[i + j:]: c2 = iter.nextchar() assert c2 == c - py.test.raises(StopIteration, iter.nextchar) + py.test.raises(StopIteration, iter.nextchar) def test_find_int(): rope, st = make_random_string() From cfbolz at codespeak.net Sun Dec 2 20:59:45 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 2 Dec 2007 20:59:45 +0100 (CET) Subject: [pypy-svn] r49294 - pypy/dist/pypy/rlib Message-ID: <20071202195945.407EF812C@code0.codespeak.net> Author: cfbolz Date: Sun Dec 2 20:59:44 2007 New Revision: 49294 Modified: pypy/dist/pypy/rlib/rope.py Log: off by one error Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Sun Dec 2 20:59:44 2007 @@ -484,7 +484,7 @@ twopower = node number = 1 result = None - while number < times: + while number <= times: if number & times: if result is None: result = twopower From cfbolz at codespeak.net Sun Dec 2 21:02:13 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 2 Dec 2007 21:02:13 +0100 (CET) Subject: [pypy-svn] r49295 - pypy/dist/pypy/rlib/parsing/test Message-ID: <20071202200213.4B0328150@code0.codespeak.net> Author: cfbolz Date: Sun Dec 2 21:02:12 2007 New Revision: 49295 Modified: pypy/dist/pypy/rlib/parsing/test/test_pypackrat.py Log: just for the fun of it: write a future parser with the python packrat parser generator. it's not really any more readable than the one we now use, actually. Modified: pypy/dist/pypy/rlib/parsing/test/test_pypackrat.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/test/test_pypackrat.py (original) +++ pypy/dist/pypy/rlib/parsing/test/test_pypackrat.py Sun Dec 2 21:02:12 2007 @@ -472,3 +472,80 @@ assert expected == ['a', 'x', 'y'] + def test_python_future(self): + class parser(PackratParser): + r""" + comment: + `#[^\r\n]*` lineend; + lineend: + `(\r|\n)+`; + docstring: + `(\"\"\"[^\\]*(\\[^\\]+)*\"\"\")|(\'\'\'[^\\]*(\\[^\\]+)*\'\'\')` + ignore* + | `(\"[^\\]*(\\[^\\]+)*\")|(\'[^\\]*(\\[^\\]+)*\')` + ignore*; + ignore: + `[ \t]+`; + ignoreline: + `[ \t]*[\r\n]+`; + fromimport: + 'from' ignore+ + '__future__' ignore+ + 'import' ignore+ + what; + identifier: + `[a-zA-Z0-9_]+`; + what: + '(' ignoreline* + g = group + ignoreline* + rest = ([',' ignoreline*] group)* + ')' + return {[g] + rest} + | g = group + rest = ([',' ignore*] group)* + return {[g] + rest}; + group: + name = identifier ignore+ 'as' ignore+ identifier ignore* + return {name} + | name = identifier ignore* + return {name}; + line: + comment + return {None} + | docstring lineend + return {None} + | ignore lineend + return {None} + | t = fromimport + ignore* + lineend + return {t}; + header: + l = line* + return {[elt for sublist in l if sublist is not None for elt in sublist]}; + """ + p = parser("#\n") + lines = p.header() + assert lines == [] + p = parser('''"abc"\n''') + lines = p.header() + assert lines == [] + p = parser(''''abc'\n''') + lines = p.header() + assert lines == [] + p = parser(''''abc'\n''') + lines = p.header() + assert lines == [] + p = parser('''from __future__ import division\n''') + lines = p.header() + assert lines == ['division'] + p = parser('''from __future__ import division, generators\n''') + lines = p.fromimport() + assert lines == ['division', 'generators'] + p = parser('''from __future__ import (division, \ngenerators)\n''') + lines = p.fromimport() + assert lines == ['division', 'generators'] + p = parser('''from __future__ import (division as d, \ngenerators)\n''') + lines = p.fromimport() + assert lines == ['division', 'generators'] From cfbolz at codespeak.net Sun Dec 2 21:05:09 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 2 Dec 2007 21:05:09 +0100 (CET) Subject: [pypy-svn] r49298 - in pypy/dist/pypy/rlib: . test Message-ID: <20071202200509.1F3FC8169@code0.codespeak.net> Author: cfbolz Date: Sun Dec 2 21:05:08 2007 New Revision: 49298 Modified: pypy/dist/pypy/rlib/rope.py pypy/dist/pypy/rlib/test/test_rope.py Log: completely rewrite the seekable rope iterator to be conceptually more right: seeking now is O(log(distance)), not O(distance). Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Sun Dec 2 21:05:08 2007 @@ -970,48 +970,64 @@ class SeekableItemIterator(object): def __init__(self, node): - self.iter = SeekableFringeIterator(node) - self.node = self.nextnode() - self.nodelength = self.node.length() - self.index = 0 + self.stack = [] + self.tookleft = [] + self.find_downward(node) + assert False not in self.tookleft + + def find_downward(self, node, items=0): + assert 0 <= items < node.length() + while isinstance(node, BinaryConcatNode): + self.stack.append(node) + right = node.right + left = node.left + if items >= left.length(): + items -= left.length() + node = node.right + self.tookleft.append(False) + else: + node = node.left + self.tookleft.append(True) + assert len(self.tookleft) == len(self.stack) + self.node = node + self.nodelength = node.length() + self.index = items + return self.node def nextnode(self): - while 1: - node = self.node = self.iter.next() - nodelength = self.nodelength = node.length() - if nodelength != 0: - break - self.index = 0 - return node - + below = self.node + while self.stack: + tookleft = self.tookleft.pop() + if tookleft: + node = self.stack[-1] + assert isinstance(node, BinaryConcatNode) + self.tookleft.append(False) + self.find_downward(node.right) + return self.node + self.stack.pop() + raise StopIteration + + def getnode(self): + if self.index == self.node.length(): + return self.nextnode() + return self.node - def advance_index(self): - if self.index == self.nodelength - 1: - self.node = None - self.index += 1 - def nextchar(self): - node = self.node - if node is None: - node = self.nextnode() - result = self.node.getchar(self.index) - self.advance_index() + node = self.getnode() + result = node.getchar(self.index) + self.index += 1 return result def nextunichar(self): - node = self.node - if node is None: - node = self.nextnode() - result = self.node.getunichar(self.index) - self.advance_index() + node = self.getnode() + result = node.getunichar(self.index) + self.index += 1 return result def nextint(self): - node = self.node - if node is None: - node = self.nextnode() - result = self.node.getint(self.index) - self.advance_index() + node = self.getnode() + result = node.getint(self.index) + self.index += 1 return result def seekforward(self, numchars): @@ -1019,36 +1035,42 @@ self.index += numchars return numchars -= self.nodelength - self.index - while 1: - node = self.iter.next() - length = node.length() - if length <= numchars: - numchars -= length - else: - self.index = numchars - self.node = node - self.nodelength = node.length() - return + while self.stack: + tookleft = self.tookleft.pop() + if tookleft: + node = self.stack[-1] + assert isinstance(node, BinaryConcatNode) + right = node.right + if right.length() > numchars: + break + numchars -= right.length() + self.stack.pop() + else: + raise StopIteration + self.tookleft.append(False) + self.find_downward(right, numchars) + def seekback(self, numchars): if numchars <= self.index: self.index -= numchars - if self.node is None: - self.iter.seekback() - self.node = self.iter.next() return numchars -= self.index - self.iter.seekback() # for first item - while 1: - node = self.iter.seekback() - length = node.length() - if length < numchars: - numchars -= length - else: - self.index = length - numchars - self.node = self.iter.next() - self.nodelength = self.node.length() - return + while self.stack: + tookleft = self.tookleft.pop() + if not tookleft: + node = self.stack[-1] + assert isinstance(node, BinaryConcatNode) + left = node.left + if left.length() >= numchars: + break + numchars -= left.length() + self.stack.pop() + else: + raise StopIteration + self.tookleft.append(True) + self.find_downward(left, left.length() - numchars) + class FindIterator(object): def __init__(self, node, sub, start=0, stop=-1): Modified: pypy/dist/pypy/rlib/test/test_rope.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rope.py (original) +++ pypy/dist/pypy/rlib/test/test_rope.py Sun Dec 2 21:05:08 2007 @@ -391,8 +391,6 @@ for j in range(len(result) - 1): for i in range(len(result) - 1 - j): iter = SeekableItemIterator(rope) -# if (j, i) == (3, 1): -# import pdb; pdb.set_trace() for c in result[:j]: c2 = iter.nextchar() assert c2 == c From cfbolz at codespeak.net Sun Dec 2 21:22:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 2 Dec 2007 21:22:23 +0100 (CET) Subject: [pypy-svn] r49300 - pypy/dist/pypy/rlib Message-ID: <20071202202223.D99AC8187@code0.codespeak.net> Author: cfbolz Date: Sun Dec 2 21:22:21 2007 New Revision: 49300 Modified: pypy/dist/pypy/rlib/rope.py Log: small beautifications Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Sun Dec 2 21:22:21 2007 @@ -1042,13 +1042,12 @@ assert isinstance(node, BinaryConcatNode) right = node.right if right.length() > numchars: - break + self.tookleft.append(False) + self.find_downward(right, numchars) + return numchars -= right.length() self.stack.pop() - else: - raise StopIteration - self.tookleft.append(False) - self.find_downward(right, numchars) + raise StopIteration def seekback(self, numchars): @@ -1063,13 +1062,12 @@ assert isinstance(node, BinaryConcatNode) left = node.left if left.length() >= numchars: - break + self.tookleft.append(True) + self.find_downward(left, left.length() - numchars) + return numchars -= left.length() self.stack.pop() - else: - raise StopIteration - self.tookleft.append(True) - self.find_downward(left, left.length() - numchars) + raise StopIteration class FindIterator(object): From cfbolz at codespeak.net Sun Dec 2 21:37:50 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 2 Dec 2007 21:37:50 +0100 (CET) Subject: [pypy-svn] r49303 - pypy/dist/pypy/translator/microbench Message-ID: <20071202203750.8AF48817D@code0.codespeak.net> Author: cfbolz Date: Sun Dec 2 21:37:50 2007 New Revision: 49303 Added: pypy/dist/pypy/translator/microbench/test_unicode.py (contents, props changed) Log: a very simple unicode finding benchmark Added: pypy/dist/pypy/translator/microbench/test_unicode.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/microbench/test_unicode.py Sun Dec 2 21:37:50 2007 @@ -0,0 +1,5 @@ +N = (2 ** 19 - 1) + +u1 = (u"not the xyz" * N) +def test_find_worstcase(): + u1.find(u"not there") From cfbolz at codespeak.net Mon Dec 3 00:09:49 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 3 Dec 2007 00:09:49 +0100 (CET) Subject: [pypy-svn] r49305 - in pypy/dist/pypy/rlib: . test Message-ID: <20071202230949.B0EA3817D@code0.codespeak.net> Author: cfbolz Date: Mon Dec 3 00:09:49 2007 New Revision: 49305 Modified: pypy/dist/pypy/rlib/rope.py pypy/dist/pypy/rlib/test/test_rope.py Log: don't search when the pattern string is longer than the search string Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Mon Dec 3 00:09:49 2007 @@ -698,6 +698,8 @@ if (stop - start) < 0: return -1 return start + if len2 >= stop - start: + return -1 restart = construct_restart_positions_node(subnode) return _find_node(node, subnode, start, stop, restart) Modified: pypy/dist/pypy/rlib/test/test_rope.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rope.py (original) +++ pypy/dist/pypy/rlib/test/test_rope.py Mon Dec 3 00:09:49 2007 @@ -437,6 +437,8 @@ LiteralStringNode("bacbb")) pos = find(node, LiteralStringNode("a"), 0, node.length()) assert pos == 6 + pos = find(node, LiteralStringNode("aaa"), 0, 2) + assert pos == -1 def test_find_unicode(): From cfbolz at codespeak.net Mon Dec 3 00:18:44 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 3 Dec 2007 00:18:44 +0100 (CET) Subject: [pypy-svn] r49306 - in pypy/dist/pypy/rlib: . test Message-ID: <20071202231844.335D18188@code0.codespeak.net> Author: cfbolz Date: Mon Dec 3 00:18:42 2007 New Revision: 49306 Modified: pypy/dist/pypy/rlib/rope.py pypy/dist/pypy/rlib/test/test_rope.py Log: an off-by-one error. also optimize the FindIterator for the case where the pattern is longer than the bit of the text string where I am searching in. Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Mon Dec 3 00:18:42 2007 @@ -698,7 +698,7 @@ if (stop - start) < 0: return -1 return start - if len2 >= stop - start: + if len2 > stop - start: return -1 restart = construct_restart_positions_node(subnode) return _find_node(node, subnode, start, stop, restart) @@ -1079,16 +1079,20 @@ len1 = self.length = node.length() len2 = sub.length() self.search_length = len2 + self.start = start + if stop == -1 or stop > len1: + stop = len1 + self.stop = stop if len2 == 0: self.restart_positions = None elif len2 == 1: self.restart_positions = None + elif len2 > stop - start: + self.restart_positions = None + # ensure that a StopIteration is immediately raised + self.stop = self.start else: self.restart_positions = construct_restart_positions_node(sub) - self.start = start - if stop == -1 or stop > len1: - stop = len1 - self.stop = stop def next(self): if self.search_length == 0: Modified: pypy/dist/pypy/rlib/test/test_rope.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rope.py (original) +++ pypy/dist/pypy/rlib/test/test_rope.py Mon Dec 3 00:18:42 2007 @@ -439,6 +439,8 @@ assert pos == 6 pos = find(node, LiteralStringNode("aaa"), 0, 2) assert pos == -1 + pos = find(node, LiteralStringNode("btf"), 0, 3) + assert pos == 0 def test_find_unicode(): From niko at codespeak.net Mon Dec 3 10:33:11 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Mon, 3 Dec 2007 10:33:11 +0100 (CET) Subject: [pypy-svn] r49308 - in pypy/dist/pypy/translator/jvm: src/pypy test Message-ID: <20071203093311.F2DEC8195@code0.codespeak.net> Author: niko Date: Mon Dec 3 10:33:10 2007 New Revision: 49308 Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java pypy/dist/pypy/translator/jvm/test/test_builtin.py Log: implement unlink, mkdir, frexp, and add a test or two Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Mon Dec 3 10:33:10 2007 @@ -978,11 +978,48 @@ return x % y; } - /* TODO - public double ll_math_frexp(double x) { - } - */ + public Object ll_math_frexp(double x) { + /* + Return the mantissa and exponent of x as the pair (m, e). m + is a float and e is an integer such that x == m * 2**e + exactly. If x is zero, returns (0.0, 0), otherwise 0.5 <= + abs(m) < 1. This is used to "pick apart" the internal + representation of a float in a portable way. + */ + + // NaN: Python returns (NaN, 0) + if (Double.isNaN(x)) + return interlink.recordFloatSigned(x, 0); + + // Infinity: Python throws exception + if (Double.isInfinite(x)) + interlink.throwOverflowError(); + // Extract the various parts of the format: + final long e=11, f=52; // number of bits in IEEE format + long bits = Double.doubleToLongBits(x); + long bits_mantissa = bits & ((1 << f) - 1); + int bits_exponent = (int)((bits >> f) & ((1 << e) - 1)); + int bits_sign = (int)(bits >> (e+f)); + + // [+-]0 + if (bits_exponent == 0 && bits_mantissa == 0) + return interlink.recordFloatSigned(x, 0); + + // TODO: Non-looping impl + double mantissa = x; + int exponent = 0; + while (mantissa > 1) { + mantissa /= 2; + exponent += 1; + } + while (mantissa < 0.5) { + mantissa *= 2; + exponent -= 1; + } + return interlink.recordFloatSigned(mantissa, exponent); + } + public double ll_math_ldexp(double v, int w) { return check(v * Math.pow(2.0, w)); } @@ -996,8 +1033,8 @@ return Math.exp(x); } - public double ll_math_log(double x, double base) { - return Math.log10(x) / Math.log10(base); + public double ll_math_log(double x) { + return Math.log(x); } public double ll_math_log10(double v) { Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Mon Dec 3 10:33:10 2007 @@ -357,6 +357,29 @@ return text.length(); } + public void ll_os_mkdir(String path, int mode) { + File f = new File(path); + if (f.exists()) + throwOSError(PyPy.EEXIST, "File exists: '"+path+"'"); + if (!f.mkdir()) + throwOSError(PyPy.EPERM, "Operation not permitted: '"+path+"'"); + } + + public void ll_os_unlink(String path) { + if (STRACE) strace("ll_os_unlink: "+path); + + File f = new File(path); + + if (!f.exists()) + throwOSError(PyPy.ENOENT, "No such file or directory: '"+path+"'"); + + if (f.isDirectory()) + throwOSError(PyPy.EPERM, "Operation not permitted: '"+path+"'"); + + if (!f.delete()) + throwOSError(PyPy.EPERM, "Operation not permitted: '"+path+"'"); + } + public boolean ll_os_isatty(int x) { // XXX: this is not the right behaviour, but it's needed Modified: pypy/dist/pypy/translator/jvm/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_builtin.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_builtin.py Mon Dec 3 10:33:10 2007 @@ -14,12 +14,6 @@ skip_win() BaseTestBuiltin.test_os_write_magic(self) - def test_builtin_math_frexp(self): - py.test.skip("metavm.py needs to be updated to handle this math op; graphless extrernal") - - def test_builtin_math_modf(self): - py.test.skip("metavm.py needs to be updated to handle this math op; graphless extrernal") - def test_os_path_exists(self): py.test.skip("fails in annotation stage, unrelated to JVM I think") From niko at codespeak.net Mon Dec 3 10:41:32 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Mon, 3 Dec 2007 10:41:32 +0100 (CET) Subject: [pypy-svn] r49309 - in pypy/dist/pypy: rpython/test translator/jvm/src/pypy translator/oosupport/test_template Message-ID: <20071203094132.89F848194@code0.codespeak.net> Author: niko Date: Mon Dec 3 10:41:32 2007 New Revision: 49309 Modified: pypy/dist/pypy/rpython/test/test_rbuiltin.py pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java pypy/dist/pypy/translator/oosupport/test_template/builtin.py Log: fix a bug in our frexp for negative numbers, and expand the rpython test to include a wider range of testing values Modified: pypy/dist/pypy/rpython/test/test_rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rbuiltin.py (original) +++ pypy/dist/pypy/rpython/test/test_rbuiltin.py Mon Dec 3 10:41:32 2007 @@ -157,9 +157,12 @@ import math def fn(f): return math.frexp(f) - res = self.interpret(fn, [10/3.0]) - mantissa, exponent = math.frexp(10/3.0) - assert self.float_eq(res.item0, mantissa) and self.float_eq(res.item1, exponent) + for x in (.5, 1, 1.5, 10/3.0): + for y in (1, -1): + res = self.interpret(fn, [x*y]) + mantissa, exponent = math.frexp(x*y) + assert (self.float_eq(res.item0, mantissa) and + self.float_eq(res.item1, exponent)) def test_builtin_math_ldexp(self): import math @@ -289,8 +292,8 @@ return os.path.exists(fn) filename = self.string_to_ll(str(py.magic.autopath())) assert self.interpret(f, [filename]) == True - assert self.interpret(f, [ - self.string_to_ll("strange_filename_that_looks_improbable.sde")]) == False + #assert self.interpret(f, [ + # self.string_to_ll("strange_filename_that_looks_improbable.sde")]) == False def test_os_isdir(self): self._skip_llinterpreter("os.stat()") Modified: pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/PyPy.java Mon Dec 3 10:41:32 2007 @@ -1007,9 +1007,9 @@ return interlink.recordFloatSigned(x, 0); // TODO: Non-looping impl - double mantissa = x; + double mantissa = Math.abs(x); int exponent = 0; - while (mantissa > 1) { + while (mantissa >= 1.0) { mantissa /= 2; exponent += 1; } @@ -1017,6 +1017,7 @@ mantissa *= 2; exponent -= 1; } + mantissa = (x < 0 ? -mantissa : mantissa); return interlink.recordFloatSigned(mantissa, exponent); } Modified: pypy/dist/pypy/translator/oosupport/test_template/builtin.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/test_template/builtin.py (original) +++ pypy/dist/pypy/translator/oosupport/test_template/builtin.py Mon Dec 3 10:41:32 2007 @@ -148,7 +148,6 @@ assert act_res.item0 == exp_res[0] assert act_res.item1 == exp_res[1] - class BaseTestTime(llBaseTestTime): def test_time_clock(self): From fijal at codespeak.net Mon Dec 3 17:36:42 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 3 Dec 2007 17:36:42 +0100 (CET) Subject: [pypy-svn] r49316 - pypy/dist/pypy/lib/ctypes Message-ID: <20071203163642.19AB78155@code0.codespeak.net> Author: fijal Date: Mon Dec 3 17:36:40 2007 New Revision: 49316 Modified: pypy/dist/pypy/lib/ctypes/ (props changed) Log: fixeol From fijal at codespeak.net Mon Dec 3 17:38:54 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 3 Dec 2007 17:38:54 +0100 (CET) Subject: [pypy-svn] r49317 - in pypy/dist/pypy/lib: app_test test2 Message-ID: <20071203163854.698DC8155@code0.codespeak.net> Author: fijal Date: Mon Dec 3 17:38:54 2007 New Revision: 49317 Removed: pypy/dist/pypy/lib/test2/test_binascii.py Modified: pypy/dist/pypy/lib/app_test/test_binascii.py Log: Move test away from using AppTest Modified: pypy/dist/pypy/lib/app_test/test_binascii.py ============================================================================== --- pypy/dist/pypy/lib/app_test/test_binascii.py (original) +++ pypy/dist/pypy/lib/app_test/test_binascii.py Mon Dec 3 17:38:54 2007 @@ -146,3 +146,7 @@ f = getattr(binascii, n) f('') binascii.crc_hqx('', 0) + +def test_incorrect_padding(): + import binascii + raises(binascii.Error, "'x'.decode('base64')") From cfbolz at codespeak.net Mon Dec 3 17:43:15 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 3 Dec 2007 17:43:15 +0100 (CET) Subject: [pypy-svn] r49318 - pypy/dist/pypy/translator/microbench Message-ID: <20071203164315.71D158155@code0.codespeak.net> Author: cfbolz Date: Mon Dec 3 17:43:14 2007 New Revision: 49318 Modified: pypy/dist/pypy/translator/microbench/test_unicode.py Log: some more find microbenches. python2.4 is marginally faster than pypy here and python2.5 is tons faster. we should steal their algorithm: http://effbot.org/zone/stringlib.htm Modified: pypy/dist/pypy/translator/microbench/test_unicode.py ============================================================================== --- pypy/dist/pypy/translator/microbench/test_unicode.py (original) +++ pypy/dist/pypy/translator/microbench/test_unicode.py Mon Dec 3 17:43:14 2007 @@ -3,3 +3,38 @@ u1 = (u"not the xyz" * N) def test_find_worstcase(): u1.find(u"not there") + +def test_count_worstcase(): + u1.count(u"not there") + +u2 = (u"aaa" * 1000) +def test_find_pattern16(): + i = 1 + while i < N: + i += 1 + u2.find(u"bbbbbbbbbbbbbbbb") + +def test_find_pattern8(): + i = 1 + while i < N: + i += 1 + u2.find(u"bbbbbbbb") + +def test_find_pattern4(): + i = 1 + while i < N: + i += 1 + u2.find(u"bbbb") + +def test_find_pattern2(): + i = 1 + while i < N: + i += 1 + u2.find(u"bb") + + +def test_find_pattern1(): + i = 1 + while i < N: + i += 1 + u2.find(u"b") From fijal at codespeak.net Mon Dec 3 17:45:00 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 3 Dec 2007 17:45:00 +0100 (CET) Subject: [pypy-svn] r49319 - in pypy/dist/pypy/lib: app_test test2 Message-ID: <20071203164500.51DF68155@code0.codespeak.net> Author: fijal Date: Mon Dec 3 17:44:59 2007 New Revision: 49319 Added: pypy/dist/pypy/lib/test2/test_binascii.py - copied unchanged from r49316, pypy/dist/pypy/lib/test2/test_binascii.py Modified: pypy/dist/pypy/lib/app_test/test_binascii.py Log: Revert last checkin, was bogus Modified: pypy/dist/pypy/lib/app_test/test_binascii.py ============================================================================== --- pypy/dist/pypy/lib/app_test/test_binascii.py (original) +++ pypy/dist/pypy/lib/app_test/test_binascii.py Mon Dec 3 17:44:59 2007 @@ -146,7 +146,3 @@ f = getattr(binascii, n) f('') binascii.crc_hqx('', 0) - -def test_incorrect_padding(): - import binascii - raises(binascii.Error, "'x'.decode('base64')") From cfbolz at codespeak.net Mon Dec 3 17:55:01 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 3 Dec 2007 17:55:01 +0100 (CET) Subject: [pypy-svn] r49320 - pypy/dist/pypy/translator/microbench Message-ID: <20071203165501.6F810817D@code0.codespeak.net> Author: cfbolz Date: Mon Dec 3 17:55:01 2007 New Revision: 49320 Modified: pypy/dist/pypy/translator/microbench/test_unicode.py Log: a find microbench where pypy is both faster than cpy 2.4 and 2.5. I think I am hitting the case here where our algorithm is O(len(text)) and theirs is O(len(text) * len(pattern)) Modified: pypy/dist/pypy/translator/microbench/test_unicode.py ============================================================================== --- pypy/dist/pypy/translator/microbench/test_unicode.py (original) +++ pypy/dist/pypy/translator/microbench/test_unicode.py Mon Dec 3 17:55:01 2007 @@ -38,3 +38,11 @@ while i < N: i += 1 u2.find(u"b") + +u3 = u"baaaaaaaaaaaaaaa" * 1000 +def test_bad_case_python2_5(): + p = u"a" * 16 + u"b" + u"a" * 16 + i = 0 + while i < 10000: + i += 1 + u3.find(p) From fijal at codespeak.net Mon Dec 3 18:00:54 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 3 Dec 2007 18:00:54 +0100 (CET) Subject: [pypy-svn] r49321 - pypy/dist/pypy/translator/llvm Message-ID: <20071203170054.D23328169@code0.codespeak.net> Author: fijal Date: Mon Dec 3 18:00:54 2007 New Revision: 49321 Modified: pypy/dist/pypy/translator/llvm/opwriter.py Log: Add ullong_is_true operation Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Mon Dec 3 18:00:54 2007 @@ -282,6 +282,7 @@ opr.argrefs[0], "0") uint_is_true = int_is_true llong_is_true = int_is_true + ullong_is_true = int_is_true def float_is_true(self, opr): self.codewriter.binaryop("fcmp une", opr.retref, opr.argtypes[0], From fijal at codespeak.net Mon Dec 3 20:01:11 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 3 Dec 2007 20:01:11 +0100 (CET) Subject: [pypy-svn] r49322 - pypy/dist/pypy/objspace/std Message-ID: <20071203190111.100BD8164@code0.codespeak.net> Author: fijal Date: Mon Dec 3 20:01:09 2007 New Revision: 49322 Modified: pypy/dist/pypy/objspace/std/formatting.py Log: A cheap way to cut time a bit. This really needs another approach though. Modified: pypy/dist/pypy/objspace/std/formatting.py ============================================================================== --- pypy/dist/pypy/objspace/std/formatting.py (original) +++ pypy/dist/pypy/objspace/std/formatting.py Mon Dec 3 20:01:09 2007 @@ -317,16 +317,21 @@ def std_wp(self, r): length = len(r) prec = self.prec + if prec == -1 and self.width == 0: + # fast path + self.result.append(const(r)) + return if prec >= 0 and prec < length: length = prec # ignore the end of the string if too long result = self.result padding = self.width - length - if not self.f_ljust: + if not self.f_ljust and padding > 0: result.append(const(' ' * padding)) # add any padding at the left of 'r' padding = 0 result.append(const(r[:length])) # add 'r' itself - result.append(const(' ' * padding)) + if padding > 0: + result.append(const(' ' * padding)) # add any remaining padding at the right std_wp._annspecialcase_ = 'specialize:argtype(1)' From cfbolz at codespeak.net Mon Dec 3 21:26:33 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 3 Dec 2007 21:26:33 +0100 (CET) Subject: [pypy-svn] r49323 - pypy/dist/pypy/rlib Message-ID: <20071203202633.1096B8158@code0.codespeak.net> Author: cfbolz Date: Mon Dec 3 21:26:31 2007 New Revision: 49323 Modified: pypy/dist/pypy/rlib/rope.py Log: wtf? Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Mon Dec 3 21:26:31 2007 @@ -167,8 +167,6 @@ if what >= 256: return -1 result = self.s.find(chr(what), start, stop) - if result == -1: - return -1 return result def literal_concat(self, other): @@ -244,8 +242,6 @@ def find_int(self, what, start, stop): result = self.u.find(unichr(what), start, stop) - if result == -1: - return -1 return result def literal_concat(self, other): From cfbolz at codespeak.net Mon Dec 3 21:43:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 3 Dec 2007 21:43:23 +0100 (CET) Subject: [pypy-svn] r49325 - pypy/dist/pypy/doc/discussion Message-ID: <20071203204323.AAA718142@code0.codespeak.net> Author: cfbolz Date: Mon Dec 3 21:43:23 2007 New Revision: 49325 Added: pypy/dist/pypy/doc/discussion/parsing-ideas.txt (contents, props changed) Log: add a discussion file with feature requests for the parsing stuff Added: pypy/dist/pypy/doc/discussion/parsing-ideas.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/discussion/parsing-ideas.txt Mon Dec 3 21:43:23 2007 @@ -0,0 +1,5 @@ +add a way to modularize regular expressions: + +_HEXNUM = "..."; +_DECNUM = "..."; +NUM = "{_HEXNUM}|{_DECNUM}"; From cfbolz at codespeak.net Tue Dec 4 00:01:00 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 4 Dec 2007 00:01:00 +0100 (CET) Subject: [pypy-svn] r49328 - pypy/dist/pypy/rlib Message-ID: <20071203230100.1A6C18139@code0.codespeak.net> Author: cfbolz Date: Tue Dec 4 00:00:59 2007 New Revision: 49328 Modified: pypy/dist/pypy/rlib/rope.py Log: add an XXX Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Tue Dec 4 00:00:59 2007 @@ -864,6 +864,7 @@ self._advance_to(start) def _advance_to(self, index): + # XXX this is O(index), should be O(log(index)) assert index > 0 assert self.index == 0 while 1: From niko at codespeak.net Tue Dec 4 10:07:35 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Tue, 4 Dec 2007 10:07:35 +0100 (CET) Subject: [pypy-svn] r49329 - pypy/dist/pypy/translator/jvm/src/pypy Message-ID: <20071204090735.4B69C819F@code0.codespeak.net> Author: niko Date: Tue Dec 4 10:07:34 2007 New Revision: 49329 Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Log: implement getpid, symlink, using jna.jar. Right now if jna.jar is not found these operations fail with EPERM and the message "jna.jar required". Modified: pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/ll_os.java Tue Dec 4 10:07:34 2007 @@ -8,6 +8,9 @@ import java.util.Iterator; import java.util.Arrays; +import com.sun.jna.Library; +import com.sun.jna.Native; + abstract class FileWrapper { public abstract void write(String buffer); @@ -160,6 +163,25 @@ public class ll_os implements Constants { + /** + * JNA Interface: allows access to functions we don't normally + * have in the Java standard lib + */ + static public interface Libc extends Library { + public int getpid(); + public int symlink(String path1, String path2); + } + static final Libc libc; + static { + Libc res; + try { + res = (Libc) Native.loadLibrary("c", Libc.class); + } catch (Throwable t) { + res = null; + } + libc = res; + } + // NB: these values are those used by Windows and they differs // from the Unix ones; the os module is patched with these // values before flowgraphing to make sure we get the very @@ -365,21 +387,26 @@ throwOSError(PyPy.EPERM, "Operation not permitted: '"+path+"'"); } - public void ll_os_unlink(String path) { - if (STRACE) strace("ll_os_unlink: "+path); - + public void delete(String path, boolean should_be_dir) { File f = new File(path); - if (!f.exists()) throwOSError(PyPy.ENOENT, "No such file or directory: '"+path+"'"); - - if (f.isDirectory()) + if (f.isDirectory() != should_be_dir) throwOSError(PyPy.EPERM, "Operation not permitted: '"+path+"'"); - if (!f.delete()) throwOSError(PyPy.EPERM, "Operation not permitted: '"+path+"'"); } + public void ll_os_rmdir(String path) { + if (STRACE) strace("ll_os_rmdir: "+path); + delete(path, true); + } + + public void ll_os_unlink(String path) { + if (STRACE) strace("ll_os_unlink: "+path); + delete(path, false); + } + public boolean ll_os_isatty(int x) { // XXX: this is not the right behaviour, but it's needed @@ -464,4 +491,23 @@ throwOSError(PyPy.ENOENT, "No such file or directory: '"+path+"'"); return null; // never reached } + + public void checkLibc() { + if (libc == null) + throwOSError(EPERM, "jna.jar required"); + } + + public int ll_os_getpid() + { + checkLibc(); + return libc.getpid(); + } + + public void ll_os_symlink(String path1, String path2) + { + checkLibc(); + int res = libc.symlink(path1, path2); + if (res != 0) + throwOSError(res, ""); + } } From antocuni at codespeak.net Tue Dec 4 13:50:00 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 4 Dec 2007 13:50:00 +0100 (CET) Subject: [pypy-svn] r49331 - pypy/dist/pypy/translator Message-ID: <20071204125000.81020817D@code0.codespeak.net> Author: antocuni Date: Tue Dec 4 13:49:59 2007 New Revision: 49331 Modified: pypy/dist/pypy/translator/driver.py Log: uname -o doesn't work on OS/X Modified: pypy/dist/pypy/translator/driver.py ============================================================================== --- pypy/dist/pypy/translator/driver.py (original) +++ pypy/dist/pypy/translator/driver.py Tue Dec 4 13:49:59 2007 @@ -664,7 +664,7 @@ f = file(newexename, 'w') f.write("""#!/bin/bash LEDIT=`type -p ledit` -if [ `uname -o` = 'Cygwin' ]; then MONO=; else MONO=mono; fi +if [ `uname -s` = 'Cygwin' ]; then MONO=; else MONO=mono; fi $LEDIT $MONO "$(dirname $0)/$(basename $0)-data/%s" "$@" # XXX doesn't work if it's placed in PATH """ % main_exe_name) f.close() From xoraxax at codespeak.net Tue Dec 4 14:08:28 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 4 Dec 2007 14:08:28 +0100 (CET) Subject: [pypy-svn] r49332 - in pypy/dist/pypy: rpython/test translator/cli/test translator/llvm/test Message-ID: <20071204130828.4023F8158@code0.codespeak.net> Author: xoraxax Date: Tue Dec 4 14:08:27 2007 New Revision: 49332 Modified: pypy/dist/pypy/rpython/test/tool.py pypy/dist/pypy/translator/cli/test/runtest.py pypy/dist/pypy/translator/llvm/test/runtest.py Log: Reduce pointless code duplication, fix frexp tests on cli and llvm. Modified: pypy/dist/pypy/rpython/test/tool.py ============================================================================== --- pypy/dist/pypy/rpython/test/tool.py (original) +++ pypy/dist/pypy/rpython/test/tool.py Tue Dec 4 14:08:27 2007 @@ -3,6 +3,8 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.test.test_llinterp import gengraph, interpret, interpret_raises +FLOAT_PRECISION = 8 + class BaseRtypingTest(object): def gengraph(self, func, argtypes=[], viewbefore='auto', policy=None, @@ -19,6 +21,10 @@ def float_eq(self, x, y): return x == y + def float_eq_approx(self, x, y): + diff = abs(x-y) + return diff < 10**-FLOAT_PRECISION + def is_of_type(self, x, type_): return type(x) is type_ Modified: pypy/dist/pypy/translator/cli/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/runtest.py (original) +++ pypy/dist/pypy/translator/cli/test/runtest.py Tue Dec 4 14:08:27 2007 @@ -23,7 +23,6 @@ from pypy.translator.cli.entrypoint import BaseEntryPoint from pypy.translator.oosupport.support import patch_os, unpatch_os -FLOAT_PRECISION = 8 def format_object(TYPE, cts, ilasm): if TYPE is ootype.Void: @@ -292,9 +291,7 @@ else: assert False, 'function did raise no exception at all' - def float_eq(self, x, y): - diff = abs(x-y) - return diff/x < 10**-FLOAT_PRECISION + float_eq = BaseRtypingTest.float_eq_approx def is_of_type(self, x, type_): return True # we can't really test the type Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Tue Dec 4 14:08:27 2007 @@ -10,7 +10,6 @@ optimize_tests = False native_llvm_backend = True MINIMUM_LLVM_VERSION = 2.0 -FLOAT_PRECISION = 8 # prevents resource leaking use_isolate = True @@ -219,10 +218,8 @@ return True else: assert False, 'function did raise no exception at all' - - def float_eq(self, x, y): - diff = abs(x-y) - return diff/x < 10**-FLOAT_PRECISION + + float_eq = BaseRtypingTest.float_eq_approx def is_of_type(self, x, type_): return True # we can't really test the type From cfbolz at codespeak.net Tue Dec 4 14:16:31 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 4 Dec 2007 14:16:31 +0100 (CET) Subject: [pypy-svn] r49333 - pypy/dist/pypy/doc Message-ID: <20071204131631.2740A816A@code0.codespeak.net> Author: cfbolz Date: Tue Dec 4 14:16:31 2007 New Revision: 49333 Modified: pypy/dist/pypy/doc/cleanup-todo.txt Log: some of these were done Modified: pypy/dist/pypy/doc/cleanup-todo.txt ============================================================================== --- pypy/dist/pypy/doc/cleanup-todo.txt (original) +++ pypy/dist/pypy/doc/cleanup-todo.txt Tue Dec 4 14:16:31 2007 @@ -20,10 +20,6 @@ https://codespeak.net/issue/pypy-dev/issue303 and the fact that we can have more than one translator/annotator around (with the timeshifter) - - unicode strings in RPython - - finish rctypes removal - - think about approaches to id, especially concerning boehm, where the id will - keep the object alive and concerning a moving GC interpreter ----------- From cfbolz at codespeak.net Tue Dec 4 14:17:47 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 4 Dec 2007 14:17:47 +0100 (CET) Subject: [pypy-svn] r49335 - pypy/dist/pypy/doc Message-ID: <20071204131747.9AB208185@code0.codespeak.net> Author: cfbolz Date: Tue Dec 4 14:17:46 2007 New Revision: 49335 Removed: pypy/dist/pypy/doc/rctypes.txt Modified: pypy/dist/pypy/doc/standalone-howto.txt Log: remove rctypes docs Modified: pypy/dist/pypy/doc/standalone-howto.txt ============================================================================== --- pypy/dist/pypy/doc/standalone-howto.txt (original) +++ pypy/dist/pypy/doc/standalone-howto.txt Tue Dec 4 14:17:46 2007 @@ -9,6 +9,8 @@ way to compile it is not as described below, but by writing a target file as described in the `FAQ entries`_.) +**Warning:** the bits of this howto that describe rctypes are outdated + ======================================== First, see the note above. @@ -99,7 +101,7 @@ Now we can sum with different kinds of lists, eg. ``sum([1,2,3])`` and ``sum([1.0,2.0,3.0])``. -Here is an example of using rctypes_:: +Here is an example of using rctypes:: import ctypes from ctypes import c_int, c_char_p From fijal at codespeak.net Tue Dec 4 14:26:41 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 4 Dec 2007 14:26:41 +0100 (CET) Subject: [pypy-svn] r49336 - pypy/dist/pypy/doc Message-ID: <20071204132641.AEB8D8185@code0.codespeak.net> Author: fijal Date: Tue Dec 4 14:26:40 2007 New Revision: 49336 Modified: pypy/dist/pypy/doc/cleanup-todo.txt Log: This is all done Modified: pypy/dist/pypy/doc/cleanup-todo.txt ============================================================================== --- pypy/dist/pypy/doc/cleanup-todo.txt (original) +++ pypy/dist/pypy/doc/cleanup-todo.txt Tue Dec 4 14:26:40 2007 @@ -27,13 +27,4 @@ - review the things implemented at applevel whether they are performance- critical - - rewrite the following rctypes modules using rffi: - - - _ssl - - - move the following from using old-style C-api - - - signal module - - strtod (RPyton level) - - review CPython regression test suite, enable running tests, fix bugs From regmee at codespeak.net Tue Dec 4 15:40:48 2007 From: regmee at codespeak.net (regmee at codespeak.net) Date: Tue, 4 Dec 2007 15:40:48 +0100 (CET) Subject: [pypy-svn] r49338 - in pypy/branch/clr-module-improvements/pypy/module/clr: . test Message-ID: <20071204144048.99D308158@code0.codespeak.net> Author: regmee Date: Tue Dec 4 15:40:47 2007 New Revision: 49338 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py Log: checking in failing test case for GetEnumerator when assigned to __iter__ Modified: pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py Tue Dec 4 15:40:47 2007 @@ -86,27 +86,31 @@ else: type.__setattr__(cls, name, value) +#class Dummy(object): +# def __init__(self, iterObj): +# self.iterObj = iterObj +# self.index = 0 +# +# def next(self): +# temp = self.index +# if self.index == self.iterObj.Count: +# raise StopIteration +# self.index = self.index + 1 +# return self.iterObj.__getitem__(temp) + + class CliClassWrapper(object): __slots__ = ('__cliobj__',) def __init__(self, *args): import clr self.__cliobj__ = clr._CliObject_internal(self.__cliclass__, args) -# self.index = self.__cliobj__.__len__(self) -# self.index = self.__cliobj__.call_method('Count',1) print self.__cliobj__ -# self.index = self.Count - - def __iter__(self): - self.index = self.Count - return self - - def next(self): - if self.index == 0: - raise StopIteration - self.index = self.index - 1 - return self.this[self.index] + print self.Count +# def __iter__(self): +# return Dummy(self) +# return Dummy(self.Count, self.__getitem__(self.Count - 1)) def build_wrapper(namespace, classname, staticmethods, methods, properties, indexers): fullname = '%s.%s' % (namespace, classname) @@ -122,7 +126,9 @@ if method == "GetEnumerator": print "Enumerator found .. Hurray !!!!!" # now add the __iter__ method to the class -# d['__iter__'] = sampleIter().__iter__ + d['__iter__'] = d['GetEnumerator'] +# d['next'] = d['MoveNext'] + assert len(indexers) <= 1 if indexers: @@ -153,3 +159,8 @@ setattr(cls, name, prop) return cls + + + + + Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py Tue Dec 4 15:40:47 2007 @@ -148,3 +148,17 @@ assert Environment.CurrentDirectory == os.getcwd() Environment.CurrentDirectory == '/' assert Environment.CurrentDirectory == os.getcwd() + + def test_GetEnumerator(self): + import clr + ArrayList = clr.load_cli_class('System.Collections', 'ArrayList') + x = ArrayList() + x.Add(1) + x.Add(6) + x.Add(31) + x.Add(2) + for i in x: + print i + + + From exarkun at codespeak.net Tue Dec 4 16:26:58 2007 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Tue, 4 Dec 2007 16:26:58 +0100 (CET) Subject: [pypy-svn] r49340 - pypy/dist/pypy/rpython/module/test Message-ID: <20071204152658.D5E658160@code0.codespeak.net> Author: exarkun Date: Tue Dec 4 16:26:58 2007 New Revision: 49340 Modified: pypy/dist/pypy/rpython/module/test/execve_tests.py pypy/dist/pypy/rpython/module/test/test_ll_os.py Log: Change test_ll_os.test_execve so that it works even if pypy is not importable based on $PYTHONPATH Modified: pypy/dist/pypy/rpython/module/test/execve_tests.py ============================================================================== --- pypy/dist/pypy/rpython/module/test/execve_tests.py (original) +++ pypy/dist/pypy/rpython/module/test/execve_tests.py Tue Dec 4 16:26:58 2007 @@ -6,6 +6,7 @@ """ import os, sys +sys.path.append(sys.argv[1]) from pypy.rpython.module.test.test_ll_os import EXECVE_ENV, getllimpl @@ -22,4 +23,4 @@ execve("/usr/bin/env", ["/usr/bin/env"], EXECVE_ENV) if __name__ == '__main__': - globals()[sys.argv[1]]() + globals()[sys.argv[2]]() Modified: pypy/dist/pypy/rpython/module/test/test_ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/test/test_ll_os.py (original) +++ pypy/dist/pypy/rpython/module/test/test_ll_os.py Tue Dec 4 16:26:58 2007 @@ -1,6 +1,7 @@ import os from py.path import local +import pypy from pypy.tool.udir import udir from pypy.translator.c.test.test_genc import compile @@ -68,7 +69,11 @@ def test_execve(): if os.name != 'posix': py.test.skip('posix specific function') - base = sys.executable + " " + execve_tests + " " + base = " ".join([ + sys.executable, + execve_tests, + str(local(pypy.__file__).join('..', '..')), + '']) # Test exit status and code result = os.system(base + "execve_true") From cfbolz at codespeak.net Tue Dec 4 16:59:20 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 4 Dec 2007 16:59:20 +0100 (CET) Subject: [pypy-svn] r49343 - in pypy/dist/pypy/rlib: . test Message-ID: <20071204155920.CCAA18182@code0.codespeak.net> Author: cfbolz Date: Tue Dec 4 16:59:20 2007 New Revision: 49343 Modified: pypy/dist/pypy/rlib/rope.py pypy/dist/pypy/rlib/test/test_rope.py Log: some refactorings about the FringeIterator. also contains the beginnings about some crazy ideas about extremely fast string searching. Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Tue Dec 4 16:59:20 2007 @@ -53,6 +53,7 @@ class StringNode(object): hash_cache = 0 + charbitmask = 0 def length(self): raise NotImplementedError("base class") @@ -86,6 +87,9 @@ def getslice(self, start, stop): raise NotImplementedError("abstract base class") + def can_contain_int(self, value): + return True #conservative default + def view(self): view([self]) @@ -115,9 +119,13 @@ assert isinstance(s, str) self.s = s is_ascii = True + charbitmask = 0 for c in s: - if ord(c) >= 128: + ordc = ord(c) + if ordc >= 128: is_ascii = False + charbitmask |= 1 << (ordc & 0x1F) + self.charbitmask = charbitmask self._is_ascii = is_ascii def length(self): @@ -158,16 +166,22 @@ def getrope(self, index): return LiteralStringNode.PREBUILT[ord(self.s[index])] + def can_contain_int(self, value): + if value > 255: + return False + if self.is_ascii() and value > 127: + return False + return (1 << (value & 0x1f)) & self.charbitmask + def getslice(self, start, stop): assert 0 <= start <= stop return LiteralStringNode(self.s[start:stop]) def find_int(self, what, start, stop): - if what >= 256: + if not self.can_contain_int(what): return -1 - result = self.s.find(chr(what), start, stop) - return result + return self.s.find(chr(what), start, stop) def literal_concat(self, other): if (isinstance(other, LiteralStringNode) and @@ -198,6 +212,13 @@ def __init__(self, u): assert isinstance(u, unicode) self.u = u + charbitmask = 0 + for c in u: + ordc = ord(c) + if ordc >= 128: + charbitmask |= 1 # be compatible with LiteralStringNode + charbitmask |= 1 << (ordc & 0x1F) + self.charbitmask = charbitmask def length(self): return len(self.u) @@ -236,13 +257,17 @@ return self return LiteralUnicodeNode(unichr(ch)) + def can_contain_int(self, value): + return (1 << (value & 0x1f)) & self.charbitmask + def getslice(self, start, stop): assert 0 <= start <= stop return LiteralUnicodeNode(self.u[start:stop]) def find_int(self, what, start, stop): - result = self.u.find(unichr(what), start, stop) - return result + if not self.can_contain_int(what): + return -1 + return self.u.find(unichr(what), start, stop) def literal_concat(self, other): if (isinstance(other, LiteralUnicodeNode) and @@ -278,6 +303,7 @@ self.balanced = False self._is_ascii = left.is_ascii() and right.is_ascii() self._is_bytestring = left.is_bytestring() and right.is_bytestring() + self.charbitmask = left.charbitmask | right.charbitmask def is_ascii(self): return self._is_ascii @@ -335,6 +361,13 @@ else: return self.left.getrope(index) + def can_contain_int(self, value): + if self.is_bytestring() and value > 255: + return False + if self.is_ascii() and value > 127: + return False + return (1 << (value & 0x1f)) & self.charbitmask + def flatten_string(self): f = fringe(self) return "".join([node.flatten_string() for node in f]) @@ -650,11 +683,6 @@ length = node.length() if stop == -1: stop = length - if start != 0 or stop != length: - newstart, newstop, node = find_straddling(node, start, stop) - offset = start - newstart - start = newstart - stop = newstop assert 0 <= start <= stop if isinstance(node, LiteralNode): pos = node.find_int(what, start, stop) @@ -662,6 +690,9 @@ return pos return pos + offset iter = FringeIterator(node) + newstart = iter._seekforward(start) + offset += start - newstart + start = newstart #import pdb; pdb.set_trace() i = 0 while i < stop: @@ -675,6 +706,8 @@ continue searchstart = max(0, start - i) searchstop = min(stop - i, nodelength) + if searchstop <= 0: + return -1 assert isinstance(fringenode, LiteralNode) pos = fringenode.find_int(what, searchstart, searchstop) if pos != -1: @@ -809,6 +842,29 @@ return curr raise StopIteration + def _seekforward(self, length): + """seek forward up to n characters, returning the number remaining chars. + experimental api""" + curr = None + while self.stack: + curr = self.stack.pop() + if length < curr.length(): + break + length -= curr.length() + else: + raise StopIteration + while isinstance(curr, BinaryConcatNode): + left_length = curr.left.length() + if length < left_length: + self.stack.append(curr.right) + curr = curr.left + else: + length -= left_length + curr = curr.right + self.stack.append(curr) + return length + + def fringe(node): result = [] iter = FringeIterator(node) @@ -834,25 +890,6 @@ return curr raise StopIteration -class SeekableFringeIterator(FringeIterator): - def __init__(self, node): - FringeIterator.__init__(self, node) - self.fringestack = [] - self.fringe = [] - - def next(self): - if self.fringestack: - result = self.fringestack.pop() - else: - result = FringeIterator.next(self) - self.fringe.append(result) - return result - - def seekback(self): - result = self.fringe.pop() - self.fringestack.append(result) - return result - class ItemIterator(object): def __init__(self, node, start=0): @@ -864,19 +901,9 @@ self._advance_to(start) def _advance_to(self, index): - # XXX this is O(index), should be O(log(index)) - assert index > 0 - assert self.index == 0 - while 1: - node = self.iter.next() - length = node.length() - if index < length: - self.index = index - self.node = node - self.nodelength = length - break - index -= length - assert index >= 0 + self.index = self.iter._seekforward(index) + self.node = self.iter.next() + self.nodelength = self.node.length() def getnode(self): node = self.node Modified: pypy/dist/pypy/rlib/test/test_rope.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rope.py (original) +++ pypy/dist/pypy/rlib/test/test_rope.py Tue Dec 4 16:59:20 2007 @@ -350,36 +350,26 @@ n = iter.next() assert n is GHI py.test.raises(StopIteration, iter.next) + iter = FringeIterator(rope) -def test_seekable_fringe_iterator(): +def test_fringe_iterator_seekforward(): ABC = LiteralStringNode("abc") DEF = LiteralStringNode("def") GHI = LiteralStringNode("ghi") rope = BinaryConcatNode(BinaryConcatNode(ABC, DEF), GHI) - iter = SeekableFringeIterator(rope) - n = iter.next() - assert n is ABC - n = iter.seekback() - assert n is ABC + iter = FringeIterator(rope) n = iter.next() assert n is ABC + i = iter._seekforward(5) + assert i == 2 n = iter.next() - assert n is DEF - n = iter.next() - assert n is GHI - n = iter.seekback() assert n is GHI - n = iter.seekback() - assert n is DEF - n = iter.seekback() - assert n is ABC - n = iter.next() - assert n is ABC - n = iter.next() - assert n is DEF + py.test.raises(StopIteration, iter.next) + iter = FringeIterator(rope) + i = iter._seekforward(7) + assert i == 1 n = iter.next() assert n is GHI - py.test.raises(StopIteration, iter.next) def test_seekforward(): From fijal at codespeak.net Tue Dec 4 17:24:36 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 4 Dec 2007 17:24:36 +0100 (CET) Subject: [pypy-svn] r49345 - in pypy/dist/pypy: jit/codegen/llgraph jit/timeshifter/test module/thread rlib rpython rpython/lltypesystem rpython/memory/gc rpython/memory/gctransform rpython/numpy rpython/test Message-ID: <20071204162436.DAF7A8174@code0.codespeak.net> Author: fijal Date: Tue Dec 4 17:24:36 2007 New Revision: 49345 Added: pypy/dist/pypy/rlib/debug.py (contents, props changed) Modified: pypy/dist/pypy/jit/codegen/llgraph/rgenop.py pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py pypy/dist/pypy/module/thread/ll_thread.py pypy/dist/pypy/rlib/objectmodel.py pypy/dist/pypy/rpython/extfunc.py pypy/dist/pypy/rpython/lltypesystem/llarena.py pypy/dist/pypy/rpython/lltypesystem/rlist.py pypy/dist/pypy/rpython/lltypesystem/rstr.py pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/gc/marksweep.py pypy/dist/pypy/rpython/memory/gc/semispace.py pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/numpy/rarray.py pypy/dist/pypy/rpython/rlist.py pypy/dist/pypy/rpython/test/test_rbuiltin.py Log: Rename objectmodel.debug_assert to debug.ll_assert and objectmodel.debug_llinterpcall to debug.llinterpcall Modified: pypy/dist/pypy/jit/codegen/llgraph/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/llgraph/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/llgraph/rgenop.py Tue Dec 4 17:24:36 2007 @@ -1,4 +1,5 @@ -from pypy.rlib.objectmodel import specialize, debug_assert +from pypy.rlib.objectmodel import specialize +from pypy.rlib.debug import ll_assert from pypy.rpython.lltypesystem import lltype, llmemory from pypy.jit.codegen.model import AbstractRGenOp, GenLabel, GenBuilder from pypy.jit.codegen.model import GenVar, GenConst, CodeGenSwitch @@ -72,7 +73,7 @@ l_case = llimpl.add_case(self.b, gv_case.v) b = llimpl.closelinktofreshblock(l_case, self.args_gv, self.l_default) builder = LLBuilder(self.rgenop, self.gv_f, b) - debug_assert(self.rgenop.currently_writing is None or + ll_assert(self.rgenop.currently_writing is None or # special case: we stop replaying and add a case after # a call to flexswitch() on a replay builder self.rgenop.currently_writing.is_default_builder, @@ -85,7 +86,7 @@ self.l_default = l_default b = llimpl.closelinktofreshblock(l_default, self.args_gv, None) builder = LLBuilder(self.rgenop, self.gv_f, b) - debug_assert(self.rgenop.currently_writing is None, + ll_assert(self.rgenop.currently_writing is None, "_add_default: currently_writing elsewhere") self.rgenop.currently_writing = builder builder.is_default_builder = True @@ -101,19 +102,19 @@ self.b = block def end(self): - debug_assert(self.rgenop.currently_writing is None, + ll_assert(self.rgenop.currently_writing is None, "end: currently_writing") llimpl.end(self.gv_f) @specialize.arg(1) def genop1(self, opname, gv_arg): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genop1: bad currently_writing") return LLVar(llimpl.genop(self.b, opname, [gv_arg], llimpl.guess)) @specialize.arg(1) def genraisingop1(self, opname, gv_arg): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genraisingop1: bad currently_writing") gv_res = LLVar(llimpl.genop(self.b, opname, [gv_arg], llimpl.guess)) gv_exc = LLVar(llimpl.genop(self.b, "check_and_clear_exc", [], @@ -122,14 +123,14 @@ @specialize.arg(1) def genop2(self, opname, gv_arg1, gv_arg2): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genop2: bad currently_writing") return LLVar(llimpl.genop(self.b, opname, [gv_arg1, gv_arg2], llimpl.guess)) @specialize.arg(1) def genraisingop2(self, opname, gv_arg1, gv_arg2): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genraisingop2: bad currently_writing") gv_res = LLVar(llimpl.genop(self.b, opname, [gv_arg1, gv_arg2], llimpl.guess)) @@ -138,7 +139,7 @@ return gv_res, gv_exc def genop_call(self, (ARGS_gv, gv_RESULT, _), gv_callable, args_gv): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genop_call: bad currently_writing") vars_gv = [gv_callable] j = 0 @@ -157,7 +158,7 @@ return LLVar(v) def genop_getfield(self, (gv_name, gv_PTRTYPE, gv_FIELDTYPE), gv_ptr): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genop_getfield: bad currently_writing") return LLVar(llimpl.gengetfield(self.b, gv_ptr.v, gv_PTRTYPE.v, gv_name.v)) @@ -167,7 +168,7 @@ def genop_setfield(self, (gv_name, gv_PTRTYPE, gv_FIELDTYPE), gv_ptr, gv_value): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genop_setfield: bad currently_writing") v_value = llimpl.cast(self.b, gv_FIELDTYPE.v, gv_value.v) llimpl.gensetfield(self.b, gv_ptr.v, gv_PTRTYPE.v, gv_name.v, v_value) @@ -178,7 +179,7 @@ # gv_Void.v)) def genop_getsubstruct(self, (gv_name, gv_PTRTYPE, gv_FIELDTYPE), gv_ptr): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genop_getsubstruct: bad currently_writing") return LLVar(llimpl.gengetsubstruct(self.b, gv_ptr.v, gv_PTRTYPE.v, gv_name.v)) @@ -187,7 +188,7 @@ # gv_FIELDTYPE.v)) def genop_getarrayitem(self, gv_ITEMTYPE, gv_ptr, gv_index): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genop_getarrayitem: bad currently_writing") return LLVar(llimpl.gengetarrayitem(self.b, gv_ITEMTYPE.v, gv_ptr.v, gv_index.v)) @@ -196,7 +197,7 @@ # gv_ITEMTYPE.v)) def genop_getarraysubstruct(self, gv_ITEMTYPE, gv_ptr, gv_index): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genop_getarraysubstruct: bad currently_writing") return LLVar(llimpl.gengetarraysubstruct(self.b, gv_ptr.v, gv_index.v)) #vars_gv = [gv_ptr.v, gv_index.v] @@ -204,53 +205,53 @@ # gv_ITEMTYPE.v)) def genop_setarrayitem(self, gv_ITEMTYPE, gv_ptr, gv_index, gv_value): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genop_setarrayitem: bad currently_writing") llimpl.gensetarrayitem(self.b, gv_ptr.v, gv_index.v, gv_value.v) #vars_gv = [gv_ptr.v, gv_index.v, gv_value.v] #llimpl.genop(self.b, 'setarrayitem', vars_gv, gv_Void.v) def genop_getarraysize(self, gv_ITEMTYPE, gv_ptr): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genop_getarraysize: bad currently_writing") return LLVar(llimpl.gengetarraysize(self.b, gv_ptr.v)) #return LLVar(llimpl.genop(self.b, 'getarraysize', [gv_ptr.v], # gv_Signed.v)) def genop_malloc_fixedsize(self, (gv_TYPE, gv_PTRTYPE)): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genop_malloc_fixedsize: bad currently_writing") vars_gv = [gv_TYPE.v, gv_flavor_gc.v] return LLVar(llimpl.genop(self.b, 'malloc', vars_gv, gv_PTRTYPE.v)) def genop_malloc_varsize(self, (gv_TYPE, gv_PTRTYPE), gv_length): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genop_malloc_varsize: bad currently_writing") vars_gv = [gv_TYPE.v, gv_flavor_gc.v, gv_length.v] return LLVar(llimpl.genop(self.b, 'malloc_varsize', vars_gv, gv_PTRTYPE.v)) def genop_same_as(self, gv_TYPE, gv_value): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genop_same_as: bad currently_writing") gv_value = llimpl.cast(self.b, gv_TYPE.v, gv_value.v) return LLVar(llimpl.genop(self.b, 'same_as', [gv_value], gv_TYPE.v)) def genop_ptr_iszero(self, gv_PTRTYPE, gv_ptr): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genop_ptr_iszero: bad currently_writing") gv_ptr = llimpl.cast(self.b, gv_PTRTYPE.v, gv_ptr.v) return LLVar(llimpl.genop(self.b, 'ptr_iszero', [gv_ptr], gv_Bool.v)) def genop_ptr_nonzero(self, gv_PTRTYPE, gv_ptr): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genop_ptr_nonzero: bad currently_writing") gv_ptr = llimpl.cast(self.b, gv_PTRTYPE.v, gv_ptr.v) return LLVar(llimpl.genop(self.b, 'ptr_nonzero', [gv_ptr], gv_Bool.v)) def genop_ptr_eq(self, gv_PTRTYPE, gv_ptr1, gv_ptr2): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genop_ptr_eq: bad currently_writing") gv_ptr1 = llimpl.cast(self.b, gv_PTRTYPE.v, gv_ptr1.v) gv_ptr2 = llimpl.cast(self.b, gv_PTRTYPE.v, gv_ptr2.v) @@ -258,7 +259,7 @@ gv_Bool.v)) def genop_ptr_ne(self, gv_PTRTYPE, gv_ptr1, gv_ptr2): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genop_ptr_ne: bad currently_writing") gv_ptr1 = llimpl.cast(self.b, gv_PTRTYPE.v, gv_ptr1.v) gv_ptr2 = llimpl.cast(self.b, gv_PTRTYPE.v, gv_ptr2.v) @@ -266,7 +267,7 @@ gv_Bool.v)) def genop_cast_int_to_ptr(self, gv_PTRTYPE, gv_int): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genop_cast_int_to_ptr: bad currently_writing") return LLVar(llimpl.genop(self.b, 'cast_int_to_ptr', [gv_int], gv_PTRTYPE.v)) @@ -276,7 +277,7 @@ return [LLVar(llimpl.geninputarg(newb, kind.v)) for kind in kinds] def enter_next_block(self, kinds, args_gv): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "enter_next_block: bad currently_writing") lnk = llimpl.closeblock1(self.b) newb_args_gv = self._newblock(kinds) @@ -297,7 +298,7 @@ self._close() def _jump(self, l_jump, l_no_jump, args_for_jump_gv): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "_jump: bad currently_writing") self.b = llimpl.closelinktofreshblock(l_no_jump, None, None) b2 = llimpl.closelinktofreshblock(l_jump, args_for_jump_gv, None) @@ -321,7 +322,7 @@ return (flexswitch, flexswitch._add_default()) def _close(self): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "_close: bad currently_writing") self.rgenop.currently_writing = None self.b = llimpl.nullblock @@ -333,7 +334,7 @@ assert self.later_block != llimpl.nullblock self.b = self.later_block self.later_block = llimpl.nullblock - debug_assert(self.rgenop.currently_writing is None, + ll_assert(self.rgenop.currently_writing is None, "start_writing: currently_writing") self.rgenop.currently_writing = self @@ -352,18 +353,18 @@ # read_frame_var support def genop_get_frame_base(self): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "genop_get_frame_base: bad currently_writing") return LLVar(llimpl.genop(self.b, 'get_frame_base', [], gv_Address.v)) def get_frame_info(self, vars): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "get_frame_info: bad currently_writing") return llimpl.get_frame_info(self.b, vars) def alloc_frame_place(self, gv_TYPE, gv_initial_value=None): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "alloc_frame_place: bad currently_writing") if gv_initial_value is None: gv_initial_value = self.rgenop.genzeroconst(gv_TYPE) @@ -373,9 +374,9 @@ return LLPlace(v, llimpl.get_frame_info(self.b, [v])) def genop_absorb_place(self, gv_TYPE, place): - debug_assert(self.rgenop.currently_writing is self, + ll_assert(self.rgenop.currently_writing is self, "alloc_frame_place: bad currently_writing") - debug_assert(not place.absorbed, "place already absorbed") + ll_assert(not place.absorbed, "place already absorbed") place.absorbed = True return place.v @@ -449,7 +450,7 @@ def replay(self, label, kinds): builder = LLBuilder(self, label.g, llimpl.nullblock) args_gv = builder._newblock(kinds) - debug_assert(self.currently_writing is None, + ll_assert(self.currently_writing is None, "replay: currently_writing") self.currently_writing = builder return builder, args_gv Modified: pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py Tue Dec 4 17:24:36 2007 @@ -10,7 +10,8 @@ from pypy.objspace.flow.model import summary, Variable from pypy.rpython.lltypesystem import lltype, llmemory, rstr from pypy.rlib.jit import hint -from pypy.rlib.objectmodel import keepalive_until_here, debug_assert +from pypy.rlib.objectmodel import keepalive_until_here +from pypy.rlib.debug import ll_assert from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rarithmetic import ovfcheck from pypy.rpython.annlowlevel import PseudoHighLevelCallable, cachedtype @@ -1379,7 +1380,7 @@ return s def g(s): # assumes that s is not null here - debug_assert(bool(s), "please don't give me a null") + ll_assert(bool(s), "please don't give me a null") return 5 def f(m): s = h() Modified: pypy/dist/pypy/module/thread/ll_thread.py ============================================================================== --- pypy/dist/pypy/module/thread/ll_thread.py (original) +++ pypy/dist/pypy/module/thread/ll_thread.py Tue Dec 4 17:24:36 2007 @@ -10,7 +10,7 @@ from pypy.rpython.extregistry import ExtRegistryEntry from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem.lltype import typeOf -from pypy.rlib.objectmodel import debug_assert +from pypy.rlib.debug import ll_assert from pypy.tool import autopath from distutils import sysconfig python_inc = sysconfig.get_python_inc() @@ -154,11 +154,11 @@ return bool(c_thread_acquirelock_NOAUTO(self._lock, int(flag))) def release(self): - debug_assert(not self.acquire(False), "Lock_NOAUTO was not held!") + ll_assert(not self.acquire(False), "Lock_NOAUTO was not held!") c_thread_releaselock_NOAUTO(self._lock) def fused_release_acquire(self): - debug_assert(not self.acquire(False), "Lock_NOAUTO was not held!") + ll_assert(not self.acquire(False), "Lock_NOAUTO was not held!") c_thread_fused_releaseacquirelock_NOAUTO(self._lock) def __del__(self): Added: pypy/dist/pypy/rlib/debug.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rlib/debug.py Tue Dec 4 17:24:36 2007 @@ -0,0 +1,59 @@ + +from pypy.rpython.extregistry import ExtRegistryEntry + +def ll_assert(x, msg): + """After translation to C, this becomes an RPyAssert.""" + assert x, msg + +class Entry(ExtRegistryEntry): + _about_ = ll_assert + + def compute_result_annotation(self, s_x, s_msg): + assert s_msg.is_constant(), ("ll_assert(x, msg): " + "the msg must be constant") + return None + + def specialize_call(self, hop): + from pypy.rpython.lltypesystem import lltype + vlist = hop.inputargs(lltype.Bool, lltype.Void) + hop.genop('debug_assert', vlist) + + +def llinterpcall(RESTYPE, pythonfunction, *args): + """When running on the llinterp, this causes the llinterp to call to + the provided Python function with the run-time value of the given args. + The Python function should return a low-level object of type RESTYPE. + This should never be called after translation: use this only if + running_on_llinterp is true. + """ + raise NotImplementedError + +class Entry(ExtRegistryEntry): + _about_ = llinterpcall + + def compute_result_annotation(self, s_RESTYPE, s_pythonfunction, *args_s): + from pypy.annotation import model as annmodel + from pypy.rpython.lltypesystem import lltype + assert s_RESTYPE.is_constant() + assert s_pythonfunction.is_constant() + s_result = s_RESTYPE.const + if isinstance(s_result, lltype.LowLevelType): + s_result = annmodel.lltype_to_annotation(s_result) + assert isinstance(s_result, annmodel.SomeObject) + return s_result + + def specialize_call(self, hop): + from pypy.annotation import model as annmodel + from pypy.rpython.lltypesystem import lltype + RESTYPE = hop.args_s[0].const + if not isinstance(RESTYPE, lltype.LowLevelType): + assert isinstance(RESTYPE, annmodel.SomeObject) + r_result = hop.rtyper.getrepr(RESTYPE) + RESTYPE = r_result.lowleveltype + pythonfunction = hop.args_s[1].const + c_pythonfunction = hop.inputconst(lltype.Void, pythonfunction) + args_v = [hop.inputarg(hop.args_r[i], arg=i) + for i in range(2, hop.nb_args)] + return hop.genop('debug_llinterpcall', [c_pythonfunction] + args_v, + resulttype=RESTYPE) + Modified: pypy/dist/pypy/rlib/objectmodel.py ============================================================================== --- pypy/dist/pypy/rlib/objectmodel.py (original) +++ pypy/dist/pypy/rlib/objectmodel.py Tue Dec 4 17:24:36 2007 @@ -201,65 +201,6 @@ raise TyperError("current_object_addr_as_int() cannot be applied to" " %r" % (vobj.concretetype,)) -# ____________________________________________________________ - -def debug_assert(x, msg): - """After translation to C, this becomes an RPyAssert.""" - assert x, msg - -class Entry(ExtRegistryEntry): - _about_ = debug_assert - - def compute_result_annotation(self, s_x, s_msg): - assert s_msg.is_constant(), ("debug_assert(x, msg): " - "the msg must be constant") - return None - - def specialize_call(self, hop): - from pypy.rpython.lltypesystem import lltype - vlist = hop.inputargs(lltype.Bool, lltype.Void) - hop.genop('debug_assert', vlist) - - -def debug_llinterpcall(RESTYPE, pythonfunction, *args): - """When running on the llinterp, this causes the llinterp to call to - the provided Python function with the run-time value of the given args. - The Python function should return a low-level object of type RESTYPE. - This should never be called after translation: use this only if - running_on_llinterp is true. - """ - raise NotImplementedError - -class Entry(ExtRegistryEntry): - _about_ = debug_llinterpcall - - def compute_result_annotation(self, s_RESTYPE, s_pythonfunction, *args_s): - from pypy.annotation import model as annmodel - from pypy.rpython.lltypesystem import lltype - assert s_RESTYPE.is_constant() - assert s_pythonfunction.is_constant() - s_result = s_RESTYPE.const - if isinstance(s_result, lltype.LowLevelType): - s_result = annmodel.lltype_to_annotation(s_result) - assert isinstance(s_result, annmodel.SomeObject) - return s_result - - def specialize_call(self, hop): - from pypy.annotation import model as annmodel - from pypy.rpython.lltypesystem import lltype - RESTYPE = hop.args_s[0].const - if not isinstance(RESTYPE, lltype.LowLevelType): - assert isinstance(RESTYPE, annmodel.SomeObject) - r_result = hop.rtyper.getrepr(RESTYPE) - RESTYPE = r_result.lowleveltype - pythonfunction = hop.args_s[1].const - c_pythonfunction = hop.inputconst(lltype.Void, pythonfunction) - args_v = [hop.inputarg(hop.args_r[i], arg=i) - for i in range(2, hop.nb_args)] - return hop.genop('debug_llinterpcall', [c_pythonfunction] + args_v, - resulttype=RESTYPE) - - def hlinvoke(repr, llcallable, *args): raise TypeError, "hlinvoke is meant to be rtyped and not called direclty" Modified: pypy/dist/pypy/rpython/extfunc.py ============================================================================== --- pypy/dist/pypy/rpython/extfunc.py (original) +++ pypy/dist/pypy/rpython/extfunc.py Tue Dec 4 17:24:36 2007 @@ -182,12 +182,12 @@ # If we have both an {ll,oo}impl and a {ll,oo}fakeimpl, # we need a wrapper that selects the proper one and calls it from pypy.rlib.objectmodel import running_on_llinterp - from pypy.rlib.objectmodel import debug_llinterpcall + from pypy.rlib.debug import llinterpcall from pypy.tool.sourcetools import func_with_new_name original_impl = impl def ll_wrapper(*args): if running_on_llinterp: - return debug_llinterpcall(s_result, fakeimpl, *args) + return llinterpcall(s_result, fakeimpl, *args) else: return original_impl(*args) impl = func_with_new_name(ll_wrapper, name + '_wrapper') Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llarena.py Tue Dec 4 17:24:36 2007 @@ -276,7 +276,7 @@ import os, sys from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.extfunc import register_external -from pypy.rlib.objectmodel import debug_assert, CDefinedIntSymbolic +from pypy.rlib.objectmodel import CDefinedIntSymbolic if os.name == 'posix': READ_MAX = (sys.maxint//4) + 1 # upper bound on reads to avoid surprises Modified: pypy/dist/pypy/rpython/lltypesystem/rlist.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rlist.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rlist.py Tue Dec 4 17:24:36 2007 @@ -16,8 +16,11 @@ Bool, nullptr, typeMethod from pypy.rpython.lltypesystem import rstr from pypy.rpython import robject -from pypy.rlib.objectmodel import debug_assert +from pypy.rlib.debug import ll_assert from pypy.rlib.rarithmetic import ovfcheck +from pypy.rpython.lltypesystem.llmemory import cast_ptr_to_adr, raw_memclear,\ + raw_memcopy, sizeof, itemoffsetof +from pypy.rpython.lltypesystem import rffi # ____________________________________________________________ # @@ -284,7 +287,7 @@ # system malloc(). # The growth pattern is: 0, 4, 8, 16, 25, 35, 46, 58, 72, 88, ... if newsize <= 0: - debug_assert(newsize == 0, "negative list length") + ll_assert(newsize == 0, "negative list length") new_allocated = 0 else: if newsize < 9: @@ -305,11 +308,11 @@ else: p = new_allocated - 1 while p >= 0: - newitems[p] = items[p] - ITEM = typeOf(l).TO.ITEM - if isinstance(ITEM, Ptr): - items[p] = nullptr(ITEM.TO) - p -= 1 + newitems[p] = items[p] + ITEM = typeOf(l).TO.ITEM + if isinstance(ITEM, Ptr): + items[p] = nullptr(ITEM.TO) + p -= 1 l.length = newsize l.items = newitems _ll_list_resize_really._annenforceargs_ = (None, int) @@ -356,7 +359,7 @@ # Accessor methods def ll_newlist(LIST, length): - debug_assert(length >= 0, "negative list length") + ll_assert(length >= 0, "negative list length") l = malloc(LIST) l.length = length l.items = malloc(LIST.items.TO, length) @@ -391,17 +394,17 @@ return l.items def ll_getitem_fast(l, index): - debug_assert(index < l.length, "getitem out of bounds") + ll_assert(index < l.length, "getitem out of bounds") return l.ll_items()[index] def ll_setitem_fast(l, index, item): - debug_assert(index < l.length, "setitem out of bounds") + ll_assert(index < l.length, "setitem out of bounds") l.ll_items()[index] = item # fixed size versions def ll_fixed_newlist(LIST, length): - debug_assert(length >= 0, "negative fixed list length") + ll_assert(length >= 0, "negative fixed list length") l = malloc(LIST, length) return l ll_fixed_newlist = typeMethod(ll_fixed_newlist) @@ -418,11 +421,11 @@ return l def ll_fixed_getitem_fast(l, index): - debug_assert(index < len(l), "fixed getitem out of bounds") + ll_assert(index < len(l), "fixed getitem out of bounds") return l[index] def ll_fixed_setitem_fast(l, index, item): - debug_assert(index < len(l), "fixed setitem out of bounds") + ll_assert(index < len(l), "fixed setitem out of bounds") l[index] = item def newlist(llops, r_list, items_v): Modified: pypy/dist/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rstr.py Tue Dec 4 17:24:36 2007 @@ -2,7 +2,7 @@ from pypy.tool.pairtype import pairtype from pypy.rpython.error import TyperError from pypy.rlib.objectmodel import malloc_zero_filled, we_are_translated -from pypy.rlib.objectmodel import debug_assert +from pypy.rlib.debug import ll_assert from pypy.rpython.robject import PyObjRepr, pyobj_repr from pypy.rlib.rarithmetic import _hash_string from pypy.rpython.rmodel import inputconst, IntegerRepr @@ -30,7 +30,7 @@ def new_malloc(TP): def mallocstr(length): - debug_assert(length >= 0, "negative string length") + ll_assert(length >= 0, "negative string length") r = malloc(TP, length) if not we_are_translated() or not malloc_zero_filled: r.hash = 0 @@ -228,8 +228,8 @@ def ll_stritem_nonneg(s, i): chars = s.chars - debug_assert(i>=0, "negative str getitem index") - debug_assert(i=0, "negative str getitem index") + ll_assert(i 0, "bogus weakref in compute_id()") # record this entry in the dict adr = llmemory.cast_ptr_to_adr(target) @@ -179,7 +179,7 @@ # not found wr = llmemory.weakref_create(ptr) if freeentry < 0: - debug_assert(end == len(lst), "unexpected lst growth in gc_id") + ll_assert(end == len(lst), "unexpected lst growth in gc_id") i = end lst.append(wr) else: Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Tue Dec 4 17:24:36 2007 @@ -3,7 +3,8 @@ GCFLAG_IMMORTAL from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage from pypy.rpython.lltypesystem import lltype, llmemory, llarena -from pypy.rlib.objectmodel import free_non_gc_object, debug_assert +from pypy.rlib.objectmodel import free_non_gc_object +from pypy.rlib.debug import ll_assert from pypy.rpython.lltypesystem.lloperation import llop # The following flag is never set on young objects, i.e. the ones living @@ -60,7 +61,7 @@ contains_weakptr=False): if (has_finalizer or not can_collect or raw_malloc_usage(size) >= self.nursery_size // 2): - debug_assert(not contains_weakptr, "wrong case for mallocing weakref") + ll_assert(not contains_weakptr, "wrong case for mallocing weakref") # "non-simple" case or object too big: don't use the nursery return SemiSpaceGC.malloc_fixedsize(self, typeid, size, can_collect, has_finalizer, @@ -174,7 +175,7 @@ if self.nursery_size > self.top_of_space - self.free: # the semispace is running out, do a full collect self.obtain_free_space(self.nursery_size) - debug_assert(self.nursery_size <= self.top_of_space - self.free, + ll_assert(self.nursery_size <= self.top_of_space - self.free, "obtain_free_space failed to do its job") if self.nursery: if DEBUG_PRINT: @@ -303,7 +304,7 @@ self.remember_young_pointer(addr_struct, newvalue) def remember_young_pointer(self, addr_struct, addr): - debug_assert(not self.is_in_nursery(addr_struct), + ll_assert(not self.is_in_nursery(addr_struct), "nursery object with GCFLAG_NO_YOUNG_PTRS") if self.is_in_nursery(addr): oldhdr = self.header(addr_struct) Modified: pypy/dist/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/dist/pypy/rpython/memory/gc/marksweep.py Tue Dec 4 17:24:36 2007 @@ -4,7 +4,7 @@ from pypy.rpython.memory.support import get_address_linked_list from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rlib.objectmodel import free_non_gc_object, debug_assert +from pypy.rlib.objectmodel import free_non_gc_object from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck from pypy.rpython.memory.gc.base import GCBase @@ -102,6 +102,7 @@ #llop.debug_print(lltype.Void, 'malloc typeid', typeid, # '->', llmemory.cast_adr_to_int(result)) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) + malloc_fixedsize.dont_inline = True def malloc_fixedsize_clear(self, typeid, size, can_collect, has_finalizer=False, contains_weakptr=False): @@ -135,6 +136,7 @@ #llop.debug_print(lltype.Void, 'malloc typeid', typeid, # '->', llmemory.cast_adr_to_int(result)) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) + malloc_fixedsize_clear.dont_inline = True def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, can_collect, has_finalizer=False): @@ -169,6 +171,7 @@ # 'typeid', typeid, # '->', llmemory.cast_adr_to_int(result)) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) + malloc_varsize.dont_inline = True def malloc_varsize_clear(self, typeid, length, size, itemsize, offset_to_length, can_collect, @@ -205,6 +208,7 @@ # 'typeid', typeid, # '->', llmemory.cast_adr_to_int(result)) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) + malloc_varsize_clear.dont_inline = True def collect(self): # 1. mark from the roots, and also the objects that objects-with-del Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Tue Dec 4 17:24:36 2007 @@ -4,7 +4,8 @@ from pypy.rpython.memory.support import get_address_linked_list from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rpython.lltypesystem import lltype, llmemory, llarena -from pypy.rlib.objectmodel import free_non_gc_object, debug_assert +from pypy.rlib.objectmodel import free_non_gc_object +from pypy.rlib.debug import ll_assert from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck from pypy.rpython.memory.gc.base import MovingGCBase @@ -37,10 +38,10 @@ def setup(self): self.tospace = llarena.arena_malloc(self.space_size, True) - debug_assert(bool(self.tospace), "couldn't allocate tospace") + ll_assert(bool(self.tospace), "couldn't allocate tospace") self.top_of_space = self.tospace + self.space_size self.fromspace = llarena.arena_malloc(self.space_size, True) - debug_assert(bool(self.fromspace), "couldn't allocate fromspace") + ll_assert(bool(self.fromspace), "couldn't allocate fromspace") self.free = self.tospace self.objects_with_finalizers = self.AddressLinkedList() self.run_finalizers = self.AddressLinkedList() @@ -134,7 +135,7 @@ while self.space_size < proposed_size: if not self.double_space_size(): return False - debug_assert(needed <= self.top_of_space - self.free, + ll_assert(needed <= self.top_of_space - self.free, "double_space_size() failed to do its job") return True @@ -166,7 +167,7 @@ # because doing arena_free(self.fromspace) would crash self.fromspace = self.tospace + self.space_size self.top_of_space = self.fromspace - debug_assert(self.free <= self.top_of_space, + ll_assert(self.free <= self.top_of_space, "unexpected growth of GC space usage during collect") return False # out of memory Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Tue Dec 4 17:24:36 2007 @@ -7,7 +7,7 @@ from pypy.rpython.memory.gc import marksweep from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rlib.rarithmetic import ovfcheck -from pypy.rlib.objectmodel import debug_assert +from pypy.rlib.debug import ll_assert from pypy.translator.backendopt import graphanalyze from pypy.translator.backendopt.support import var_needsgc from pypy.annotation import model as annmodel @@ -129,39 +129,39 @@ TYPE_INFO_TABLE = lltype.Array(TYPE_INFO) def q_is_varsize(typeid): - debug_assert(typeid > 0, "invalid type_id") + ll_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].isvarsize def q_finalizer(typeid): - debug_assert(typeid > 0, "invalid type_id") + ll_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].finalizer def q_offsets_to_gc_pointers(typeid): - debug_assert(typeid > 0, "invalid type_id") + ll_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].ofstoptrs def q_fixed_size(typeid): - debug_assert(typeid > 0, "invalid type_id") + ll_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].fixedsize def q_varsize_item_sizes(typeid): - debug_assert(typeid > 0, "invalid type_id") + ll_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].varitemsize def q_varsize_offset_to_variable_part(typeid): - debug_assert(typeid > 0, "invalid type_id") + ll_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].ofstovar def q_varsize_offset_to_length(typeid): - debug_assert(typeid > 0, "invalid type_id") + ll_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].ofstolength def q_varsize_offsets_to_gcpointers_in_var_part(typeid): - debug_assert(typeid > 0, "invalid type_id") + ll_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].varofstoptrs def q_weakpointer_offset(typeid): - debug_assert(typeid > 0, "invalid type_id") + ll_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].weakptrofs self.layoutbuilder = TransformerLayoutBuilder(self) @@ -389,7 +389,7 @@ _alloc_flavor_ = 'raw' def setup_root_stack(): stackbase = llmemory.raw_malloc(rootstacksize) - debug_assert(bool(stackbase), "could not allocate root stack") + ll_assert(bool(stackbase), "could not allocate root stack") llmemory.raw_memclear(stackbase, rootstacksize) gcdata.root_stack_top = stackbase gcdata.root_stack_base = stackbase Modified: pypy/dist/pypy/rpython/numpy/rarray.py ============================================================================== --- pypy/dist/pypy/rpython/numpy/rarray.py (original) +++ pypy/dist/pypy/rpython/numpy/rarray.py Tue Dec 4 17:24:36 2007 @@ -5,7 +5,7 @@ from pypy.tool.pairtype import pairtype, pair from pypy.rlib.unroll import unrolling_iterable from pypy.annotation import listdef -from pypy.rlib.objectmodel import debug_assert +from pypy.rlib.debug import ll_assert from pypy.rpython.annlowlevel import ADTInterface from pypy.rpython.memory.lltypelayout import sizeof from pypy.rpython.rmodel import Repr, FloatRepr, inputconst @@ -136,8 +136,8 @@ # Suffix of ao.shape must match target_ao.shape # (suffix starts at the first non-1 entry in ao.shape.) # ao.shape must be no longer than target_ao.shape. - debug_assert(ao.ndim <= ndim, "ao.ndim <= ndim") - debug_assert(target_ao.ndim == ndim, "target_ao.ndim == ndim") + ll_assert(ao.ndim <= ndim, "ao.ndim <= ndim") + ll_assert(target_ao.ndim == ndim, "target_ao.ndim == ndim") # XX check suffix condition here... ? broadcast = ao.ndim < ndim i = 0 @@ -147,7 +147,7 @@ i += 1 if broadcast: return iter_broadcast_to_shape(ITER, ao, target_ao) - debug_assert(ao.ndim == ndim, "ao.ndim == ndim") + ll_assert(ao.ndim == ndim, "ao.ndim == ndim") it = malloc(ITER) it.nd_m1 = ndim - 1 it.size = ll_mul_list(ao.shape, ndim) @@ -165,7 +165,7 @@ def ll_iter_broadcast_to_shape(ITER, ao, target_ao): "iterate over but broadcast to the shape of " - debug_assert(target_ao.ndim == ndim, "target_ao.ndim == ndim") + ll_assert(target_ao.ndim == ndim, "target_ao.ndim == ndim") delta = j = ndim - ao.ndim shape = target_ao.shape for i in range(ao.ndim): @@ -198,7 +198,7 @@ def ll_array_set(ITEM, it0, it1): if it0.size == 0: return # empty LHS.. - debug_assert(it0.size == it1.size, "it0.size == it1.size") + ll_assert(it0.size == it1.size, "it0.size == it1.size") while it0.index < it0.size: it0.dataptr[0] = cast_primitive(ITEM, it1.dataptr[0]) it0.ll_next() @@ -365,8 +365,8 @@ #______________________________________________________________________________ def ll_array_binop(it0, it1, it2, binop): - debug_assert(it0.size == it1.size, "it0.size == it1.size") - debug_assert(it1.size == it2.size, "it0.size == it1.size") + ll_assert(it0.size == it1.size, "it0.size == it1.size") + ll_assert(it1.size == it2.size, "it0.size == it1.size") while it0.index < it0.size: # We don't need a cast here, because it0.dataptr[0] is always # big enough to contain the result. @@ -423,7 +423,7 @@ #______________________________________________________________________________ def ll_array_inplace_binop(ITEM, it0, it1, binop): - debug_assert(it0.size == it1.size, "it0.size == it1.size") + ll_assert(it0.size == it1.size, "it0.size == it1.size") while it0.index < it0.size: it0.dataptr[0] = cast_primitive(ITEM, binop(it0.dataptr[0], it1.dataptr[0])) it0.ll_next() @@ -538,7 +538,7 @@ array.strides[tgt_i] = ao.strides[src_i] tgt_i += 1 src_i += 1 - debug_assert(tgt_i == ndim, "tgt_i == ndim") + ll_assert(tgt_i == ndim, "tgt_i == ndim") array.dataptr = dataptr array.data = ao.data # keep a ref return array Modified: pypy/dist/pypy/rpython/rlist.py ============================================================================== --- pypy/dist/pypy/rpython/rlist.py (original) +++ pypy/dist/pypy/rpython/rlist.py Tue Dec 4 17:24:36 2007 @@ -8,7 +8,8 @@ from pypy.rpython.lltypesystem.lltype import typeOf, Ptr, Void, Signed, Bool from pypy.rpython.lltypesystem.lltype import nullptr, Char, UniChar from pypy.rpython import robject -from pypy.rlib.objectmodel import malloc_zero_filled, debug_assert +from pypy.rlib.objectmodel import malloc_zero_filled +from pypy.rlib.debug import ll_assert from pypy.rlib.rarithmetic import ovfcheck from pypy.rpython.annlowlevel import ADTInterface @@ -582,8 +583,8 @@ def ll_insert_nonneg(l, index, newitem): length = l.ll_length() - debug_assert(0 <= index, "negative list insertion index") - debug_assert(index <= length, "list insertion index out of bound") + ll_assert(0 <= index, "negative list insertion index") + ll_assert(index <= length, "list insertion index out of bound") l._ll_resize_ge(length+1) # see "a note about overflows" above dst = length while dst > index: @@ -594,12 +595,12 @@ ll_insert_nonneg.oopspec = 'list.insert(l, index, newitem)' def ll_pop_nonneg(func, l, index): - debug_assert(index >= 0, "unexpectedly negative list pop index") + ll_assert(index >= 0, "unexpectedly negative list pop index") if func is dum_checkidx: if index >= l.ll_length(): raise IndexError else: - debug_assert(index < l.ll_length(), "list pop index out of bound") + ll_assert(index < l.ll_length(), "list pop index out of bound") res = l.ll_getitem_fast(index) ll_delitem_nonneg(dum_nocheck, l, index) return res @@ -609,7 +610,7 @@ length = l.ll_length() if func is dum_checkidx and (length == 0): raise IndexError - debug_assert(length > 0, "pop from empty list") + ll_assert(length > 0, "pop from empty list") index = length - 1 newlength = index res = l.ll_getitem_fast(index) @@ -624,7 +625,7 @@ length = l.ll_length() if func is dum_checkidx and (length == 0): raise IndexError - debug_assert(length > 0, "pop(0) from empty list") + ll_assert(length > 0, "pop(0) from empty list") newlength = length - 1 res = l.ll_getitem_fast(0) j = 0 @@ -648,8 +649,8 @@ if index < 0 or index >= length: raise IndexError else: - debug_assert(index >= 0, "negative list pop index out of bound") - debug_assert(index < length, "list pop index out of bound") + ll_assert(index >= 0, "negative list pop index out of bound") + ll_assert(index < length, "list pop index out of bound") res = l.ll_getitem_fast(index) ll_delitem_nonneg(dum_nocheck, l, index) return res @@ -668,12 +669,12 @@ ll_reverse.oopspec = 'list.reverse(l)' def ll_getitem_nonneg(func, l, index): - debug_assert(index >= 0, "unexpectedly negative list getitem index") + ll_assert(index >= 0, "unexpectedly negative list getitem index") if func is dum_checkidx: if index >= l.ll_length(): raise IndexError else: - debug_assert(index < l.ll_length(), "list getitem index out of bound") + ll_assert(index < l.ll_length(), "list getitem index out of bound") return l.ll_getitem_fast(index) ll_getitem_nonneg.oopspec = 'list.getitem(l, index)' ll_getitem_nonneg.oopargcheck = lambda l, index: (bool(l) and @@ -687,20 +688,20 @@ if index < 0 or index >= length: raise IndexError else: - debug_assert(index >= 0, "negative list getitem index out of bound") - debug_assert(index < length, "list getitem index out of bound") + ll_assert(index >= 0, "negative list getitem index out of bound") + ll_assert(index < length, "list getitem index out of bound") return l.ll_getitem_fast(index) ll_getitem.oopspec = 'list.getitem(l, index)' ll_getitem.oopargcheck = lambda l, index: (bool(l) and -l.ll_length() <= index < l.ll_length()) def ll_setitem_nonneg(func, l, index, newitem): - debug_assert(index >= 0, "unexpectedly negative list setitem index") + ll_assert(index >= 0, "unexpectedly negative list setitem index") if func is dum_checkidx: if index >= l.ll_length(): raise IndexError else: - debug_assert(index < l.ll_length(), "list setitem index out of bound") + ll_assert(index < l.ll_length(), "list setitem index out of bound") l.ll_setitem_fast(index, newitem) ll_setitem_nonneg.oopspec = 'list.setitem(l, index, newitem)' @@ -712,19 +713,19 @@ if index < 0 or index >= length: raise IndexError else: - debug_assert(index >= 0, "negative list setitem index out of bound") - debug_assert(index < length, "list setitem index out of bound") + ll_assert(index >= 0, "negative list setitem index out of bound") + ll_assert(index < length, "list setitem index out of bound") l.ll_setitem_fast(index, newitem) ll_setitem.oopspec = 'list.setitem(l, index, newitem)' def ll_delitem_nonneg(func, l, index): - debug_assert(index >= 0, "unexpectedly negative list delitem index") + ll_assert(index >= 0, "unexpectedly negative list delitem index") length = l.ll_length() if func is dum_checkidx: if index >= length: raise IndexError else: - debug_assert(index < length, "list delitem index out of bound") + ll_assert(index < length, "list delitem index out of bound") newlength = length - 1 j = index j1 = j+1 @@ -747,8 +748,8 @@ if i < 0 or i >= length: raise IndexError else: - debug_assert(i >= 0, "negative list delitem index out of bound") - debug_assert(i < length, "list delitem index out of bound") + ll_assert(i >= 0, "negative list delitem index out of bound") + ll_assert(i < length, "list delitem index out of bound") ll_delitem_nonneg(dum_nocheck, l, i) ll_delitem.oopspec = 'list.delitem(l, i)' @@ -775,7 +776,7 @@ len1 = lst.ll_length() len2 = getstrlen(s) count2 = len2 - start - debug_assert(start >= 0, "unexpectedly negative str slice start") + ll_assert(start >= 0, "unexpectedly negative str slice start") assert count2 >= 0, "str slice start larger than str length" try: newlength = ovfcheck(len1 + count2) @@ -797,8 +798,8 @@ stop = slice.stop len1 = lst.ll_length() len2 = getstrlen(s) - debug_assert(start >= 0, "unexpectedly negative str slice start") - debug_assert(start <= len2, "str slice start larger than str length") + ll_assert(start >= 0, "unexpectedly negative str slice start") + ll_assert(start <= len2, "str slice start larger than str length") if stop > len2: stop = len2 count2 = stop - start @@ -855,8 +856,8 @@ def ll_listslice_startonly(RESLIST, l1, start): len1 = l1.ll_length() - debug_assert(start >= 0, "unexpectedly negative list slice start") - debug_assert(start <= len1, "list slice start larger than list length") + ll_assert(start >= 0, "unexpectedly negative list slice start") + ll_assert(start <= len1, "list slice start larger than list length") newlength = len1 - start l = RESLIST.ll_newlist(newlength) j = 0 @@ -871,9 +872,9 @@ start = slice.start stop = slice.stop length = l1.ll_length() - debug_assert(start >= 0, "unexpectedly negative list slice start") - debug_assert(start <= length, "list slice start larger than list length") - debug_assert(stop >= start, "list slice stop smaller than start") + ll_assert(start >= 0, "unexpectedly negative list slice start") + ll_assert(start <= length, "list slice start larger than list length") + ll_assert(stop >= start, "list slice stop smaller than start") if stop > length: stop = length newlength = stop - start @@ -888,7 +889,7 @@ def ll_listslice_minusone(RESLIST, l1): newlength = l1.ll_length() - 1 - debug_assert(newlength >= 0, "empty list is sliced with [:-1]") + ll_assert(newlength >= 0, "empty list is sliced with [:-1]") l = RESLIST.ll_newlist(newlength) j = 0 while j < newlength: @@ -897,8 +898,8 @@ return l def ll_listdelslice_startonly(l, start): - debug_assert(start >= 0, "del l[start:] with unexpectedly negative start") - debug_assert(start <= l.ll_length(), "del l[start:] with start > len(l)") + ll_assert(start >= 0, "del l[start:] with unexpectedly negative start") + ll_assert(start <= l.ll_length(), "del l[start:] with start > len(l)") newlength = start null = ll_null_item(l) if null is not None: @@ -912,9 +913,9 @@ start = slice.start stop = slice.stop length = l.ll_length() - debug_assert(start >= 0, "del l[start:x] with unexpectedly negative start") - debug_assert(start <= length, "del l[start:x] with start > len(l)") - debug_assert(stop >= start, "del l[x:y] with x > y") + ll_assert(start >= 0, "del l[start:x] with unexpectedly negative start") + ll_assert(start <= length, "del l[start:x] with start > len(l)") + ll_assert(stop >= start, "del l[x:y] with x > y") if stop > length: stop = length newlength = length - (stop-start) @@ -935,9 +936,9 @@ def ll_listsetslice(l1, slice, l2): count = l2.ll_length() start = slice.start - debug_assert(start >= 0, "l[start:x] = l with unexpectedly negative start") - debug_assert(start <= l1.ll_length(), "l[start:x] = l with start > len(l)") - debug_assert(count == slice.stop - start, + ll_assert(start >= 0, "l[start:x] = l with unexpectedly negative start") + ll_assert(start <= l1.ll_length(), "l[start:x] = l with start > len(l)") + ll_assert(count == slice.stop - start, "setslice cannot resize lists in RPython") # XXX but it should be easy enough to support, soon j = start Modified: pypy/dist/pypy/rpython/test/test_rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rbuiltin.py (original) +++ pypy/dist/pypy/rpython/test/test_rbuiltin.py Tue Dec 4 17:24:36 2007 @@ -1,7 +1,8 @@ from pypy.translator.translator import graphof from pypy.rpython.test import test_llinterp from pypy.rlib.objectmodel import instantiate, we_are_translated -from pypy.rlib.objectmodel import running_on_llinterp, debug_llinterpcall +from pypy.rlib.objectmodel import running_on_llinterp +from pypy.rlib.debug import llinterpcall from pypy.rpython.lltypesystem import lltype from pypy.tool import udir from pypy.rlib.rarithmetic import r_uint, intmask, r_longlong @@ -422,7 +423,7 @@ return s def fn(n): if running_on_llinterp: - return debug_llinterpcall(SPTR, foo, n).m + return llinterpcall(SPTR, foo, n).m else: return 321 res = self.interpret(fn, [7]) From cfbolz at codespeak.net Tue Dec 4 17:38:00 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 4 Dec 2007 17:38:00 +0100 (CET) Subject: [pypy-svn] r49346 - pypy/dist/pypy/rlib Message-ID: <20071204163800.BB27A8189@code0.codespeak.net> Author: cfbolz Date: Tue Dec 4 17:38:00 2007 New Revision: 49346 Modified: pypy/dist/pypy/rlib/rope.py Log: trying to give the searching of single char strings an extremely good best case: O(log(n)) if the char is there and O(1) if the char is not there. Worst case is still O(n), of course. Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Tue Dec 4 17:38:00 2007 @@ -683,24 +683,36 @@ length = node.length() if stop == -1: stop = length + if start != 0 or stop != length: + newstart, newstop, node = find_straddling(node, start, stop) + offset = start - newstart + start = newstart + stop = newstop assert 0 <= start <= stop if isinstance(node, LiteralNode): pos = node.find_int(what, start, stop) if pos == -1: return pos return pos + offset - iter = FringeIterator(node) - newstart = iter._seekforward(start) - offset += start - newstart - start = newstart + if not node.can_contain_int(what): + return -1 + # invariant: stack should only contain nodes that can contain the int what + stack = [node] #import pdb; pdb.set_trace() i = 0 - while i < stop: - try: - fringenode = iter.next() - except StopIteration: - return -1 - nodelength = fringenode.length() + while stack: + curr = stack.pop() + while isinstance(curr, BinaryConcatNode): + if curr.left.can_contain_int(what): + if curr.right.can_contain_int(what): + stack.append(curr.right) + curr = curr.left + else: + i += curr.left.length() + # if left cannot contain what, then right must contain it + curr = curr.right + nodelength = curr.length() + fringenode = curr if i + nodelength <= start: i += nodelength continue From simonb at codespeak.net Tue Dec 4 17:57:22 2007 From: simonb at codespeak.net (simonb at codespeak.net) Date: Tue, 4 Dec 2007 17:57:22 +0100 (CET) Subject: [pypy-svn] r49347 - pypy/dist/pypy/doc Message-ID: <20071204165722.517488198@code0.codespeak.net> Author: simonb Date: Tue Dec 4 17:57:22 2007 New Revision: 49347 Modified: pypy/dist/pypy/doc/standalone-howto.txt Log: hack off the part describing rctypes Modified: pypy/dist/pypy/doc/standalone-howto.txt ============================================================================== --- pypy/dist/pypy/doc/standalone-howto.txt (original) +++ pypy/dist/pypy/doc/standalone-howto.txt Tue Dec 4 17:57:22 2007 @@ -9,8 +9,6 @@ way to compile it is not as described below, but by writing a target file as described in the `FAQ entries`_.) -**Warning:** the bits of this howto that describe rctypes are outdated - ======================================== First, see the note above. @@ -101,33 +99,9 @@ Now we can sum with different kinds of lists, eg. ``sum([1,2,3])`` and ``sum([1.0,2.0,3.0])``. -Here is an example of using rctypes:: - - import ctypes - from ctypes import c_int, c_char_p - import pypy.rpython.rctypes.implementation - - libc = ctypes.cdll.LoadLibrary('libc.so.6') - - write = libc.write - write.argtypes = [c_int, c_char_p, c_int] - write.restype = c_int - - def main(argv): - - msg = "Hello world!\n" - write(1, msg, len(msg)) - - return 0 - -PyPy implements the `ctypes semantics`_ as the corresponding native c-code -equivalents. - .. _`FAQ Entries`: faq.html#pypy-translation-tool-chain .. _`RPython`: coding-guide.html#restricted-python .. _`Annotator`: dynamic-language-translation.html#annotator .. _`RTyper`: dynamic-language-translation.html#rtyper -.. _`rctypes`: rctypes.html -.. _`ctypes semantics`: http://docs.python.org/dev/lib/module-ctypes.html From cfbolz at codespeak.net Tue Dec 4 18:03:28 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 4 Dec 2007 18:03:28 +0100 (CET) Subject: [pypy-svn] r49348 - pypy/dist/pypy/rlib/test Message-ID: <20071204170328.9D9B28198@code0.codespeak.net> Author: cfbolz Date: Tue Dec 4 18:03:28 2007 New Revision: 49348 Modified: pypy/dist/pypy/rlib/test/test_rope.py Log: seems some more testing of string finding is needed Modified: pypy/dist/pypy/rlib/test/test_rope.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rope.py (original) +++ pypy/dist/pypy/rlib/test/test_rope.py Tue Dec 4 18:03:28 2007 @@ -432,6 +432,20 @@ pos = find(node, LiteralStringNode("btf"), 0, 3) assert pos == 0 +def test_find_random(): + py.test.skip("fix me!") + rope, st = make_random_string(unicode=True) + rope = getslice_one(rope, 10, 10000) + st = st[10:10000] + for i in range(1000): + searchlength = random.randrange(2, min(len(st) - 1, 1001)) + start = random.randrange(len(st) - searchlength) + searchstart = random.randrange(len(st)) + searchstop = random.randrange(searchstart, len(st)) + p = st[start:start+searchlength] + rp = getslice_one(rope, start, start + searchlength) + pos = find(rope, rp, searchstart, searchstop) + assert pos == st.find(p, searchstart, searchstop) def test_find_unicode(): node = BinaryConcatNode(LiteralUnicodeNode(u"\uaaaa\ubbbb\uaaaa"), From arigo at codespeak.net Tue Dec 4 18:12:48 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 4 Dec 2007 18:12:48 +0100 (CET) Subject: [pypy-svn] r49349 - in pypy/dist/pypy: interpreter module/__builtin__ Message-ID: <20071204171248.ECDFA81A0@code0.codespeak.net> Author: arigo Date: Tue Dec 4 18:12:48 2007 New Revision: 49349 Modified: pypy/dist/pypy/interpreter/function.py pypy/dist/pypy/interpreter/typedef.py pypy/dist/pypy/module/__builtin__/__init__.py pypy/dist/pypy/module/__builtin__/descriptor.py Log: Move ClassMethod together with StaticMethod into function.py, where it is more easily importable. It should be possible to just use it to create TypeDefs with classmethods. Modified: pypy/dist/pypy/interpreter/function.py ============================================================================== --- pypy/dist/pypy/interpreter/function.py (original) +++ pypy/dist/pypy/interpreter/function.py Tue Dec 4 18:12:48 2007 @@ -451,8 +451,7 @@ return space.newtuple([new_inst, space.newtuple(tup)]) class StaticMethod(Wrappable): - """A static method. Note that there is one class staticmethod at - app-level too currently; this is only used for __new__ methods.""" + """The staticmethod objects.""" def __init__(self, w_function): self.w_function = w_function @@ -464,6 +463,24 @@ def descr_staticmethod__new__(space, w_type, w_function): return space.wrap(StaticMethod(w_function)) +class ClassMethod(Wrappable): + """The classmethod objects.""" + + def __init__(self, w_function): + self.w_function = w_function + + def descr_classmethod_get(self, space, w_obj, w_klass=None): + if space.is_w(w_klass, space.w_None): + w_klass = space.type(w_obj) + return space.wrap(Method(space, self.w_function, w_klass, space.w_None)) + + def descr_classmethod__new__(space, w_type, w_function): + if not space.is_true(space.callable(w_function)): + typename = space.type(w_function).getname(space, '?') + raise OperationError(space.w_TypeError, space.wrap( + "'%s' object is not callable" % typename)) + return space.wrap(ClassMethod(w_function)) + class BuiltinFunction(Function): def __init__(self, func): Modified: pypy/dist/pypy/interpreter/typedef.py ============================================================================== --- pypy/dist/pypy/interpreter/typedef.py (original) +++ pypy/dist/pypy/interpreter/typedef.py Tue Dec 4 18:12:48 2007 @@ -482,6 +482,7 @@ from pypy.interpreter.pyopcode import SuspendedUnroller from pypy.interpreter.module import Module from pypy.interpreter.function import Function, Method, StaticMethod +from pypy.interpreter.function import ClassMethod from pypy.interpreter.function import BuiltinFunction, descr_function_get from pypy.interpreter.pytraceback import PyTraceback from pypy.interpreter.generator import GeneratorIterator @@ -696,6 +697,30 @@ unwrap_spec = [ObjSpace, W_Root, W_Root]), ) +ClassMethod.typedef = TypeDef( + 'classmethod', + __new__ = interp2app(ClassMethod.descr_classmethod__new__.im_func, + unwrap_spec = [ObjSpace, W_Root, W_Root]), + __get__ = interp2app(ClassMethod.descr_classmethod_get, + unwrap_spec = ['self', ObjSpace, W_Root, W_Root]), + __doc__ = """classmethod(function) -> class method + +Convert a function to be a class method. + +A class method receives the class as implicit first argument, +just like an instance method receives the instance. +To declare a class method, use this idiom: + + class C: + def f(cls, arg1, arg2, ...): ... + f = classmethod(f) + +It can be called either on the class (e.g. C.f()) or on an instance +(e.g. C().f()). The instance is ignored except for its class. +If a class method is called for a derived class, the derived class +object is passed as the implied first argument.""", +) + def always_none(self, obj): return None BuiltinFunction.typedef = TypeDef("builtin_function",**Function.typedef.rawdict) Modified: pypy/dist/pypy/module/__builtin__/__init__.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/__init__.py (original) +++ pypy/dist/pypy/module/__builtin__/__init__.py Tue Dec 4 18:12:48 2007 @@ -124,7 +124,7 @@ 'any' : 'functional.any', 'super' : 'descriptor.W_Super', 'staticmethod' : 'descriptor.StaticMethod', - 'classmethod' : 'descriptor.W_ClassMethod', + 'classmethod' : 'descriptor.ClassMethod', 'property' : 'descriptor.W_Property', } Modified: pypy/dist/pypy/module/__builtin__/descriptor.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/descriptor.py (original) +++ pypy/dist/pypy/module/__builtin__/descriptor.py Tue Dec 4 18:12:48 2007 @@ -5,7 +5,7 @@ from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError from pypy.interpreter.callmethod import object_getattribute -from pypy.interpreter.function import StaticMethod, Method +from pypy.interpreter.function import StaticMethod, ClassMethod from pypy.interpreter.typedef import GetSetProperty, descr_get_dict, \ descr_set_dict, interp_attrproperty_w @@ -92,46 +92,6 @@ super(C, self).meth(arg)""" ) -class W_ClassMethod(Wrappable): - def __init__(self, w_function): - self.w_function = w_function - - def new(space, w_type, w_function): - if not space.is_true(space.callable(w_function)): - name = space.getattr(space.type(w_function), space.wrap('__name__')) - raise OperationError(space.w_TypeError, space.wrap( - "'%s' object is not callable" % name)) - return W_ClassMethod(w_function) - - def get(self, space, w_obj, w_klass=None): - if space.is_w(w_klass, space.w_None): - w_klass = space.type(w_obj) - return space.wrap(Method(space, self.w_function, w_klass, space.w_None)) - -W_ClassMethod.typedef = TypeDef( - 'classmethod', - __new__ = interp2app(W_ClassMethod.new.im_func, - unwrap_spec=[ObjSpace, W_Root, W_Root]), - __get__ = interp2app(W_ClassMethod.get, - unwrap_spec=['self', ObjSpace, W_Root, W_Root]), - __doc__ = """classmethod(function) -> class method - -Convert a function to be a class method. - -A class method receives the class as implicit first argument, -just like an instance method receives the instance. -To declare a class method, use this idiom: - - class C: - def f(cls, arg1, arg2, ...): ... - f = classmethod(f) - -It can be called either on the class (e.g. C.f()) or on an instance -(e.g. C().f()). The instance is ignored except for its class. -If a class method is called for a derived class, the derived class -object is passed as the implied first argument.""", -) - class W_Property(Wrappable): def __init__(self, space, w_fget, w_fset, w_fdel, w_doc): self.w_fget = w_fget From cfbolz at codespeak.net Tue Dec 4 23:25:14 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 4 Dec 2007 23:25:14 +0100 (CET) Subject: [pypy-svn] r49350 - pypy/dist/pypy/rlib/parsing Message-ID: <20071204222514.4123C811F@codespeak.net> Author: cfbolz Date: Tue Dec 4 23:25:12 2007 New Revision: 49350 Modified: pypy/dist/pypy/rlib/parsing/tree.py Log: prevent double dict lookup in the tree visitor dispatch Modified: pypy/dist/pypy/rlib/parsing/tree.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/tree.py (original) +++ pypy/dist/pypy/rlib/parsing/tree.py Tue Dec 4 23:25:12 2007 @@ -88,17 +88,19 @@ **dispatch_table): def dispatch(self, node): if isinstance(node, Nonterminal): - if node.symbol not in dispatch_table: + func = dispatch_table.get(node.symbol, None) + if func is None: if __general_nonterminal_visit: return __general_nonterminal_visit(self, node) else: - return dispatch_table[node.symbol](self, node) + return func(self, node) elif isinstance(node, Symbol): - if node.symbol not in dispatch_table: + func = dispatch_table.get(node.symbol, None) + if func is None: if __general_symbol_visit: return __general_symbol_visit(self, node) else: - return dispatch_table[node.symbol](self, node) + return func(self, node) if __general_visit: return __general_visit(self, node) raise VisitError(node) From arigo at codespeak.net Wed Dec 5 08:01:00 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 08:01:00 +0100 (CET) Subject: [pypy-svn] r49358 - in pypy/dist/pypy/rpython/memory: gc test Message-ID: <20071205070100.5DF6C810C@codespeak.net> Author: arigo Date: Wed Dec 5 08:00:58 2007 New Revision: 49358 Modified: pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: Ouuuuups! Most var-sized objects were never allocated from the nursery at all. This is the minimal fix with a test. Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Wed Dec 5 08:00:58 2007 @@ -125,6 +125,9 @@ itemsize, offset_to_length, True, False) + malloc_fixedsize_clear = malloc_fixedsize + malloc_varsize_clear = malloc_varsize + # override the init_gc_object methods to change the default value of 'flags', # used by objects that are directly created outside the nursery by the SemiSpaceGC. # These objects must have the GCFLAG_NO_YOUNG_PTRS flag set immediately. Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Wed Dec 5 08:00:58 2007 @@ -9,6 +9,7 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.memory.gc.marksweep import X_CLONE, X_POOL, X_POOL_PTR from pypy.rlib.objectmodel import compute_unique_id +from pypy.rlib.debug import ll_assert from pypy import conftest INT_SIZE = struct.calcsize("i") # only for estimates @@ -861,3 +862,37 @@ run = self.runner(f, nbargs=0) run([]) + +class TestGenerationalNoFullCollectGC(GCTest): + # test that nursery is doing its job and that no full collection + # is needed when most allocated objects die quickly + + gcname = "generation" + + class gcpolicy(gc.FrameworkGcPolicy): + class transformerclass(framework.FrameworkGCTransformer): + from pypy.rpython.memory.gc.generation import GenerationGC + class GCClass(GenerationGC): + def semispace_collect(self, size_changing=False): + ll_assert(False, + "no full collect should occur in this test") + GC_PARAMS = {'space_size': 2048, + 'nursery_size': 512} + root_stack_depth = 200 + + def test_working_nursery(self): + def f(): + total = 0 + i = 0 + while i < 40: + lst = [] + j = 0 + while j < 5: + lst.append(i*j) + j += 1 + total += len(lst) + i += 1 + return total + run = self.runner(f, nbargs=0) + res = run([]) + assert res == 40 * 5 From arigo at codespeak.net Wed Dec 5 08:30:44 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 08:30:44 +0100 (CET) Subject: [pypy-svn] r49359 - in pypy/dist/pypy/rpython/memory: . gc gctransform Message-ID: <20071205073044.63B76811A@codespeak.net> Author: arigo Date: Wed Dec 5 08:30:44 2007 New Revision: 49359 Modified: pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/gc/semispace.py pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gcwrapper.py Log: Arguably a clean-up: this removes the need for the error-prone lines that were missing in GenerationGC (see previous check-in). It adds fallback rules in case methods are not defined at all, as documented in GCBase.malloc(). Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Wed Dec 5 08:30:44 2007 @@ -39,8 +39,14 @@ def malloc(self, typeid, length=0, zero=False, coallocator=None): """For testing. The interface used by the gctransformer is the four malloc_[fixed,var]size[_clear]() functions. - And (if they exist) to the coalloc_[fixed,var]size functios + And (if they exist) to the coalloc_[fixed,var]size_clear functions """ + # Rules about fallbacks in case of missing malloc methods: + # * malloc_fixedsize_clear() and malloc_varsize_clear() are mandatory + # * malloc_fixedsize() and malloc_varsize() fallback to the above + # * coalloc_fixedsize_clear() and coalloc_varsize_clear() are optional + # There is no non-clear version of coalloc for now. + size = self.fixed_size(typeid) needs_finalizer = bool(self.getfinalizer(typeid)) weakptr_offset = self.weakpointer_offset(typeid) @@ -56,28 +62,31 @@ assert not contains_weakptr itemsize = self.varsize_item_sizes(typeid) offset_to_length = self.varsize_offset_to_length(typeid) - if zero: - malloc_varsize = self.malloc_varsize_clear - else: - malloc_varsize = self.malloc_varsize - if coallocator is not None and hasattr(self, "coalloc_varsize"): + if (coallocator is not None and + hasattr(self, "coalloc_varsize_clear")): assert not needs_finalizer coallocator = llmemory.cast_ptr_to_adr(coallocator) - ref = self.coalloc_varsize(coallocator, typeid, length, size, - itemsize, offset_to_length) - else: + ref = self.coalloc_varsize_clear(coallocator, typeid, + length, size, + itemsize, offset_to_length) + else: + if zero or not hasattr(self, 'malloc_varsize'): + malloc_varsize = self.malloc_varsize_clear + else: + malloc_varsize = self.malloc_varsize ref = malloc_varsize(typeid, length, size, itemsize, offset_to_length, True, needs_finalizer) else: - if zero: - malloc_fixedsize = self.malloc_fixedsize_clear - else: - malloc_fixedsize = self.malloc_fixedsize - if coallocator is not None and hasattr(self, "coalloc_fixedsize"): + if (coallocator is not None and + hasattr(self, "coalloc_fixedsize_clear")): assert not needs_finalizer coallocator = llmemory.cast_ptr_to_adr(coallocator) - ref = self.coalloc_fixedsize(coallocator, typeid, size) + ref = self.coalloc_fixedsize_clear(coallocator, typeid, size) else: + if zero or not hasattr(self, 'malloc_fixedsize'): + malloc_fixedsize = self.malloc_fixedsize_clear + else: + malloc_fixedsize = self.malloc_fixedsize ref = malloc_fixedsize(typeid, size, True, needs_finalizer, contains_weakptr) # lots of cast and reverse-cast around... Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Wed Dec 5 08:30:44 2007 @@ -57,15 +57,16 @@ def is_in_nursery(self, addr): return self.nursery <= addr < self.nursery_top - def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, - contains_weakptr=False): + def malloc_fixedsize_clear(self, typeid, size, can_collect, + has_finalizer=False, contains_weakptr=False): if (has_finalizer or not can_collect or raw_malloc_usage(size) >= self.nursery_size // 2): ll_assert(not contains_weakptr, "wrong case for mallocing weakref") # "non-simple" case or object too big: don't use the nursery - return SemiSpaceGC.malloc_fixedsize(self, typeid, size, - can_collect, has_finalizer, - contains_weakptr) + return SemiSpaceGC.malloc_fixedsize_clear(self, typeid, size, + can_collect, + has_finalizer, + contains_weakptr) size_gc_header = self.gcheaderbuilder.size_gc_header totalsize = size_gc_header + size result = self.nursery_free @@ -79,27 +80,29 @@ self.young_objects_with_weakrefs.append(result + size_gc_header) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) - def coalloc_fixedsize(self, coallocator, typeid, size): + def coalloc_fixedsize_clear(self, coallocator, typeid, size): # note: a coallocated object can never return a weakref, since the # coallocation analysis is done at a time where weakrefs are # represented as opaque objects which aren't allocated using malloc but # with weakref_create if self.is_in_nursery(coallocator): - return self.malloc_fixedsize(typeid, size, True, False, False) + return self.malloc_fixedsize_clear(typeid, size, + True, False, False) else: - return SemiSpaceGC.malloc_fixedsize(self, typeid, size, True, - False, False) + return SemiSpaceGC.malloc_fixedsize_clear(self, typeid, size, + True, False, False) - def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, - can_collect, has_finalizer=False): + def malloc_varsize_clear(self, typeid, length, size, itemsize, + offset_to_length, can_collect, + has_finalizer=False): # only use the nursery if there are not too many items if (has_finalizer or not can_collect or (raw_malloc_usage(itemsize) and length > self.nursery_size // 4 // raw_malloc_usage(itemsize)) or raw_malloc_usage(size) > self.nursery_size // 4): - return SemiSpaceGC.malloc_varsize(self, typeid, length, size, - itemsize, offset_to_length, - can_collect, has_finalizer) + return SemiSpaceGC.malloc_varsize_clear(self, typeid, length, size, + itemsize, offset_to_length, + can_collect, has_finalizer) # with the above checks we know now that totalsize cannot be more # than about half of the nursery size; in particular, the + and * # cannot overflow @@ -115,18 +118,16 @@ self.nursery_free = result + llarena.round_up_for_allocation(totalsize) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) - def coalloc_varsize(self, coallocator, typeid, length, size, itemsize, - offset_to_length): + def coalloc_varsize_clear(self, coallocator, typeid, + length, size, itemsize, + offset_to_length): if self.is_in_nursery(coallocator): - return self.malloc_varsize(typeid, length, size, itemsize, - offset_to_length, True, False) + return self.malloc_varsize_clear(typeid, length, size, itemsize, + offset_to_length, True, False) else: - return SemiSpaceGC.malloc_varsize(self, typeid, length, size, - itemsize, offset_to_length, - True, False) - - malloc_fixedsize_clear = malloc_fixedsize - malloc_varsize_clear = malloc_varsize + return SemiSpaceGC.malloc_varsize_clear(self, typeid, length, size, + itemsize, offset_to_length, + True, False) # override the init_gc_object methods to change the default value of 'flags', # used by objects that are directly created outside the nursery by the SemiSpaceGC. Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Wed Dec 5 08:30:44 2007 @@ -57,8 +57,11 @@ if self.run_finalizers.non_empty(): self.execute_finalizers() - def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, - contains_weakptr=False): + # This class only defines the malloc_{fixed,var}size_clear() methods + # because the spaces are filled with zeroes in advance. + + def malloc_fixedsize_clear(self, typeid, size, can_collect, + has_finalizer=False, contains_weakptr=False): size_gc_header = self.gcheaderbuilder.size_gc_header totalsize = size_gc_header + size result = self.free @@ -75,8 +78,9 @@ self.objects_with_weakrefs.append(result + size_gc_header) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) - def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, - can_collect, has_finalizer=False): + def malloc_varsize_clear(self, typeid, length, size, itemsize, + offset_to_length, can_collect, + has_finalizer=False): size_gc_header = self.gcheaderbuilder.size_gc_header nonvarsize = size_gc_header + size try: @@ -97,13 +101,9 @@ self.objects_with_finalizers.append(result + size_gc_header) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) - # for now, the spaces are filled with zeroes in advance - malloc_fixedsize_clear = malloc_fixedsize - malloc_varsize_clear = malloc_varsize - def obtain_free_space(self, needed): # a bit of tweaking to maximize the performance and minimize the - # amount of code in an inlined version of malloc_fixedsize() + # amount of code in an inlined version of malloc_fixedsize_clear() if not self.try_obtain_free_space(needed): raise memoryError return self.free Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Wed Dec 5 08:30:44 2007 @@ -264,20 +264,27 @@ classdef = bk.getuniqueclassdef(GCClass) s_gc = annmodel.SomeInstance(classdef) s_gcref = annmodel.SomePtr(llmemory.GCREF) - self.malloc_fixedsize_ptr = getfn( - GCClass.malloc_fixedsize.im_func, - [s_gc, annmodel.SomeInteger(nonneg=True), - annmodel.SomeInteger(nonneg=True), - annmodel.SomeBool(), annmodel.SomeBool(), - annmodel.SomeBool()], s_gcref, - inline = False) + + malloc_fixedsize_clear_meth = GCClass.malloc_fixedsize_clear.im_func self.malloc_fixedsize_clear_ptr = getfn( - GCClass.malloc_fixedsize_clear.im_func, + malloc_fixedsize_clear_meth, [s_gc, annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True), annmodel.SomeBool(), annmodel.SomeBool(), annmodel.SomeBool()], s_gcref, inline = False) + if hasattr(GCClass, 'malloc_fixedsize'): + malloc_fixedsize_meth = GCClass.malloc_fixedsize.im_func + self.malloc_fixedsize_ptr = getfn( + malloc_fixedsize_meth, + [s_gc, annmodel.SomeInteger(nonneg=True), + annmodel.SomeInteger(nonneg=True), + annmodel.SomeBool(), annmodel.SomeBool(), + annmodel.SomeBool()], s_gcref, + inline = False) + else: + malloc_fixedsize_meth = None + self.malloc_fixedsize_ptr = self.malloc_fixedsize_clear_ptr ## self.malloc_varsize_ptr = getfn( ## GCClass.malloc_varsize.im_func, ## [s_gc] + [annmodel.SomeInteger(nonneg=True) for i in range(5)] @@ -294,8 +301,14 @@ if getattr(GCClass, 'inline_simple_malloc', False): # make a copy of this function so that it gets annotated # independently and the constants are folded inside + if malloc_fixedsize_meth is None: + malloc_fast_meth = malloc_fixedsize_clear_meth + self.malloc_fast_is_clearing = True + else: + malloc_fast_meth = malloc_fixedsize_meth + self.malloc_fast_is_clearing = False malloc_fast = func_with_new_name( - GCClass.malloc_fixedsize.im_func, + malloc_fast_meth, "malloc_fast") s_False = annmodel.SomeBool(); s_False.const = False s_True = annmodel.SomeBool(); s_True .const = True @@ -307,7 +320,7 @@ s_False], s_gcref, inline = True) else: - self.malloc_fast_ptr = self.malloc_fixedsize_ptr + self.malloc_fast_ptr = None if GCClass.moving_gc: self.id_ptr = getfn(GCClass.id.im_func, @@ -326,15 +339,15 @@ inline=True) else: self.write_barrier_ptr = None - if hasattr(GCClass, "coalloc_fixedsize"): + if hasattr(GCClass, "coalloc_fixedsize_clear"): self.coalloc_clear_ptr = getfn( - GCClass.coalloc_fixedsize.im_func, + GCClass.coalloc_fixedsize_clear.im_func, [s_gc, annmodel.SomeAddress(), annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True)], s_gcref, inline=True) self.coalloc_varsize_clear_ptr = getfn( - GCClass.coalloc_varsize.im_func, + GCClass.coalloc_varsize_clear.im_func, [s_gc, annmodel.SomeAddress()] + [annmodel.SomeInteger(nonneg=True) for i in range(5)], s_gcref, inline=True) @@ -552,10 +565,12 @@ if not op.opname.endswith('_varsize'): #malloc_ptr = self.malloc_fixedsize_ptr zero = flags.get('zero', False) - if zero: - malloc_ptr = self.malloc_fixedsize_clear_ptr - elif c_can_collect.value and not c_has_finalizer.value: + if (self.malloc_fast_ptr is not None and + c_can_collect.value and not c_has_finalizer.value and + (self.malloc_fast_is_clearing or not zero)): malloc_ptr = self.malloc_fast_ptr + elif zero: + malloc_ptr = self.malloc_fixedsize_clear_ptr else: malloc_ptr = self.malloc_fixedsize_ptr args = [self.c_const_gc, c_type_id, c_size, c_can_collect, Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Wed Dec 5 08:30:44 2007 @@ -63,7 +63,7 @@ return lltype.free(TYPE, flavor=flavor) def coalloc(self, TYPE, coallocator, size=None, zero=False): - if hasattr(self.gc, "coalloc_fixedsize"): + if hasattr(self.gc, "coalloc_fixedsize_clear"): typeid = self.get_type_id(TYPE) addr = self.gc.malloc(typeid, size, zero=zero, coallocator=coallocator) From arigo at codespeak.net Wed Dec 5 08:40:32 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 08:40:32 +0100 (CET) Subject: [pypy-svn] r49361 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20071205074032.4CAC18115@codespeak.net> Author: arigo Date: Wed Dec 5 08:40:31 2007 New Revision: 49361 Modified: pypy/dist/pypy/rpython/memory/gc/base.py Log: Also document this. Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Wed Dec 5 08:40:31 2007 @@ -46,6 +46,8 @@ # * malloc_fixedsize() and malloc_varsize() fallback to the above # * coalloc_fixedsize_clear() and coalloc_varsize_clear() are optional # There is no non-clear version of coalloc for now. + # XXX: as of r49360, gctransformer.framework never inserts calls + # to malloc_varsize(), but always uses malloc_varsize_clear() size = self.fixed_size(typeid) needs_finalizer = bool(self.getfinalizer(typeid)) From arigo at codespeak.net Wed Dec 5 09:41:47 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 09:41:47 +0100 (CET) Subject: [pypy-svn] r49364 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071205084147.AA6A2811E@codespeak.net> Author: arigo Date: Wed Dec 5 09:41:46 2007 New Revision: 49364 Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py Log: Fix for a shallow failure in test_weakref_across_minor_collection. Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py Wed Dec 5 09:41:46 2007 @@ -580,6 +580,8 @@ return '<%s>' % (self,) def __str__(self): return 'gctransformed_wref(%s)' % (self._ptr,) + def _normalizedcontainer(self): + return self._ptr._obj # ____________________________________________________________ From fijal at codespeak.net Wed Dec 5 10:35:17 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 5 Dec 2007 10:35:17 +0100 (CET) Subject: [pypy-svn] r49368 - pypy/dist/pypy/rlib Message-ID: <20071205093517.812D98126@codespeak.net> Author: fijal Date: Wed Dec 5 10:35:15 2007 New Revision: 49368 Modified: pypy/dist/pypy/rlib/debug.py Log: New operation, rlib.debug.check_assertion, which will perform a translation- time check if annotations are as we expect. Modified: pypy/dist/pypy/rlib/debug.py ============================================================================== --- pypy/dist/pypy/rlib/debug.py (original) +++ pypy/dist/pypy/rlib/debug.py Wed Dec 5 10:35:15 2007 @@ -57,3 +57,28 @@ return hop.genop('debug_llinterpcall', [c_pythonfunction] + args_v, resulttype=RESTYPE) + +def check_annotation(arg, checker): + """ Function checking if annotation is as expected when translating, + does nothing when just run. Checker is supposed to be a constant + callable which checks if annotation is as expected, + arguments passed are (current annotation, bookkeeper) + """ + pass + +class Entry(ExtRegistryEntry): + _about_ = check_annotation + + def compute_result_annotation(self, s_arg, s_checker): + if not s_checker.is_constant(): + raise ValueError("Second argument of check_annotation must be constant") + checker = s_checker.const + checker(s_arg, self.bookkeeper) + from pypy.annotation import model + return s_arg + + def specialize_call(self, hop): + hop.exception_cannot_occur() + vlist = [hop.inputarg(hop.args_r[0], arg=0)] + return hop.genop("same_as", vlist, resulttype=hop.r_result) + From arigo at codespeak.net Wed Dec 5 10:53:22 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 10:53:22 +0100 (CET) Subject: [pypy-svn] r49369 - pypy/dist/pypy/rlib Message-ID: <20071205095322.45CE6813B@codespeak.net> Author: arigo Date: Wed Dec 5 10:53:20 2007 New Revision: 49369 Modified: pypy/dist/pypy/rlib/debug.py Log: Simplify. Modified: pypy/dist/pypy/rlib/debug.py ============================================================================== --- pypy/dist/pypy/rlib/debug.py (original) +++ pypy/dist/pypy/rlib/debug.py Wed Dec 5 10:53:20 2007 @@ -74,11 +74,9 @@ raise ValueError("Second argument of check_annotation must be constant") checker = s_checker.const checker(s_arg, self.bookkeeper) - from pypy.annotation import model return s_arg def specialize_call(self, hop): hop.exception_cannot_occur() - vlist = [hop.inputarg(hop.args_r[0], arg=0)] - return hop.genop("same_as", vlist, resulttype=hop.r_result) + return hop.inputarg(hop.args_r[0], arg=0) From fijal at codespeak.net Wed Dec 5 12:57:05 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 5 Dec 2007 12:57:05 +0100 (CET) Subject: [pypy-svn] r49374 - pypy/dist/pypy/rlib/test Message-ID: <20071205115705.9D93C1683EC@codespeak.net> Author: fijal Date: Wed Dec 5 12:57:04 2007 New Revision: 49374 Added: pypy/dist/pypy/rlib/test/test_debug.py (contents, props changed) Log: Forgotten test. Added: pypy/dist/pypy/rlib/test/test_debug.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rlib/test/test_debug.py Wed Dec 5 12:57:04 2007 @@ -0,0 +1,29 @@ + +import py +from pypy.rlib.debug import check_annotation +from pypy.rpython.test.test_llinterp import interpret + +def test_check_annotation(): + class Error(Exception): + pass + + def checker(ann, bk): + from pypy.annotation.model import SomeList, SomeInteger + if not isinstance(ann, SomeList): + raise Error() + if not isinstance(ann.listdef.listitem.s_value, SomeInteger): + raise Error() + + def f(x): + result = [x] + check_annotation(result, checker) + return result + + interpret(f, [3]) + + def g(x): + check_annotation(x, checker) + return x + + py.test.raises(Error, "interpret(g, [3])") + From fijal at codespeak.net Wed Dec 5 13:04:29 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 5 Dec 2007 13:04:29 +0100 (CET) Subject: [pypy-svn] r49375 - pypy/dist/pypy/rpython/test Message-ID: <20071205120429.B902816844F@codespeak.net> Author: fijal Date: Wed Dec 5 13:04:29 2007 New Revision: 49375 Modified: pypy/dist/pypy/rpython/test/test_llinterp.py Log: Allow the very same hack for unicode as for string Modified: pypy/dist/pypy/rpython/test/test_llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_llinterp.py (original) +++ pypy/dist/pypy/rpython/test/test_llinterp.py Wed Dec 5 13:04:29 2007 @@ -86,6 +86,8 @@ return object elif T == Ptr(rstr.STR): return str + elif T == Ptr(rstr.UNICODE): + return unicode else: return lltype_to_annotation(T) From fijal at codespeak.net Wed Dec 5 13:15:48 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 5 Dec 2007 13:15:48 +0100 (CET) Subject: [pypy-svn] r49376 - in pypy/dist/pypy/rpython: . module ootypesystem test Message-ID: <20071205121548.C802616846F@codespeak.net> Author: fijal Date: Wed Dec 5 13:15:48 2007 New Revision: 49376 Modified: pypy/dist/pypy/rpython/module/support.py pypy/dist/pypy/rpython/ootypesystem/ootype.py pypy/dist/pypy/rpython/rlist.py pypy/dist/pypy/rpython/test/test_llinterp.py pypy/dist/pypy/rpython/test/test_rstr.py pypy/dist/pypy/rpython/test/test_runicode.py pypy/dist/pypy/rpython/test/tool.py Log: Add a few hacks to be able to test one-line diff to support inplace_add of list of unichars and unicode string. Modified: pypy/dist/pypy/rpython/module/support.py ============================================================================== --- pypy/dist/pypy/rpython/module/support.py (original) +++ pypy/dist/pypy/rpython/module/support.py Wed Dec 5 13:15:48 2007 @@ -56,6 +56,10 @@ def to_rstr(s): return ootype.oostring(s, -1) to_rstr = staticmethod(to_rstr) + + def to_runicode(u): + return ootype.oounicode(u, -1) + to_runicode = staticmethod(to_runicode) def from_rstr(rs): if not rs: # null pointer Modified: pypy/dist/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/ootype.py Wed Dec 5 13:15:48 2007 @@ -1571,7 +1571,6 @@ """ assert base == -1 if isinstance(obj, unicode): - assert len(obj) == 1 return make_unicode(obj) elif isinstance(obj, _string): s = unicode(obj._str) Modified: pypy/dist/pypy/rpython/rlist.py ============================================================================== --- pypy/dist/pypy/rpython/rlist.py (original) +++ pypy/dist/pypy/rpython/rlist.py Wed Dec 5 13:15:48 2007 @@ -359,7 +359,7 @@ if r_lst1.item_repr.lowleveltype not in (Char, UniChar): raise TyperError('"lst += string" only supported with a list ' 'of chars or unichars') - string_repr = r_lst1.rtyper.type_system.rstr.string_repr + string_repr = r_str2.repr v_lst1, v_str2 = hop.inputargs(r_lst1, string_repr) c_strlen = hop.inputconst(Void, string_repr.ll.ll_strlen) c_stritem = hop.inputconst(Void, string_repr.ll.ll_stritem_nonneg) Modified: pypy/dist/pypy/rpython/test/test_llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_llinterp.py (original) +++ pypy/dist/pypy/rpython/test/test_llinterp.py Wed Dec 5 13:15:48 2007 @@ -11,6 +11,7 @@ from pypy.annotation import model as annmodel from pypy.annotation.model import lltype_to_annotation from pypy.rlib.rarithmetic import r_uint, ovfcheck +from pypy.rpython.ootypesystem import ootype from pypy import conftest # switch on logging of interp to show more info on failing tests @@ -84,9 +85,9 @@ T = typeOf(x) if T == Ptr(PyObject) and someobjects: return object - elif T == Ptr(rstr.STR): + elif T == Ptr(rstr.STR) or T == ootype.String: return str - elif T == Ptr(rstr.UNICODE): + elif T == Ptr(rstr.UNICODE) or T == ootype.Unicode: return unicode else: return lltype_to_annotation(T) Modified: pypy/dist/pypy/rpython/test/test_rstr.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rstr.py (original) +++ pypy/dist/pypy/rpython/test/test_rstr.py Wed Dec 5 13:15:48 2007 @@ -779,6 +779,18 @@ if conftest.option.view: t.view() assert summary(fgraph) == {} + + def test_inplace_add(self): + const = self.const + def f(x, y): + if x > 0: + l = [const('a'), const('b')] + else: + l = [const('a')] + l += y + return const('').join(l) + + assert self.ll_to_string(self.interpret(f, [1, self.string_to_ll(const('abc'))])) == 'ababc' def FIXME_test_str_to_pystringobj(): Modified: pypy/dist/pypy/rpython/test/test_runicode.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_runicode.py (original) +++ pypy/dist/pypy/rpython/test/test_runicode.py Wed Dec 5 13:15:48 2007 @@ -215,6 +215,19 @@ py.test.skip("We should think how to solve this problem") test_rfind_empty_string = test_find_empty_string + + def test_inplace_add(self): + const = self.const + def f(x, y): + if x > 0: + l = [const('a'), const('b')] + else: + l = [const('a')] + l += y + return const('').join(l) + + assert self.ll_to_unicode(self.interpret(f, [1, self.unicode_to_ll(const('abc'))])) == 'ababc' + class TestLLtype(BaseTestRUnicode, LLRtypeMixin): EMPTY_STRING_HASH = -1 Modified: pypy/dist/pypy/rpython/test/tool.py ============================================================================== --- pypy/dist/pypy/rpython/test/tool.py (original) +++ pypy/dist/pypy/rpython/test/tool.py Wed Dec 5 13:15:48 2007 @@ -94,6 +94,10 @@ from pypy.rpython.module.support import OOSupport return OOSupport.to_rstr(s) + def unicode_to_ll(self, u): + from pypy.rpython.module.support import OOSupport + return OOSupport.to_runicode(u) + def ll_to_list(self, l): return l._list[:] From fijal at codespeak.net Wed Dec 5 13:53:04 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 5 Dec 2007 13:53:04 +0100 (CET) Subject: [pypy-svn] r49378 - pypy/dist/pypy/module/marshal Message-ID: <20071205125304.BA60F168474@codespeak.net> Author: fijal Date: Wed Dec 5 13:53:02 2007 New Revision: 49378 Modified: pypy/dist/pypy/module/marshal/interp_marshal.py Log: Mute warning Modified: pypy/dist/pypy/module/marshal/interp_marshal.py ============================================================================== --- pypy/dist/pypy/module/marshal/interp_marshal.py (original) +++ pypy/dist/pypy/module/marshal/interp_marshal.py Wed Dec 5 13:53:02 2007 @@ -69,6 +69,8 @@ def finished(self): pass + def read(self, n): + raise NotImplementedError("Purely abstract method") class FileWriter(AbstractReaderWriter): def __init__(self, space, w_f): From fijal at codespeak.net Wed Dec 5 13:53:20 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 5 Dec 2007 13:53:20 +0100 (CET) Subject: [pypy-svn] r49379 - pypy/dist/pypy/module/marshal Message-ID: <20071205125320.E411E168474@codespeak.net> Author: fijal Date: Wed Dec 5 13:53:20 2007 New Revision: 49379 Modified: pypy/dist/pypy/module/marshal/interp_marshal.py Log: Mute another warning Modified: pypy/dist/pypy/module/marshal/interp_marshal.py ============================================================================== --- pypy/dist/pypy/module/marshal/interp_marshal.py (original) +++ pypy/dist/pypy/module/marshal/interp_marshal.py Wed Dec 5 13:53:20 2007 @@ -72,6 +72,9 @@ def read(self, n): raise NotImplementedError("Purely abstract method") + def write(self, data): + raise NotImplementedError("Purely abstract method") + class FileWriter(AbstractReaderWriter): def __init__(self, space, w_f): AbstractReaderWriter.__init__(self, space) From fijal at codespeak.net Wed Dec 5 15:03:04 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 5 Dec 2007 15:03:04 +0100 (CET) Subject: [pypy-svn] r49388 - pypy/dist/pypy/objspace/std Message-ID: <20071205140304.6A50116845E@codespeak.net> Author: fijal Date: Wed Dec 5 15:03:03 2007 New Revision: 49388 Modified: pypy/dist/pypy/objspace/std/formatting.py Log: * Make formatting faster, by keeping the list of chars/unichars instead of list of strings * Make assertion whether it's true or not, translation will explode if someone will try to mess with that Modified: pypy/dist/pypy/objspace/std/formatting.py ============================================================================== --- pypy/dist/pypy/objspace/std/formatting.py (original) +++ pypy/dist/pypy/objspace/std/formatting.py Wed Dec 5 15:03:03 2007 @@ -5,7 +5,7 @@ from pypy.rlib.rarithmetic import ovfcheck, formatd_overflow, isnan, isinf from pypy.interpreter.error import OperationError from pypy.tool.sourcetools import func_with_new_name - +from pypy.rlib.debug import check_annotation class BaseStringFormatter(object): def __init__(self, space, values_w, w_valuedict): @@ -267,9 +267,9 @@ break i += 1 else: - result.append(const(fmt[i0:])) + result += const(fmt[i0:]) break # end of 'fmt' string - result.append(const(fmt[i0:i])) + result += const(fmt[i0:i]) self.fmtpos = i + 1 # interpret the next formatter @@ -277,7 +277,7 @@ c = self.peekchr() self.forward() if c == '%': - self.std_wp('%') + self.std_wp(const('%')) continue if w_value is None: w_value = self.nextinputvalue() @@ -319,19 +319,19 @@ prec = self.prec if prec == -1 and self.width == 0: # fast path - self.result.append(const(r)) + self.result += const(r) return if prec >= 0 and prec < length: length = prec # ignore the end of the string if too long result = self.result padding = self.width - length if not self.f_ljust and padding > 0: - result.append(const(' ' * padding)) + result += const(' ') * padding # add any padding at the left of 'r' padding = 0 - result.append(const(r[:length])) # add 'r' itself + result += const(r[:length]) # add 'r' itself if padding > 0: - result.append(const(' ' * padding)) + result += const(' ') * padding # add any remaining padding at the right std_wp._annspecialcase_ = 'specialize:argtype(1)' @@ -359,15 +359,15 @@ padnumber = '>' if padnumber == '>': - result.append(const(' ' * padding)) # pad with spaces on the left + result += const(' ') * padding # pad with spaces on the left if sign: - result.append(const(r[0])) # the sign - result.append(const(prefix)) # the prefix + result += const(r[0]) # the sign + result += const(prefix) # the prefix if padnumber == '0': - result.append(const('0' * padding)) # pad with zeroes - result.append(const(r[int(sign):])) # the rest of the number + result += const('0') * padding # pad with zeroes + result += const(r[int(sign):]) # the rest of the number if padnumber == '<': # spaces on the right - result.append(const(' ' * padding)) + result += const(' ') * padding def fmt_s(self, w_value): space = self.space @@ -437,6 +437,11 @@ [_name[-1] for _name in dir(StringFormatter) if len(_name) == 5 and _name.startswith('fmt_')]) +def is_list_of_chars_or_unichars(ann, bk): + from pypy.annotation.model import SomeChar, SomeUnicodeCodePoint + if not isinstance(ann.listdef.listitem.s_value, + (SomeChar, SomeUnicodeCodePoint)): + raise TypeError("Formatter should return as a result a list of chars or unichars, otherwise we miss important optimization") def format(space, w_fmt, values_w, w_valuedict=None, do_unicode=False): "Entry point" @@ -449,11 +454,13 @@ # fall through to the unicode case fmt = unicode(fmt) else: + check_annotation(result, is_list_of_chars_or_unichars) return space.wrap(''.join(result)) else: fmt = space.unicode_w(w_fmt) formatter = UnicodeFormatter(space, fmt, values_w, w_valuedict) result = formatter.format() + check_annotation(result, is_list_of_chars_or_unichars) return space.wrap(u''.join(result)) def mod_format(space, w_format, w_values, do_unicode=False): From fijal at codespeak.net Wed Dec 5 15:52:56 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 5 Dec 2007 15:52:56 +0100 (CET) Subject: [pypy-svn] r49389 - in pypy/dist/pypy/rlib: . test Message-ID: <20071205145256.B5884168467@codespeak.net> Author: fijal Date: Wed Dec 5 15:52:55 2007 New Revision: 49389 Modified: pypy/dist/pypy/rlib/objectmodel.py pypy/dist/pypy/rlib/test/test_objectmodel.py Log: A special hint that allows one to specify expected size of a list. Modified: pypy/dist/pypy/rlib/objectmodel.py ============================================================================== --- pypy/dist/pypy/rlib/objectmodel.py (original) +++ pypy/dist/pypy/rlib/objectmodel.py Wed Dec 5 15:52:55 2007 @@ -350,3 +350,24 @@ self.key = key self.hash = hash +# ------------------------- optimization hints ------------------------------ + +def newlist(sizehint=0): + return [] + +class Entry(ExtRegistryEntry): + _about_ = newlist + + def compute_result_annotation(self, s_sizehint): + return self.bookkeeper.newlist() + + def specialize_call(self, orig_hop, i_sizehint): + from pypy.rpython.rlist import rtype_newlist + from pypy.rpython.lltypesystem import lltype + # fish a bit hop + hop = orig_hop.copy() + v = hop.args_v[0] + r, s = hop.r_s_popfirstarg() + if s.is_constant(): + v = hop.inputconst(r, s.const) + return rtype_newlist(hop, v_sizehint=v) Modified: pypy/dist/pypy/rlib/test/test_objectmodel.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_objectmodel.py (original) +++ pypy/dist/pypy/rlib/test/test_objectmodel.py Wed Dec 5 15:52:55 2007 @@ -2,6 +2,7 @@ from pypy.rlib.objectmodel import * from pypy.translator.translator import TranslationContext, graphof from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin +from pypy.conftest import option def strange_key_eq(key1, key2): return key1[0] == key2[0] # only the 1st character is relevant @@ -302,3 +303,46 @@ specialize.arg(1)(f) assert f._annspecialcase_ == 'specialize:arg(1)' + +def getgraph(f, argtypes): + from pypy.translator.translator import TranslationContext, graphof + from pypy.translator.backendopt.all import backend_optimizations + t = TranslationContext() + a = t.buildannotator() + typer = t.buildrtyper() + a.build_types(f, argtypes) + typer.specialize() + backend_optimizations(t) + graph = graphof(t, f) + if option.view: + graph.show() + return graph + +def test_newlist(): + from pypy.annotation.model import SomeInteger + def f(z): + x = newlist(sizehint=38) + if z < 0: + x.append(1) + return len(x) + + graph = getgraph(f, [SomeInteger()]) + for llop in graph.startblock.operations: + if llop.opname == 'malloc_varsize': + break + assert llop.args[2].value == 38 + +def test_newlist_nonconst(): + from pypy.annotation.model import SomeInteger + from pypy.objspace.flow.model import Variable + def f(z): + x = newlist(sizehint=z) + return len(x) + + graph = getgraph(f, [SomeInteger()]) + for llop in graph.startblock.operations: + if llop.opname == 'malloc_varsize': + break + assert llop.args[2] is graph.startblock.inputargs[0] + + From antocuni at codespeak.net Wed Dec 5 17:45:44 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 5 Dec 2007 17:45:44 +0100 (CET) Subject: [pypy-svn] r49392 - in pypy/branch/clr-module-improvements/pypy/module/clr: . test Message-ID: <20071205164544.28E6A16846E@codespeak.net> Author: antocuni Date: Wed Dec 5 17:45:43 2007 New Revision: 49392 Added: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_interp_clr.py (contents, props changed) Modified: pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py Log: now we can iterate over .NET objects that implement IEnumerable Modified: pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py Wed Dec 5 17:45:43 2007 @@ -16,8 +16,8 @@ 'load_cli_class': 'interp_clr.load_cli_class', } - def setup_after_space_initialization(self): - self.space.appexec([self], """(clr_module): - import sys - sys.meta_path.append(clr_module.dotnetimporter()) - """) +## def setup_after_space_initialization(self): +## self.space.appexec([self], """(clr_module): +## import sys +## sys.meta_path.append(clr_module.dotnetimporter()) +## """) Modified: pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py Wed Dec 5 17:45:43 2007 @@ -9,7 +9,6 @@ def __call__(self, *args): import clr - # now call the .NET static method call_staticmethod in interp_cl return clr.call_staticmethod(self.class_name, self.meth_name, args) def __repr__(self): @@ -67,8 +66,9 @@ return self.im_self.__cliobj__.call_method(self.im_name, args) def __repr__(self): - return '' % (self.im_self.__class__.__cliclass__, self.im_name, self.im_self) - + return '' % (self.im_self.__class__.__cliclass__, + self.im_name, + self.im_self) class StaticProperty(object): def __init__(self, fget=None, fset=None): @@ -86,31 +86,34 @@ else: type.__setattr__(cls, name, value) -#class Dummy(object): -# def __init__(self, iterObj): -# self.iterObj = iterObj -# self.index = 0 -# -# def next(self): -# temp = self.index -# if self.index == self.iterObj.Count: -# raise StopIteration -# self.index = self.index + 1 -# return self.iterObj.__getitem__(temp) - - class CliClassWrapper(object): __slots__ = ('__cliobj__',) def __init__(self, *args): import clr self.__cliobj__ = clr._CliObject_internal(self.__cliclass__, args) - print self.__cliobj__ - print self.Count -# def __iter__(self): -# return Dummy(self) -# return Dummy(self.Count, self.__getitem__(self.Count - 1)) + +class IEnumeratorWrapper(object): + def __init__(self, enumerator): + self.__enumerator__ = enumerator + + def __iter__(self): + return self + + def next(self): + if not self.__enumerator__.MoveNext(): + raise StopIteration + return self.__enumerator__.Current + +# this method need to be attached only to classes that implements IEnumerable (see build_wrapper) +def __iter__(self): + return IEnumeratorWrapper(self.GetEnumerator()) + +def wrapper_from_cliobj(cls, cliobj): + obj = cls.__new__(cls) + obj.__cliobj__ = cliobj + return obj def build_wrapper(namespace, classname, staticmethods, methods, properties, indexers): fullname = '%s.%s' % (namespace, classname) @@ -121,20 +124,17 @@ for name in methods: d[name] = MethodWrapper(name) - # check if there is GetEnumerator() method + # check if there is GetEnumerator() method + # XXX: use .NET Reflection for this, checking for method name is not safe for method in methods: if method == "GetEnumerator": - print "Enumerator found .. Hurray !!!!!" # now add the __iter__ method to the class - d['__iter__'] = d['GetEnumerator'] -# d['next'] = d['MoveNext'] - + d['__iter__'] = __iter__ assert len(indexers) <= 1 if indexers: name, getter, setter, is_static = indexers[0] assert not is_static - print " Indexers Tuple -----> (%s,%s,%s,%s) "%(name, getter, setter, is_static) if getter: d['__getitem__'] = d[getter] if setter: @@ -144,8 +144,6 @@ # we must add properties *after* the class has been created # because we need to store UnboundMethods as getters and setters for (name, getter, setter, is_static) in properties: - print " Properties Tuple -----> (%s,%s,%s,%s) "%(name, getter, setter, is_static) - fget = None fset = None if getter: @@ -159,8 +157,3 @@ setattr(cls, name, prop) return cls - - - - - Modified: pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py Wed Dec 5 17:45:43 2007 @@ -109,8 +109,16 @@ strval = unbox(b_obj, ootype.String) return space.wrap(strval) else: - msg = "Can't convert object %s to Python" % str(b_obj.ToString()) - raise OperationError(space.w_TypeError, space.wrap(msg)) + namespace, classname = split_fullname(b_type.ToString()) + w_cls = load_cli_class(space, namespace, classname) + cliobj = W_CliObject(space, b_obj) + return wrapper_from_cliobj(space, w_cls, cliobj) + +def split_fullname(name): + lastdot = name.rfind('.') + if lastdot == -1: + return '', name + return name[:lastdot], name[lastdot+1:] def wrap_list_of_tuples(space, lst): list_w = [] @@ -188,9 +196,6 @@ - classname: the name of the class in the specified namespace (e.g. ``ArrayList``). """ - #import sys - #for module in sys.modules: - # print "mod ----> %s"%module fullname = '%s.%s' % (namespace, classname) w_cls = CliClassCache.get(fullname) if w_cls is None: @@ -246,3 +251,4 @@ app = ApplevelClass(file(app_clr).read()) del path, app_clr build_wrapper = app.interphook("build_wrapper") +wrapper_from_cliobj = app.interphook("wrapper_from_cliobj") Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py Wed Dec 5 17:45:43 2007 @@ -153,12 +153,19 @@ import clr ArrayList = clr.load_cli_class('System.Collections', 'ArrayList') x = ArrayList() + enum = x.GetEnumerator() + assert enum.MoveNext() is False + + def test_iteratrion(self): + import clr + ArrayList = clr.load_cli_class('System.Collections', 'ArrayList') + x = ArrayList() x.Add(1) - x.Add(6) - x.Add(31) x.Add(2) + x.Add(3) + x.Add(4) + sum = 0 for i in x: - print i - - + sum += i + assert sum == 1+2+3+4 Added: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_interp_clr.py ============================================================================== --- (empty file) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_interp_clr.py Wed Dec 5 17:45:43 2007 @@ -0,0 +1,9 @@ +from pypy.module.clr.interp_clr import split_fullname + +def test_split_fullname(): + split = split_fullname + assert split('Foo') == ('', 'Foo') + assert split('System.Foo') == ('System', 'Foo') + assert split('System.Foo.Bar') == ('System.Foo', 'Bar') + assert split('System.Foo.A+B') == ('System.Foo', 'A+B') + assert split('System.') == ('System', '') From fijal at codespeak.net Wed Dec 5 19:10:21 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 5 Dec 2007 19:10:21 +0100 (CET) Subject: [pypy-svn] r49393 - in pypy/dist/pypy/rpython: . lltypesystem ootypesystem test Message-ID: <20071205181021.60527168469@codespeak.net> Author: fijal Date: Wed Dec 5 19:10:20 2007 New Revision: 49393 Modified: pypy/dist/pypy/rpython/lltypesystem/rlist.py pypy/dist/pypy/rpython/ootypesystem/rlist.py pypy/dist/pypy/rpython/rlist.py pypy/dist/pypy/rpython/test/test_rlist.py Log: Support for newlist(sizehint=xxx), with a test! Modified: pypy/dist/pypy/rpython/lltypesystem/rlist.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rlist.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rlist.py Wed Dec 5 19:10:20 2007 @@ -188,6 +188,7 @@ ("items", Ptr(ITEMARRAY)), adtmeths = ADTIList({ "ll_newlist": ll_newlist, + "ll_newlist_hint": ll_newlist_hint, "ll_newemptylist": ll_newemptylist, "ll_length": ll_length, "ll_items": ll_items, @@ -367,6 +368,15 @@ ll_newlist = typeMethod(ll_newlist) ll_newlist.oopspec = 'newlist(length)' +def ll_newlist_hint(LIST, lengthhint): + ll_assert(lengthhint >= 0, "negative list length") + l = malloc(LIST) + l.length = 0 + l.items = malloc(LIST.items.TO, lengthhint) + return l +ll_newlist_hint = typeMethod(ll_newlist_hint) +ll_newlist_hint.oopspec = 'newlist(length)' + # should empty lists start with no allocated memory, or with a preallocated # minimal number of entries? XXX compare memory usage versus speed, and # check how many always-empty lists there are in a typical pypy-c run... @@ -428,10 +438,13 @@ ll_assert(index < len(l), "fixed setitem out of bounds") l[index] = item -def newlist(llops, r_list, items_v): +def newlist(llops, r_list, items_v, v_sizehint=None): LIST = r_list.LIST if len(items_v) == 0: - v_result = llops.gendirectcall(LIST.ll_newemptylist) + if v_sizehint is None: + v_result = llops.gendirectcall(LIST.ll_newemptylist) + else: + v_result = llops.gendirectcall(LIST.ll_newlist_hint, v_sizehint) else: cno = inputconst(Signed, len(items_v)) v_result = llops.gendirectcall(LIST.ll_newlist, cno) Modified: pypy/dist/pypy/rpython/ootypesystem/rlist.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rlist.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rlist.py Wed Dec 5 19:10:20 2007 @@ -85,7 +85,8 @@ -def newlist(llops, r_list, items_v): +def newlist(llops, r_list, items_v, v_sizehint=None): + # XXX do something about v_sizehint c_list = inputconst(ootype.Void, r_list.lowleveltype) v_result = llops.genop("new", [c_list], resulttype=r_list.lowleveltype) c_resize = inputconst(ootype.Void, "_ll_resize") Modified: pypy/dist/pypy/rpython/rlist.py ============================================================================== --- pypy/dist/pypy/rpython/rlist.py (original) +++ pypy/dist/pypy/rpython/rlist.py Wed Dec 5 19:10:20 2007 @@ -312,7 +312,7 @@ return hop.genop('bool_not', [flag], resulttype=Bool) -def rtype_newlist(hop): +def rtype_newlist(hop, v_sizehint=None): nb_args = hop.nb_args r_list = hop.r_result if r_list == robject.pyobj_repr: # special case: SomeObject lists! @@ -326,7 +326,8 @@ return v_result r_listitem = r_list.item_repr items_v = [hop.inputarg(r_listitem, arg=i) for i in range(nb_args)] - return hop.rtyper.type_system.rlist.newlist(hop.llops, r_list, items_v) + return hop.rtyper.type_system.rlist.newlist(hop.llops, r_list, items_v, + v_sizehint=v_sizehint) def rtype_alloc_and_set(hop): r_list = hop.r_result Modified: pypy/dist/pypy/rpython/test/test_rlist.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rlist.py (original) +++ pypy/dist/pypy/rpython/test/test_rlist.py Wed Dec 5 19:10:20 2007 @@ -1389,7 +1389,17 @@ for i in range(3): lis = self.interpret(fnpop, [i]) assert list_is_clear(lis, 3-i) - + + def test_hints(self): + from pypy.rlib.objectmodel import newlist + + def f(z): + x = newlist(sizehint=13) + x += z + return ''.join(x) + + res = self.interpret(f, [self.string_to_ll('abc')]) + assert self.ll_to_string(res) == 'abc' class TestOOtype(BaseTestRlist, OORtypeMixin): rlist = oo_rlist From arigo at codespeak.net Wed Dec 5 19:12:19 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 19:12:19 +0100 (CET) Subject: [pypy-svn] r49394 - in pypy/dist/pypy/interpreter: . test Message-ID: <20071205181219.37C28168467@codespeak.net> Author: arigo Date: Wed Dec 5 19:12:18 2007 New Revision: 49394 Modified: pypy/dist/pypy/interpreter/gateway.py pypy/dist/pypy/interpreter/test/test_gateway.py Log: Support for classmethods in TypeDef(). Kill a bit of old code. Modified: pypy/dist/pypy/interpreter/gateway.py ============================================================================== --- pypy/dist/pypy/interpreter/gateway.py (original) +++ pypy/dist/pypy/interpreter/gateway.py Wed Dec 5 19:12:18 2007 @@ -14,7 +14,7 @@ from pypy.tool.sourcetools import func_with_new_name from pypy.interpreter.error import OperationError from pypy.interpreter import eval -from pypy.interpreter.function import Function, Method +from pypy.interpreter.function import Function, Method, ClassMethod from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable from pypy.interpreter.baseobjspace import Wrappable, SpaceCache, DescrMismatch from pypy.interpreter.argument import Arguments, AbstractArguments @@ -656,7 +656,7 @@ NOT_RPYTHON_ATTRIBUTES = ['_staticdefs'] def __init__(self, f, app_name=None, unwrap_spec = None, - descrmismatch=None): + descrmismatch=None, as_classmethod=False): "NOT_RPYTHON" Wrappable.__init__(self) # f must be a function whose name does NOT start with 'app_' @@ -676,6 +676,7 @@ descrmismatch=descrmismatch) self.__name__ = f.func_name self.name = app_name + self.as_classmethod = as_classmethod self._staticdefs = list(f.func_defaults or ()) def _getdefaults(self, space): @@ -701,20 +702,6 @@ def getcache(self, space): return space.fromcache(GatewayCache) - def get_method(self, obj): - # to bind this as a method out of an instance, we build a - # Function and get it. - # the object space is implicitely fetched out of the instance - assert self._code.ismethod, ( - 'global built-in function %r used as method' % - self._code.func) - - space = obj.space - fn = self.get_function(space) - w_obj = space.wrap(obj) - return Method(space, space.wrap(fn), - w_obj, space.type(w_obj)) - class GatewayCache(SpaceCache): def build(cache, gateway): @@ -723,6 +710,8 @@ defs = gateway._getdefaults(space) # needs to be implemented by subclass code = gateway._code fn = Function(space, code, None, defs, forcename = gateway.name) + if gateway.as_classmethod: + fn = ClassMethod(space.wrap(fn)) return fn Modified: pypy/dist/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_gateway.py (original) +++ pypy/dist/pypy/interpreter/test/test_gateway.py Wed Dec 5 19:12:18 2007 @@ -325,3 +325,18 @@ assert space.eq_w(space.call_function(w_app_g_id,w("foo")),w("foo")) assert len(l) == 1 assert space.eq_w(l[0], w("foo")) + + def test_interp2app_classmethod(self): + space = self.space + w = space.wrap + def g_run(space, w_type): + assert space.is_w(w_type, space.w_str) + return w(42) + + app_g_run = gateway.interp2app_temp(g_run, + unwrap_spec=[gateway.ObjSpace, + gateway.W_Root], + as_classmethod=True) + w_app_g_run = space.wrap(app_g_run) + w_bound = space.get(w_app_g_run, w("hello"), space.w_str) + assert space.eq_w(space.call_function(w_bound), w(42)) From arigo at codespeak.net Wed Dec 5 19:19:30 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 19:19:30 +0100 (CET) Subject: [pypy-svn] r49395 - in pypy/branch/pypy-interp-file: . interpreter interpreter/test module module/_file module/marshal module/sys Message-ID: <20071205181930.7D62F168458@codespeak.net> Author: arigo Date: Wed Dec 5 19:19:29 2007 New Revision: 49395 Added: pypy/branch/pypy-interp-file/ - copied from r49388, pypy/dist/pypy/ pypy/branch/pypy-interp-file/interpreter/gateway.py - copied unchanged from r49394, pypy/dist/pypy/interpreter/gateway.py pypy/branch/pypy-interp-file/interpreter/test/test_gateway.py - copied unchanged from r49394, pypy/dist/pypy/interpreter/test/test_gateway.py pypy/branch/pypy-interp-file/module/ - copied from r49393, pypy/dist/pypy/module/ pypy/branch/pypy-interp-file/module/_file/ - copied from r49388, pypy/dist/pypy/module/_file/ pypy/branch/pypy-interp-file/module/_file/interp_stream.py - copied, changed from r49345, pypy/dist/pypy/module/_file/interp_file.py Removed: pypy/branch/pypy-interp-file/module/_file/interp_file.py Modified: pypy/branch/pypy-interp-file/interpreter/baseobjspace.py pypy/branch/pypy-interp-file/interpreter/module.py pypy/branch/pypy-interp-file/module/_file/__init__.py pypy/branch/pypy-interp-file/module/_file/app_file.py pypy/branch/pypy-interp-file/module/marshal/interp_marshal.py pypy/branch/pypy-interp-file/module/sys/__init__.py pypy/branch/pypy-interp-file/module/sys/app.py Log: Branch in which I'm moving app_file to interpreter-level code. Modified: pypy/branch/pypy-interp-file/interpreter/baseobjspace.py ============================================================================== --- pypy/dist/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/pypy-interp-file/interpreter/baseobjspace.py Wed Dec 5 19:19:29 2007 @@ -206,7 +206,7 @@ for w_modname in self.unpackiterable( self.sys.get('builtin_module_names')): modname = self.str_w(w_modname) - mod = self.getbuiltinmodule(modname) + mod = self.interpclass_w(self.getbuiltinmodule(modname)) if isinstance(mod, Module): mod.startup(self) @@ -214,12 +214,12 @@ w_exitfunc = self.sys.getdictvalue_w(self, 'exitfunc') if w_exitfunc is not None: self.call_function(w_exitfunc) - w_exithandlers = self.sys.getdictvalue_w(self, 'pypy__exithandlers__') - if w_exithandlers is not None: - while self.is_true(w_exithandlers): - w_key_value = self.call_method(w_exithandlers, 'popitem') - w_key, w_value = self.unpacktuple(w_key_value, 2) - self.call_function(w_value) + for w_modname in self.unpackiterable( + self.sys.get('builtin_module_names')): + modname = self.str_w(w_modname) + mod = self.interpclass_w(self.getbuiltinmodule(modname)) + if isinstance(mod, Module): + mod.shutdown(self) if self.config.objspace.std.withdictmeasurement: from pypy.objspace.std.dictmultiobject import report report() Modified: pypy/branch/pypy-interp-file/interpreter/module.py ============================================================================== --- pypy/dist/pypy/interpreter/module.py (original) +++ pypy/branch/pypy-interp-file/interpreter/module.py Wed Dec 5 19:19:29 2007 @@ -24,6 +24,11 @@ """This is called at runtime before the space gets uses to allow the module to do initialization at runtime. """ + + def shutdown(self, space): + """This is called when the space is shut down, just after + sys.exitfunc(). + """ def getdict(self): return self.w_dict Modified: pypy/branch/pypy-interp-file/module/_file/__init__.py ============================================================================== --- pypy/dist/pypy/module/_file/__init__.py (original) +++ pypy/branch/pypy-interp-file/module/_file/__init__.py Wed Dec 5 19:19:29 2007 @@ -4,11 +4,21 @@ class Module(MixedModule): appleveldefs = { - "file": "app_file.file", } interpleveldefs = { - "open_file_as_stream": "interp_file.open_file_as_stream", - "fdopen_as_stream": "interp_file.fdopen_as_stream", + "file": "interp_file.W_File", } + def shutdown(self, space): + # at shutdown, flush all open streams + from pypy.module._file.interp_file import getopenstreams + openstreams = getopenstreams(space) + while openstreams: + for stream in openstreams.keys(): + try: + del openstreams[stream] + except KeyError: + pass # key was removed in the meantime + else: + stream.flush() Modified: pypy/branch/pypy-interp-file/module/_file/app_file.py ============================================================================== --- pypy/dist/pypy/module/_file/app_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/app_file.py Wed Dec 5 19:19:29 2007 @@ -25,13 +25,13 @@ _closed = True # Until the file is successfully opened - def __init__(self, name, mode='r', buffering=-1): + #def __init__(self, name, mode='r', buffering=-1): stream = _file.open_file_as_stream(name, mode, buffering) fd = stream.try_to_find_file_descriptor() assert fd != -1 self._fdopenstream(fd, mode, buffering, name, stream) - def fdopen(cls, fd, mode='r', buffering=-1): + #def fdopen(cls, fd, mode='r', buffering=-1): f = cls.__new__(cls) stream = _file.fdopen_as_stream(fd, mode, buffering) f._fdopenstream(fd, mode, buffering, '', stream) @@ -88,7 +88,7 @@ newlines = property(lambda self: self.getnewlines(), doc = "end-of-line convention used in this file") - def read(self, n=-1): + #def read(self, n=-1): """read([size]) -> read at most size bytes, returned as a string. If the size argument is negative or omitted, read until EOF is reached. @@ -114,7 +114,7 @@ self.stream.unlock() return ''.join(result) - def readline(self, size=-1): + #def readline(self, size=-1): """readline([size]) -> next line from the file, as a string. Retain newline. A non-negative size argument limits the maximum @@ -149,7 +149,7 @@ self.stream.unlock() return ''.join(result) - def readlines(self, size=0): + #def readlines(self, size=0): """readlines([size]) -> list of strings, each a line from the file. Call readline() repeatedly and return a list of the lines so read. @@ -177,7 +177,7 @@ self.stream.unlock() return result - def write(self, data): + #def write(self, data): """write(str) -> None. Write string str to file. Note that due to buffering, flush() or close() may be needed before @@ -254,7 +254,7 @@ raise ValueError('I/O operation on closed file') self.stream.flush() - def close(self): + #def close(self): """close() -> None or (perhaps) an integer. Close the file. Sets data attribute .closed to True. A closed file cannot be used for Modified: pypy/branch/pypy-interp-file/module/marshal/interp_marshal.py ============================================================================== --- pypy/dist/pypy/module/marshal/interp_marshal.py (original) +++ pypy/branch/pypy-interp-file/module/marshal/interp_marshal.py Wed Dec 5 19:19:29 2007 @@ -1,7 +1,7 @@ from pypy.interpreter.baseobjspace import ObjSpace from pypy.interpreter.error import OperationError from pypy.rlib.rarithmetic import intmask -from pypy.module._file.interp_file import file2stream +#from pypy.module._file.interp_file import file2stream import sys # Py_MARSHAL_VERSION = 2 Modified: pypy/branch/pypy-interp-file/module/sys/__init__.py ============================================================================== --- pypy/dist/pypy/module/sys/__init__.py (original) +++ pypy/branch/pypy-interp-file/module/sys/__init__.py Wed Dec 5 19:19:29 2007 @@ -76,7 +76,6 @@ '__excepthook__' : 'app.excepthook', 'exit' : 'app.exit', 'exitfunc' : 'app.exitfunc', - 'pypy__exithandlers__' : 'app.pypy__exithandlers__', # internal 'getfilesystemencoding' : 'app.getfilesystemencoding', 'callstats' : 'app.callstats', } Modified: pypy/branch/pypy-interp-file/module/sys/app.py ============================================================================== --- pypy/dist/pypy/module/sys/app.py (original) +++ pypy/branch/pypy-interp-file/module/sys/app.py Wed Dec 5 19:19:29 2007 @@ -24,8 +24,6 @@ def exitfunc(): """Placeholder for sys.exitfunc(), which is called when PyPy exits.""" -pypy__exithandlers__ = {} - #import __builtin__ def getfilesystemencoding(): From arigo at codespeak.net Wed Dec 5 19:21:54 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 19:21:54 +0100 (CET) Subject: [pypy-svn] r49396 - pypy/branch/pypy-interp-file/module/_file Message-ID: <20071205182154.6FE97168467@codespeak.net> Author: arigo Date: Wed Dec 5 19:21:54 2007 New Revision: 49396 Added: pypy/branch/pypy-interp-file/module/_file/interp_file.py (contents, props changed) Log: Forgot this file. Added: pypy/branch/pypy-interp-file/module/_file/interp_file.py ============================================================================== --- (empty file) +++ pypy/branch/pypy-interp-file/module/_file/interp_file.py Wed Dec 5 19:21:54 2007 @@ -0,0 +1,311 @@ +import py +from pypy.rlib import streamio +from pypy.module._file.interp_stream import W_AbstractStream +from pypy.module._file.interp_stream import StreamErrors, wrap_streamerror +from pypy.interpreter.error import OperationError +from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import interp2app + + +class W_File(W_AbstractStream): + """An interp-level file object. This implements the same interface than + the app-level files, with the following differences: + + * method names are prefixed with 'file_' + * the 'normal' app-level constructor is implemented by file___init__(). + * the methods with the 'direct_' prefix should be used if the caller + locks and unlocks the file itself, and takes care of StreamErrors. + """ + + # Default values until the file is successfully opened + stream = None + name = "" + mode = "" + encoding = None + fd = -1 + + def __init__(self, space): + self.space = space + + def __del__(self): + # assume that the file and stream objects are only visible in the + # thread that runs __del__, so no race condition should be possible + if self.stream is not None: + self.direct_close() + + def fdopenstream(self, stream, fd, mode, name): + self.fd = fd + self.name = name + self.softspace = 0 # Required according to file object docs + self.encoding = None # This is not used internally by file objects + self.mode = mode + self.stream = stream + if stream.flushable(): + getopenstreams(self.space)[stream] = None + + def check_mode_ok(self, mode): + if not mode or mode[0] not in ['r', 'w', 'a', 'U']: + space = self.space + raise OperationError(space.w_IOError, + space.wrap('invalid mode : "%s"' % mode)) + + def getstream(self): + """Return self.stream or raise an app-level ValueError if missing + (i.e. if the file is closed).""" + stream = self.stream + if stream is None: + space = self.space + raise OperationError(space.w_ValueError, + space.wrap('I/O operation on closed file')) + return stream + + # ____________________________________________________________ + # + # The 'direct_' methods assume that the caller already acquired the + # file lock. They don't convert StreamErrors to OperationErrors, too. + + def direct___init__(self, name, mode='r', buffering=-1): + self.direct_close() + self.check_mode_ok(mode) + stream = streamio.open_file_as_stream(name, mode, buffering) + fd = stream.try_to_find_file_descriptor() + self.fdopenstream(stream, fd, mode, name) + + def direct_fdopen(self, fd, mode='r', buffering=-1): + self.direct_close() + self.check_mode_ok(mode) + stream = streamio.fdopen_as_stream(fd, mode, buffering) + self.fdopenstream(stream, fd, mode, '') + + def direct_close(self): + space = self.space + stream = self.stream + if stream is not None: + self.stream = None + openstreams = getopenstreams(self.space) + try: + del openstreams[stream] + except KeyError: + pass + stream.close() + + def direct_read(self, n=-1): + stream = self.getstream() + if n < 0: + return stream.readall() + else: + result = [] + while n > 0: + data = stream.read(n) + if not data: + break + n -= len(data) + result.append(data) + return ''.join(result) + + def direct_readline(self, size=-1): + stream = self.getstream() + if size < 0: + return stream.readline() + else: + # very inefficient unless there is a peek() + result = [] + while size > 0: + # "peeks" on the underlying stream to see how many chars + # we can safely read without reading past an end-of-line + peeked = stream.peek() + pn = peeked.find("\n", 0, size) + if pn < 0: + pn = min(size-1, len(peeked)) + c = stream.read(pn + 1) + if not c: + break + result.append(c) + if c.endswith('\n'): + break + size -= len(c) + return ''.join(result) + + def direct_readlines(self, size=0): + stream = self.getstream() + # NB. this implementation is very inefficient for unbuffered + # streams, but ok if stream.readline() is efficient. + if size <= 0: + result = [] + while True: + line = stream.readline() + if not line: + break + result.append(line) + size -= len(line) + else: + result = [] + while size > 0: + line = stream.readline() + if not line: + break + result.append(line) + size -= len(line) + return result + + def direct_write(self, data): + self.getstream().write(data) + + # ____________________________________________________________ + # + # The 'file_' methods are the one exposed to app-level. + + _exposed_method_names = [] + _exposed_classmethod_names = [] + + def _decl(class_scope, name, unwrap_spec, docstring, + as_classmethod=False, wrapresult="space.wrap(result)"): + # hack hack to build a wrapper around the direct_xxx methods. + # The wrapper adds lock/unlock calls and a space.wrap() on + # the result, conversion of stream errors to OperationErrors, + # and has the specified docstring and unwrap_spec. + direct_fn = class_scope['direct_' + name] + co = direct_fn.func_code + argnames = co.co_varnames[:co.co_argcount] + defaults = direct_fn.func_defaults or () + + args = [] + for i, argname in enumerate(argnames): + try: + default = defaults[-len(argnames) + i] + except IndexError: + args.append(argname) + else: + args.append('%s=%r' % (argname, default)) + sig = ', '.join(args) + assert argnames[0] == 'self' + callsig = ', '.join(argnames[1:]) + + src = py.code.Source(""" + def file_%(name)s(%(sig)s): + %(docstring)r + space = self.space + self.lock() + try: + try: + result = self.direct_%(name)s(%(callsig)s) + except StreamErrors, e: + raise wrap_streamerror(space, e) + finally: + self.unlock() + return %(wrapresult)s + """ % locals()) + exec str(src) in globals(), class_scope + class_scope['file_' + name].unwrap_spec = unwrap_spec + interp2app + if as_classmethod: + class_scope['_exposed_classmethod_names'].append(name) + else: + class_scope['_exposed_method_names'].append(name) + + + _decl(locals(), "__init__", ['self', str, str, int], + """Opens a file.""") + + _decl(locals(), "close", ['self'], + """close() -> None or (perhaps) an integer. Close the file. + +Sets data attribute .closed to True. A closed file cannot be used for +further I/O operations. close() may be called more than once without +error. Some kinds of file objects (for example, opened by popen()) +may return an exit status upon closing.""") + # NB. close() need to use the stream lock to avoid double-closes or + # close-while-another-thread-uses-it. + + _decl(locals(), "read", ['self', int], + """read([size]) -> read at most size bytes, returned as a string. + +If the size argument is negative or omitted, read until EOF is reached. +Notice that when in non-blocking mode, less data than what was requested +may be returned, even if no size parameter was given.""") + + _decl(locals(), "readline", ['self', int], + """readlines([size]) -> list of strings, each a line from the file. + +Call readline() repeatedly and return a list of the lines so read. +The optional size argument, if given, is an approximate bound on the +total number of bytes in the lines returned.""") + + _decl(locals(), "readlines", ['self', int], + """readlines([size]) -> list of strings, each a line from the file. + +Call readline() repeatedly and return a list of the lines so read. +The optional size argument, if given, is an approximate bound on the +total number of bytes in the lines returned.""", + wrapresult = "wrap_list_of_str(space, result)") + + _decl(locals(), "write", ['self', str], + """write(str) -> None. Write string str to file. + +Note that due to buffering, flush() or close() may be needed before +the file on disk reflects the data written.""") + + +# ____________________________________________________________ + + +def descr_file__new__(space, w_subtype, args): + file = space.allocate_instance(W_File, w_subtype) + W_File.__init__(file, space) + return space.wrap(file) +descr_file__new__.unwrap_spec = [ObjSpace, W_Root, Arguments] + +def descr_file_fdopen(space, w_subtype, fd, mode='r', buffering=-1): + file = space.allocate_instance(W_File, w_subtype) + W_File.__init__(file, space) + try: + file.direct_fdopen(fd, mode, buffering) + except StreamErrors, e: + raise wrap_streamerror(space, e) + return space.wrap(file) +descr_file_fdopen.unwrap_spec = [ObjSpace, W_Root, int, str, int] + + +W_File.typedef = TypeDef( + "file", + __doc__ = """file(name[, mode[, buffering]]) -> file object + +Open a file. The mode can be 'r', 'w' or 'a' for reading (default), +writing or appending. The file will be created if it doesn't exist +when opened for writing or appending; it will be truncated when +opened for writing. Add a 'b' to the mode for binary files. +Add a '+' to the mode to allow simultaneous reading and writing. +If the buffering argument is given, 0 means unbuffered, 1 means line +buffered, and larger numbers specify the buffer size. +Add a 'U' to mode to open the file for input with universal newline +support. Any line ending in the input file will be seen as a '\n' +in Python. Also, a file so opened gains the attribute 'newlines'; +the value for this attribute is one of None (no newline read yet), +'\r', '\n', '\r\n' or a tuple containing all the newline types seen. + +Note: open() is an alias for file(). +""", + __new__ = interp2app(descr_file__new__), + fdopen = interp2app(descr_file_fdopen, as_classmethod=True), + **dict([(name, interp2app(getattr(W_File, 'file_' + name))) + for name in W_File._exposed_method_names]) + ) + +# ____________________________________________________________ + +def fdopen_as_stream(space, fd, mode="r", buffering=-1): + is_mode_ok(space, mode) + return space.wrap(W_Stream( + space, streamio.fdopen_as_stream(fd, mode, buffering))) +fdopen_as_stream.unwrap_spec = [ObjSpace, int, str, int] + +def wrap_list_of_str(space, lst): + return space.newlist([space.wrap(s) for s in lst]) + +class FileState: + def __init__(self, space): + self.openstreams = {} + +def getopenstreams(space): + return space.fromcache(FileState).openstreams From arigo at codespeak.net Wed Dec 5 19:24:32 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 19:24:32 +0100 (CET) Subject: [pypy-svn] r49397 - pypy/branch/pypy-interp-file/module/_file Message-ID: <20071205182432.44E69168467@codespeak.net> Author: arigo Date: Wed Dec 5 19:24:31 2007 New Revision: 49397 Modified: pypy/branch/pypy-interp-file/module/_file/app_file.py pypy/branch/pypy-interp-file/module/_file/interp_file.py Log: file.seek() Modified: pypy/branch/pypy-interp-file/module/_file/app_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/app_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/app_file.py Wed Dec 5 19:24:31 2007 @@ -202,7 +202,7 @@ raise ValueError('I/O operation on closed file') return self.stream.tell() - def seek(self, offset, whence=0): + #def seek(self, offset, whence=0): """seek(offset[, whence]) -> None. Move to new file position. Argument offset is a byte count. Optional argument whence defaults to Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/interp_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/interp_file.py Wed Dec 5 19:24:31 2007 @@ -149,6 +149,9 @@ size -= len(line) return result + def direct_seek(self, offset, whence=0): + self.getstream().seek(offset, whence) + def direct_write(self, data): self.getstream().write(data) @@ -240,6 +243,18 @@ total number of bytes in the lines returned.""", wrapresult = "wrap_list_of_str(space, result)") + _decl(locals(), "seek", ['self', int, int], + """seek(offset[, whence]) -> None. Move to new file position. + +Argument offset is a byte count. Optional argument whence defaults to +0 (offset from start of file, offset should be >= 0); other values are 1 +(move relative to current position, positive or negative), and 2 (move +relative to end of file, usually negative, although many platforms allow +seeking beyond the end of a file). If the file is opened in text mode, +only offsets returned by tell() are legal. Use of other offsets causes +undefined behavior. +Note that not all file objects are seekable.""") + _decl(locals(), "write", ['self', str], """write(str) -> None. Write string str to file. From fijal at codespeak.net Wed Dec 5 19:26:55 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 5 Dec 2007 19:26:55 +0100 (CET) Subject: [pypy-svn] r49398 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071205182655.2C33C168458@codespeak.net> Author: fijal Date: Wed Dec 5 19:26:54 2007 New Revision: 49398 Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py Log: Remove dead code Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Wed Dec 5 19:26:54 2007 @@ -281,20 +281,6 @@ setter = llexternal(setter_name, [TYPE], lltype.Void, compilation_info=new_eci, sandboxsafe=sandboxsafe) return getter, setter - -## # XXX THIS IS ONLY A QUICK HACK TO MAKE IT WORK -## # In general, we need to re-think a few things to be more consistent, -## # e.g. what if a CStruct, COpaque or CExternVariable requires -## # some #include... -## assert not isinstance(TYPE, lltype.ContainerType) -## CTYPE = lltype.FixedSizeArray(TYPE, 1) -## c_variable_ref = _CConstantClass('(&%s)' % (name,), lltype.Ptr(CTYPE)) -## def getter(): -## return c_variable_ref[0] -## def setter(newvalue): -## c_variable_ref[0] = newvalue -## return (func_with_new_name(getter, '%s_getter' % (name,)), -## func_with_new_name(setter, '%s_setter' % (name,))) # char, represented as a Python character # (use SIGNEDCHAR or UCHAR for the small integer types) From arigo at codespeak.net Wed Dec 5 19:30:18 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 19:30:18 +0100 (CET) Subject: [pypy-svn] r49399 - pypy/branch/pypy-interp-file/module/_file Message-ID: <20071205183018.A43331684CA@codespeak.net> Author: arigo Date: Wed Dec 5 19:30:18 2007 New Revision: 49399 Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py Log: file.newlines property. Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/interp_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/interp_file.py Wed Dec 5 19:30:18 2007 @@ -4,7 +4,7 @@ from pypy.module._file.interp_stream import StreamErrors, wrap_streamerror from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments -from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.gateway import interp2app @@ -281,6 +281,25 @@ return space.wrap(file) descr_file_fdopen.unwrap_spec = [ObjSpace, W_Root, int, str, int] +def descr_file_newlines(space, file): + newlines = file.getstream().getnewlines() + if newlines == 0: + return space.w_None + elif newlines == 1: + return space.wrap("\r") + elif newlines == 2: + return space.wrap("\n") + elif newlines == 4: + return space.wrap("\r\n") + result = [] + if newlines & 1: + result.append(space.wrap('\r')) + if newlines & 2: + result.append(space.wrap('\n')) + if newlines & 4: + result.append(space.wrap('\r\n')) + return space.newtuple(result) + W_File.typedef = TypeDef( "file", @@ -303,6 +322,8 @@ """, __new__ = interp2app(descr_file__new__), fdopen = interp2app(descr_file_fdopen, as_classmethod=True), + newlines = GetSetProperty(descr_file_newlines, cls=W_File, + doc="end-of-line convention used in this file"), **dict([(name, interp2app(getattr(W_File, 'file_' + name))) for name in W_File._exposed_method_names]) ) From arigo at codespeak.net Wed Dec 5 19:36:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 19:36:24 +0100 (CET) Subject: [pypy-svn] r49405 - in pypy/branch/pypy-interp-file: interpreter module/_file module/sys Message-ID: <20071205183624.977DE1684CE@codespeak.net> Author: arigo Date: Wed Dec 5 19:36:24 2007 New Revision: 49405 Modified: pypy/branch/pypy-interp-file/interpreter/baseobjspace.py pypy/branch/pypy-interp-file/module/_file/interp_file.py pypy/branch/pypy-interp-file/module/sys/state.py Log: module/_file tests pass now -- which just shows that they're incomplete :-/ Modified: pypy/branch/pypy-interp-file/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/pypy-interp-file/interpreter/baseobjspace.py (original) +++ pypy/branch/pypy-interp-file/interpreter/baseobjspace.py Wed Dec 5 19:36:24 2007 @@ -214,6 +214,7 @@ w_exitfunc = self.sys.getdictvalue_w(self, 'exitfunc') if w_exitfunc is not None: self.call_function(w_exitfunc) + from pypy.interpreter.module import Module for w_modname in self.unpackiterable( self.sys.get('builtin_module_names')): modname = self.str_w(w_modname) Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/interp_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/interp_file.py Wed Dec 5 19:36:24 2007 @@ -159,6 +159,12 @@ # # The 'file_' methods are the one exposed to app-level. + def file_fdopen(self, fd, mode="r", buffering=-1): + try: + self.direct_fdopen(fd, mode, buffering) + except StreamErrors, e: + raise wrap_streamerror(self.space, e) + _exposed_method_names = [] _exposed_classmethod_names = [] @@ -274,10 +280,7 @@ def descr_file_fdopen(space, w_subtype, fd, mode='r', buffering=-1): file = space.allocate_instance(W_File, w_subtype) W_File.__init__(file, space) - try: - file.direct_fdopen(fd, mode, buffering) - except StreamErrors, e: - raise wrap_streamerror(space, e) + file.file_fdopen(fd, mode, buffering) return space.wrap(file) descr_file_fdopen.unwrap_spec = [ObjSpace, W_Root, int, str, int] Modified: pypy/branch/pypy-interp-file/module/sys/state.py ============================================================================== --- pypy/branch/pypy-interp-file/module/sys/state.py (original) +++ pypy/branch/pypy-interp-file/module/sys/state.py Wed Dec 5 19:36:24 2007 @@ -76,25 +76,23 @@ class IOState: def __init__(self, space): + from pypy.module._file.interp_file import W_File self.space = space - w_fdopen = space.getattr(space.builtin.get('file'), - space.wrap("fdopen")) - self.w_stdin = space.call_function( - w_fdopen, space.wrap(0), space.wrap("r"), - space.wrap(1)) - space.setattr(self.w_stdin, space.wrap("_name"), - space.wrap("")) - self.w_stdout = space.call_function( - w_fdopen, space.wrap(1), space.wrap("w"), - space.wrap(1)) - space.setattr(self.w_stdout, space.wrap("_name"), - space.wrap("")) - self.w_stderr = space.call_function( - w_fdopen, space.wrap(2), space.wrap("w"), - space.wrap(0)) - space.setattr(self.w_stderr, space.wrap("_name"), - space.wrap("")) + stdin = W_File(space) + stdin.file_fdopen(0, "r", 1) + stdin.name = '' + self.w_stdin = space.wrap(stdin) + + stdout = W_File(space) + stdout.file_fdopen(1, "w", 1) + stdout.name = '' + self.w_stdout = space.wrap(stdout) + + stderr = W_File(space) + stderr.file_fdopen(2, "w", 0) + stderr.name = '' + self.w_stderr = space.wrap(stderr) def getio(space): return space.fromcache(IOState) From arigo at codespeak.net Wed Dec 5 19:47:44 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 19:47:44 +0100 (CET) Subject: [pypy-svn] r49406 - pypy/branch/pypy-interp-file/module/_file/test Message-ID: <20071205184744.884911684D0@codespeak.net> Author: arigo Date: Wed Dec 5 19:47:42 2007 New Revision: 49406 Modified: pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Log: In the process of resurrecting this test file. Modified: pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py (original) +++ pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Wed Dec 5 19:47:42 2007 @@ -1,8 +1,6 @@ import os, random, sys import pypy.tool.udir import py -py.test.skip("XXX convert this to a normal applevel test!") -from pypy.interpreter.mixedmodule import testmodule udir = pypy.tool.udir.udir.ensure('test_file_extra', dir=1) @@ -15,11 +13,7 @@ def setup_module(mod): - mod._file = testmodule("_file") udir.join('sample').write(SAMPLE) - # workaround for testing _file on top of CPython - if not hasattr(sys, 'pypy_objspaceclass'): - sys.pypy__exithandlers__ = {} class BaseROTests: @@ -74,13 +68,6 @@ def test_isatty(self): assert not self.file.isatty() - try: - f = _file.file('/dev/tty') - except IOError: - pass - else: - assert f.isatty() - f.close() def test_next(self): lines = self.expected_lines() @@ -213,23 +200,28 @@ extra_args = () def setup_method(self, method): - self.file = _file.file(self.expected_filename, - self.expected_mode, - *self.extra_args) + from pypy.module._file.interp_file import W_File + space = self.space + w_filetype = space.gettypeobject(W_File.typedef) + self.w_file = space.call_function( + w_filetype, + space.wrap(self.expected_filename), + space.wrap(self.expected_mode), + *[space.wrap(a) for a in self.extra_args]) def teardown_method(self, method): - self.file.close() + self.space.call_method(self.w_file, 'close') -class TestUnbufferedFile(TestFile): +class AppTestUnbufferedFile(AppTestFile): extra_args = (0,) -class TestLineBufferedFile(TestFile): +class AppTestLineBufferedFile(AppTestFile): extra_args = (1,) -class TestLargeBufferFile(TestFile): +class AppTestLargeBufferFile(AppTestFile): extra_args = (len(SAMPLE),) @@ -238,18 +230,21 @@ # Check on top of CPython -class TestWithCPython(TestFile): +class TestWithCPython(BaseROTests): + expected_filename = str(udir.join('sample')) + expected_mode = 'rb' + def setup_method(self, method): - self.file = open(self.expected_filename, - self.expected_mode, - *self.extra_args) + self.file = open(self.expected_filename, self.expected_mode) + def teardown_method(self, method): + self.file.close() # ____________________________________________________________ # # Files built with fdopen() -class TestFdOpen(BaseROTests): +class AppTestFdOpen(BaseROTests): expected_filename = None expected_mode = 'rb' extra_args = () @@ -265,15 +260,15 @@ self.file.close() -class TestUnbufferedFdOpen(TestFdOpen): +class AppTestUnbufferedFdOpen(AppTestFdOpen): extra_args = (0,) -class TestLineBufferedFdOpen(TestFdOpen): +class AppTestLineBufferedFdOpen(AppTestFdOpen): extra_args = (1,) -class TestLargeBufferFdOpen(TestFdOpen): +class AppTestLargeBufferFdOpen(AppTestFdOpen): extra_args = (len(SAMPLE),) @@ -281,7 +276,7 @@ # # Universal newlines -class TestUniversalNewlines(TestFile): +class AppTestUniversalNewlines(AppTestFile): expected_mode = 'rU' sample = '\n'.join(SAMPLE.splitlines(False)) @@ -291,15 +286,15 @@ test_tell = test_seek -class TestUnbufferedUniversal(TestUniversalNewlines): +class AppTestUnbufferedUniversal(AppTestUniversalNewlines): extra_args = (0,) -class TestLineBufferedUniversal(TestUniversalNewlines): +class AppTestLineBufferedUniversal(AppTestUniversalNewlines): extra_args = (1,) -class TestLargeBufferUniversal(TestUniversalNewlines): +class AppTestLargeBufferUniversal(AppTestUniversalNewlines): extra_args = (len(SAMPLE),) @@ -307,7 +302,7 @@ # # A few extra tests -def test_case_readonly(): +def app_test_case_readonly(): fn = str(udir.join('temptestfile')) f = _file.file(fn, 'w') assert f.name == fn @@ -316,7 +311,7 @@ assert f.encoding == None # Fix when we find out what this is py.test.raises((TypeError, AttributeError), setattr, f, 'name', 42) -def test_readlines(): +def app_test_readlines(): fn = str(udir.join('temptestfile')) lines = ['line%d\n' % i for i in range(1000)] f=_file.file(fn, 'w') @@ -328,7 +323,11 @@ assert len(somelines) > 200 assert somelines == lines[:len(somelines)] -def stresstest(flags, checkflags, eolstyles): +def app_test_rw_bin(): + flags = 'w+b' + checkflags = 'rb' + eolstyles = [('', ''), ('\n', '\n'), + ('\r', '\r'), ('\r\n', '\r\n')] fn = str(udir.join('temptestfile')) f = _file.file(fn, flags) expected = '' @@ -372,11 +371,7 @@ g.close() assert buf == expected -def test_rw_bin(): - stresstest('w+b', 'rb', [('', ''), ('\n', '\n'), - ('\r', '\r'), ('\r\n', '\r\n')]) - -def test_rw(): +def app_test_rw(): fn = str(udir.join('temptestfile')) f = _file.file(fn, 'w+') f.write('hello\nworld\n') @@ -384,7 +379,7 @@ assert f.read() == 'hello\nworld\n' f.close() -def test_r_universal(): +def app_test_r_universal(): fn = str(udir.join('temptestfile')) f = open(fn, 'wb') f.write('hello\r\nworld\r\n') @@ -393,7 +388,7 @@ assert f.read() == 'hello\nworld\n' f.close() -def test_flush(): +def app_test_flush(): fn = str(udir.join('temptestfile')) f = _file.file(fn, 'w', 0) f.write('x') @@ -423,3 +418,12 @@ assert os.stat(fn).st_size == 3 f.close() assert os.stat(fn).st_size == 3 + +def app_test_isatty(): + try: + f = file('/dev/tty') + except IOError: + pass + else: + assert f.isatty() + f.close() From arigo at codespeak.net Wed Dec 5 19:51:23 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 19:51:23 +0100 (CET) Subject: [pypy-svn] r49407 - in pypy/branch/pypy-interp-file/module/_file: . test Message-ID: <20071205185123.37C291684D6@codespeak.net> Author: arigo Date: Wed Dec 5 19:51:22 2007 New Revision: 49407 Modified: pypy/branch/pypy-interp-file/module/_file/app_file.py pypy/branch/pypy-interp-file/module/_file/interp_file.py pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Log: file.tell(). More test fixing. Modified: pypy/branch/pypy-interp-file/module/_file/app_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/app_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/app_file.py Wed Dec 5 19:51:22 2007 @@ -38,7 +38,7 @@ return f fdopen = classmethod(fdopen) - def tmpfile(cls, stream): + #def tmpfile(cls, stream): f = cls.__new__(cls) fd = stream.try_to_find_file_descriptor() assert fd != -1 @@ -46,7 +46,7 @@ return f tmpfile = classmethod(tmpfile) - def _fdopenstream(self, fd, mode, buffering, name, stream): + #def _fdopenstream(self, fd, mode, buffering, name, stream): self.fd = fd self._name = name self.softspace = 0 # Required according to file object docs @@ -57,7 +57,7 @@ if stream.flushable(): sys.pypy__exithandlers__[stream] = stream.flush - def getnewlines(self): + #def getnewlines(self): "end-of-line convention used in this file" newlines = self.stream.getnewlines() @@ -85,8 +85,8 @@ name = property(lambda self: self._name, doc = "file name") closed = property(lambda self: self._closed, doc = "True if the file is closed") - newlines = property(lambda self: self.getnewlines(), - doc = "end-of-line convention used in this file") + #newlines = property(lambda self: self.getnewlines(), + # doc = "end-of-line convention used in this file") #def read(self, n=-1): """read([size]) -> read at most size bytes, returned as a string. @@ -196,7 +196,7 @@ for line in sequence_of_strings: self.stream.write(line) - def tell(self): + #def tell(self): """tell() -> current file position, an integer (may be a long integer).""" if self._closed: raise ValueError('I/O operation on closed file') Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/interp_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/interp_file.py Wed Dec 5 19:51:22 2007 @@ -152,6 +152,9 @@ def direct_seek(self, offset, whence=0): self.getstream().seek(offset, whence) + def direct_tell(self): + return self.getstream().tell() + def direct_write(self, data): self.getstream().write(data) @@ -261,6 +264,9 @@ undefined behavior. Note that not all file objects are seekable.""") + _decl(locals(), "tell", ['self'], + "tell() -> current file position, an integer (may be a long integer).") + _decl(locals(), "write", ['self', str], """write(str) -> None. Write string str to file. Modified: pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py (original) +++ pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Wed Dec 5 19:51:22 2007 @@ -19,7 +19,7 @@ class BaseROTests: sample = SAMPLE - def expected_lines(self): + def get_expected_lines(self): lines = self.sample.split('\n') for i in range(len(lines)-1): lines[i] += '\n' @@ -33,7 +33,7 @@ assert data1 == self.sample def test_readline(self): - lines = self.expected_lines() + lines = self.expected_lines for sampleline in lines: inputline = self.file.readline() assert inputline == sampleline @@ -42,6 +42,7 @@ assert inputline == "" def test_readline_max(self): + import random i = 0 stop = 0 while stop < 5: @@ -58,7 +59,7 @@ def test_iter(self): inputlines = list(self.file) - assert inputlines == self.expected_lines() + assert inputlines == self.expected_lines def test_repr(self): r = repr(self.file) @@ -70,7 +71,7 @@ assert not self.file.isatty() def test_next(self): - lines = self.expected_lines() + lines = self.expected_lines for sampleline in lines: inputline = self.file.next() assert inputline == sampleline @@ -78,6 +79,7 @@ py.test.raises(StopIteration, self.file.next) def test_read(self): + import random i = 0 stop = 0 while stop < 5: @@ -91,12 +93,13 @@ def test_readlines(self): lines = self.file.readlines() - assert lines == self.expected_lines() + assert lines == self.expected_lines def test_readlines_max(self): + import random i = 0 stop = 0 - samplelines = self.expected_lines() + samplelines = self.expected_lines while stop < 5: morelines = self.file.readlines(random.randrange(1, 300)) for inputline in morelines: @@ -110,6 +113,7 @@ # to endless loops def test_seek(self): + import random for i in range(100): position = random.randrange(0, len(self.sample)) self.file.seek(position) @@ -129,6 +133,7 @@ prevpos = position + 1 def test_tell(self): + import random for i in range(100): position = random.randrange(0, len(self.sample)+1) self.file.seek(position) @@ -148,6 +153,7 @@ prevpos = position def test_tell_and_seek_back(self): + import random i = 0 stop = 0 secondpass = [] @@ -208,6 +214,8 @@ space.wrap(self.expected_filename), space.wrap(self.expected_mode), *[space.wrap(a) for a in self.extra_args]) + self.w_sample = space.wrap(SAMPLE) + self.w_expected_lines = space.wrap(self.get_expected_lines()) def teardown_method(self, method): self.space.call_method(self.w_file, 'close') From arigo at codespeak.net Wed Dec 5 19:56:08 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 19:56:08 +0100 (CET) Subject: [pypy-svn] r49408 - pypy/branch/pypy-interp-file/module/_file Message-ID: <20071205185608.CC6861684D7@codespeak.net> Author: arigo Date: Wed Dec 5 19:56:08 2007 New Revision: 49408 Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py Log: Iteration. Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/interp_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/interp_file.py Wed Dec 5 19:56:08 2007 @@ -90,6 +90,12 @@ pass stream.close() + def direct_next(self): + line = self.getstream().readline() + if line == '': + raise OperationError(self.space.w_StopIteration, self.space.w_None) + return line + def direct_read(self, n=-1): stream = self.getstream() if n < 0: @@ -158,6 +164,11 @@ def direct_write(self, data): self.getstream().write(data) + def direct___iter__(self): + self.getstream() + return self + direct_xreadlines = direct___iter__ + # ____________________________________________________________ # # The 'file_' methods are the one exposed to app-level. @@ -230,6 +241,9 @@ # NB. close() need to use the stream lock to avoid double-closes or # close-while-another-thread-uses-it. + _decl(locals(), "next", ['self'], + """next() -> the next line in the file, or raise StopIteration""") + _decl(locals(), "read", ['self', int], """read([size]) -> read at most size bytes, returned as a string. @@ -273,6 +287,16 @@ Note that due to buffering, flush() or close() may be needed before the file on disk reflects the data written.""") + _decl(locals(), "__iter__", ['self'], + """Iterating over files, as in 'for line in f:', returns each line of +the file one by one.""") + + _decl(locals(), "xreadlines", ['self'], + """xreadlines() -> returns self. + +For backward compatibility. File objects now include the performance +optimizations previously implemented in the xreadlines module.""") + # ____________________________________________________________ From arigo at codespeak.net Wed Dec 5 20:05:20 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 20:05:20 +0100 (CET) Subject: [pypy-svn] r49409 - in pypy/branch/pypy-interp-file: interpreter module/_file Message-ID: <20071205190520.AC49B1684CF@codespeak.net> Author: arigo Date: Wed Dec 5 20:05:19 2007 New Revision: 49409 Modified: pypy/branch/pypy-interp-file/interpreter/typedef.py pypy/branch/pypy-interp-file/module/_file/app_file.py pypy/branch/pypy-interp-file/module/_file/interp_file.py Log: file.__repr__, file.mode, file.name. Modified: pypy/branch/pypy-interp-file/interpreter/typedef.py ============================================================================== --- pypy/branch/pypy-interp-file/interpreter/typedef.py (original) +++ pypy/branch/pypy-interp-file/interpreter/typedef.py Wed Dec 5 20:05:19 2007 @@ -380,13 +380,13 @@ def descr_get_objclass(space, property): return property.objclass_getter(space) -def interp_attrproperty(name, cls): +def interp_attrproperty(name, cls, doc=None): "NOT_RPYTHON: initialization-time only" def fget(space, obj): return space.wrap(getattr(obj, name)) - return GetSetProperty(fget, cls=cls) + return GetSetProperty(fget, cls=cls, doc=doc) -def interp_attrproperty_w(name, cls): +def interp_attrproperty_w(name, cls, doc=None): "NOT_RPYTHON: initialization-time only" def fget(space, obj): w_value = getattr(obj, name) @@ -395,7 +395,7 @@ else: return w_value - return GetSetProperty(fget, cls=cls) + return GetSetProperty(fget, cls=cls, doc=doc) GetSetProperty.typedef = TypeDef( "getset_descriptor", Modified: pypy/branch/pypy-interp-file/module/_file/app_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/app_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/app_file.py Wed Dec 5 20:05:19 2007 @@ -82,7 +82,7 @@ mode = property(lambda self: self._mode, doc = "file mode ('r', 'U', 'w', 'a', " "possibly with 'b' or '+' added)") - name = property(lambda self: self._name, doc = "file name") + #name = property(lambda self: self._name, doc = "file name") closed = property(lambda self: self._closed, doc = "True if the file is closed") #newlines = property(lambda self: self.getnewlines(), @@ -217,15 +217,15 @@ raise ValueError('I/O operation on closed file') self.stream.seek(offset, whence) - def __iter__(self): + #def __iter__(self): """Iterating over files, as in 'for line in f:', returns each line of the file one by one.""" if self._closed: raise ValueError('I/O operation on closed file') return self - xreadlines = __iter__ + #xreadlines = __iter__ - def next(self): + #def next(self): """next() -> the next line in the file, or raise StopIteration""" if self._closed: raise ValueError('I/O operation on closed file') @@ -303,7 +303,7 @@ import os return os.isatty(self.fd) - def __repr__(self): + #def __repr__(self): return "<%s file '%s', mode %r at 0x%x>" % ( self._closed and 'closed' or 'open', self._name, Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/interp_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/interp_file.py Wed Dec 5 20:05:19 2007 @@ -5,6 +5,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.interpreter.typedef import interp_attrproperty from pypy.interpreter.gateway import interp2app @@ -297,6 +298,15 @@ For backward compatibility. File objects now include the performance optimizations previously implemented in the xreadlines module.""") + def file__repr__(self): + if self.stream is None: + head = "closed" + else: + head = "open" + info = "%s file '%s', mode '%s'" % (head, self.name, self.mode) + return self.getrepr(self.space, info) + file__repr__.unwrap_spec = ['self'] + # ____________________________________________________________ @@ -355,8 +365,11 @@ """, __new__ = interp2app(descr_file__new__), fdopen = interp2app(descr_file_fdopen, as_classmethod=True), + name = interp_attrproperty('name', cls=W_File), + mode = interp_attrproperty('mode', cls=W_File), newlines = GetSetProperty(descr_file_newlines, cls=W_File, doc="end-of-line convention used in this file"), + __repr__ = interp2app(W_File.file__repr__), **dict([(name, interp2app(getattr(W_File, 'file_' + name))) for name in W_File._exposed_method_names]) ) From arigo at codespeak.net Wed Dec 5 20:06:11 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 20:06:11 +0100 (CET) Subject: [pypy-svn] r49410 - pypy/branch/pypy-interp-file/module/_file Message-ID: <20071205190611.F2E621684CF@codespeak.net> Author: arigo Date: Wed Dec 5 20:06:11 2007 New Revision: 49410 Modified: pypy/branch/pypy-interp-file/module/_file/app_file.py pypy/branch/pypy-interp-file/module/_file/interp_file.py Log: forgot to check in docstrings. Modified: pypy/branch/pypy-interp-file/module/_file/app_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/app_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/app_file.py Wed Dec 5 20:06:11 2007 @@ -79,7 +79,7 @@ result.append('\r\n') return tuple(result) - mode = property(lambda self: self._mode, + #mode = property(lambda self: self._mode, doc = "file mode ('r', 'U', 'w', 'a', " "possibly with 'b' or '+' added)") #name = property(lambda self: self._name, doc = "file name") Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/interp_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/interp_file.py Wed Dec 5 20:06:11 2007 @@ -365,8 +365,10 @@ """, __new__ = interp2app(descr_file__new__), fdopen = interp2app(descr_file_fdopen, as_classmethod=True), - name = interp_attrproperty('name', cls=W_File), - mode = interp_attrproperty('mode', cls=W_File), + name = interp_attrproperty('name', cls=W_File, doc="file name"), + mode = interp_attrproperty('mode', cls=W_File, + doc = "file mode ('r', 'U', 'w', 'a', " + "possibly with 'b' or '+' added)"), newlines = GetSetProperty(descr_file_newlines, cls=W_File, doc="end-of-line convention used in this file"), __repr__ = interp2app(W_File.file__repr__), From arigo at codespeak.net Wed Dec 5 20:07:54 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 20:07:54 +0100 (CET) Subject: [pypy-svn] r49411 - pypy/branch/pypy-interp-file/module/_file Message-ID: <20071205190754.6297B1684D8@codespeak.net> Author: arigo Date: Wed Dec 5 20:07:54 2007 New Revision: 49411 Modified: pypy/branch/pypy-interp-file/module/_file/app_file.py pypy/branch/pypy-interp-file/module/_file/interp_file.py Log: file.isatty(). Modified: pypy/branch/pypy-interp-file/module/_file/app_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/app_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/app_file.py Wed Dec 5 20:07:54 2007 @@ -296,7 +296,7 @@ raise ValueError('I/O operation on closed file') return self.fd - def isatty(self): + #def isatty(self): """isatty() -> true or false. True if the file is connected to a tty device.""" if self._closed: raise ValueError('I/O operation on closed file') Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/interp_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/interp_file.py Wed Dec 5 20:07:54 2007 @@ -1,4 +1,5 @@ import py +import os from pypy.rlib import streamio from pypy.module._file.interp_stream import W_AbstractStream from pypy.module._file.interp_stream import StreamErrors, wrap_streamerror @@ -84,6 +85,7 @@ stream = self.stream if stream is not None: self.stream = None + self.fd = -1 openstreams = getopenstreams(self.space) try: del openstreams[stream] @@ -170,6 +172,9 @@ return self direct_xreadlines = direct___iter__ + def direct_isatty(self): + return os.isatty(self.fd) + # ____________________________________________________________ # # The 'file_' methods are the one exposed to app-level. @@ -242,6 +247,9 @@ # NB. close() need to use the stream lock to avoid double-closes or # close-while-another-thread-uses-it. + _decl(locals(), "isatty", ['self'], + """isatty() -> true or false. True if the file is connected to a tty device.""") + _decl(locals(), "next", ['self'], """next() -> the next line in the file, or raise StopIteration""") From arigo at codespeak.net Wed Dec 5 20:13:02 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 20:13:02 +0100 (CET) Subject: [pypy-svn] r49412 - in pypy/branch/pypy-interp-file/module/_file: . test Message-ID: <20071205191302.105111684DD@codespeak.net> Author: arigo Date: Wed Dec 5 20:13:02 2007 New Revision: 49412 Modified: pypy/branch/pypy-interp-file/module/_file/app_file.py pypy/branch/pypy-interp-file/module/_file/interp_file.py pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Log: Fix test. More file properties. Modified: pypy/branch/pypy-interp-file/module/_file/app_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/app_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/app_file.py Wed Dec 5 20:13:02 2007 @@ -83,7 +83,7 @@ doc = "file mode ('r', 'U', 'w', 'a', " "possibly with 'b' or '+' added)") #name = property(lambda self: self._name, doc = "file name") - closed = property(lambda self: self._closed, + #closed = property(lambda self: self._closed, doc = "True if the file is closed") #newlines = property(lambda self: self.getnewlines(), # doc = "end-of-line convention used in this file") Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/interp_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/interp_file.py Wed Dec 5 20:13:02 2007 @@ -332,6 +332,9 @@ return space.wrap(file) descr_file_fdopen.unwrap_spec = [ObjSpace, W_Root, int, str, int] +def descr_file_closed(space, file): + return space.wrap(file.stream is None) + def descr_file_newlines(space, file): newlines = file.getstream().getnewlines() if newlines == 0: @@ -351,6 +354,13 @@ result.append(space.wrap('\r\n')) return space.newtuple(result) +def descr_file_softspace(space, file): + return space.wrap(file.softspace) + +def descr_file_setsoftspace(space, file, w_newvalue): + file.softspace = space.int_w(w_newvalue) + +# ____________________________________________________________ W_File.typedef = TypeDef( "file", @@ -377,8 +387,14 @@ mode = interp_attrproperty('mode', cls=W_File, doc = "file mode ('r', 'U', 'w', 'a', " "possibly with 'b' or '+' added)"), + closed = GetSetProperty(descr_file_closed, cls=W_File, + doc="True if the file is closed"), newlines = GetSetProperty(descr_file_newlines, cls=W_File, doc="end-of-line convention used in this file"), + softspace= GetSetProperty(descr_file_softspace, + descr_file_setsoftspace, + cls=W_File, + doc="Support for 'print'."), __repr__ = interp2app(W_File.file__repr__), **dict([(name, interp2app(getattr(W_File, 'file_' + name))) for name in W_File._exposed_method_names]) Modified: pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py (original) +++ pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Wed Dec 5 20:13:02 2007 @@ -76,7 +76,7 @@ inputline = self.file.next() assert inputline == sampleline for i in range(5): - py.test.raises(StopIteration, self.file.next) + raises(StopIteration, self.file.next) def test_read(self): import random @@ -185,10 +185,10 @@ assert f.mode == self.expected_mode assert f.closed == False assert not f.softspace - py.test.raises((TypeError, AttributeError), 'f.name = 42') - py.test.raises((TypeError, AttributeError), 'f.name = "stuff"') - py.test.raises((TypeError, AttributeError), 'f.mode = "r"') - py.test.raises((TypeError, AttributeError), 'f.closed = True') + raises((TypeError, AttributeError), 'f.name = 42') + raises((TypeError, AttributeError), 'f.name = "stuff"') + raises((TypeError, AttributeError), 'f.mode = "r"') + raises((TypeError, AttributeError), 'f.closed = True') f.softspace = True assert f.softspace f.softspace = False @@ -215,6 +215,8 @@ space.wrap(self.expected_mode), *[space.wrap(a) for a in self.extra_args]) self.w_sample = space.wrap(SAMPLE) + self.w_expected_filename = space.wrap(self.expected_filename) + self.w_expected_mode = space.wrap(self.expected_mode) self.w_expected_lines = space.wrap(self.get_expected_lines()) def teardown_method(self, method): @@ -317,7 +319,7 @@ assert f.mode == 'w' assert f.closed == False assert f.encoding == None # Fix when we find out what this is - py.test.raises((TypeError, AttributeError), setattr, f, 'name', 42) + raises((TypeError, AttributeError), setattr, f, 'name', 42) def app_test_readlines(): fn = str(udir.join('temptestfile')) From arigo at codespeak.net Wed Dec 5 20:15:31 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 20:15:31 +0100 (CET) Subject: [pypy-svn] r49413 - pypy/branch/pypy-interp-file/module/_file/test Message-ID: <20071205191531.005E01684DD@codespeak.net> Author: arigo Date: Wed Dec 5 20:15:31 2007 New Revision: 49413 Modified: pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Log: Fix more tests. Modified: pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py (original) +++ pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Wed Dec 5 20:15:31 2007 @@ -260,14 +260,24 @@ extra_args = () def setup_method(self, method): + from pypy.module._file.interp_file import W_File + space = self.space O_BINARY = getattr(os, "O_BINARY", 0) - fd = os.open(TestFile.expected_filename, os.O_RDONLY | O_BINARY) - self.file = _file.file.fdopen(fd, - self.expected_mode, - *self.extra_args) + w_filetype = space.gettypeobject(W_File.typedef) + fd = os.open(AppTestFile.expected_filename, os.O_RDONLY | O_BINARY) + self.w_file = space.call_method( + w_filetype, + 'fdopen', + space.wrap(fd), + space.wrap(self.expected_mode), + *[space.wrap(a) for a in self.extra_args]) + self.w_sample = space.wrap(SAMPLE) + self.w_expected_filename = space.wrap(self.expected_filename) + self.w_expected_mode = space.wrap(self.expected_mode) + self.w_expected_lines = space.wrap(self.get_expected_lines()) def teardown_method(self, method): - self.file.close() + self.space.call_method(self.w_file, 'close') class AppTestUnbufferedFdOpen(AppTestFdOpen): From arigo at codespeak.net Wed Dec 5 20:23:50 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 20:23:50 +0100 (CET) Subject: [pypy-svn] r49414 - in pypy/branch/pypy-interp-file/module/_file: . test Message-ID: <20071205192350.A98641684CE@codespeak.net> Author: arigo Date: Wed Dec 5 20:23:50 2007 New Revision: 49414 Modified: pypy/branch/pypy-interp-file/module/_file/app_file.py pypy/branch/pypy-interp-file/module/_file/interp_file.py pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Log: Progress. Modified: pypy/branch/pypy-interp-file/module/_file/app_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/app_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/app_file.py Wed Dec 5 20:23:50 2007 @@ -186,7 +186,7 @@ raise ValueError('I/O operation on closed file') return self.stream.write(data) - def writelines(self, sequence_of_strings): + #def writelines(self, sequence_of_strings): """writelines(sequence_of_strings) -> None. Write the strings to the file. Note that newlines are not added. The sequence can be any iterable object Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/interp_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/interp_file.py Wed Dec 5 20:23:50 2007 @@ -167,6 +167,19 @@ def direct_write(self, data): self.getstream().write(data) + def direct_writelines(self, w_lines): # note: a wrapped list! + stream = self.getstream() + space = self.space + w_iterator = space.iter(w_lines) + while True: + try: + w_line = space.next(w_iterator) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + break # done + stream.write(space.str_w(w_line)) + def direct___iter__(self): self.getstream() return self @@ -296,6 +309,12 @@ Note that due to buffering, flush() or close() may be needed before the file on disk reflects the data written.""") + _decl(locals(), "writelines", ['self', W_Root], + """writelines(sequence_of_strings) -> None. Write the strings to the file. + +Note that newlines are not added. The sequence can be any iterable object +producing strings. This is equivalent to calling write() for each string.""") + _decl(locals(), "__iter__", ['self'], """Iterating over files, as in 'for line in f:', returns each line of the file one by one.""") @@ -387,6 +406,7 @@ mode = interp_attrproperty('mode', cls=W_File, doc = "file mode ('r', 'U', 'w', 'a', " "possibly with 'b' or '+' added)"), + encoding = interp_attrproperty('encoding', cls=W_File), closed = GetSetProperty(descr_file_closed, cls=W_File, doc="True if the file is closed"), newlines = GetSetProperty(descr_file_newlines, cls=W_File, Modified: pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py (original) +++ pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Wed Dec 5 20:23:50 2007 @@ -246,6 +246,7 @@ def setup_method(self, method): self.file = open(self.expected_filename, self.expected_mode) + self.expected_lines = self.get_expected_lines() def teardown_method(self, method): self.file.close() @@ -322,128 +323,135 @@ # # A few extra tests -def app_test_case_readonly(): - fn = str(udir.join('temptestfile')) - f = _file.file(fn, 'w') - assert f.name == fn - assert f.mode == 'w' - assert f.closed == False - assert f.encoding == None # Fix when we find out what this is - raises((TypeError, AttributeError), setattr, f, 'name', 42) - -def app_test_readlines(): - fn = str(udir.join('temptestfile')) - lines = ['line%d\n' % i for i in range(1000)] - f=_file.file(fn, 'w') - f.writelines(lines) - f.close() - assert open(fn, 'r').readlines() == lines - assert _file.file(fn, 'r').readlines() == lines - somelines = _file.file(fn, 'r').readlines(2000) - assert len(somelines) > 200 - assert somelines == lines[:len(somelines)] - -def app_test_rw_bin(): - flags = 'w+b' - checkflags = 'rb' - eolstyles = [('', ''), ('\n', '\n'), - ('\r', '\r'), ('\r\n', '\r\n')] - fn = str(udir.join('temptestfile')) - f = _file.file(fn, flags) - expected = '' - pos = 0 - for i in range(5000): - x = random.random() - if x < 0.4: - l = int(x*100) - buf = f.read(l) - assert buf == expected[pos:pos+l] - pos += len(buf) - elif x < 0.75: - writeeol, expecteol = random.choice(eolstyles) - x = str(x) - buf1 = x+writeeol - buf2 = x+expecteol - f.write(buf1) - expected = expected[:pos] + buf2 + expected[pos+len(buf2):] - pos += len(buf2) - elif x < 0.80: - pos = random.randint(0, len(expected)) - f.seek(pos) - elif x < 0.85: - pos = random.randint(0, len(expected)) - f.seek(pos - len(expected), 2) - elif x < 0.90: - currentpos = pos - pos = random.randint(0, len(expected)) - f.seek(pos - currentpos, 1) - elif x < 0.95: - assert f.tell() == pos - else: - f.flush() - g = open(fn, checkflags) - buf = g.read() - g.close() - assert buf == expected - f.close() - g = open(fn, checkflags) - buf = g.read() - g.close() - assert buf == expected - -def app_test_rw(): - fn = str(udir.join('temptestfile')) - f = _file.file(fn, 'w+') - f.write('hello\nworld\n') - f.seek(0) - assert f.read() == 'hello\nworld\n' - f.close() - -def app_test_r_universal(): - fn = str(udir.join('temptestfile')) - f = open(fn, 'wb') - f.write('hello\r\nworld\r\n') - f.close() - f = _file.file(fn, 'rU') - assert f.read() == 'hello\nworld\n' - f.close() - -def app_test_flush(): - fn = str(udir.join('temptestfile')) - f = _file.file(fn, 'w', 0) - f.write('x') - assert os.stat(fn).st_size == 1 - f.close() - - f = _file.file(fn, 'wb', 1) - f.write('x') - assert os.stat(fn).st_size == 0 - f.write('\n') - assert os.stat(fn).st_size == 2 - f.write('x') - assert os.stat(fn).st_size == 2 - f.flush() - assert os.stat(fn).st_size == 3 - f.close() - assert os.stat(fn).st_size == 3 - - f = _file.file(fn, 'wb', 100) - f.write('x') - assert os.stat(fn).st_size == 0 - f.write('\n') - assert os.stat(fn).st_size == 0 - f.write('x') - assert os.stat(fn).st_size == 0 - f.flush() - assert os.stat(fn).st_size == 3 - f.close() - assert os.stat(fn).st_size == 3 - -def app_test_isatty(): - try: - f = file('/dev/tty') - except IOError: - pass - else: - assert f.isatty() +class AppTestAFewExtra: + + def setup_method(self, method): + fn = str(udir.join('temptestfile')) + self.w_temptestfile = self.space.wrap(fn) + + def test_case_readonly(self): + fn = self.temptestfile + f = file(fn, 'w') + assert f.name == fn + assert f.mode == 'w' + assert f.closed == False + assert f.encoding == None # Fix when we find out what this is + raises((TypeError, AttributeError), setattr, f, 'name', 42) + + def test_readlines(self): + fn = self.temptestfile + lines = ['line%d\n' % i for i in range(1000)] + f = file(fn, 'w') + f.writelines(lines) + f.close() + assert open(fn, 'r').readlines() == lines + assert file(fn, 'r').readlines() == lines + somelines = file(fn, 'r').readlines(2000) + assert len(somelines) > 200 + assert somelines == lines[:len(somelines)] + + def test_rw_bin(self): + import random + flags = 'w+b' + checkflags = 'rb' + eolstyles = [('', ''), ('\n', '\n'), + ('\r', '\r'), ('\r\n', '\r\n')] + fn = self.temptestfile + f = file(fn, flags) + expected = '' + pos = 0 + for i in range(5000): + x = random.random() + if x < 0.4: + l = int(x*100) + buf = f.read(l) + assert buf == expected[pos:pos+l] + pos += len(buf) + elif x < 0.75: + writeeol, expecteol = random.choice(eolstyles) + x = str(x) + buf1 = x+writeeol + buf2 = x+expecteol + f.write(buf1) + expected = expected[:pos] + buf2 + expected[pos+len(buf2):] + pos += len(buf2) + elif x < 0.80: + pos = random.randint(0, len(expected)) + f.seek(pos) + elif x < 0.85: + pos = random.randint(0, len(expected)) + f.seek(pos - len(expected), 2) + elif x < 0.90: + currentpos = pos + pos = random.randint(0, len(expected)) + f.seek(pos - currentpos, 1) + elif x < 0.95: + assert f.tell() == pos + else: + f.flush() + g = open(fn, checkflags) + buf = g.read() + g.close() + assert buf == expected + f.close() + g = open(fn, checkflags) + buf = g.read() + g.close() + assert buf == expected + + def test_rw(self): + fn = self.temptestfile + f = file(fn, 'w+') + f.write('hello\nworld\n') + f.seek(0) + assert f.read() == 'hello\nworld\n' + f.close() + + def test_r_universal(self): + fn = self.temptestfile + f = open(fn, 'wb') + f.write('hello\r\nworld\r\n') + f.close() + f = file(fn, 'rU') + assert f.read() == 'hello\nworld\n' f.close() + + def test_flush(self): + fn = self.temptestfile + f = file(fn, 'w', 0) + f.write('x') + assert os.stat(fn).st_size == 1 + f.close() + + f = file(fn, 'wb', 1) + f.write('x') + assert os.stat(fn).st_size == 0 + f.write('\n') + assert os.stat(fn).st_size == 2 + f.write('x') + assert os.stat(fn).st_size == 2 + f.flush() + assert os.stat(fn).st_size == 3 + f.close() + assert os.stat(fn).st_size == 3 + + f = file(fn, 'wb', 100) + f.write('x') + assert os.stat(fn).st_size == 0 + f.write('\n') + assert os.stat(fn).st_size == 0 + f.write('x') + assert os.stat(fn).st_size == 0 + f.flush() + assert os.stat(fn).st_size == 3 + f.close() + assert os.stat(fn).st_size == 3 + + def test_isatty(): + try: + f = file('/dev/tty') + except IOError: + pass + else: + assert f.isatty() + f.close() From arigo at codespeak.net Wed Dec 5 20:31:26 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 20:31:26 +0100 (CET) Subject: [pypy-svn] r49415 - in pypy/branch/pypy-interp-file/module/_file: . test Message-ID: <20071205193126.3B5A61684D3@codespeak.net> Author: arigo Date: Wed Dec 5 20:31:25 2007 New Revision: 49415 Modified: pypy/branch/pypy-interp-file/module/_file/app_file.py pypy/branch/pypy-interp-file/module/_file/interp_file.py pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Log: Many tests pass. Missing (and untested): truncate(), readinto(), fileno(). Failures in AppTestUniversalNewlines. Modified: pypy/branch/pypy-interp-file/module/_file/app_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/app_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/app_file.py Wed Dec 5 20:31:25 2007 @@ -248,7 +248,7 @@ finally: self.stream.unlock() - def flush(self): + #def flush(self): """flush() -> None. Flush the internal I/O buffer.""" if self._closed: raise ValueError('I/O operation on closed file') Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/interp_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/interp_file.py Wed Dec 5 20:31:25 2007 @@ -93,6 +93,9 @@ pass stream.close() + def direct_flush(self): + self.getstream().flush() + def direct_next(self): line = self.getstream().readline() if line == '': @@ -257,9 +260,13 @@ further I/O operations. close() may be called more than once without error. Some kinds of file objects (for example, opened by popen()) may return an exit status upon closing.""") - # NB. close() need to use the stream lock to avoid double-closes or + # NB. close() needs to use the stream lock to avoid double-closes or # close-while-another-thread-uses-it. + + _decl(locals(), "flush", ['self'], + """flush() -> None. Flush the internal I/O buffer.""") + _decl(locals(), "isatty", ['self'], """isatty() -> true or false. True if the file is connected to a tty device.""") Modified: pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py (original) +++ pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Wed Dec 5 20:31:25 2007 @@ -417,6 +417,7 @@ f.close() def test_flush(self): + import os fn = self.temptestfile f = file(fn, 'w', 0) f.write('x') @@ -447,7 +448,7 @@ f.close() assert os.stat(fn).st_size == 3 - def test_isatty(): + def test_isatty(self): try: f = file('/dev/tty') except IOError: From arigo at codespeak.net Wed Dec 5 20:32:43 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 20:32:43 +0100 (CET) Subject: [pypy-svn] r49416 - pypy/branch/pypy-interp-file/module/_file/test Message-ID: <20071205193243.561131684E3@codespeak.net> Author: arigo Date: Wed Dec 5 20:32:42 2007 New Revision: 49416 Modified: pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Log: The bug was in the test. Modified: pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py (original) +++ pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Wed Dec 5 20:32:42 2007 @@ -214,7 +214,7 @@ space.wrap(self.expected_filename), space.wrap(self.expected_mode), *[space.wrap(a) for a in self.extra_args]) - self.w_sample = space.wrap(SAMPLE) + self.w_sample = space.wrap(self.sample) self.w_expected_filename = space.wrap(self.expected_filename) self.w_expected_mode = space.wrap(self.expected_mode) self.w_expected_lines = space.wrap(self.get_expected_lines()) @@ -272,7 +272,7 @@ space.wrap(fd), space.wrap(self.expected_mode), *[space.wrap(a) for a in self.extra_args]) - self.w_sample = space.wrap(SAMPLE) + self.w_sample = space.wrap(self.sample) self.w_expected_filename = space.wrap(self.expected_filename) self.w_expected_mode = space.wrap(self.expected_mode) self.w_expected_lines = space.wrap(self.get_expected_lines()) From arigo at codespeak.net Wed Dec 5 20:36:50 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 20:36:50 +0100 (CET) Subject: [pypy-svn] r49417 - pypy/branch/pypy-interp-file/module/_file/test Message-ID: <20071205193650.B172A1684E3@codespeak.net> Author: arigo Date: Wed Dec 5 20:36:50 2007 New Revision: 49417 Modified: pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Log: Leftover. Modified: pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py (original) +++ pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Wed Dec 5 20:36:50 2007 @@ -302,7 +302,7 @@ sample = '\n'.join(SAMPLE.splitlines(False)) def test_seek(self): - py.test.skip("does not apply in universal newlines mode") + skip("does not apply in universal newlines mode") test_tell = test_seek From arigo at codespeak.net Wed Dec 5 20:42:02 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 20:42:02 +0100 (CET) Subject: [pypy-svn] r49418 - in pypy/branch/pypy-interp-file/module/_file: . test Message-ID: <20071205194202.E66CA1684D9@codespeak.net> Author: arigo Date: Wed Dec 5 20:42:02 2007 New Revision: 49418 Modified: pypy/branch/pypy-interp-file/module/_file/app_file.py pypy/branch/pypy-interp-file/module/_file/interp_file.py pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Log: file.truncate(). Modified: pypy/branch/pypy-interp-file/module/_file/app_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/app_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/app_file.py Wed Dec 5 20:42:02 2007 @@ -234,7 +234,7 @@ raise StopIteration return line - def truncate(self, size=None): + #def truncate(self, size=None): """truncate([size]) -> None. Truncate the file to at most size bytes. Size defaults to the current file position, as returned by tell().""" Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/interp_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/interp_file.py Wed Dec 5 20:42:02 2007 @@ -167,6 +167,15 @@ def direct_tell(self): return self.getstream().tell() + def direct_truncate(self, w_size=None): # note: a wrapped size! + stream = self.getstream() + space = self.space + if w_size is None or space.is_w(w_size, space.w_None): + size = stream.tell() + else: + size = space.int_w(w_size) + stream.truncate(size) + def direct_write(self, data): self.getstream().write(data) @@ -310,6 +319,11 @@ _decl(locals(), "tell", ['self'], "tell() -> current file position, an integer (may be a long integer).") + _decl(locals(), "truncate", ['self', W_Root], + """truncate([size]) -> None. Truncate the file to at most size bytes. + +Size defaults to the current file position, as returned by tell().""") + _decl(locals(), "write", ['self', str], """write(str) -> None. Write string str to file. Modified: pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py (original) +++ pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Wed Dec 5 20:42:02 2007 @@ -456,3 +456,20 @@ else: assert f.isatty() f.close() + + def test_truncate(self): + fn = self.temptestfile + f = open(fn, 'w+b') + f.write('hello world') + f.seek(7) + f.truncate() + f.seek(0) + data = f.read() + assert data == 'hello w' + f.seek(0, 2) + assert f.tell() == 7 + f.seek(0) + f.truncate(3) + data = f.read(123) + assert data == 'hel' + f.close() From arigo at codespeak.net Wed Dec 5 20:44:13 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 5 Dec 2007 20:44:13 +0100 (CET) Subject: [pypy-svn] r49419 - in pypy/branch/pypy-interp-file/module/_file: . test Message-ID: <20071205194413.D6B041684D9@codespeak.net> Author: arigo Date: Wed Dec 5 20:44:11 2007 New Revision: 49419 Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Log: file.fileno() Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/interp_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/interp_file.py Wed Dec 5 20:44:11 2007 @@ -93,6 +93,9 @@ pass stream.close() + def direct_fileno(self): + return self.fd + def direct_flush(self): self.getstream().flush() @@ -273,6 +276,11 @@ # close-while-another-thread-uses-it. + _decl(locals(), "fileno", ['self'], + '''fileno() -> integer "file descriptor". + +This is needed for lower-level file interfaces, such os.read().''') + _decl(locals(), "flush", ['self'], """flush() -> None. Flush the internal I/O buffer.""") Modified: pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py (original) +++ pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Wed Dec 5 20:44:11 2007 @@ -272,6 +272,7 @@ space.wrap(fd), space.wrap(self.expected_mode), *[space.wrap(a) for a in self.extra_args]) + self.w_fd = space.wrap(fd) self.w_sample = space.wrap(self.sample) self.w_expected_filename = space.wrap(self.expected_filename) self.w_expected_mode = space.wrap(self.expected_mode) @@ -280,6 +281,9 @@ def teardown_method(self, method): self.space.call_method(self.w_file, 'close') + def test_fileno(self): + assert self.file.fileno() == self.fd + class AppTestUnbufferedFdOpen(AppTestFdOpen): extra_args = (0,) From regmee at codespeak.net Wed Dec 5 20:57:19 2007 From: regmee at codespeak.net (regmee at codespeak.net) Date: Wed, 5 Dec 2007 20:57:19 +0100 (CET) Subject: [pypy-svn] r49420 - in pypy/branch/clr-module-improvements/pypy/module/clr: . test Message-ID: <20071205195719.F3ADF1684E8@codespeak.net> Author: regmee Date: Wed Dec 5 20:57:18 2007 New Revision: 49420 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py Log: Fix for check if a class implements IEnumerable and addition of test cases for iterations on ArrayList and Stack objects Modified: pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py Wed Dec 5 20:57:18 2007 @@ -115,7 +115,7 @@ obj.__cliobj__ = cliobj return obj -def build_wrapper(namespace, classname, staticmethods, methods, properties, indexers): +def build_wrapper(namespace, classname, staticmethods, methods, properties, indexers, hasIEnumerable): fullname = '%s.%s' % (namespace, classname) d = {'__cliclass__': fullname, '__module__': namespace} @@ -124,12 +124,9 @@ for name in methods: d[name] = MethodWrapper(name) - # check if there is GetEnumerator() method - # XXX: use .NET Reflection for this, checking for method name is not safe - for method in methods: - if method == "GetEnumerator": - # now add the __iter__ method to the class - d['__iter__'] = __iter__ + # check if IEnumerable is implemented + if hasIEnumerable: + d['__iter__'] = __iter__ assert len(indexers) <= 1 if indexers: Modified: pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py Wed Dec 5 20:57:18 2007 @@ -206,6 +206,15 @@ def build_cli_class(space, namespace, classname, fullname): b_type = System.Type.GetType(fullname) + hasIEnumerable = 0 # flag + + # this is where we locate the interfaces inherited by the class + # set the flag hasIEnumerable if IEnumerable interface has been by the class + ifaces = b_type.GetInterfaces() + for interface in ifaces: + if interface.ToString() == "System.Collections.IEnumerable": + hasIEnumerable = 1 + w_staticmethods, w_methods = get_methods(space, b_type) w_properties, w_indexers = get_properties(space, b_type) return build_wrapper(space, @@ -214,7 +223,8 @@ w_staticmethods, w_methods, w_properties, - w_indexers) + w_indexers, + space.wrap(hasIEnumerable)) class W_CliObject(Wrappable): Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py Wed Dec 5 20:57:18 2007 @@ -158,6 +158,8 @@ def test_iteratrion(self): import clr + + # test iteration in ArrayList ArrayList = clr.load_cli_class('System.Collections', 'ArrayList') x = ArrayList() x.Add(1) @@ -169,3 +171,20 @@ sum += i assert sum == 1+2+3+4 + # test iteration in Stack + Stack = clr.load_cli_class('System.Collections', 'Stack') + obj = Stack() + obj.Push(1) + obj.Push(54) + obj.Push(21) + for i in obj: + sum += i + assert sum == 1+54+21 + + + + + + + + From fijal at codespeak.net Wed Dec 5 21:08:35 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 5 Dec 2007 21:08:35 +0100 (CET) Subject: [pypy-svn] r49421 - pypy/dist/pypy/doc Message-ID: <20071205200835.36A6916844F@codespeak.net> Author: fijal Date: Wed Dec 5 21:08:32 2007 New Revision: 49421 Modified: pypy/dist/pypy/doc/coding-guide.txt Log: Fix rctypes link. Modified: pypy/dist/pypy/doc/coding-guide.txt ============================================================================== --- pypy/dist/pypy/doc/coding-guide.txt (original) +++ pypy/dist/pypy/doc/coding-guide.txt Wed Dec 5 21:08:32 2007 @@ -726,11 +726,11 @@ namespace. Sometimes it is necessary to really write some functions in C (or -whatever target language). See `rctypes`_ and `external functions +whatever target language). See `rffi`_ and `external functions documentation`_ for details. The latter approach is cumbersome and being phased out and former has currently quite a few rough edges. -.. _`rctypes`: rctypes.html +.. _`rffi`: rffi.html .. _`external functions documentation`: translation.html#extfunccalls application level definitions From fijal at codespeak.net Wed Dec 5 21:13:27 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 5 Dec 2007 21:13:27 +0100 (CET) Subject: [pypy-svn] r49422 - in pypy/dist/pypy/doc: . config Message-ID: <20071205201327.25A3C1684D9@codespeak.net> Author: fijal Date: Wed Dec 5 21:13:26 2007 New Revision: 49422 Modified: pypy/dist/pypy/doc/config/objspace.usemodules._socket.txt pypy/dist/pypy/doc/config/objspace.usemodules.rctime.txt pypy/dist/pypy/doc/redirections Log: Remove rctypes links Modified: pypy/dist/pypy/doc/config/objspace.usemodules._socket.txt ============================================================================== --- pypy/dist/pypy/doc/config/objspace.usemodules._socket.txt (original) +++ pypy/dist/pypy/doc/config/objspace.usemodules._socket.txt Wed Dec 5 21:13:26 2007 @@ -2,6 +2,6 @@ This is our implementation of '_socket', the Python builtin module exposing socket primitives, which is wrapped and used by the standard -library 'socket.py' module. It is based on `rctypes`_. +library 'socket.py' module. It is based on `rffi`_. -.. _`rctypes`: ../rctypes.html +.. _`rffi`: ../rffi.html Modified: pypy/dist/pypy/doc/config/objspace.usemodules.rctime.txt ============================================================================== --- pypy/dist/pypy/doc/config/objspace.usemodules.rctime.txt (original) +++ pypy/dist/pypy/doc/config/objspace.usemodules.rctime.txt Wed Dec 5 21:13:26 2007 @@ -1,7 +1,7 @@ Use the 'rctime' module. -'rctime' is our `rctypes`_ based implementation of the builtin 'time' module. +'rctime' is our `rffi`_ based implementation of the builtin 'time' module. It supersedes the less complete :config:`objspace.usemodules.time`, at least for C-like targets (the C and LLVM backends). -.. _`rctypes`: ../rctypes.html +.. _`rffi`: ../rffi.html Modified: pypy/dist/pypy/doc/redirections ============================================================================== --- pypy/dist/pypy/doc/redirections (original) +++ pypy/dist/pypy/doc/redirections Wed Dec 5 21:13:26 2007 @@ -32,8 +32,6 @@ 'draft-dynamic-language-translation.html': 'dynamic-language-translation.html', 'draft-low-level-encapsulation.html': 'low-level-encapsulation.html', 'draft-memory-management-threading-model.html': 'translation-aspects.html', - 'ctypes-integration.html': 'rctypes.html', - 'cpython-ctypes-behaviour.html': 'rctypes.html#cpython-behavior', 'proxy.html': 'objspace-proxies.html#tproxy', } From rxe at codespeak.net Wed Dec 5 23:10:17 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 5 Dec 2007 23:10:17 +0100 (CET) Subject: [pypy-svn] r49424 - pypy/dist/pypy/translator/llvm Message-ID: <20071205221017.D3FDB1684D7@codespeak.net> Author: rxe Date: Wed Dec 5 23:10:16 2007 New Revision: 49424 Modified: pypy/dist/pypy/translator/llvm/arraynode.py pypy/dist/pypy/translator/llvm/codewriter.py pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/externs2ll.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/modwrapper.py pypy/dist/pypy/translator/llvm/node.py pypy/dist/pypy/translator/llvm/opaquenode.py pypy/dist/pypy/translator/llvm/opwriter.py pypy/dist/pypy/translator/llvm/structnode.py pypy/dist/pypy/translator/llvm/typedefnode.py Log: rewrite of pbc code. it is quite simple now. the old code was written before knowledge of parentlink() and was somehow stitched up to work for the last 2.5 years - but i had no idea how it worked and i wrote it. so just threw it all away... :-) the emitted code is now one getelementptr instead of a nested list of them that could extend across about 100 lines so store %structtype_object_vtable* getelementptr(%structtype_pypy.translator.goal.richards.TaskState_vtable* getelementptr(%structtype_pypy.translator.goal.richards.Task_vtable* getelementptr(%structtype_pypy.translator.goal.richards.IdleTask_vtable* @s_inst_pypy.translator.goal.richards.IdleTask_vtable, i32 0, i32 0), i32 0, i32 0), i32 0, i32 0), %structtype_object_vtable** %tmp_17 now looks like store %structtype_object_vtable* getelementptr(%structtype_IdleTaskRec_vtable* @s_inst_IdleTaskRec_vtable, i32 0, i32 0, i32 0), %structtype_object_ vtable** %tmp_99 also some random simplications. Modified: pypy/dist/pypy/translator/llvm/arraynode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/arraynode.py (original) +++ pypy/dist/pypy/translator/llvm/arraynode.py Wed Dec 5 23:10:16 2007 @@ -15,12 +15,6 @@ def constantvalue(self): return '%s c"%s\\00"' % (self.get_typerepr(), self.value) - - def get_childref(self, index): - return "getelementptr(%s* %s, i32 0, i32 %s)" % ( - self.get_typerepr(), - self.name, - index) def writeglobalconstants(self, codewriter): codewriter.globalinstance(self.ref, self.constantvalue()) @@ -69,33 +63,6 @@ typeval = self.db.repr_type(self.arraytype) return "{ %s, [%s x %s] }" % (self.db.get_machine_word(), arraylen, typeval) - - def get_ref(self): - typeval = self.db.repr_type(lltype.typeOf(self.value)) - p, c = lltype.parentlink(self.value) - if p is None: - ref = self.name - else: - ref = self.db.get_childref(p, c) - - ref = "bitcast(%s* %s to %s*)" % (self.get_typerepr(), - ref, - typeval) - return ref - - def get_pbcref(self, toptr): - p, c = lltype.parentlink(self.value) - assert p is None, "child PBC arrays are NOT needed by rtyper" - - fromptr = "%s*" % self.get_typerepr() - ref = "bitcast(%s %s to %s)" % (fromptr, self.name, toptr) - return ref - - def get_childref(self, index): - return "getelementptr(%s* %s, i32 0, i32 1, i32 %s)" % ( - self.get_typerepr(), - self.name, - index) def constantvalue(self): physicallen, arrayrepr = self.get_arrayvalue() @@ -108,20 +75,13 @@ typeval, arrayrepr) - s = "%s {%s}" % (self.get_typerepr(), value) - return s + return "%s {%s}" % (self.get_typerepr(), value) class ArrayNoLengthNode(ArrayNode): def get_typerepr(self): arraylen = self.get_arrayvalue()[0] typeval = self.db.repr_type(self.arraytype) return "[%s x %s]" % (arraylen, typeval) - - def get_childref(self, index): - return "getelementptr(%s* %s, i32 0, i32 %s)" %( - self.get_typerepr(), - self.name, - index) def constantvalue(self): physicallen, arrayrepr = self.get_arrayvalue() @@ -164,6 +124,9 @@ name = '' #str(value).split()[1] self.make_name(name) + def get_typerepr(self): + return '[%s x i8]' % self.get_length() + def constantvalue(self): return "{ %s } {%s %s}" % (self.db.get_machine_word(), self.db.get_machine_word(), Modified: pypy/dist/pypy/translator/llvm/codewriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/codewriter.py (original) +++ pypy/dist/pypy/translator/llvm/codewriter.py Wed Dec 5 23:10:16 2007 @@ -172,6 +172,7 @@ self._indent("free %s %s" % (vartype, varref)) def debug_print(self, s): + XXX # fixme var = self.db.repr_tmpvar() node = self.db.create_debug_string(s) self.call(var, "i32", "@write", Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Wed Dec 5 23:10:16 2007 @@ -1,4 +1,3 @@ - import sys from pypy.translator.llvm.log import log @@ -22,6 +21,17 @@ log = log.database +def var_size_type(T): + " returns None if T not varsize " + if not T._is_varsize(): + return None + elif isinstance(T, lltype.Array): + return T.OF + elif isinstance(T, lltype.Struct): + return T._arrayfld + else: + assert False, "unknown type" + class Database(object): def __init__(self, genllvm, translator): self.genllvm = genllvm @@ -43,6 +53,7 @@ #_______debuggging______________________________________ def dump_pbcs(self): + XXX#FIXME r = "" for k, v in self.obj2node.iteritems(): @@ -58,13 +69,10 @@ p, _ = lltype.parentlink(k) ref = v.get_ref() type_ = self.repr_type(lltype.Ptr(lltype.typeOf(k))) - pbc_ref = v.get_pbcref(type_) - r += "\ndump_pbcs %s (%s)\n" \ "parent %s\n" \ "type %s\n" \ - "getref -> %s \n" \ - "pbcref -> %s \n" % (v, k, p, type_, ref, pbc_ref) + "getref -> %s \n" % (v, k, p, type_, ref) return r #_______setting up and preparation______________________________ @@ -113,7 +121,7 @@ assert key not in self.obj2node, ( "node with key %r already known!" %(key,)) - + #log("added to pending nodes:", type(key), node) self.obj2node[key] = node @@ -242,6 +250,87 @@ # __________________________________________________________ # Representing variables and constants in LLVM source code + def to_getelementptr(self, value): + # so we build the thing up instead + p = value + children = [] + while True: + p, c = lltype.parentlink(p) + if p is None: + break + children.append((p, c)) + + children.reverse() + + TYPE = lltype.typeOf(children[0][0]) + parentnode = self.obj2node[children[0][0]] + + indices = [("i32", 0)] + + for _, ii in children: + name = None + + # this is because FixedSizeArray can sometimes be accessed like an + # Array and then sometimes a Struct + if isinstance(ii, str): + name = ii + assert name in list(TYPE._names) + fieldnames = TYPE._names_without_voids() + indexref = fieldnames.index(name) + else: + indexref = ii + + if isinstance(TYPE, lltype.FixedSizeArray): + indices.append(("i32", indexref)) + TYPE = TYPE.OF + + elif isinstance(TYPE, lltype.Array): + if not TYPE._hints.get("nolength", False): + indices.append(("i32", 1)) + indices.append(("i32", indexref)) + TYPE = TYPE.OF + + elif isinstance(TYPE, lltype.Struct): + assert name is not None + TYPE = getattr(TYPE, name) + indices.append(("i32", indexref)) + + else: + raise Exception("unsupported type: %s" % TYPE) + + indices_str = ', '.join ([('%s %s' % (x,y)) for x, y in indices]) + ref = "getelementptr(%s* %s, %s)" % ( + parentnode.get_typerepr(), + parentnode.ref, + indices_str) + + return ref + + def get_ref(self, value): + node = self.obj2node[value] + T = lltype.typeOf(value) + p, c = lltype.parentlink(value) + if p is None: + ref = node.ref + VT = var_size_type(T) + if VT and VT is not lltype.Void: + ref = "bitcast(%s* %s to %s*)" % (node.get_typerepr(), + ref, + self.repr_type(T)) + else: + ref = self.to_getelementptr(value) + + if isinstance(node, FixedSizeArrayNode): + assert isinstance(value, lltype._subarray) + + # XXX UGLY (but needs fixing outside of genllvm) + # ptr -> array of len 1 (for now, since operations expect this) + ref = "bitcast(%s* %s to %s*)" % (self.repr_type(T.OF), + ref, + self.repr_type(T)) + + return ref + def repr_arg(self, arg): if isinstance(arg, Constant): if isinstance(arg.concretetype, lltype.Primitive): @@ -251,8 +340,7 @@ if not arg.value: return 'null' else: - node = self.obj2node[arg.value._obj] - return node.get_ref() + return self.get_ref(arg.value._obj) else: assert isinstance(arg, Variable) return "%" + str(arg) @@ -298,7 +386,7 @@ return None, "%s null" % toptr node = self.obj2node[value] - ref = node.get_pbcref(toptr) + ref = self.get_ref(value) return node, "%s %s" % (toptr, ref) elif isinstance(type_, (lltype.Array, lltype.Struct)): @@ -331,10 +419,6 @@ return True return False - def get_childref(self, parent, child): - node = self.obj2node[parent] - return node.get_childref(child) - def create_debug_string(self, s): r = DebugStrNode(s) self.debugstringnodes.append(r) Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Wed Dec 5 23:10:16 2007 @@ -65,7 +65,7 @@ ccode = [] if standalone: - ccode.append('#define __ENTRY_POINT__ %s' % entrynode.get_ref()[1:]) + ccode.append('#define __ENTRY_POINT__ %s' % entrynode.ref[1:]) ccode.append('#define ENTRY_POINT_DEFINED 1') sio = StringIO() Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Wed Dec 5 23:10:16 2007 @@ -154,9 +154,9 @@ return codewriter def setup_externs(self, c_db, db): - # XXX + # XXX this should be done via augmenting entrypoint exctransformer = c_db.exctransformer - for obj in [exctransformer._rpyexc_occured_ptr.value, + for obj in [exctransformer.rpyexc_occured_ptr.value, exctransformer.rpyexc_fetch_type_ptr.value, exctransformer.rpyexc_clear_ptr.value]: db.prepare_constant(lltype.typeOf(obj), obj) Modified: pypy/dist/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/dist/pypy/translator/llvm/modwrapper.py (original) +++ pypy/dist/pypy/translator/llvm/modwrapper.py Wed Dec 5 23:10:16 2007 @@ -14,9 +14,9 @@ _c = ctypes.CDLL(join(dirname(realpath(__file__)), "%s")) -rpyexc_occured = _c.pypy__rpyexc_occured +rpyexc_occured = _c.pypy_rpyexc_occured rpyexc_occured.argtypes = [] -rpyexc_occured.restype = ctypes.c_int +rpyexc_occured.restype = ctypes.c_byte rpyexc_fetch_type = _c.pypy_rpyexc_fetch_type rpyexc_fetch_type.argtypes = [] Modified: pypy/dist/pypy/translator/llvm/node.py ============================================================================== --- pypy/dist/pypy/translator/llvm/node.py (original) +++ pypy/dist/pypy/translator/llvm/node.py Wed Dec 5 23:10:16 2007 @@ -8,12 +8,13 @@ nodename_count = {} def mangle(self, name): if name not in self.nodename_count: - result = name self.nodename_count[name] = 1 - return result + return name else: result = '%s_%d' % (name, self.nodename_count[name]) self.nodename_count[name] += 1 + # this ensures (a) doesn exist yet, and (b) adds it to the + # dictionary just to prevent some function called xxx_42() and clashing return self.mangle(result) def make_name(self, name=''): @@ -44,14 +45,6 @@ class FuncNode(Node): - # XXX proof that the whole llvm is hanging on a bunch of loose stitches - def get_ref(self): - return self.ref - - # XXX proof that the whole llvm is hanging on a bunch of loose stitches - def get_pbcref(self, _): - return self.ref - def writedecl(self, codewriter): " write function forward declarations " pass @@ -63,18 +56,6 @@ class ConstantNode(Node): __slots__ = "".split() - def get_ref(self): - # XXX tmp - return self.ref - - def get_childref(self, index): - """ Returns a reference as used for operations in blocks for internals of a pbc. """ - raise AttributeError("Must be implemented in subclass") - - def get_pbcref(self, toptr): - """ Returns a reference as a pointer used per pbc. """ - return self.ref - # ______________________________________________________________________ # entry points from genllvm Modified: pypy/dist/pypy/translator/llvm/opaquenode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opaquenode.py (original) +++ pypy/dist/pypy/translator/llvm/opaquenode.py Wed Dec 5 23:10:16 2007 @@ -15,7 +15,6 @@ def __init__(self, db, value): self.db = db self.value = value - self._get_ref_cache = None name = str(value).split()[1] self.make_name(name) @@ -23,18 +22,6 @@ # ______________________________________________________________________ # main entry points from genllvm - def get_ref(self): - """ Returns a reference as used for operations in blocks. """ - if self._get_ref_cache: - return self._get_ref_cache - p, c = lltype.parentlink(self.value) - if p is None: - ref = self.name - else: - ref = self.db.get_childref(p, c) - self._get_ref_cache = ref - return ref - def writeglobalconstants(self, codewriter): pass Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Wed Dec 5 23:10:16 2007 @@ -26,18 +26,6 @@ self.rettype = "%s (%s)*" % (self.rettype, ", ".join(self.argtypes[1:])) -class OpReprInvoke(OpReprCall): - __slots__ = "db op retref rettype argrefs argtypes functionref".split() - def __init__(self, op, db): - super(OpReprInvoke, self).__init__(op, db) - - if op.opname in ('direct_call', 'indirect_call'): - self.functionref = self.argrefs[0] - self.argrefs = self.argrefs[1:] - self.argtypes = self.argtypes[1:] - else: - self.functionref = '%pypyop_' + op.opname - class OpWriter(object): shift_operations = { @@ -123,7 +111,6 @@ return indices def write_operation(self, op): - if self.db.genllvm.config.translation.llvm.debug: self.codewriter.comment(str(op)) #self.codewriter.debug_print(str(op) + "\n") Modified: pypy/dist/pypy/translator/llvm/structnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/structnode.py (original) +++ pypy/dist/pypy/translator/llvm/structnode.py Wed Dec 5 23:10:16 2007 @@ -12,12 +12,7 @@ return index class StructNode(ConstantNode): - """ A struct constant. Can simply contain - a primitive, - a struct, - pointer to struct/array - """ - __slots__ = "db value structtype _get_ref_cache _get_types".split() + __slots__ = "db value structtype _get_types".split() prefix = '@s_inst_' @@ -25,8 +20,8 @@ self.db = db self.value = value self.structtype = self.value._TYPE - name = str(value).split()[1] - self._get_ref_cache = None + parts = str(value).split()[1] + name = parts.split('.')[-1] self._get_types = self._compute_types() self.make_name(name) @@ -53,38 +48,6 @@ def get_typerepr(self): return self.db.repr_type(self.structtype) - - def get_childref(self, index): - pos = 0 - found = False - for name in self.structtype._names_without_voids(): - if name == index: - found = True - break - pos += 1 - - return "getelementptr(%s* %s, i32 0, i32 %s)" %( - self.get_typerepr(), - self.get_ref(), - pos) - - def get_ref(self): - """ Returns a reference as used for operations in blocks. """ - # XXX cache here is **dangerous** considering it can return different values :-( - # XXX should write a test to prove this - #if self._get_ref_cache: - # return self._get_ref_cache - p, c = lltype.parentlink(self.value) - if p is None: - ref = self.name - else: - ref = self.db.get_childref(p, c) - #XXXself._get_ref_cache = ref - return ref - - def get_pbcref(self, toptr): - """ Returns a reference as used per pbc. """ - return self.get_ref() def constantvalue(self): """ Returns the constant representation for this node. """ @@ -95,7 +58,7 @@ else: all_values = ", ".join(values) return "%s { %s }" % (self.get_typerepr(), all_values) - + class FixedSizeArrayNode(StructNode): prefix = '@fa_inst_' @@ -110,40 +73,8 @@ all_values = ",\n ".join(values) return "%s [\n %s\n ]\n" % (self.get_typerepr(), all_values) - def get_ref(self): - p, c = lltype.parentlink(self.value) - if p is None: - ref = self.name - else: - ref = self.db.get_childref(p, c) - if isinstance(self.value, lltype._subarray): - # ptr -> array of len 1 - ref = "bitcast(%s* %s to %s*)" % (self.db.repr_type(self.arraytype), - ref, - self.db.repr_type(lltype.typeOf(self.value))) - return ref - - def get_childref(self, index): - if isinstance(index, str): - pos = 0 - found = False - for name in self.structtype._names_without_voids(): - if name == index: - found = True - break - pos += 1 - else: - pos = index - - return "getelementptr(%s* %s, i32 0, i32 %s)" % ( - self.get_typerepr(), - self.get_ref(), - pos) - def setup(self): if isinstance(self.value, lltype._subarray): - # XXX what is this? - # self.value._parentstructure() p, c = lltype.parentlink(self.value) if p is not None: self.db.prepare_constant(lltype.typeOf(p), p) @@ -151,17 +82,6 @@ super(FixedSizeArrayNode, self).setup() class StructVarsizeNode(StructNode): - """ A varsize struct constant. Can simply contain - a primitive, - a struct, - pointer to struct/array - - and the last element *must* be - an array - OR - a series of embedded structs, which has as its last element an array. - """ - prefix = '@sv_inst_' def _getvalues(self): @@ -199,39 +119,3 @@ result = "{%s}" % ", ".join(types_repr) self._get_typerepr_cache = result return result - - def get_childref(self, index): - pos = 0 - found = False - for name in self.structtype._names_without_voids(): - if name == index: - found = True - break - pos += 1 - assert found - - ref = "getelementptr(%s* %s, i32 0, i32 %s)" %( - self.get_typerepr(), - super(StructVarsizeNode, self).get_ref(), - pos) - - return ref - - def get_ref(self): - ref = super(StructVarsizeNode, self).get_ref() - typeval = self.db.repr_type(lltype.typeOf(self.value)) - ref = "bitcast(%s* %s to %s*)" % (self.get_typerepr(), - ref, - typeval) - return ref - - def get_pbcref(self, toptr): - """ Returns a reference as used per pbc. """ - ref = self.name - p, c = lltype.parentlink(self.value) - assert p is None, "child varsize struct are NOT needed by rtyper" - fromptr = "%s*" % self.get_typerepr() - refptr = "getelementptr(%s %s, i32 0)" % (fromptr, ref) - ref = "bitcast(%s %s to %s)" % (fromptr, refptr, toptr) - return ref - Modified: pypy/dist/pypy/translator/llvm/typedefnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/typedefnode.py (original) +++ pypy/dist/pypy/translator/llvm/typedefnode.py Wed Dec 5 23:10:16 2007 @@ -64,7 +64,9 @@ assert isinstance(STRUCT, lltype.Struct) self.db = db self.STRUCT = STRUCT - self.make_name(self.STRUCT._name) + parts = self.STRUCT._name.split('.') + name = parts[-1] + self.make_name(name) def _fields(self): return [getattr(self.STRUCT, name) From fijal at codespeak.net Wed Dec 5 23:35:32 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 5 Dec 2007 23:35:32 +0100 (CET) Subject: [pypy-svn] r49425 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071205223532.2AA2B168505@codespeak.net> Author: fijal Date: Wed Dec 5 23:35:31 2007 New Revision: 49425 Modified: pypy/dist/pypy/translator/llvm/test/test_rstr.py Log: Make py.test -k work Modified: pypy/dist/pypy/translator/llvm/test/test_rstr.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_rstr.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_rstr.py Wed Dec 5 23:35:31 2007 @@ -2,6 +2,8 @@ from pypy.rpython.test.test_rstr import BaseTestRstr from pypy.translator.llvm.test.runtest import * +# ====> ../../../rpython/test/test_rstr.py + class TestLLVMStr(LLVMTest, BaseTestRstr): EMPTY_STRING_HASH = -1 From antocuni at codespeak.net Wed Dec 5 23:46:03 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 5 Dec 2007 23:46:03 +0100 (CET) Subject: [pypy-svn] r49426 - pypy/dist/pypy/translator/goal Message-ID: <20071205224603.E482F1684EC@codespeak.net> Author: antocuni Date: Wed Dec 5 23:46:03 2007 New Revision: 49426 Modified: pypy/dist/pypy/translator/goal/bench-unix.py Log: ignore .jar files when searching for executables Modified: pypy/dist/pypy/translator/goal/bench-unix.py ============================================================================== --- pypy/dist/pypy/translator/goal/bench-unix.py (original) +++ pypy/dist/pypy/translator/goal/bench-unix.py Wed Dec 5 23:46:03 2007 @@ -82,7 +82,7 @@ def get_executables(): #sorted by revision number (highest first) exes = [] for exe in [os.path.join('.', name) for name in os.listdir('.') if name.startswith('pypy-')]: - if os.path.isdir(exe): + if os.path.isdir(exe) or exe.endswith('.jar'): continue try: exes.append( (exe.split('-')[2], exe) ) From rxe at codespeak.net Wed Dec 5 23:49:30 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 5 Dec 2007 23:49:30 +0100 (CET) Subject: [pypy-svn] r49427 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071205224930.E97F21684EC@codespeak.net> Author: rxe Date: Wed Dec 5 23:49:30 2007 New Revision: 49427 Modified: pypy/dist/pypy/translator/llvm/test/test_rstr.py Log: skip test for inplace_add Modified: pypy/dist/pypy/translator/llvm/test/test_rstr.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_rstr.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_rstr.py Wed Dec 5 23:49:30 2007 @@ -13,4 +13,6 @@ def test_float(self): py.test.skip('XXX special case me') + def test_inplace_add(self): + py.test.skip('XXX special case me') From rxe at codespeak.net Thu Dec 6 00:19:44 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 6 Dec 2007 00:19:44 +0100 (CET) Subject: [pypy-svn] r49429 - pypy/dist/pypy/rpython/lltypesystem/test Message-ID: <20071205231944.0F772168513@codespeak.net> Author: rxe Date: Thu Dec 6 00:19:44 2007 New Revision: 49429 Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Log: fix the test to be more opaque like Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Thu Dec 6 00:19:44 2007 @@ -219,13 +219,19 @@ struct stuff { char data[38]; }; - + """) + + c_source = py.code.Source(""" + #include "opaque.h" + char get(struct stuff* x) { x->data[13] = 'a'; return x->data[13]; } """) + + # if it doesn't segfault, than we probably malloced it :-) h_file = udir.join("opaque.h") h_file.write(h_source) @@ -233,7 +239,8 @@ from pypy.rpython.tool import rffi_platform eci = ExternalCompilationInfo( includes=['opaque.h'], - include_dirs=[str(udir)] + include_dirs=[str(udir)], + separate_module_sources=[c_source] ) STUFFP = COpaquePtr('struct stuff', compilation_info=eci) @@ -534,6 +541,3 @@ def test_hashdefine(self): py.test.skip("Macros cannot be called as llexternals by design, rffi does not have any special support for them") - - def test_opaque_type(self): - py.test.skip("GenLLVM handles opaque type defs incorrectly") From antocuni at codespeak.net Thu Dec 6 00:22:51 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 6 Dec 2007 00:22:51 +0100 (CET) Subject: [pypy-svn] r49430 - pypy/dist/pypy/translator Message-ID: <20071205232251.44FC2168513@codespeak.net> Author: antocuni Date: Thu Dec 6 00:22:50 2007 New Revision: 49430 Modified: pypy/dist/pypy/translator/driver.py Log: increase the maximum heap size for pypy-jvm, else microbench won't run on tuatara Modified: pypy/dist/pypy/translator/driver.py ============================================================================== --- pypy/dist/pypy/translator/driver.py (original) +++ pypy/dist/pypy/translator/driver.py Thu Dec 6 00:22:50 2007 @@ -734,7 +734,7 @@ f = file(newexename, 'w') f.write("""#!/bin/bash LEDIT=`type -p ledit` -$LEDIT java -jar $0.jar "$@" +$LEDIT java -Xmx256m -jar $0.jar "$@" """) f.close() os.chmod(newexename, 0755) From rxe at codespeak.net Thu Dec 6 00:22:55 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 6 Dec 2007 00:22:55 +0100 (CET) Subject: [pypy-svn] r49431 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071205232255.3AA6B168516@codespeak.net> Author: rxe Date: Thu Dec 6 00:22:53 2007 New Revision: 49431 Modified: pypy/dist/pypy/translator/llvm/codewriter.py pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/opwriter.py pypy/dist/pypy/translator/llvm/test/runtest.py pypy/dist/pypy/translator/llvm/test/test_genllvm.py Log: clean up some XXXs and write a half test the debug code Modified: pypy/dist/pypy/translator/llvm/codewriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/codewriter.py (original) +++ pypy/dist/pypy/translator/llvm/codewriter.py Thu Dec 6 00:22:53 2007 @@ -1,10 +1,5 @@ -from pypy.translator.llvm.log import log -from pypy.translator.llvm.buildllvm import postfix - -log = log.codewriter - class CodeWriter(object): - linkage = 'internal ' #/internal (disabled for now because of the JIT) + linkage = 'internal ' #internal/ def __init__(self, file, db, linkage=None): self.db = db @@ -27,7 +22,6 @@ for l in lines.split("\n"): if patch: l = l.replace('WORD', self.word_repr) - l = l.replace('POSTFIX', postfix()) self._append(l) def comment(self, line, indent=True): @@ -156,25 +150,24 @@ def alloca(self, targetvar, vartype): self._indent("%s = alloca %s" % (targetvar, vartype)) - def malloc(self, targetvar, vartype, numelements=1): - XXX # we should use this for raw malloc (unless it is slow) - if numelements == 1: - self._indent("%s = malloc %s" % (targetvar, vartype)) - else: - assert numelements > 1 - self._indent("%s = malloc %s, uint %s" % (targetvar, - vartype, - numelements)) +# def malloc(self, targetvar, vartype, numelements=1): +# XXX # we should use this for raw malloc (unless it is slow) +# if numelements == 1: +# self._indent("%s = malloc %s" % (targetvar, vartype)) +# else: +# assert numelements > 1 +# self._indent("%s = malloc %s, uint %s" % (targetvar, +# vartype, +# numelements)) - def free(self, vartype, varref): - XXX # we should use this for raw malloc (unless it is slow) - self._indent("free %s %s" % (vartype, varref)) +# def free(self, vartype, varref): +# XXX # we should use this for raw malloc (unless it is slow) +# self._indent("free %s %s" % (vartype, varref)) def debug_print(self, s): - XXX # fixme var = self.db.repr_tmpvar() node = self.db.create_debug_string(s) self.call(var, "i32", "@write", ['i32', 'i8*', 'i32'], - ['2', node.get_childref(0), '%d' % node.get_length()]) + ['2', node.ref, '%d' % node.get_length()]) Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Thu Dec 6 00:22:53 2007 @@ -53,7 +53,6 @@ #_______debuggging______________________________________ def dump_pbcs(self): - XXX#FIXME r = "" for k, v in self.obj2node.iteritems(): @@ -67,12 +66,11 @@ # Only dump top levels p, _ = lltype.parentlink(k) - ref = v.get_ref() type_ = self.repr_type(lltype.Ptr(lltype.typeOf(k))) r += "\ndump_pbcs %s (%s)\n" \ "parent %s\n" \ "type %s\n" \ - "getref -> %s \n" % (v, k, p, type_, ref) + "ref -> %s \n" % (v, k, p, type_, v.ref) return r #_______setting up and preparation______________________________ Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Thu Dec 6 00:22:53 2007 @@ -263,11 +263,12 @@ log('STATS %s' % str(s)) def _debug(self, codewriter): - if self.db.debugstringnodes: - codewriter.header_comment("Debug string") - for node in self.db.debugstringnodes: - node.writeglobalconstants(codewriter) + if self.config.translation.llvm.debug: + if self.db.debugstringnodes: + codewriter.header_comment("Debug string") + for node in self.db.debugstringnodes: + node.writeglobalconstants(codewriter) - #print "Start" - #print self.db.dump_pbcs() - #print "End" + print "Start" + print self.db.dump_pbcs() + print "End" Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Thu Dec 6 00:22:53 2007 @@ -311,8 +311,6 @@ self.db.gcpolicy._zeromalloc(self.codewriter, opr.retref, opr.argrefs[0], atomic=True) def boehm_register_finalizer(self, opr): - # XXX point in note - the registeree here have fastcc.... not sure if llvm is dealing with this - # because it is a pointer... - presumably tmpvar = self._tmp() self.codewriter.cast(tmpvar, opr.argtypes[1], opr.argrefs[1], 'i8 *') self.codewriter.call(None, 'void', '@pypy_register_finalizer', ['i8 *', 'i8 *'], [opr.argrefs[0], tmpvar]) Modified: pypy/dist/pypy/translator/llvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/runtest.py Thu Dec 6 00:22:53 2007 @@ -95,7 +95,7 @@ gcpolicy='boehm', # debug options - debug=True, + debug=False, logging=False, isolate=True, Modified: pypy/dist/pypy/translator/llvm/test/test_genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_genllvm.py Thu Dec 6 00:22:53 2007 @@ -533,3 +533,9 @@ # if res is still 0, then we haven't tested anything so fail. assert 0 < res <= 84 +def test_debug(): + # just tests code runs + def simple(): + return 42 + f = compile_function(simple, [], debug=True) + assert f() == 42 From rxe at codespeak.net Thu Dec 6 00:28:48 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 6 Dec 2007 00:28:48 +0100 (CET) Subject: [pypy-svn] r49432 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071205232848.16212168471@codespeak.net> Author: rxe Date: Thu Dec 6 00:28:47 2007 New Revision: 49432 Modified: pypy/dist/pypy/translator/llvm/test/test_rffi.py Log: these pass Modified: pypy/dist/pypy/translator/llvm/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_rffi.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_rffi.py Thu Dec 6 00:28:47 2007 @@ -270,7 +270,6 @@ posix = __import__(os.name) if hasattr(posix, 'execv'): def test_execv(): - py.test.skip("not working yet") filename = str(udir.join('test_execv.txt')) executable = sys.executable def does_stuff(): @@ -290,7 +289,6 @@ assert open(filename).read() == "1" def test_execv_raising(): - py.test.skip("not working yet") def does_stuff(): l = [] l.append("asddsadw32eewdfwqdqwdqwd") @@ -305,7 +303,6 @@ assert res == 1 def test_execve(): - py.test.skip("not working yet") filename = str(udir.join('test_execve.txt')) executable = sys.executable def does_stuff(): From rxe at codespeak.net Thu Dec 6 01:03:02 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 6 Dec 2007 01:03:02 +0100 (CET) Subject: [pypy-svn] r49434 - pypy/dist/pypy/rpython/module Message-ID: <20071206000302.694571684C1@codespeak.net> Author: rxe Date: Thu Dec 6 01:03:00 2007 New Revision: 49434 Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py Log: maks os.stat and friends work with genllvm again Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_stat.py Thu Dec 6 01:03:00 2007 @@ -196,7 +196,11 @@ 'lstat': '_stati64'} # no lstat on Windows c_func_name = _functions[name] else: - c_func_name = name + # because we always use _FILE_OFFSET_BITS 64 - this helps things work that are not a c compiler + _functions = {'stat': 'stat64', + 'fstat': 'fstat64', + 'lstat': 'lstat64'} + c_func_name = _functions[name] arg_is_path = (name != 'fstat') if arg_is_path: ARG1 = rffi.CCHARP From rxe at codespeak.net Thu Dec 6 01:31:17 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 6 Dec 2007 01:31:17 +0100 (CET) Subject: [pypy-svn] r49435 - pypy/dist/pypy/rpython/module Message-ID: <20071206003117.EAA4F1684FC@codespeak.net> Author: rxe Date: Thu Dec 6 01:31:17 2007 New Revision: 49435 Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py Log: fix for os/x Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_stat.py Thu Dec 6 01:31:17 2007 @@ -141,6 +141,9 @@ if sys.platform.startswith('win'): _name_struct_stat = '_stati64' INCLUDES = ['sys/types.h', 'sys/stat.h'] +elif sys.platform.startswith('darwin'): + _name_struct_stat = 'stat64' + INCLUDES = ['sys/stat.h'] else: _name_struct_stat = 'stat' INCLUDES = ['sys/types.h', 'sys/stat.h', 'unistd.h'] From fijal at codespeak.net Thu Dec 6 08:49:17 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 6 Dec 2007 08:49:17 +0100 (CET) Subject: [pypy-svn] r49436 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20071206074917.BE284168500@codespeak.net> Author: fijal Date: Thu Dec 6 08:49:16 2007 New Revision: 49436 Modified: pypy/dist/pypy/rpython/lltypesystem/rlist.py Log: Add a dummy adtmethod, just for tests. Modified: pypy/dist/pypy/rpython/lltypesystem/rlist.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rlist.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rlist.py Thu Dec 6 08:49:16 2007 @@ -74,6 +74,7 @@ ITEMARRAY = GcArray(ITEM, adtmeths = ADTIFixedList({ "ll_newlist": ll_fixed_newlist, + "ll_newlist_hint": ll_fixed_newlist, "ll_newemptylist": ll_fixed_newemptylist, "ll_length": ll_fixed_length, "ll_items": ll_fixed_items, From antocuni at codespeak.net Thu Dec 6 12:16:02 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 6 Dec 2007 12:16:02 +0100 (CET) Subject: [pypy-svn] r49446 - in pypy/dist/pypy/translator: . jvm Message-ID: <20071206111602.39C1E16852E@codespeak.net> Author: antocuni Date: Thu Dec 6 12:15:58 2007 New Revision: 49446 Modified: pypy/dist/pypy/translator/driver.py pypy/dist/pypy/translator/jvm/genjvm.py Log: intermediate checkin; automatically merge jna.jar into the jar generated by genjvm. I still have to add jna.jar to the repo Modified: pypy/dist/pypy/translator/driver.py ============================================================================== --- pypy/dist/pypy/translator/driver.py (original) +++ pypy/dist/pypy/translator/driver.py Thu Dec 6 12:15:58 2007 @@ -722,7 +722,8 @@ basename = self.exe_name % self.get_info() root = udir.join('pypy') manifest = self.create_manifest(root) - classlist = self.create_classlist(root) + jnajar = py.path.local(__file__).dirpath('jvm', 'src', 'jna.jar') + classlist = self.create_classlist(root, [jnajar]) jarfile = py.path.local(basename + '.jar') self.log.info('Creating jar file') oldpath = root.chdir() @@ -746,10 +747,19 @@ manifest.close() return filename - def create_classlist(self, root): + def create_classlist(self, root, additional_jars=[]): + from py.compat import subprocess + # first, uncompress additional jars + for jarfile in additional_jars: + oldpwd = root.chdir() + subprocess.call(['jar', 'xf', str(jarfile)]) + oldpwd.chdir() filename = root.join('classlist.txt') classlist = filename.open('w') - classfiles = root.visit('*.class', True) + classfiles = list(root.visit('*.class', True)) + classfiles += root.visit('*.so', True) + classfiles += root.visit('*.dll', True) + classfiles += root.visit('*.jnilib', True) for classfile in classfiles: print >> classlist, classfile.relto(root) classlist.close() Modified: pypy/dist/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/genjvm.py (original) +++ pypy/dist/pypy/translator/jvm/genjvm.py Thu Dec 6 12:15:58 2007 @@ -114,6 +114,7 @@ srcdir = rootdir.join('pypy') javafiles = srcdir.listdir('*.java') classfiles = srcdir.listdir('*.class') + jnajar = rootdir.join('jna.jar') recompile = True if len(classfiles) == len(javafiles): @@ -125,7 +126,12 @@ if recompile: log.red('Compiling java classes') javasrcs = [str(jf) for jf in javafiles] - self._invoke([getoption('javac'), '-nowarn', '-d', str(rootdir)] + javasrcs, True) + self._invoke([getoption('javac'), + '-nowarn', + '-d', str(rootdir), + '-classpath', str(jnajar) + ] + javasrcs, + True) # copy .class files to classdir for classfile in srcdir.listdir('*.class'): From antocuni at codespeak.net Thu Dec 6 12:37:42 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 6 Dec 2007 12:37:42 +0100 (CET) Subject: [pypy-svn] r49448 - in pypy/dist: . pypy/translator/jvm/src Message-ID: <20071206113742.20ECD16853B@codespeak.net> Author: antocuni Date: Thu Dec 6 12:37:42 2007 New Revision: 49448 Added: pypy/dist/pypy/translator/jvm/src/jna.jar (contents, props changed) Modified: pypy/dist/LICENSE Log: add jna.jar to the repo Modified: pypy/dist/LICENSE ============================================================================== --- pypy/dist/LICENSE (original) +++ pypy/dist/LICENSE Thu Dec 6 12:37:42 2007 @@ -111,3 +111,10 @@ are all copyrighted by the Python Software Foundation and licensed under the Python Software License of which you can find a copy here: http://www.python.org/doc/Copyright.html + +License for 'pypy/translator/jvm/src/jna.jar' +============================================= + +The file 'pypy/translator/jvm/src/jna.jar' is licensed under the GNU +Lesser General Public License of which you can find a copy here: +http://www.gnu.org/licenses/lgpl.html Added: pypy/dist/pypy/translator/jvm/src/jna.jar ============================================================================== Binary file. No diff available. From pypy-svn at codespeak.net Thu Dec 6 13:05:18 2007 From: pypy-svn at codespeak.net (VIAGRA ® Official Site) Date: Thu, 6 Dec 2007 13:05:18 +0100 (CET) Subject: [pypy-svn] December 74% OFF Message-ID: <20071206020702.6724.qmail@AMarseille-157-1-29-39.w90-15.abo.wanadoo.fr> An HTML attachment was scrubbed... URL: From fijal at codespeak.net Thu Dec 6 13:12:08 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 6 Dec 2007 13:12:08 +0100 (CET) Subject: [pypy-svn] r49449 - pypy/branch/lazy-write-barrier Message-ID: <20071206121208.28824168531@codespeak.net> Author: fijal Date: Thu Dec 6 13:12:07 2007 New Revision: 49449 Added: pypy/branch/lazy-write-barrier/ - copied from r49448, pypy/dist/ Log: Create a branch for some experiments on lazy write barrier. From rxe at codespeak.net Thu Dec 6 15:39:06 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 6 Dec 2007 15:39:06 +0100 (CET) Subject: [pypy-svn] r49450 - in pypy/dist/pypy/translator: . c Message-ID: <20071206143906.A7D37168532@codespeak.net> Author: rxe Date: Thu Dec 6 15:39:05 2007 New Revision: 49450 Modified: pypy/dist/pypy/translator/c/extfunc.py pypy/dist/pypy/translator/exceptiontransform.py Log: not needed anymore Modified: pypy/dist/pypy/translator/c/extfunc.py ============================================================================== --- pypy/dist/pypy/translator/c/extfunc.py (original) +++ pypy/dist/pypy/translator/c/extfunc.py Thu Dec 6 15:39:05 2007 @@ -109,7 +109,6 @@ if not db.standalone: yield ('RPYTHON_PYEXCCLASS2EXC', exceptiondata.fn_pyexcclass2exc) - yield ('_RPyExceptionOccurred', exctransformer._rpyexc_occured_ptr.value) yield ('RPyExceptionOccurred1', exctransformer.rpyexc_occured_ptr.value) yield ('RPyFetchExceptionType', exctransformer.rpyexc_fetch_type_ptr.value) yield ('RPyFetchExceptionValue', exctransformer.rpyexc_fetch_value_ptr.value) Modified: pypy/dist/pypy/translator/exceptiontransform.py ============================================================================== --- pypy/dist/pypy/translator/exceptiontransform.py (original) +++ pypy/dist/pypy/translator/exceptiontransform.py Thu Dec 6 15:39:05 2007 @@ -52,7 +52,7 @@ edata = translator.rtyper.getexceptiondata() self.lltype_of_exception_value = edata.lltype_of_exception_value self.lltype_of_exception_type = edata.lltype_of_exception_type - self.mixlevelannotator = MixLevelHelperAnnotator(translator.rtyper) + self.mixlevelannotator = MixLevelHelperAnnotator(translator.rtyper) exc_data, null_type, null_value = self.setup_excdata() rclass = translator.rtyper.type_system.rclass @@ -64,13 +64,6 @@ exc_type = exc_data.exc_type return bool(exc_type) - # XXX tmp HACK for genllvm - # llvm is strongly typed between bools and ints, which means we have no way of - # calling rpyexc_occured() from c code with lltype.Bool - def _rpyexc_occured(): - exc_type = exc_data.exc_type - return bool(exc_type) - def rpyexc_fetch_type(): return exc_data.exc_type @@ -103,12 +96,6 @@ rpyexc_occured, [], lltype.Bool) - # XXX tmp HACK for genllvm - self._rpyexc_occured_ptr = self.build_func( - "_RPyExceptionOccurred", - _rpyexc_occured, - [], lltype.Signed) - self.rpyexc_fetch_type_ptr = self.build_func( "RPyFetchExceptionType", rpyexc_fetch_type, From fijal at codespeak.net Thu Dec 6 16:55:18 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 6 Dec 2007 16:55:18 +0100 (CET) Subject: [pypy-svn] r49451 - pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/test Message-ID: <20071206155518.AAE0D168535@codespeak.net> Author: fijal Date: Thu Dec 6 16:55:17 2007 New Revision: 49451 Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/test/test_framework.py Log: Add a strange test where I try to have a static root. Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/test/test_framework.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/test/test_framework.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/test/test_framework.py Thu Dec 6 16:55:17 2007 @@ -166,3 +166,28 @@ collect_analyzer = CollectAnalyzer(t) init_stores = find_initializing_stores(collect_analyzer, t.graphs[0]) assert len(init_stores) == 5 + +def test_immutable_to_old_promotion(): + T_CHILD = lltype.Ptr(lltype.GcStruct('Childe', ('field', lltype.Signed))) + T_PARENT = lltype.Ptr(lltype.GcStruct('Parent', ('sub', T_CHILD))) + child = lltype.malloc(T_CHILD.TO) + parent = lltype.malloc(T_PARENT.TO) + parent.sub = child + child.field = 3 + d = {'x' : parent} + d['y'] = lltype.malloc(T_PARENT.TO) + d['y'].sub = lltype.malloc(T_CHILD.TO) + + def f(x, y): + t = d[y] + res = t.sub.field + x + del d[y] + return res + + t = rtype(f, [int, str]) + etrafo = ExceptionTransformer(t) + graphs = etrafo.transform_completely() + transformer = WriteBarrierTransformer(t) + transformer.finish() + graphof(t, f).show() + From pedronis at codespeak.net Thu Dec 6 17:43:28 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 6 Dec 2007 17:43:28 +0100 (CET) Subject: [pypy-svn] r49452 - pypy/dist/pypy/translator/microbench Message-ID: <20071206164328.AAB8916851E@codespeak.net> Author: pedronis Date: Thu Dec 6 17:43:27 2007 New Revision: 49452 Modified: pypy/dist/pypy/translator/microbench/microbench.py Log: sprinkle a gc collect as armin suggested Modified: pypy/dist/pypy/translator/microbench/microbench.py ============================================================================== --- pypy/dist/pypy/translator/microbench/microbench.py (original) +++ pypy/dist/pypy/translator/microbench/microbench.py Thu Dec 6 17:43:27 2007 @@ -4,7 +4,7 @@ implementations on a set of microbenchmarks. The script usally is started with "./microbench.py python ./pypy" where pypy is a symlink to you pypy exectable.""" -import os, time, sys +import os, time, sys, gc microbenches = [] for fname in os.listdir('.'): @@ -26,6 +26,7 @@ else: continue testcase = microbench + '.' + k + '()' + gc.collect() start = time.clock() n = 0 duration = 0.0 From rxe at codespeak.net Thu Dec 6 17:46:32 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 6 Dec 2007 17:46:32 +0100 (CET) Subject: [pypy-svn] r49453 - pypy/dist/pypy/translator/llvm Message-ID: <20071206164632.F385016851E@codespeak.net> Author: rxe Date: Thu Dec 6 17:46:32 2007 New Revision: 49453 Modified: pypy/dist/pypy/translator/llvm/codewriter.py pypy/dist/pypy/translator/llvm/externs2ll.py pypy/dist/pypy/translator/llvm/opwriter.py Log: fix up debug_print() again Modified: pypy/dist/pypy/translator/llvm/codewriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/codewriter.py (original) +++ pypy/dist/pypy/translator/llvm/codewriter.py Thu Dec 6 17:46:32 2007 @@ -168,6 +168,7 @@ def debug_print(self, s): var = self.db.repr_tmpvar() node = self.db.create_debug_string(s) + arg = "bitcast(%s* %s to i8*)" % (node.get_typerepr(), node.ref) self.call(var, "i32", "@write", ['i32', 'i8*', 'i32'], - ['2', node.ref, '%d' % node.get_length()]) + ['2', arg, '%d' % node.get_length()]) Modified: pypy/dist/pypy/translator/llvm/externs2ll.py ============================================================================== --- pypy/dist/pypy/translator/llvm/externs2ll.py (original) +++ pypy/dist/pypy/translator/llvm/externs2ll.py Thu Dec 6 17:46:32 2007 @@ -35,8 +35,6 @@ includestr += "-I %s " % ii return includestr -# call boehm finalizers need to be fastcc - def generate_ll(ccode, eci): filename = str(udir.join("ccode.c")) f = open(filename, "w") @@ -52,13 +50,14 @@ raise Exception("Failed to run '%s'" % cmd) llcode = open(plain + '.ll').read() - + # strip lines lines = [] for line in llcode.split('\n'): lines.append(line) - lines.append("declare ccc void @abort()") + lines.append("declare void @abort()") + lines.append("declare i32 @write(i32, i8 *, i32)") return'\n'.join(lines) def generate_c(db, entrynode, eci, standalone): Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Thu Dec 6 17:46:32 2007 @@ -111,9 +111,9 @@ return indices def write_operation(self, op): + self.codewriter.comment(str(op)) if self.db.genllvm.config.translation.llvm.debug: - self.codewriter.comment(str(op)) - #self.codewriter.debug_print(str(op) + "\n") + self.codewriter.debug_print(str(op) + "\n") if op.opname in ("direct_call", 'indirect_call'): opr = OpReprCall(op, self.db) From regmee at codespeak.net Thu Dec 6 19:52:50 2007 From: regmee at codespeak.net (regmee at codespeak.net) Date: Thu, 6 Dec 2007 19:52:50 +0100 (CET) Subject: [pypy-svn] r49454 - in pypy/branch/clr-module-improvements/pypy/module/clr: . test Message-ID: <20071206185250.BA42416853A@codespeak.net> Author: regmee Date: Thu Dec 6 19:52:49 2007 New Revision: 49454 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/app_importer.py pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py Log: import hooks implemented. More test cases added. Hardcoded .NET modules names for the check to be resolved using reflection Modified: pypy/branch/clr-module-improvements/pypy/module/clr/app_importer.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/app_importer.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/app_importer.py Thu Dec 6 19:52:49 2007 @@ -1,66 +1,117 @@ -""" Importer class - # Meta hooks are called at the start of Import Processing - # Meta hooks can override the sys.path, frozen modules , built-in modules - # to register a Meta Hook simply add importer object to sys.meta_path - # a path hook is registered by adding an Importer factory to sys.path_hooks - # sys.path_hooks is a list of Class of the HOOK. - # whose __init__ is called when the calleable in the list is obtained. - # __init__ cant return anything so some __new__ method should return - This is used to enable the "import module" mechanism for .NET classes""" +# Meta hooks are called at the start of Import Processing +# Meta hooks can override the sys.path, frozen modules , built-in modules +# To register a Meta Hook simply add importer object to sys.meta_path import imp import sys - + +DotNetModuleList = ['System', + 'System.Collections', + 'System.Collections.ArrayList', + 'System.Collections.Stack', + 'System.Collections.Queue', + 'System.Math'] + + class loader(object): + ''' + This method returns the loaded module or raises an exception, ImportError + loader is ONLY calld for proper .NET modules (check done in Importer class + ''' def __init__(self): - self.Names = [] + pass def load_module(self, fullname): - - # Now since the module was not found .. Call the Loader and load it. + ''' + The load_module() must fulfill the following *before* it runs any code: + Note that the module object *must* be in sys.modules before the + loader executes the module code. + + A If 'fullname' exists in sys.modules, the loader must use that + else the loader must create a new module object and add it to sys.modules. + + module = sys.modules.setdefault(fullname, new.module(fullname)) + + B The __file__ attribute must be set. String say "" + + C The __name__ attribute must be set. If one uses + imp.new_module() then the attribute is set automatically. + + D If it's a package, the __path__ variable must be set. This must + be a list, but may be empty if __path__ has no further + significance to the importer (more on this later). + + E It should add a __loader__ attribute to the module, set to the loader object. + + ''' + # If it is a call for a Class then return with the Class reference + code = 0 if fullname == "System.Math": + code = 1 import clr - return clr.load_cli_class('System','Math') - + sys.modules[fullname] = clr.load_cli_class('System','Math') + if fullname == "System.Collections.Queue": + code = 1 + import clr + sys.modules[fullname] = clr.load_cli_class('System.Collections','Queue') + if fullname == "System.Collections.Stack": + code = 1 + import clr + sys.modules[fullname] = clr.load_cli_class('System.Collections','Stack') if fullname == "System.Collections.ArrayList": + code = 1 import clr - return clr.load_cli_class('System.Collections','ArrayList') + sys.modules[fullname] = clr.load_cli_class('System.Collections','ArrayList') - # Now create a new module and append it at the end of the sys.modules list - mod = imp.new_module(fullname) - mod.__file__ = "<%s>" % self.__class__.__name__ - mod.__loader__ = self - mod.__name__ = fullname - #if ispkg: - #if : - # mod.__path__ = [] - #exec code in mod.__dict__''' + # if not a call for actual class assign an empty module for it. + if not code: + mod = imp.new_module(fullname) + mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) + mod.__file__ = "<%s>" % self.__class__.__name__ + mod.__loader__ = self + mod.__name__ = fullname - # add it to the modules list - sys.modules[fullname] = mod + # if it is a PACKAGE then we are to initialize the __path__ for the module + # we won't deal with Packages here - return mod + # add it to the modules list + sys.modules[fullname] = mod -class importer(object): + return sys.modules[fullname] +class importer(object): + ''' + If the importer is installed on sys.meta_path, it will + receive a second argument, which is None for a top-level module, or + package.__path__ for submodules or subpackages + + It should return a loader object if the module was found, or None if it wasn't. + If find_module() raises an exception, the caller will abort the import. + When importer.find_module("spam.eggs.ham") is called, "spam.eggs" has already + been imported and added to sys.modules. + ''' def __init__(self): self.loader = loader() - def find_module(self, fullname, path): - # path will be None for top-level Module and __path__ for sub-modules - print fullname - if path != None: - __path__ = path - try: - return sys.modules[fullname] - except KeyError: - pass - - try: - return self.loader - except ImportError: - print "Import Error exception raised hence you better quit" + def find_module(self, fullname, path = None): + #print "( fullname = %s ) + ( path = %s )"%(fullname, path) + if fullname in DotNetModuleList: + # fullname is a .Net Module + if path != None: + __path__ = path + try: + return sys.modules[fullname] + except KeyError: + pass + + try: + return self.loader + except ImportError: + return None + else: + # fullname is not a .Net Module return None + + -#def load_cli_class(space, namespace, classname): Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_importer.py Thu Dec 6 19:52:49 2007 @@ -15,6 +15,19 @@ Math = clr.load_cli_class('System', 'Math') assert Math is System.Math + import System.Collections.Stack + a = System.Collections.Stack() + a.Push(3) + a.Push(44) + sum = 0 + for i in a: + sum += i + assert sum == 3+44 + + import System.Collections.ArrayList + ArrayList = clr.load_cli_class('System.Collections', 'ArrayList') + assert ArrayList is System.Collections.ArrayList + def test_ImportError(self): skip('Fixme!') def fn(): From regmee at codespeak.net Thu Dec 6 19:54:24 2007 From: regmee at codespeak.net (regmee at codespeak.net) Date: Thu, 6 Dec 2007 19:54:24 +0100 (CET) Subject: [pypy-svn] r49455 - pypy/branch/clr-module-improvements/pypy/module/clr Message-ID: <20071206185424.97829168544@codespeak.net> Author: regmee Date: Thu Dec 6 19:54:24 2007 New Revision: 49455 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py Log: import hooks contd... missed to include file Modified: pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py Thu Dec 6 19:54:24 2007 @@ -16,8 +16,8 @@ 'load_cli_class': 'interp_clr.load_cli_class', } -## def setup_after_space_initialization(self): -## self.space.appexec([self], """(clr_module): -## import sys -## sys.meta_path.append(clr_module.dotnetimporter()) -## """) + def setup_after_space_initialization(self): + self.space.appexec([self], """(clr_module): + import sys + sys.meta_path.append(clr_module.dotnetimporter()) + """) From cfbolz at codespeak.net Thu Dec 6 20:26:15 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 6 Dec 2007 20:26:15 +0100 (CET) Subject: [pypy-svn] r49456 - in pypy/dist/pypy/rlib/parsing: . test Message-ID: <20071206192615.38A6916853B@codespeak.net> Author: cfbolz Date: Thu Dec 6 20:26:13 2007 New Revision: 49456 Modified: pypy/dist/pypy/rlib/parsing/ebnfparse.py pypy/dist/pypy/rlib/parsing/test/test_ebnfparse.py Log: whack whack whack a bit to allow epsilon productions. not really all that well tested. Modified: pypy/dist/pypy/rlib/parsing/ebnfparse.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/ebnfparse.py (original) +++ pypy/dist/pypy/rlib/parsing/ebnfparse.py Thu Dec 6 20:26:13 2007 @@ -27,7 +27,7 @@ list: element+; element: | ; regex: SYMBOLNAME ":" QUOTE ";"; - production: NONTERMINALNAME ":" body ";"; + production: NONTERMINALNAME ":" body? ";"; body: (expansion ["|"])* expansion; expansion: decorated+; decorated: enclosed "*" | @@ -125,6 +125,10 @@ def visit_production(self, node): name = node.children[0].additional_info + if len(node.children) == 3: + self.changes.append([]) + self.rules.append(Rule(name, [[]])) + return expansions = node.children[2].visit(self) changes = [] rule_expansions = [] @@ -228,6 +232,10 @@ all_changes = [] other_changes = [] for rule, changes in zip(self.rules, self.changes): + if rule.expansions == [[]]: + all_rules.append(rule) + all_changes.append([]) + continue real_changes = [] real_expansions = [] for index, (expansion, change) in enumerate( @@ -331,6 +339,10 @@ rule = self.rules[index] change = self.changes[index] self.start_block("def visit_%s(self, node):" % (rule.nonterminal, )) + if len(change) == 0: + self.emit("return [node]") + self.end_block(rule.nonterminal) + return for expansion, subchange in self.generate_conditions(index): if "<" in subchange: i = subchange.index("<") @@ -468,14 +480,27 @@ children.extend([node.children[2]]) children.extend([node.children[3]]) return [Nonterminal(node.symbol, children)] + def visit__maybe_symbol0(self, node): + children = [] + children.extend(self.visit_body(node.children[0])) + return [Nonterminal(node.symbol, children)] def visit_production(self, node): + length = len(node.children) + if length == 3: + children = [] + children.extend([node.children[0]]) + children.extend([node.children[1]]) + children.extend([node.children[2]]) + return [Nonterminal(node.symbol, children)] children = [] children.extend([node.children[0]]) children.extend([node.children[1]]) - children.extend(self.visit_body(node.children[2])) + expr = self.visit__maybe_symbol0(node.children[2]) + assert len(expr) == 1 + children.extend(expr[0].children) children.extend([node.children[3]]) return [Nonterminal(node.symbol, children)] - def visit__star_symbol0(self, node): + def visit__star_symbol1(self, node): length = len(node.children) if length == 2: children = [] @@ -483,7 +508,7 @@ return [Nonterminal(node.symbol, children)] children = [] children.extend(self.visit_expansion(node.children[0])) - expr = self.visit__star_symbol0(node.children[2]) + expr = self.visit__star_symbol1(node.children[2]) assert len(expr) == 1 children.extend(expr[0].children) return [Nonterminal(node.symbol, children)] @@ -494,7 +519,7 @@ children.extend(self.visit_expansion(node.children[0])) return [Nonterminal(node.symbol, children)] children = [] - expr = self.visit__star_symbol0(node.children[0]) + expr = self.visit__star_symbol1(node.children[0]) assert len(expr) == 1 children.extend(expr[0].children) children.extend(self.visit_expansion(node.children[1])) @@ -588,9 +613,10 @@ Rule('list', [['_plus_symbol0']]), Rule('element', [['regex'], ['production']]), Rule('regex', [['SYMBOLNAME', '__0_:', 'QUOTE', '__1_;']]), - Rule('production', [['NONTERMINALNAME', '__0_:', 'body', '__1_;']]), - Rule('_star_symbol0', [['expansion', '__2_|', '_star_symbol0'], ['expansion', '__2_|']]), - Rule('body', [['_star_symbol0', 'expansion'], ['expansion']]), + Rule('_maybe_symbol0', [['body']]), + Rule('production', [['NONTERMINALNAME', '__0_:', '_maybe_symbol0', '__1_;'], ['NONTERMINALNAME', '__0_:', '__1_;']]), + Rule('_star_symbol1', [['expansion', '__2_|', '_star_symbol1'], ['expansion', '__2_|']]), + Rule('body', [['_star_symbol1', 'expansion'], ['expansion']]), Rule('_plus_symbol1', [['decorated', '_plus_symbol1'], ['decorated']]), Rule('expansion', [['_plus_symbol1']]), Rule('decorated', [['enclosed', '__3_*'], ['enclosed', '__4_+'], ['enclosed', '__5_?'], ['enclosed']]), @@ -603,43 +629,43 @@ state = 0 while 1: if state == 0: - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 0 return ~i if char == '\t': state = 1 elif char == '\n': state = 2 - elif 'A' <= char <= 'Z': - state = 3 elif char == ' ': - state = 4 + state = 3 elif char == '#': - state = 5 + state = 4 elif char == '"': - state = 6 + state = 5 elif char == "'": - state = 7 + state = 6 elif char == ')': - state = 8 + state = 7 elif char == '(': - state = 9 + state = 8 elif char == '+': - state = 10 + state = 9 elif char == '*': - state = 11 + state = 10 elif char == ';': - state = 12 + state = 11 elif char == ':': - state = 13 + state = 12 elif char == '<': - state = 14 + state = 13 elif char == '?': - state = 15 + state = 14 elif char == '>': + state = 15 + elif 'A' <= char <= 'Z': state = 16 elif char == '[': state = 17 @@ -653,87 +679,87 @@ state = 21 else: break - if state == 3: - runner.last_matched_index = i - 1 - runner.last_matched_state = state - if i < len(input): - char = input[i] - i += 1 - else: - runner.state = 3 - return i - if 'A' <= char <= 'Z': - state = 3 - continue - elif char == '_': - state = 3 - continue - else: - break - if state == 5: - if i < len(input): + if state == 4: + try: char = input[i] i += 1 - else: - runner.state = 5 + except IndexError: + runner.state = 4 return ~i if char == '\n': state = 27 elif '\x00' <= char <= '\t': - state = 5 + state = 4 continue elif '\x0b' <= char <= '\xff': - state = 5 + state = 4 continue else: break - if state == 6: - if i < len(input): + if state == 5: + try: char = input[i] i += 1 - else: - runner.state = 6 + except IndexError: + runner.state = 5 return ~i if char == '\\': state = 24 elif char == '"': state = 25 elif '\x00' <= char <= '!': - state = 6 + state = 5 continue elif '#' <= char <= '[': - state = 6 + state = 5 continue elif ']' <= char <= '\xff': - state = 6 + state = 5 continue else: break - if state == 7: - if i < len(input): + if state == 6: + try: char = input[i] i += 1 - else: - runner.state = 7 + except IndexError: + runner.state = 6 return ~i if char == '"': state = 22 else: break - if state == 19: + if state == 16: runner.last_matched_index = i - 1 runner.last_matched_state = state - if i < len(input): + try: char = input[i] i += 1 - else: - runner.state = 19 + except IndexError: + runner.state = 16 return i if char == '_': - state = 19 + state = 16 continue elif 'A' <= char <= 'Z': - state = 3 + state = 16 + continue + else: + break + if state == 19: + runner.last_matched_index = i - 1 + runner.last_matched_state = state + try: + char = input[i] + i += 1 + except IndexError: + runner.state = 19 + return i + if 'A' <= char <= 'Z': + state = 16 + continue + elif char == '_': + state = 19 continue elif '0' <= char <= '9': state = 20 @@ -744,16 +770,16 @@ if state == 20: runner.last_matched_index = i - 1 runner.last_matched_state = state - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 20 return i - if '0' <= char <= '9': + if char == '_': state = 20 continue - elif char == '_': + elif '0' <= char <= '9': state = 20 continue elif 'a' <= char <= 'z': @@ -762,10 +788,10 @@ else: break if state == 22: - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 22 return ~i if char == "'": @@ -773,10 +799,10 @@ else: break if state == 24: - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 24 return ~i if char == '\\': @@ -785,32 +811,32 @@ elif char == '"': state = 26 elif '\x00' <= char <= '!': - state = 6 + state = 5 continue elif '#' <= char <= '[': - state = 6 + state = 5 continue elif ']' <= char <= '\xff': - state = 6 + state = 5 continue else: break if state == 26: runner.last_matched_index = i - 1 runner.last_matched_state = state - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 26 return i if char == '"': state = 25 elif '\x00' <= char <= '!': - state = 6 + state = 5 continue elif '#' <= char <= '\xff': - state = 6 + state = 5 continue else: break @@ -827,45 +853,45 @@ lexer = DummyLexer(recognize, DFA(28, {(0, '\t'): 1, (0, '\n'): 2, - (0, ' '): 4, - (0, '"'): 6, - (0, '#'): 5, - (0, "'"): 7, - (0, '('): 9, - (0, ')'): 8, - (0, '*'): 11, - (0, '+'): 10, - (0, ':'): 13, - (0, ';'): 12, - (0, '<'): 14, - (0, '>'): 16, - (0, '?'): 15, - (0, 'A'): 3, - (0, 'B'): 3, - (0, 'C'): 3, - (0, 'D'): 3, - (0, 'E'): 3, - (0, 'F'): 3, - (0, 'G'): 3, - (0, 'H'): 3, - (0, 'I'): 3, - (0, 'J'): 3, - (0, 'K'): 3, - (0, 'L'): 3, - (0, 'M'): 3, - (0, 'N'): 3, - (0, 'O'): 3, - (0, 'P'): 3, - (0, 'Q'): 3, - (0, 'R'): 3, - (0, 'S'): 3, - (0, 'T'): 3, - (0, 'U'): 3, - (0, 'V'): 3, - (0, 'W'): 3, - (0, 'X'): 3, - (0, 'Y'): 3, - (0, 'Z'): 3, + (0, ' '): 3, + (0, '"'): 5, + (0, '#'): 4, + (0, "'"): 6, + (0, '('): 8, + (0, ')'): 7, + (0, '*'): 10, + (0, '+'): 9, + (0, ':'): 12, + (0, ';'): 11, + (0, '<'): 13, + (0, '>'): 15, + (0, '?'): 14, + (0, 'A'): 16, + (0, 'B'): 16, + (0, 'C'): 16, + (0, 'D'): 16, + (0, 'E'): 16, + (0, 'F'): 16, + (0, 'G'): 16, + (0, 'H'): 16, + (0, 'I'): 16, + (0, 'J'): 16, + (0, 'K'): 16, + (0, 'L'): 16, + (0, 'M'): 16, + (0, 'N'): 16, + (0, 'O'): 16, + (0, 'P'): 16, + (0, 'Q'): 16, + (0, 'R'): 16, + (0, 'S'): 16, + (0, 'T'): 16, + (0, 'U'): 16, + (0, 'V'): 16, + (0, 'W'): 16, + (0, 'X'): 16, + (0, 'Y'): 16, + (0, 'Z'): 16, (0, '['): 17, (0, ']'): 18, (0, '_'): 19, @@ -896,33 +922,262 @@ (0, 'y'): 20, (0, 'z'): 20, (0, '|'): 21, - (3, 'A'): 3, - (3, 'B'): 3, - (3, 'C'): 3, - (3, 'D'): 3, - (3, 'E'): 3, - (3, 'F'): 3, - (3, 'G'): 3, - (3, 'H'): 3, - (3, 'I'): 3, - (3, 'J'): 3, - (3, 'K'): 3, - (3, 'L'): 3, - (3, 'M'): 3, - (3, 'N'): 3, - (3, 'O'): 3, - (3, 'P'): 3, - (3, 'Q'): 3, - (3, 'R'): 3, - (3, 'S'): 3, - (3, 'T'): 3, - (3, 'U'): 3, - (3, 'V'): 3, - (3, 'W'): 3, - (3, 'X'): 3, - (3, 'Y'): 3, - (3, 'Z'): 3, - (3, '_'): 3, + (4, '\x00'): 4, + (4, '\x01'): 4, + (4, '\x02'): 4, + (4, '\x03'): 4, + (4, '\x04'): 4, + (4, '\x05'): 4, + (4, '\x06'): 4, + (4, '\x07'): 4, + (4, '\x08'): 4, + (4, '\t'): 4, + (4, '\n'): 27, + (4, '\x0b'): 4, + (4, '\x0c'): 4, + (4, '\r'): 4, + (4, '\x0e'): 4, + (4, '\x0f'): 4, + (4, '\x10'): 4, + (4, '\x11'): 4, + (4, '\x12'): 4, + (4, '\x13'): 4, + (4, '\x14'): 4, + (4, '\x15'): 4, + (4, '\x16'): 4, + (4, '\x17'): 4, + (4, '\x18'): 4, + (4, '\x19'): 4, + (4, '\x1a'): 4, + (4, '\x1b'): 4, + (4, '\x1c'): 4, + (4, '\x1d'): 4, + (4, '\x1e'): 4, + (4, '\x1f'): 4, + (4, ' '): 4, + (4, '!'): 4, + (4, '"'): 4, + (4, '#'): 4, + (4, '$'): 4, + (4, '%'): 4, + (4, '&'): 4, + (4, "'"): 4, + (4, '('): 4, + (4, ')'): 4, + (4, '*'): 4, + (4, '+'): 4, + (4, ','): 4, + (4, '-'): 4, + (4, '.'): 4, + (4, '/'): 4, + (4, '0'): 4, + (4, '1'): 4, + (4, '2'): 4, + (4, '3'): 4, + (4, '4'): 4, + (4, '5'): 4, + (4, '6'): 4, + (4, '7'): 4, + (4, '8'): 4, + (4, '9'): 4, + (4, ':'): 4, + (4, ';'): 4, + (4, '<'): 4, + (4, '='): 4, + (4, '>'): 4, + (4, '?'): 4, + (4, '@'): 4, + (4, 'A'): 4, + (4, 'B'): 4, + (4, 'C'): 4, + (4, 'D'): 4, + (4, 'E'): 4, + (4, 'F'): 4, + (4, 'G'): 4, + (4, 'H'): 4, + (4, 'I'): 4, + (4, 'J'): 4, + (4, 'K'): 4, + (4, 'L'): 4, + (4, 'M'): 4, + (4, 'N'): 4, + (4, 'O'): 4, + (4, 'P'): 4, + (4, 'Q'): 4, + (4, 'R'): 4, + (4, 'S'): 4, + (4, 'T'): 4, + (4, 'U'): 4, + (4, 'V'): 4, + (4, 'W'): 4, + (4, 'X'): 4, + (4, 'Y'): 4, + (4, 'Z'): 4, + (4, '['): 4, + (4, '\\'): 4, + (4, ']'): 4, + (4, '^'): 4, + (4, '_'): 4, + (4, '`'): 4, + (4, 'a'): 4, + (4, 'b'): 4, + (4, 'c'): 4, + (4, 'd'): 4, + (4, 'e'): 4, + (4, 'f'): 4, + (4, 'g'): 4, + (4, 'h'): 4, + (4, 'i'): 4, + (4, 'j'): 4, + (4, 'k'): 4, + (4, 'l'): 4, + (4, 'm'): 4, + (4, 'n'): 4, + (4, 'o'): 4, + (4, 'p'): 4, + (4, 'q'): 4, + (4, 'r'): 4, + (4, 's'): 4, + (4, 't'): 4, + (4, 'u'): 4, + (4, 'v'): 4, + (4, 'w'): 4, + (4, 'x'): 4, + (4, 'y'): 4, + (4, 'z'): 4, + (4, '{'): 4, + (4, '|'): 4, + (4, '}'): 4, + (4, '~'): 4, + (4, '\x7f'): 4, + (4, '\x80'): 4, + (4, '\x81'): 4, + (4, '\x82'): 4, + (4, '\x83'): 4, + (4, '\x84'): 4, + (4, '\x85'): 4, + (4, '\x86'): 4, + (4, '\x87'): 4, + (4, '\x88'): 4, + (4, '\x89'): 4, + (4, '\x8a'): 4, + (4, '\x8b'): 4, + (4, '\x8c'): 4, + (4, '\x8d'): 4, + (4, '\x8e'): 4, + (4, '\x8f'): 4, + (4, '\x90'): 4, + (4, '\x91'): 4, + (4, '\x92'): 4, + (4, '\x93'): 4, + (4, '\x94'): 4, + (4, '\x95'): 4, + (4, '\x96'): 4, + (4, '\x97'): 4, + (4, '\x98'): 4, + (4, '\x99'): 4, + (4, '\x9a'): 4, + (4, '\x9b'): 4, + (4, '\x9c'): 4, + (4, '\x9d'): 4, + (4, '\x9e'): 4, + (4, '\x9f'): 4, + (4, '\xa0'): 4, + (4, '\xa1'): 4, + (4, '\xa2'): 4, + (4, '\xa3'): 4, + (4, '\xa4'): 4, + (4, '\xa5'): 4, + (4, '\xa6'): 4, + (4, '\xa7'): 4, + (4, '\xa8'): 4, + (4, '\xa9'): 4, + (4, '\xaa'): 4, + (4, '\xab'): 4, + (4, '\xac'): 4, + (4, '\xad'): 4, + (4, '\xae'): 4, + (4, '\xaf'): 4, + (4, '\xb0'): 4, + (4, '\xb1'): 4, + (4, '\xb2'): 4, + (4, '\xb3'): 4, + (4, '\xb4'): 4, + (4, '\xb5'): 4, + (4, '\xb6'): 4, + (4, '\xb7'): 4, + (4, '\xb8'): 4, + (4, '\xb9'): 4, + (4, '\xba'): 4, + (4, '\xbb'): 4, + (4, '\xbc'): 4, + (4, '\xbd'): 4, + (4, '\xbe'): 4, + (4, '\xbf'): 4, + (4, '\xc0'): 4, + (4, '\xc1'): 4, + (4, '\xc2'): 4, + (4, '\xc3'): 4, + (4, '\xc4'): 4, + (4, '\xc5'): 4, + (4, '\xc6'): 4, + (4, '\xc7'): 4, + (4, '\xc8'): 4, + (4, '\xc9'): 4, + (4, '\xca'): 4, + (4, '\xcb'): 4, + (4, '\xcc'): 4, + (4, '\xcd'): 4, + (4, '\xce'): 4, + (4, '\xcf'): 4, + (4, '\xd0'): 4, + (4, '\xd1'): 4, + (4, '\xd2'): 4, + (4, '\xd3'): 4, + (4, '\xd4'): 4, + (4, '\xd5'): 4, + (4, '\xd6'): 4, + (4, '\xd7'): 4, + (4, '\xd8'): 4, + (4, '\xd9'): 4, + (4, '\xda'): 4, + (4, '\xdb'): 4, + (4, '\xdc'): 4, + (4, '\xdd'): 4, + (4, '\xde'): 4, + (4, '\xdf'): 4, + (4, '\xe0'): 4, + (4, '\xe1'): 4, + (4, '\xe2'): 4, + (4, '\xe3'): 4, + (4, '\xe4'): 4, + (4, '\xe5'): 4, + (4, '\xe6'): 4, + (4, '\xe7'): 4, + (4, '\xe8'): 4, + (4, '\xe9'): 4, + (4, '\xea'): 4, + (4, '\xeb'): 4, + (4, '\xec'): 4, + (4, '\xed'): 4, + (4, '\xee'): 4, + (4, '\xef'): 4, + (4, '\xf0'): 4, + (4, '\xf1'): 4, + (4, '\xf2'): 4, + (4, '\xf3'): 4, + (4, '\xf4'): 4, + (4, '\xf5'): 4, + (4, '\xf6'): 4, + (4, '\xf7'): 4, + (4, '\xf8'): 4, + (4, '\xf9'): 4, + (4, '\xfa'): 4, + (4, '\xfb'): 4, + (4, '\xfc'): 4, + (4, '\xfd'): 4, + (4, '\xfe'): 4, + (4, '\xff'): 4, (5, '\x00'): 5, (5, '\x01'): 5, (5, '\x02'): 5, @@ -933,7 +1188,7 @@ (5, '\x07'): 5, (5, '\x08'): 5, (5, '\t'): 5, - (5, '\n'): 27, + (5, '\n'): 5, (5, '\x0b'): 5, (5, '\x0c'): 5, (5, '\r'): 5, @@ -957,7 +1212,7 @@ (5, '\x1f'): 5, (5, ' '): 5, (5, '!'): 5, - (5, '"'): 5, + (5, '"'): 25, (5, '#'): 5, (5, '$'): 5, (5, '%'): 5, @@ -1015,7 +1270,7 @@ (5, 'Y'): 5, (5, 'Z'): 5, (5, '['): 5, - (5, '\\'): 5, + (5, '\\'): 24, (5, ']'): 5, (5, '^'): 5, (5, '_'): 5, @@ -1179,263 +1434,34 @@ (5, '\xfd'): 5, (5, '\xfe'): 5, (5, '\xff'): 5, - (6, '\x00'): 6, - (6, '\x01'): 6, - (6, '\x02'): 6, - (6, '\x03'): 6, - (6, '\x04'): 6, - (6, '\x05'): 6, - (6, '\x06'): 6, - (6, '\x07'): 6, - (6, '\x08'): 6, - (6, '\t'): 6, - (6, '\n'): 6, - (6, '\x0b'): 6, - (6, '\x0c'): 6, - (6, '\r'): 6, - (6, '\x0e'): 6, - (6, '\x0f'): 6, - (6, '\x10'): 6, - (6, '\x11'): 6, - (6, '\x12'): 6, - (6, '\x13'): 6, - (6, '\x14'): 6, - (6, '\x15'): 6, - (6, '\x16'): 6, - (6, '\x17'): 6, - (6, '\x18'): 6, - (6, '\x19'): 6, - (6, '\x1a'): 6, - (6, '\x1b'): 6, - (6, '\x1c'): 6, - (6, '\x1d'): 6, - (6, '\x1e'): 6, - (6, '\x1f'): 6, - (6, ' '): 6, - (6, '!'): 6, - (6, '"'): 25, - (6, '#'): 6, - (6, '$'): 6, - (6, '%'): 6, - (6, '&'): 6, - (6, "'"): 6, - (6, '('): 6, - (6, ')'): 6, - (6, '*'): 6, - (6, '+'): 6, - (6, ','): 6, - (6, '-'): 6, - (6, '.'): 6, - (6, '/'): 6, - (6, '0'): 6, - (6, '1'): 6, - (6, '2'): 6, - (6, '3'): 6, - (6, '4'): 6, - (6, '5'): 6, - (6, '6'): 6, - (6, '7'): 6, - (6, '8'): 6, - (6, '9'): 6, - (6, ':'): 6, - (6, ';'): 6, - (6, '<'): 6, - (6, '='): 6, - (6, '>'): 6, - (6, '?'): 6, - (6, '@'): 6, - (6, 'A'): 6, - (6, 'B'): 6, - (6, 'C'): 6, - (6, 'D'): 6, - (6, 'E'): 6, - (6, 'F'): 6, - (6, 'G'): 6, - (6, 'H'): 6, - (6, 'I'): 6, - (6, 'J'): 6, - (6, 'K'): 6, - (6, 'L'): 6, - (6, 'M'): 6, - (6, 'N'): 6, - (6, 'O'): 6, - (6, 'P'): 6, - (6, 'Q'): 6, - (6, 'R'): 6, - (6, 'S'): 6, - (6, 'T'): 6, - (6, 'U'): 6, - (6, 'V'): 6, - (6, 'W'): 6, - (6, 'X'): 6, - (6, 'Y'): 6, - (6, 'Z'): 6, - (6, '['): 6, - (6, '\\'): 24, - (6, ']'): 6, - (6, '^'): 6, - (6, '_'): 6, - (6, '`'): 6, - (6, 'a'): 6, - (6, 'b'): 6, - (6, 'c'): 6, - (6, 'd'): 6, - (6, 'e'): 6, - (6, 'f'): 6, - (6, 'g'): 6, - (6, 'h'): 6, - (6, 'i'): 6, - (6, 'j'): 6, - (6, 'k'): 6, - (6, 'l'): 6, - (6, 'm'): 6, - (6, 'n'): 6, - (6, 'o'): 6, - (6, 'p'): 6, - (6, 'q'): 6, - (6, 'r'): 6, - (6, 's'): 6, - (6, 't'): 6, - (6, 'u'): 6, - (6, 'v'): 6, - (6, 'w'): 6, - (6, 'x'): 6, - (6, 'y'): 6, - (6, 'z'): 6, - (6, '{'): 6, - (6, '|'): 6, - (6, '}'): 6, - (6, '~'): 6, - (6, '\x7f'): 6, - (6, '\x80'): 6, - (6, '\x81'): 6, - (6, '\x82'): 6, - (6, '\x83'): 6, - (6, '\x84'): 6, - (6, '\x85'): 6, - (6, '\x86'): 6, - (6, '\x87'): 6, - (6, '\x88'): 6, - (6, '\x89'): 6, - (6, '\x8a'): 6, - (6, '\x8b'): 6, - (6, '\x8c'): 6, - (6, '\x8d'): 6, - (6, '\x8e'): 6, - (6, '\x8f'): 6, - (6, '\x90'): 6, - (6, '\x91'): 6, - (6, '\x92'): 6, - (6, '\x93'): 6, - (6, '\x94'): 6, - (6, '\x95'): 6, - (6, '\x96'): 6, - (6, '\x97'): 6, - (6, '\x98'): 6, - (6, '\x99'): 6, - (6, '\x9a'): 6, - (6, '\x9b'): 6, - (6, '\x9c'): 6, - (6, '\x9d'): 6, - (6, '\x9e'): 6, - (6, '\x9f'): 6, - (6, '\xa0'): 6, - (6, '\xa1'): 6, - (6, '\xa2'): 6, - (6, '\xa3'): 6, - (6, '\xa4'): 6, - (6, '\xa5'): 6, - (6, '\xa6'): 6, - (6, '\xa7'): 6, - (6, '\xa8'): 6, - (6, '\xa9'): 6, - (6, '\xaa'): 6, - (6, '\xab'): 6, - (6, '\xac'): 6, - (6, '\xad'): 6, - (6, '\xae'): 6, - (6, '\xaf'): 6, - (6, '\xb0'): 6, - (6, '\xb1'): 6, - (6, '\xb2'): 6, - (6, '\xb3'): 6, - (6, '\xb4'): 6, - (6, '\xb5'): 6, - (6, '\xb6'): 6, - (6, '\xb7'): 6, - (6, '\xb8'): 6, - (6, '\xb9'): 6, - (6, '\xba'): 6, - (6, '\xbb'): 6, - (6, '\xbc'): 6, - (6, '\xbd'): 6, - (6, '\xbe'): 6, - (6, '\xbf'): 6, - (6, '\xc0'): 6, - (6, '\xc1'): 6, - (6, '\xc2'): 6, - (6, '\xc3'): 6, - (6, '\xc4'): 6, - (6, '\xc5'): 6, - (6, '\xc6'): 6, - (6, '\xc7'): 6, - (6, '\xc8'): 6, - (6, '\xc9'): 6, - (6, '\xca'): 6, - (6, '\xcb'): 6, - (6, '\xcc'): 6, - (6, '\xcd'): 6, - (6, '\xce'): 6, - (6, '\xcf'): 6, - (6, '\xd0'): 6, - (6, '\xd1'): 6, - (6, '\xd2'): 6, - (6, '\xd3'): 6, - (6, '\xd4'): 6, - (6, '\xd5'): 6, - (6, '\xd6'): 6, - (6, '\xd7'): 6, - (6, '\xd8'): 6, - (6, '\xd9'): 6, - (6, '\xda'): 6, - (6, '\xdb'): 6, - (6, '\xdc'): 6, - (6, '\xdd'): 6, - (6, '\xde'): 6, - (6, '\xdf'): 6, - (6, '\xe0'): 6, - (6, '\xe1'): 6, - (6, '\xe2'): 6, - (6, '\xe3'): 6, - (6, '\xe4'): 6, - (6, '\xe5'): 6, - (6, '\xe6'): 6, - (6, '\xe7'): 6, - (6, '\xe8'): 6, - (6, '\xe9'): 6, - (6, '\xea'): 6, - (6, '\xeb'): 6, - (6, '\xec'): 6, - (6, '\xed'): 6, - (6, '\xee'): 6, - (6, '\xef'): 6, - (6, '\xf0'): 6, - (6, '\xf1'): 6, - (6, '\xf2'): 6, - (6, '\xf3'): 6, - (6, '\xf4'): 6, - (6, '\xf5'): 6, - (6, '\xf6'): 6, - (6, '\xf7'): 6, - (6, '\xf8'): 6, - (6, '\xf9'): 6, - (6, '\xfa'): 6, - (6, '\xfb'): 6, - (6, '\xfc'): 6, - (6, '\xfd'): 6, - (6, '\xfe'): 6, - (6, '\xff'): 6, - (7, '"'): 22, + (6, '"'): 22, + (16, 'A'): 16, + (16, 'B'): 16, + (16, 'C'): 16, + (16, 'D'): 16, + (16, 'E'): 16, + (16, 'F'): 16, + (16, 'G'): 16, + (16, 'H'): 16, + (16, 'I'): 16, + (16, 'J'): 16, + (16, 'K'): 16, + (16, 'L'): 16, + (16, 'M'): 16, + (16, 'N'): 16, + (16, 'O'): 16, + (16, 'P'): 16, + (16, 'Q'): 16, + (16, 'R'): 16, + (16, 'S'): 16, + (16, 'T'): 16, + (16, 'U'): 16, + (16, 'V'): 16, + (16, 'W'): 16, + (16, 'X'): 16, + (16, 'Y'): 16, + (16, 'Z'): 16, + (16, '_'): 16, (19, '0'): 20, (19, '1'): 20, (19, '2'): 20, @@ -1446,32 +1472,32 @@ (19, '7'): 20, (19, '8'): 20, (19, '9'): 20, - (19, 'A'): 3, - (19, 'B'): 3, - (19, 'C'): 3, - (19, 'D'): 3, - (19, 'E'): 3, - (19, 'F'): 3, - (19, 'G'): 3, - (19, 'H'): 3, - (19, 'I'): 3, - (19, 'J'): 3, - (19, 'K'): 3, - (19, 'L'): 3, - (19, 'M'): 3, - (19, 'N'): 3, - (19, 'O'): 3, - (19, 'P'): 3, - (19, 'Q'): 3, - (19, 'R'): 3, - (19, 'S'): 3, - (19, 'T'): 3, - (19, 'U'): 3, - (19, 'V'): 3, - (19, 'W'): 3, - (19, 'X'): 3, - (19, 'Y'): 3, - (19, 'Z'): 3, + (19, 'A'): 16, + (19, 'B'): 16, + (19, 'C'): 16, + (19, 'D'): 16, + (19, 'E'): 16, + (19, 'F'): 16, + (19, 'G'): 16, + (19, 'H'): 16, + (19, 'I'): 16, + (19, 'J'): 16, + (19, 'K'): 16, + (19, 'L'): 16, + (19, 'M'): 16, + (19, 'N'): 16, + (19, 'O'): 16, + (19, 'P'): 16, + (19, 'Q'): 16, + (19, 'R'): 16, + (19, 'S'): 16, + (19, 'T'): 16, + (19, 'U'): 16, + (19, 'V'): 16, + (19, 'W'): 16, + (19, 'X'): 16, + (19, 'Y'): 16, + (19, 'Z'): 16, (19, '_'): 19, (19, 'a'): 20, (19, 'b'): 20, @@ -1537,527 +1563,526 @@ (20, 'y'): 20, (20, 'z'): 20, (22, "'"): 23, - (24, '\x00'): 6, - (24, '\x01'): 6, - (24, '\x02'): 6, - (24, '\x03'): 6, - (24, '\x04'): 6, - (24, '\x05'): 6, - (24, '\x06'): 6, - (24, '\x07'): 6, - (24, '\x08'): 6, - (24, '\t'): 6, - (24, '\n'): 6, - (24, '\x0b'): 6, - (24, '\x0c'): 6, - (24, '\r'): 6, - (24, '\x0e'): 6, - (24, '\x0f'): 6, - (24, '\x10'): 6, - (24, '\x11'): 6, - (24, '\x12'): 6, - (24, '\x13'): 6, - (24, '\x14'): 6, - (24, '\x15'): 6, - (24, '\x16'): 6, - (24, '\x17'): 6, - (24, '\x18'): 6, - (24, '\x19'): 6, - (24, '\x1a'): 6, - (24, '\x1b'): 6, - (24, '\x1c'): 6, - (24, '\x1d'): 6, - (24, '\x1e'): 6, - (24, '\x1f'): 6, - (24, ' '): 6, - (24, '!'): 6, + (24, '\x00'): 5, + (24, '\x01'): 5, + (24, '\x02'): 5, + (24, '\x03'): 5, + (24, '\x04'): 5, + (24, '\x05'): 5, + (24, '\x06'): 5, + (24, '\x07'): 5, + (24, '\x08'): 5, + (24, '\t'): 5, + (24, '\n'): 5, + (24, '\x0b'): 5, + (24, '\x0c'): 5, + (24, '\r'): 5, + (24, '\x0e'): 5, + (24, '\x0f'): 5, + (24, '\x10'): 5, + (24, '\x11'): 5, + (24, '\x12'): 5, + (24, '\x13'): 5, + (24, '\x14'): 5, + (24, '\x15'): 5, + (24, '\x16'): 5, + (24, '\x17'): 5, + (24, '\x18'): 5, + (24, '\x19'): 5, + (24, '\x1a'): 5, + (24, '\x1b'): 5, + (24, '\x1c'): 5, + (24, '\x1d'): 5, + (24, '\x1e'): 5, + (24, '\x1f'): 5, + (24, ' '): 5, + (24, '!'): 5, (24, '"'): 26, - (24, '#'): 6, - (24, '$'): 6, - (24, '%'): 6, - (24, '&'): 6, - (24, "'"): 6, - (24, '('): 6, - (24, ')'): 6, - (24, '*'): 6, - (24, '+'): 6, - (24, ','): 6, - (24, '-'): 6, - (24, '.'): 6, - (24, '/'): 6, - (24, '0'): 6, - (24, '1'): 6, - (24, '2'): 6, - (24, '3'): 6, - (24, '4'): 6, - (24, '5'): 6, - (24, '6'): 6, - (24, '7'): 6, - (24, '8'): 6, - (24, '9'): 6, - (24, ':'): 6, - (24, ';'): 6, - (24, '<'): 6, - (24, '='): 6, - (24, '>'): 6, - (24, '?'): 6, - (24, '@'): 6, - (24, 'A'): 6, - (24, 'B'): 6, - (24, 'C'): 6, - (24, 'D'): 6, - (24, 'E'): 6, - (24, 'F'): 6, - (24, 'G'): 6, - (24, 'H'): 6, - (24, 'I'): 6, - (24, 'J'): 6, - (24, 'K'): 6, - (24, 'L'): 6, - (24, 'M'): 6, - (24, 'N'): 6, - (24, 'O'): 6, - (24, 'P'): 6, - (24, 'Q'): 6, - (24, 'R'): 6, - (24, 'S'): 6, - (24, 'T'): 6, - (24, 'U'): 6, - (24, 'V'): 6, - (24, 'W'): 6, - (24, 'X'): 6, - (24, 'Y'): 6, - (24, 'Z'): 6, - (24, '['): 6, + (24, '#'): 5, + (24, '$'): 5, + (24, '%'): 5, + (24, '&'): 5, + (24, "'"): 5, + (24, '('): 5, + (24, ')'): 5, + (24, '*'): 5, + (24, '+'): 5, + (24, ','): 5, + (24, '-'): 5, + (24, '.'): 5, + (24, '/'): 5, + (24, '0'): 5, + (24, '1'): 5, + (24, '2'): 5, + (24, '3'): 5, + (24, '4'): 5, + (24, '5'): 5, + (24, '6'): 5, + (24, '7'): 5, + (24, '8'): 5, + (24, '9'): 5, + (24, ':'): 5, + (24, ';'): 5, + (24, '<'): 5, + (24, '='): 5, + (24, '>'): 5, + (24, '?'): 5, + (24, '@'): 5, + (24, 'A'): 5, + (24, 'B'): 5, + (24, 'C'): 5, + (24, 'D'): 5, + (24, 'E'): 5, + (24, 'F'): 5, + (24, 'G'): 5, + (24, 'H'): 5, + (24, 'I'): 5, + (24, 'J'): 5, + (24, 'K'): 5, + (24, 'L'): 5, + (24, 'M'): 5, + (24, 'N'): 5, + (24, 'O'): 5, + (24, 'P'): 5, + (24, 'Q'): 5, + (24, 'R'): 5, + (24, 'S'): 5, + (24, 'T'): 5, + (24, 'U'): 5, + (24, 'V'): 5, + (24, 'W'): 5, + (24, 'X'): 5, + (24, 'Y'): 5, + (24, 'Z'): 5, + (24, '['): 5, (24, '\\'): 24, - (24, ']'): 6, - (24, '^'): 6, - (24, '_'): 6, - (24, '`'): 6, - (24, 'a'): 6, - (24, 'b'): 6, - (24, 'c'): 6, - (24, 'd'): 6, - (24, 'e'): 6, - (24, 'f'): 6, - (24, 'g'): 6, - (24, 'h'): 6, - (24, 'i'): 6, - (24, 'j'): 6, - (24, 'k'): 6, - (24, 'l'): 6, - (24, 'm'): 6, - (24, 'n'): 6, - (24, 'o'): 6, - (24, 'p'): 6, - (24, 'q'): 6, - (24, 'r'): 6, - (24, 's'): 6, - (24, 't'): 6, - (24, 'u'): 6, - (24, 'v'): 6, - (24, 'w'): 6, - (24, 'x'): 6, - (24, 'y'): 6, - (24, 'z'): 6, - (24, '{'): 6, - (24, '|'): 6, - (24, '}'): 6, - (24, '~'): 6, - (24, '\x7f'): 6, - (24, '\x80'): 6, - (24, '\x81'): 6, - (24, '\x82'): 6, - (24, '\x83'): 6, - (24, '\x84'): 6, - (24, '\x85'): 6, - (24, '\x86'): 6, - (24, '\x87'): 6, - (24, '\x88'): 6, - (24, '\x89'): 6, - (24, '\x8a'): 6, - (24, '\x8b'): 6, - (24, '\x8c'): 6, - (24, '\x8d'): 6, - (24, '\x8e'): 6, - (24, '\x8f'): 6, - (24, '\x90'): 6, - (24, '\x91'): 6, - (24, '\x92'): 6, - (24, '\x93'): 6, - (24, '\x94'): 6, - (24, '\x95'): 6, - (24, '\x96'): 6, - (24, '\x97'): 6, - (24, '\x98'): 6, - (24, '\x99'): 6, - (24, '\x9a'): 6, - (24, '\x9b'): 6, - (24, '\x9c'): 6, - (24, '\x9d'): 6, - (24, '\x9e'): 6, - (24, '\x9f'): 6, - (24, '\xa0'): 6, - (24, '\xa1'): 6, - (24, '\xa2'): 6, - (24, '\xa3'): 6, - (24, '\xa4'): 6, - (24, '\xa5'): 6, - (24, '\xa6'): 6, - (24, '\xa7'): 6, - (24, '\xa8'): 6, - (24, '\xa9'): 6, - (24, '\xaa'): 6, - (24, '\xab'): 6, - (24, '\xac'): 6, - (24, '\xad'): 6, - (24, '\xae'): 6, - (24, '\xaf'): 6, - (24, '\xb0'): 6, - (24, '\xb1'): 6, - (24, '\xb2'): 6, - (24, '\xb3'): 6, - (24, '\xb4'): 6, - (24, '\xb5'): 6, - (24, '\xb6'): 6, - (24, '\xb7'): 6, - (24, '\xb8'): 6, - (24, '\xb9'): 6, - (24, '\xba'): 6, - (24, '\xbb'): 6, - (24, '\xbc'): 6, - (24, '\xbd'): 6, - (24, '\xbe'): 6, - (24, '\xbf'): 6, - (24, '\xc0'): 6, - (24, '\xc1'): 6, - (24, '\xc2'): 6, - (24, '\xc3'): 6, - (24, '\xc4'): 6, - (24, '\xc5'): 6, - (24, '\xc6'): 6, - (24, '\xc7'): 6, - (24, '\xc8'): 6, - (24, '\xc9'): 6, - (24, '\xca'): 6, - (24, '\xcb'): 6, - (24, '\xcc'): 6, - (24, '\xcd'): 6, - (24, '\xce'): 6, - (24, '\xcf'): 6, - (24, '\xd0'): 6, - (24, '\xd1'): 6, - (24, '\xd2'): 6, - (24, '\xd3'): 6, - (24, '\xd4'): 6, - (24, '\xd5'): 6, - (24, '\xd6'): 6, - (24, '\xd7'): 6, - (24, '\xd8'): 6, - (24, '\xd9'): 6, - (24, '\xda'): 6, - (24, '\xdb'): 6, - (24, '\xdc'): 6, - (24, '\xdd'): 6, - (24, '\xde'): 6, - (24, '\xdf'): 6, - (24, '\xe0'): 6, - (24, '\xe1'): 6, - (24, '\xe2'): 6, - (24, '\xe3'): 6, - (24, '\xe4'): 6, - (24, '\xe5'): 6, - (24, '\xe6'): 6, - (24, '\xe7'): 6, - (24, '\xe8'): 6, - (24, '\xe9'): 6, - (24, '\xea'): 6, - (24, '\xeb'): 6, - (24, '\xec'): 6, - (24, '\xed'): 6, - (24, '\xee'): 6, - (24, '\xef'): 6, - (24, '\xf0'): 6, - (24, '\xf1'): 6, - (24, '\xf2'): 6, - (24, '\xf3'): 6, - (24, '\xf4'): 6, - (24, '\xf5'): 6, - (24, '\xf6'): 6, - (24, '\xf7'): 6, - (24, '\xf8'): 6, - (24, '\xf9'): 6, - (24, '\xfa'): 6, - (24, '\xfb'): 6, - (24, '\xfc'): 6, - (24, '\xfd'): 6, - (24, '\xfe'): 6, - (24, '\xff'): 6, - (26, '\x00'): 6, - (26, '\x01'): 6, - (26, '\x02'): 6, - (26, '\x03'): 6, - (26, '\x04'): 6, - (26, '\x05'): 6, - (26, '\x06'): 6, - (26, '\x07'): 6, - (26, '\x08'): 6, - (26, '\t'): 6, - (26, '\n'): 6, - (26, '\x0b'): 6, - (26, '\x0c'): 6, - (26, '\r'): 6, - (26, '\x0e'): 6, - (26, '\x0f'): 6, - (26, '\x10'): 6, - (26, '\x11'): 6, - (26, '\x12'): 6, - (26, '\x13'): 6, - (26, '\x14'): 6, - (26, '\x15'): 6, - (26, '\x16'): 6, - (26, '\x17'): 6, - (26, '\x18'): 6, - (26, '\x19'): 6, - (26, '\x1a'): 6, - (26, '\x1b'): 6, - (26, '\x1c'): 6, - (26, '\x1d'): 6, - (26, '\x1e'): 6, - (26, '\x1f'): 6, - (26, ' '): 6, - (26, '!'): 6, + (24, ']'): 5, + (24, '^'): 5, + (24, '_'): 5, + (24, '`'): 5, + (24, 'a'): 5, + (24, 'b'): 5, + (24, 'c'): 5, + (24, 'd'): 5, + (24, 'e'): 5, + (24, 'f'): 5, + (24, 'g'): 5, + (24, 'h'): 5, + (24, 'i'): 5, + (24, 'j'): 5, + (24, 'k'): 5, + (24, 'l'): 5, + (24, 'm'): 5, + (24, 'n'): 5, + (24, 'o'): 5, + (24, 'p'): 5, + (24, 'q'): 5, + (24, 'r'): 5, + (24, 's'): 5, + (24, 't'): 5, + (24, 'u'): 5, + (24, 'v'): 5, + (24, 'w'): 5, + (24, 'x'): 5, + (24, 'y'): 5, + (24, 'z'): 5, + (24, '{'): 5, + (24, '|'): 5, + (24, '}'): 5, + (24, '~'): 5, + (24, '\x7f'): 5, + (24, '\x80'): 5, + (24, '\x81'): 5, + (24, '\x82'): 5, + (24, '\x83'): 5, + (24, '\x84'): 5, + (24, '\x85'): 5, + (24, '\x86'): 5, + (24, '\x87'): 5, + (24, '\x88'): 5, + (24, '\x89'): 5, + (24, '\x8a'): 5, + (24, '\x8b'): 5, + (24, '\x8c'): 5, + (24, '\x8d'): 5, + (24, '\x8e'): 5, + (24, '\x8f'): 5, + (24, '\x90'): 5, + (24, '\x91'): 5, + (24, '\x92'): 5, + (24, '\x93'): 5, + (24, '\x94'): 5, + (24, '\x95'): 5, + (24, '\x96'): 5, + (24, '\x97'): 5, + (24, '\x98'): 5, + (24, '\x99'): 5, + (24, '\x9a'): 5, + (24, '\x9b'): 5, + (24, '\x9c'): 5, + (24, '\x9d'): 5, + (24, '\x9e'): 5, + (24, '\x9f'): 5, + (24, '\xa0'): 5, + (24, '\xa1'): 5, + (24, '\xa2'): 5, + (24, '\xa3'): 5, + (24, '\xa4'): 5, + (24, '\xa5'): 5, + (24, '\xa6'): 5, + (24, '\xa7'): 5, + (24, '\xa8'): 5, + (24, '\xa9'): 5, + (24, '\xaa'): 5, + (24, '\xab'): 5, + (24, '\xac'): 5, + (24, '\xad'): 5, + (24, '\xae'): 5, + (24, '\xaf'): 5, + (24, '\xb0'): 5, + (24, '\xb1'): 5, + (24, '\xb2'): 5, + (24, '\xb3'): 5, + (24, '\xb4'): 5, + (24, '\xb5'): 5, + (24, '\xb6'): 5, + (24, '\xb7'): 5, + (24, '\xb8'): 5, + (24, '\xb9'): 5, + (24, '\xba'): 5, + (24, '\xbb'): 5, + (24, '\xbc'): 5, + (24, '\xbd'): 5, + (24, '\xbe'): 5, + (24, '\xbf'): 5, + (24, '\xc0'): 5, + (24, '\xc1'): 5, + (24, '\xc2'): 5, + (24, '\xc3'): 5, + (24, '\xc4'): 5, + (24, '\xc5'): 5, + (24, '\xc6'): 5, + (24, '\xc7'): 5, + (24, '\xc8'): 5, + (24, '\xc9'): 5, + (24, '\xca'): 5, + (24, '\xcb'): 5, + (24, '\xcc'): 5, + (24, '\xcd'): 5, + (24, '\xce'): 5, + (24, '\xcf'): 5, + (24, '\xd0'): 5, + (24, '\xd1'): 5, + (24, '\xd2'): 5, + (24, '\xd3'): 5, + (24, '\xd4'): 5, + (24, '\xd5'): 5, + (24, '\xd6'): 5, + (24, '\xd7'): 5, + (24, '\xd8'): 5, + (24, '\xd9'): 5, + (24, '\xda'): 5, + (24, '\xdb'): 5, + (24, '\xdc'): 5, + (24, '\xdd'): 5, + (24, '\xde'): 5, + (24, '\xdf'): 5, + (24, '\xe0'): 5, + (24, '\xe1'): 5, + (24, '\xe2'): 5, + (24, '\xe3'): 5, + (24, '\xe4'): 5, + (24, '\xe5'): 5, + (24, '\xe6'): 5, + (24, '\xe7'): 5, + (24, '\xe8'): 5, + (24, '\xe9'): 5, + (24, '\xea'): 5, + (24, '\xeb'): 5, + (24, '\xec'): 5, + (24, '\xed'): 5, + (24, '\xee'): 5, + (24, '\xef'): 5, + (24, '\xf0'): 5, + (24, '\xf1'): 5, + (24, '\xf2'): 5, + (24, '\xf3'): 5, + (24, '\xf4'): 5, + (24, '\xf5'): 5, + (24, '\xf6'): 5, + (24, '\xf7'): 5, + (24, '\xf8'): 5, + (24, '\xf9'): 5, + (24, '\xfa'): 5, + (24, '\xfb'): 5, + (24, '\xfc'): 5, + (24, '\xfd'): 5, + (24, '\xfe'): 5, + (24, '\xff'): 5, + (26, '\x00'): 5, + (26, '\x01'): 5, + (26, '\x02'): 5, + (26, '\x03'): 5, + (26, '\x04'): 5, + (26, '\x05'): 5, + (26, '\x06'): 5, + (26, '\x07'): 5, + (26, '\x08'): 5, + (26, '\t'): 5, + (26, '\n'): 5, + (26, '\x0b'): 5, + (26, '\x0c'): 5, + (26, '\r'): 5, + (26, '\x0e'): 5, + (26, '\x0f'): 5, + (26, '\x10'): 5, + (26, '\x11'): 5, + (26, '\x12'): 5, + (26, '\x13'): 5, + (26, '\x14'): 5, + (26, '\x15'): 5, + (26, '\x16'): 5, + (26, '\x17'): 5, + (26, '\x18'): 5, + (26, '\x19'): 5, + (26, '\x1a'): 5, + (26, '\x1b'): 5, + (26, '\x1c'): 5, + (26, '\x1d'): 5, + (26, '\x1e'): 5, + (26, '\x1f'): 5, + (26, ' '): 5, + (26, '!'): 5, (26, '"'): 25, - (26, '#'): 6, - (26, '$'): 6, - (26, '%'): 6, - (26, '&'): 6, - (26, "'"): 6, - (26, '('): 6, - (26, ')'): 6, - (26, '*'): 6, - (26, '+'): 6, - (26, ','): 6, - (26, '-'): 6, - (26, '.'): 6, - (26, '/'): 6, - (26, '0'): 6, - (26, '1'): 6, - (26, '2'): 6, - (26, '3'): 6, - (26, '4'): 6, - (26, '5'): 6, - (26, '6'): 6, - (26, '7'): 6, - (26, '8'): 6, - (26, '9'): 6, - (26, ':'): 6, - (26, ';'): 6, - (26, '<'): 6, - (26, '='): 6, - (26, '>'): 6, - (26, '?'): 6, - (26, '@'): 6, - (26, 'A'): 6, - (26, 'B'): 6, - (26, 'C'): 6, - (26, 'D'): 6, - (26, 'E'): 6, - (26, 'F'): 6, - (26, 'G'): 6, - (26, 'H'): 6, - (26, 'I'): 6, - (26, 'J'): 6, - (26, 'K'): 6, - (26, 'L'): 6, - (26, 'M'): 6, - (26, 'N'): 6, - (26, 'O'): 6, - (26, 'P'): 6, - (26, 'Q'): 6, - (26, 'R'): 6, - (26, 'S'): 6, - (26, 'T'): 6, - (26, 'U'): 6, - (26, 'V'): 6, - (26, 'W'): 6, - (26, 'X'): 6, - (26, 'Y'): 6, - (26, 'Z'): 6, - (26, '['): 6, - (26, '\\'): 6, - (26, ']'): 6, - (26, '^'): 6, - (26, '_'): 6, - (26, '`'): 6, - (26, 'a'): 6, - (26, 'b'): 6, - (26, 'c'): 6, - (26, 'd'): 6, - (26, 'e'): 6, - (26, 'f'): 6, - (26, 'g'): 6, - (26, 'h'): 6, - (26, 'i'): 6, - (26, 'j'): 6, - (26, 'k'): 6, - (26, 'l'): 6, - (26, 'm'): 6, - (26, 'n'): 6, - (26, 'o'): 6, - (26, 'p'): 6, - (26, 'q'): 6, - (26, 'r'): 6, - (26, 's'): 6, - (26, 't'): 6, - (26, 'u'): 6, - (26, 'v'): 6, - (26, 'w'): 6, - (26, 'x'): 6, - (26, 'y'): 6, - (26, 'z'): 6, - (26, '{'): 6, - (26, '|'): 6, - (26, '}'): 6, - (26, '~'): 6, - (26, '\x7f'): 6, - (26, '\x80'): 6, - (26, '\x81'): 6, - (26, '\x82'): 6, - (26, '\x83'): 6, - (26, '\x84'): 6, - (26, '\x85'): 6, - (26, '\x86'): 6, - (26, '\x87'): 6, - (26, '\x88'): 6, - (26, '\x89'): 6, - (26, '\x8a'): 6, - (26, '\x8b'): 6, - (26, '\x8c'): 6, - (26, '\x8d'): 6, - (26, '\x8e'): 6, - (26, '\x8f'): 6, - (26, '\x90'): 6, - (26, '\x91'): 6, - (26, '\x92'): 6, - (26, '\x93'): 6, - (26, '\x94'): 6, - (26, '\x95'): 6, - (26, '\x96'): 6, - (26, '\x97'): 6, - (26, '\x98'): 6, - (26, '\x99'): 6, - (26, '\x9a'): 6, - (26, '\x9b'): 6, - (26, '\x9c'): 6, - (26, '\x9d'): 6, - (26, '\x9e'): 6, - (26, '\x9f'): 6, - (26, '\xa0'): 6, - (26, '\xa1'): 6, - (26, '\xa2'): 6, - (26, '\xa3'): 6, - (26, '\xa4'): 6, - (26, '\xa5'): 6, - (26, '\xa6'): 6, - (26, '\xa7'): 6, - (26, '\xa8'): 6, - (26, '\xa9'): 6, - (26, '\xaa'): 6, - (26, '\xab'): 6, - (26, '\xac'): 6, - (26, '\xad'): 6, - (26, '\xae'): 6, - (26, '\xaf'): 6, - (26, '\xb0'): 6, - (26, '\xb1'): 6, - (26, '\xb2'): 6, - (26, '\xb3'): 6, - (26, '\xb4'): 6, - (26, '\xb5'): 6, - (26, '\xb6'): 6, - (26, '\xb7'): 6, - (26, '\xb8'): 6, - (26, '\xb9'): 6, - (26, '\xba'): 6, - (26, '\xbb'): 6, - (26, '\xbc'): 6, - (26, '\xbd'): 6, - (26, '\xbe'): 6, - (26, '\xbf'): 6, - (26, '\xc0'): 6, - (26, '\xc1'): 6, - (26, '\xc2'): 6, - (26, '\xc3'): 6, - (26, '\xc4'): 6, - (26, '\xc5'): 6, - (26, '\xc6'): 6, - (26, '\xc7'): 6, - (26, '\xc8'): 6, - (26, '\xc9'): 6, - (26, '\xca'): 6, - (26, '\xcb'): 6, - (26, '\xcc'): 6, - (26, '\xcd'): 6, - (26, '\xce'): 6, - (26, '\xcf'): 6, - (26, '\xd0'): 6, - (26, '\xd1'): 6, - (26, '\xd2'): 6, - (26, '\xd3'): 6, - (26, '\xd4'): 6, - (26, '\xd5'): 6, - (26, '\xd6'): 6, - (26, '\xd7'): 6, - (26, '\xd8'): 6, - (26, '\xd9'): 6, - (26, '\xda'): 6, - (26, '\xdb'): 6, - (26, '\xdc'): 6, - (26, '\xdd'): 6, - (26, '\xde'): 6, - (26, '\xdf'): 6, - (26, '\xe0'): 6, - (26, '\xe1'): 6, - (26, '\xe2'): 6, - (26, '\xe3'): 6, - (26, '\xe4'): 6, - (26, '\xe5'): 6, - (26, '\xe6'): 6, - (26, '\xe7'): 6, - (26, '\xe8'): 6, - (26, '\xe9'): 6, - (26, '\xea'): 6, - (26, '\xeb'): 6, - (26, '\xec'): 6, - (26, '\xed'): 6, - (26, '\xee'): 6, - (26, '\xef'): 6, - (26, '\xf0'): 6, - (26, '\xf1'): 6, - (26, '\xf2'): 6, - (26, '\xf3'): 6, - (26, '\xf4'): 6, - (26, '\xf5'): 6, - (26, '\xf6'): 6, - (26, '\xf7'): 6, - (26, '\xf8'): 6, - (26, '\xf9'): 6, - (26, '\xfa'): 6, - (26, '\xfb'): 6, - (26, '\xfc'): 6, - (26, '\xfd'): 6, - (26, '\xfe'): 6, - (26, '\xff'): 6}, - set([1, 2, 3, 4, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 25, 26, 27]), - set([1, 2, 3, 4, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 25, 26, 27]), - ['0, 0, 0, final*, start*, 0, 0, 1, final*, start*, 0, 0, 0, start|, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0', + (26, '#'): 5, + (26, '$'): 5, + (26, '%'): 5, + (26, '&'): 5, + (26, "'"): 5, + (26, '('): 5, + (26, ')'): 5, + (26, '*'): 5, + (26, '+'): 5, + (26, ','): 5, + (26, '-'): 5, + (26, '.'): 5, + (26, '/'): 5, + (26, '0'): 5, + (26, '1'): 5, + (26, '2'): 5, + (26, '3'): 5, + (26, '4'): 5, + (26, '5'): 5, + (26, '6'): 5, + (26, '7'): 5, + (26, '8'): 5, + (26, '9'): 5, + (26, ':'): 5, + (26, ';'): 5, + (26, '<'): 5, + (26, '='): 5, + (26, '>'): 5, + (26, '?'): 5, + (26, '@'): 5, + (26, 'A'): 5, + (26, 'B'): 5, + (26, 'C'): 5, + (26, 'D'): 5, + (26, 'E'): 5, + (26, 'F'): 5, + (26, 'G'): 5, + (26, 'H'): 5, + (26, 'I'): 5, + (26, 'J'): 5, + (26, 'K'): 5, + (26, 'L'): 5, + (26, 'M'): 5, + (26, 'N'): 5, + (26, 'O'): 5, + (26, 'P'): 5, + (26, 'Q'): 5, + (26, 'R'): 5, + (26, 'S'): 5, + (26, 'T'): 5, + (26, 'U'): 5, + (26, 'V'): 5, + (26, 'W'): 5, + (26, 'X'): 5, + (26, 'Y'): 5, + (26, 'Z'): 5, + (26, '['): 5, + (26, '\\'): 5, + (26, ']'): 5, + (26, '^'): 5, + (26, '_'): 5, + (26, '`'): 5, + (26, 'a'): 5, + (26, 'b'): 5, + (26, 'c'): 5, + (26, 'd'): 5, + (26, 'e'): 5, + (26, 'f'): 5, + (26, 'g'): 5, + (26, 'h'): 5, + (26, 'i'): 5, + (26, 'j'): 5, + (26, 'k'): 5, + (26, 'l'): 5, + (26, 'm'): 5, + (26, 'n'): 5, + (26, 'o'): 5, + (26, 'p'): 5, + (26, 'q'): 5, + (26, 'r'): 5, + (26, 's'): 5, + (26, 't'): 5, + (26, 'u'): 5, + (26, 'v'): 5, + (26, 'w'): 5, + (26, 'x'): 5, + (26, 'y'): 5, + (26, 'z'): 5, + (26, '{'): 5, + (26, '|'): 5, + (26, '}'): 5, + (26, '~'): 5, + (26, '\x7f'): 5, + (26, '\x80'): 5, + (26, '\x81'): 5, + (26, '\x82'): 5, + (26, '\x83'): 5, + (26, '\x84'): 5, + (26, '\x85'): 5, + (26, '\x86'): 5, + (26, '\x87'): 5, + (26, '\x88'): 5, + (26, '\x89'): 5, + (26, '\x8a'): 5, + (26, '\x8b'): 5, + (26, '\x8c'): 5, + (26, '\x8d'): 5, + (26, '\x8e'): 5, + (26, '\x8f'): 5, + (26, '\x90'): 5, + (26, '\x91'): 5, + (26, '\x92'): 5, + (26, '\x93'): 5, + (26, '\x94'): 5, + (26, '\x95'): 5, + (26, '\x96'): 5, + (26, '\x97'): 5, + (26, '\x98'): 5, + (26, '\x99'): 5, + (26, '\x9a'): 5, + (26, '\x9b'): 5, + (26, '\x9c'): 5, + (26, '\x9d'): 5, + (26, '\x9e'): 5, + (26, '\x9f'): 5, + (26, '\xa0'): 5, + (26, '\xa1'): 5, + (26, '\xa2'): 5, + (26, '\xa3'): 5, + (26, '\xa4'): 5, + (26, '\xa5'): 5, + (26, '\xa6'): 5, + (26, '\xa7'): 5, + (26, '\xa8'): 5, + (26, '\xa9'): 5, + (26, '\xaa'): 5, + (26, '\xab'): 5, + (26, '\xac'): 5, + (26, '\xad'): 5, + (26, '\xae'): 5, + (26, '\xaf'): 5, + (26, '\xb0'): 5, + (26, '\xb1'): 5, + (26, '\xb2'): 5, + (26, '\xb3'): 5, + (26, '\xb4'): 5, + (26, '\xb5'): 5, + (26, '\xb6'): 5, + (26, '\xb7'): 5, + (26, '\xb8'): 5, + (26, '\xb9'): 5, + (26, '\xba'): 5, + (26, '\xbb'): 5, + (26, '\xbc'): 5, + (26, '\xbd'): 5, + (26, '\xbe'): 5, + (26, '\xbf'): 5, + (26, '\xc0'): 5, + (26, '\xc1'): 5, + (26, '\xc2'): 5, + (26, '\xc3'): 5, + (26, '\xc4'): 5, + (26, '\xc5'): 5, + (26, '\xc6'): 5, + (26, '\xc7'): 5, + (26, '\xc8'): 5, + (26, '\xc9'): 5, + (26, '\xca'): 5, + (26, '\xcb'): 5, + (26, '\xcc'): 5, + (26, '\xcd'): 5, + (26, '\xce'): 5, + (26, '\xcf'): 5, + (26, '\xd0'): 5, + (26, '\xd1'): 5, + (26, '\xd2'): 5, + (26, '\xd3'): 5, + (26, '\xd4'): 5, + (26, '\xd5'): 5, + (26, '\xd6'): 5, + (26, '\xd7'): 5, + (26, '\xd8'): 5, + (26, '\xd9'): 5, + (26, '\xda'): 5, + (26, '\xdb'): 5, + (26, '\xdc'): 5, + (26, '\xdd'): 5, + (26, '\xde'): 5, + (26, '\xdf'): 5, + (26, '\xe0'): 5, + (26, '\xe1'): 5, + (26, '\xe2'): 5, + (26, '\xe3'): 5, + (26, '\xe4'): 5, + (26, '\xe5'): 5, + (26, '\xe6'): 5, + (26, '\xe7'): 5, + (26, '\xe8'): 5, + (26, '\xe9'): 5, + (26, '\xea'): 5, + (26, '\xeb'): 5, + (26, '\xec'): 5, + (26, '\xed'): 5, + (26, '\xee'): 5, + (26, '\xef'): 5, + (26, '\xf0'): 5, + (26, '\xf1'): 5, + (26, '\xf2'): 5, + (26, '\xf3'): 5, + (26, '\xf4'): 5, + (26, '\xf5'): 5, + (26, '\xf6'): 5, + (26, '\xf7'): 5, + (26, '\xf8'): 5, + (26, '\xf9'): 5, + (26, '\xfa'): 5, + (26, '\xfb'): 5, + (26, '\xfc'): 5, + (26, '\xfd'): 5, + (26, '\xfe'): 5, + (26, '\xff'): 5}, + set([1, 2, 3, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 25, 26, 27]), + set([1, 2, 3, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 25, 26, 27]), + ['0, 0, 0, final*, 0, final*, start*, 0, final*, 0, 1, final*, start*, 0, 0, 0, 0, 0, 0, start|, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0', 'IGNORE', 'IGNORE', - 'SYMBOLNAME', 'IGNORE', - '1, 0, start|, 0, final*, start*, 0, 0, 1, final|, start|, 0, final*, start*, 0, 0, final|, start|, 0, 1, final*, start*, 0', - '1, 0, final*, start*, start|, 0, final|, final*, start*, 0, 0, 0, start|, 0, 0, final*, final|, start|, 0, final|, final*, start*, 0, 0, 0, start|, 1, 0, start*, 0, final*, final|, start|, 0, 1, final*, start*, 0, 0, 0, final|, start|, 0, start*, 0, 1, final|, start|, 0, final*, start*, 0, final*, final*, 1, final|, final*, 0, start|, 0, final*, start*, final*, start*, 0, final|, start|, 0, 0, 0, final|, start|, 0, 1, final*, start*, 0, final*, final*, final|, 1, final*, 0, start|, 0, final*, start*, final*, start*, 0, final|, start|, 0, 0, 0, final*, start*, final*, start*, 0, final|, start|, 0, 0, final*, final*, 0, 1, final|, final*, 0, start|, 0, final*, start*, final*, start*, 0, final|, start|, 0, 0, final*, final*, 0, final|, 1, final*, 0, start|, 0, final*, start*, final*, start*, 0, final|, start|, 0, 0, final*, final*, 0, final*, 0, 1, final|, start|, 0, 1, final*, start*, final*, start*, 0, final|, start|, 0, 0, final*, final*, 0, final*, 0, final|, start|, 0, final*, start*, final*, start*, 0, final|, start|, 0, 0, final*, final*, 0, 1, final|, final*, 0, 1, final|, start|, 0, 1, final*, start*, final*, start*, 0, final|, start|, 0, 0, final*, final*, 0, final|, 1, final*, 0, final|, start|, 0, 1, final|, start|, 0, final*, start*, 0, final*, final*, 1, final|, final*, 0, 1, final|, start|, 0, final*, start*, final*, start*, 0, final|, start|, 0, 0, 0, final|, start|, 0, 1, final*, start*, 0, final*, final*, final|, 1, final*, 0, final|, start|, 0, 1, final*, start*, final*, start*, 0, final|, start|, 0, 0, 0, 1, final|, start|, 0, final*, start*, 0, final*, final*, final*, 0, 1, final|, start|, 0, final*, start*, final*, start*, 0, final|, start|, 0, 0, 0, final|, start|, 0, 1, final*, start*, 0, final*, final*, final*, 0, final|, start|, 0, 1, final*, start*, final*, start*, 0, final|, start|, 0, 0', + '1, final*, 0, start|, 0, final*, start*, 0, final*, 0, 1, final|, start|, 0, final*, start*, 0, final*, 0, final|, start|, 0, 1, final*, start*, 0', + '1, final*, 0, final*, start*, start|, 0, final|, final*, start*, final*, 0, 0, start|, 0, final*, 0, final*, 0, 1, final|, final*, 0, final|, final*, start*, final*, 0, 0, start|, 0, final*, start|, 0, start*, final*, 0, final*, final|, final*, 0, 1, final*, start*, final*, 0, 0, final|, start|, 0, start|, 0, start*, final*, 0, 1, final|, start|, 0, final*, start*, final*, 0, final*, 1, final|, final*, 0, start|, 0, final*, start*, final*, start*, final*, 0, final|, start|, 0, 0, final*, 0, final|, start|, 0, 1, final*, start*, final*, 0, final*, final|, 1, final*, 0, start|, 0, final*, start*, final*, start*, final*, 0, final|, start|, 0, 0, 0, final*, start*, final*, start*, final*, 0, final|, start|, 0, final*, 0, 0, final*, 1, final|, final*, 0, start|, 0, final*, start*, final*, start*, final*, 0, final|, start|, 0, final*, 0, 0, final*, final|, 1, final*, 0, start|, 0, final*, start*, final*, start*, final*, 0, final|, start|, 0, final*, 0, 0, final*, final*, 0, 1, final|, start|, 0, 1, final*, start*, final*, start*, final*, 0, final|, start|, 0, final*, 0, 0, final*, final*, 0, final|, start|, 0, final*, start*, final*, start*, final*, 0, final|, start|, 0, final*, 0, 0, final*, 1, final|, final*, 0, 1, final|, start|, 0, 1, final*, start*, final*, start*, final*, 0, final|, start|, 0, final*, 0, 0, final*, final|, 1, final*, 0, final|, start|, final*, 0, 1, final|, start|, 0, final*, start*, final*, 0, final*, 1, final|, final*, 0, 1, final|, start|, 0, final*, start*, final*, start*, final*, 0, final|, start|, 0, 0, final*, 0, final|, start|, 0, 1, final*, start*, final*, 0, final*, final|, 1, final*, 0, final|, start|, 0, 1, final*, start*, final*, start*, final*, 0, final|, start|, 0, 0, final*, 0, 1, final|, start|, 0, final*, start*, final*, 0, final*, final*, 0, 1, final|, start|, 0, final*, start*, final*, start*, final*, 0, final|, start|, 0, 0, final*, 0, final|, start|, 0, 1, final*, start*, final*, 0, final*, final*, 0, final|, start|, 0, 1, final*, start*, final*, start*, final*, 0, final|, start|, 0, 0', '1', '__11_)', '__10_(', @@ -2068,6 +2093,7 @@ '__9_<', '__5_?', '__8_>', + 'SYMBOLNAME', '__6_[', '__7_]', 'NONTERMINALNAME', @@ -2075,7 +2101,7 @@ '__2_|', '2', 'QUOTE', - '0, 1, final*, 0, final|, start|, 0, final*, 0, final|, start|, 0, 1, final*, 0, final|, start|, 0, 1, final*, start*, final*, start*, 0, final|, start|, 0, 1, 0, 0, final|, start|, 0, 1, final*, start*, 0, 1, 0, final|, start|, 0, 0, start|, 0, final*, start*, 0, final|, start|, 0, 1, 0, 0, final|, start|, 0, 1, final*, start*, 0, 1, final*, 0, final|, start|, 0, final*, 0, start|, 0, final*, 0, final|, start|, 0, 1, final*, start*, final*, start*, 0, final|, start|, 0, 1, 0, 0, final|, start|, 0, 1, final*, start*, 0, 1, final*, 0, final|, start|, 0, final*, 0, final|, start|, 0, 1, final*, 0, start|, 0, final*, start*, final*, start*, 0, final|, start|, 0, 1, 0, 0, final|, start|, 0, 1, final*, start*, 0, 1, final*, 0, final|, start|, 0, final*, 0, final|, start|, 0, 1, final*, 0, final|, start|, 0, 1, final*, start*, final*, start*, 0, final|, start|, 0, 1, 0, 0, 1, final*, 0, final|, start|, 0, final*, 0, start|, 0, final*, 0, final|, start|, 0, 1, final*, start*, final*, start*, 0, final|, start|, 0, 1, 0', + 'final*, 0, 1, final*, 0, final|, start|, 0, final*, 0, final|, start|, 0, 1, final*, 0, final|, start|, 0, 1, final*, start*, final*, start*, final*, 0, final|, start|, 0, 1, 0, final*, 0, final|, start|, 0, 1, final*, start*, final*, 0, 1, final*, 0, final|, start|, 0, 0, start|, 0, final*, start*, final*, 0, final|, start|, 0, 1, 0, final*, 0, final|, start|, 0, 1, final*, start*, final*, 0, 1, final*, 0, final|, start|, 0, final*, 0, start|, 0, final*, 0, final|, start|, 0, 1, final*, start*, final*, start*, final*, 0, final|, start|, 0, 1, 0, final*, 0, final|, start|, 0, 1, final*, start*, final*, 0, 1, final*, 0, final|, start|, 0, final*, 0, final|, start|, 0, 1, final*, 0, start|, 0, final*, start*, final*, start*, final*, 0, final|, start|, 0, 1, 0, final*, 0, final|, start|, 0, 1, final*, start*, final*, 0, 1, final*, 0, final|, start|, 0, final*, 0, final|, start|, 0, 1, final*, 0, final|, start|, 0, 1, final*, start*, final*, start*, final*, 0, final|, start|, 0, 1, 0, final*, 0, 1, final*, 0, final|, start|, 0, final*, 0, start|, 0, final*, 0, final|, start|, 0, 1, final*, start*, final*, start*, final*, 0, final|, start|, 0, 1, 0', 'QUOTE', 'QUOTE', 'IGNORE']), {'IGNORE': None}) Modified: pypy/dist/pypy/rlib/parsing/test/test_ebnfparse.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/test/test_ebnfparse.py (original) +++ pypy/dist/pypy/rlib/parsing/test/test_ebnfparse.py Thu Dec 6 20:26:13 2007 @@ -450,3 +450,20 @@ t = parse("1 2 3 4 5") t = ToAST().transform(t) +def test_empty_production(): + # this could be seen as using the transformer in the wrong way + # but I have no clue how to detect this situation + regexs, rules, ToAST = parse_ebnf(""" +IGNORE: " "; +DECIMAL: "0|[1-9][0-9]*"; +stuff: "a" >stuff< "a" | "y" | >empty<; +empty: ; + """) + parse = make_parse_function(regexs, rules) + t = parse(" ") + t = ToAST().transform(t) + assert isinstance(t, Nonterminal) + assert len(t.children) == 0 + t = parse(" a a a a a a ") + t = ToAST().transform(t) + assert len(t.children) == 6 From cfbolz at codespeak.net Thu Dec 6 20:40:13 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 6 Dec 2007 20:40:13 +0100 (CET) Subject: [pypy-svn] r49457 - pypy/dist/pypy/rlib/parsing/test Message-ID: <20071206194013.B045016853D@codespeak.net> Author: cfbolz Date: Thu Dec 6 20:40:13 2007 New Revision: 49457 Modified: pypy/dist/pypy/rlib/parsing/test/test_ebnfparse.py Log: add some more stuff to test Modified: pypy/dist/pypy/rlib/parsing/test/test_ebnfparse.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/test/test_ebnfparse.py (original) +++ pypy/dist/pypy/rlib/parsing/test/test_ebnfparse.py Thu Dec 6 20:40:13 2007 @@ -456,10 +456,11 @@ regexs, rules, ToAST = parse_ebnf(""" IGNORE: " "; DECIMAL: "0|[1-9][0-9]*"; +file: EOF; stuff: "a" >stuff< "a" | "y" | >empty<; empty: ; """) - parse = make_parse_function(regexs, rules) + parse = make_parse_function(regexs, rules, eof=True) t = parse(" ") t = ToAST().transform(t) assert isinstance(t, Nonterminal) @@ -467,3 +468,4 @@ t = parse(" a a a a a a ") t = ToAST().transform(t) assert len(t.children) == 6 + excinfo = py.test.raises(ParseError, parse, "a") From fijal at codespeak.net Thu Dec 6 20:47:26 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 6 Dec 2007 20:47:26 +0100 (CET) Subject: [pypy-svn] r49458 - pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/test Message-ID: <20071206194726.2BD67168519@codespeak.net> Author: fijal Date: Thu Dec 6 20:47:25 2007 New Revision: 49458 Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/test/test_framework.py Log: Still completely broken test. No idea why Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/test/test_framework.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/test/test_framework.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/test/test_framework.py Thu Dec 6 20:47:25 2007 @@ -168,26 +168,26 @@ assert len(init_stores) == 5 def test_immutable_to_old_promotion(): - T_CHILD = lltype.Ptr(lltype.GcStruct('Childe', ('field', lltype.Signed))) + T_CHILD = lltype.Ptr(lltype.GcStruct('Child', ('field', lltype.Signed))) T_PARENT = lltype.Ptr(lltype.GcStruct('Parent', ('sub', T_CHILD))) child = lltype.malloc(T_CHILD.TO) + child2 = lltype.malloc(T_CHILD.TO) parent = lltype.malloc(T_PARENT.TO) + parent2 = lltype.malloc(T_PARENT.TO) parent.sub = child child.field = 3 - d = {'x' : parent} - d['y'] = lltype.malloc(T_PARENT.TO) - d['y'].sub = lltype.malloc(T_CHILD.TO) + parent2.sub = child2 + child2.field = 8 - def f(x, y): - t = d[y] - res = t.sub.field + x - del d[y] - return res + T_ALL = lltype.Ptr(lltype.GcArray(T_PARENT)) + all = lltype.malloc(T_ALL.TO, 2) + all[0] = parent + all[1] = parent2 - t = rtype(f, [int, str]) - etrafo = ExceptionTransformer(t) - graphs = etrafo.transform_completely() - transformer = WriteBarrierTransformer(t) - transformer.finish() - graphof(t, f).show() + def f(x): + res = all[x] + all[x] = lltype.nullptr(T_PARENT.TO) + return res.sub.field + + t, transformer = rtype_and_transform(f, [int], WriteBarrierTransformer) From antocuni at codespeak.net Fri Dec 7 00:19:09 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 7 Dec 2007 00:19:09 +0100 (CET) Subject: [pypy-svn] r49489 - in pypy/branch/clr-module-improvements/pypy/module/clr: . test Message-ID: <20071206231909.3CA01168559@codespeak.net> Author: antocuni Date: Fri Dec 7 00:19:08 2007 New Revision: 49489 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py Log: don't crash if the type cannot be found Modified: pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py Fri Dec 7 00:19:08 2007 @@ -206,14 +206,15 @@ def build_cli_class(space, namespace, classname, fullname): b_type = System.Type.GetType(fullname) - hasIEnumerable = 0 # flag + if b_type is None: + raise OperationError(space.w_ImportError, space.wrap("Cannot load .NET type: %s" % fullname)) # this is where we locate the interfaces inherited by the class # set the flag hasIEnumerable if IEnumerable interface has been by the class - ifaces = b_type.GetInterfaces() - for interface in ifaces: + hasIEnumerable = False + for interface in b_type.GetInterfaces(i): if interface.ToString() == "System.Collections.IEnumerable": - hasIEnumerable = 1 + hasIEnumerable = True w_staticmethods, w_methods = get_methods(space, b_type) w_properties, w_indexers = get_properties(space, b_type) Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py Fri Dec 7 00:19:08 2007 @@ -17,6 +17,10 @@ ArrayList2 = clr.load_cli_class('System.Collections', 'ArrayList') assert ArrayList is ArrayList2 + def test_load_fail(self): + import clr + raises(ImportError, clr.load_cli_class, 'Foo', 'Bar') + def test_ArrayList(self): import clr ArrayList = clr.load_cli_class('System.Collections', 'ArrayList') @@ -181,10 +185,3 @@ sum += i assert sum == 1+54+21 - - - - - - - From antocuni at codespeak.net Fri Dec 7 00:29:34 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 7 Dec 2007 00:29:34 +0100 (CET) Subject: [pypy-svn] r49490 - pypy/branch/clr-module-improvements/pypy/module/clr Message-ID: <20071206232934.D4C49168527@codespeak.net> Author: antocuni Date: Fri Dec 7 00:29:34 2007 New Revision: 49490 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py Log: typo Modified: pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py Fri Dec 7 00:29:34 2007 @@ -212,7 +212,7 @@ # this is where we locate the interfaces inherited by the class # set the flag hasIEnumerable if IEnumerable interface has been by the class hasIEnumerable = False - for interface in b_type.GetInterfaces(i): + for interface in b_type.GetInterfaces(): if interface.ToString() == "System.Collections.IEnumerable": hasIEnumerable = True From antocuni at codespeak.net Fri Dec 7 11:01:27 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 7 Dec 2007 11:01:27 +0100 (CET) Subject: [pypy-svn] r49498 - pypy/dist/pypy/translator/cli/test Message-ID: <20071207100127.CD2FC168570@codespeak.net> Author: antocuni Date: Fri Dec 7 11:01:26 2007 New Revision: 49498 Added: pypy/dist/pypy/translator/cli/test/test_cts.py (contents, props changed) Log: this has been in my wc for ages - I forgot to check in Added: pypy/dist/pypy/translator/cli/test/test_cts.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/cli/test/test_cts.py Fri Dec 7 11:01:26 2007 @@ -0,0 +1,24 @@ +from pypy.translator.cli import cts + +def test_primitive(): + void = cts.CliPrimitiveType('void') + assert str(void) == void.typename() == 'void' + assert void == cts.CliPrimitiveType('void') + +def test_class(): + Math = cts.CliClassType('mscorlib', 'System.Math') + assert str(Math) == Math.typename() == 'class [mscorlib]System.Math' + assert Math.classname() == '[mscorlib]System.Math' + assert Math == cts.CliClassType('mscorlib', 'System.Math') + +def test_generic(): + Dict = cts.CliGenericType('mscorlib', 'System.Dict', 2) + assert str(Dict) == Dict.typename() == 'class [mscorlib]System.Dict`2' + + int32 = cts.CliPrimitiveType('int32') + Math = cts.CliClassType('mscorlib', 'System.Math') + MyDict = Dict.specialize(int32, Math) + assert isinstance(MyDict, cts.CliSpecializedType) + classname = '[mscorlib]System.Dict`2' + assert str(MyDict) == MyDict.typename() == 'class ' + classname + assert MyDict.classname() == classname From rxe at codespeak.net Fri Dec 7 11:38:10 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Fri, 7 Dec 2007 11:38:10 +0100 (CET) Subject: [pypy-svn] r49499 - pypy/dist/pypy/rpython/module Message-ID: <20071207103810.076D716855D@codespeak.net> Author: rxe Date: Fri Dec 7 11:38:09 2007 New Revision: 49499 Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py Log: try a different way - dont force darwin to be 64 bit. this still seems somewhat fragile - it also depends what we want (should it choose 64 if available) Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_stat.py Fri Dec 7 11:38:09 2007 @@ -141,9 +141,6 @@ if sys.platform.startswith('win'): _name_struct_stat = '_stati64' INCLUDES = ['sys/types.h', 'sys/stat.h'] -elif sys.platform.startswith('darwin'): - _name_struct_stat = 'stat64' - INCLUDES = ['sys/stat.h'] else: _name_struct_stat = 'stat' INCLUDES = ['sys/types.h', 'sys/stat.h', 'unistd.h'] @@ -198,12 +195,15 @@ 'fstat': '_fstati64', 'lstat': '_stati64'} # no lstat on Windows c_func_name = _functions[name] - else: + elif sys.platform.startswith('linux'): # because we always use _FILE_OFFSET_BITS 64 - this helps things work that are not a c compiler _functions = {'stat': 'stat64', 'fstat': 'fstat64', 'lstat': 'lstat64'} c_func_name = _functions[name] + else: + c_func_name = name + arg_is_path = (name != 'fstat') if arg_is_path: ARG1 = rffi.CCHARP From fijal at codespeak.net Fri Dec 7 12:27:18 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 7 Dec 2007 12:27:18 +0100 (CET) Subject: [pypy-svn] r49504 - in pypy/branch/lazy-write-barrier/pypy/rpython/memory: gctransform/test test Message-ID: <20071207112718.75729168578@codespeak.net> Author: fijal Date: Fri Dec 7 12:27:18 2007 New Revision: 49504 Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/test/test_framework.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/test/test_transformed_gc.py Log: Shuffle this test around, now it works (still not testing anything, but that's another level) Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/test/test_framework.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/test/test_framework.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/test/test_framework.py Fri Dec 7 12:27:18 2007 @@ -166,28 +166,4 @@ collect_analyzer = CollectAnalyzer(t) init_stores = find_initializing_stores(collect_analyzer, t.graphs[0]) assert len(init_stores) == 5 - -def test_immutable_to_old_promotion(): - T_CHILD = lltype.Ptr(lltype.GcStruct('Child', ('field', lltype.Signed))) - T_PARENT = lltype.Ptr(lltype.GcStruct('Parent', ('sub', T_CHILD))) - child = lltype.malloc(T_CHILD.TO) - child2 = lltype.malloc(T_CHILD.TO) - parent = lltype.malloc(T_PARENT.TO) - parent2 = lltype.malloc(T_PARENT.TO) - parent.sub = child - child.field = 3 - parent2.sub = child2 - child2.field = 8 - - T_ALL = lltype.Ptr(lltype.GcArray(T_PARENT)) - all = lltype.malloc(T_ALL.TO, 2) - all[0] = parent - all[1] = parent2 - - def f(x): - res = all[x] - all[x] = lltype.nullptr(T_PARENT.TO) - return res.sub.field - - t, transformer = rtype_and_transform(f, [int], WriteBarrierTransformer) Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/test/test_transformed_gc.py Fri Dec 7 12:27:18 2007 @@ -862,6 +862,30 @@ run = self.runner(f, nbargs=0) run([]) + def test_immutable_to_old_promotion(self): + T_CHILD = lltype.Ptr(lltype.GcStruct('Child', ('field', lltype.Signed))) + T_PARENT = lltype.Ptr(lltype.GcStruct('Parent', ('sub', T_CHILD))) + child = lltype.malloc(T_CHILD.TO) + child2 = lltype.malloc(T_CHILD.TO) + parent = lltype.malloc(T_PARENT.TO) + parent2 = lltype.malloc(T_PARENT.TO) + parent.sub = child + child.field = 3 + parent2.sub = child2 + child2.field = 8 + + T_ALL = lltype.Ptr(lltype.GcArray(T_PARENT)) + all = lltype.malloc(T_ALL.TO, 2) + all[0] = parent + all[1] = parent2 + + def f(x, y): + res = all[x] + all[x] = lltype.nullptr(T_PARENT.TO) + return res.sub.field + + run = self.runner(f, nbargs=2) + run([1, 4]) class TestGenerationalNoFullCollectGC(GCTest): # test that nursery is doing its job and that no full collection From antocuni at codespeak.net Fri Dec 7 13:58:41 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 7 Dec 2007 13:58:41 +0100 (CET) Subject: [pypy-svn] r49508 - in pypy/dist/pypy: rpython/test translator/cli/test translator/jvm/test Message-ID: <20071207125841.3234E168449@codespeak.net> Author: antocuni Date: Fri Dec 7 13:58:40 2007 New Revision: 49508 Modified: pypy/dist/pypy/rpython/test/tool.py pypy/dist/pypy/translator/cli/test/test_builtin.py pypy/dist/pypy/translator/jvm/test/runtest.py Log: - a much better way to test for float equality - skip os.dup test in cli Modified: pypy/dist/pypy/rpython/test/tool.py ============================================================================== --- pypy/dist/pypy/rpython/test/tool.py (original) +++ pypy/dist/pypy/rpython/test/tool.py Fri Dec 7 13:58:40 2007 @@ -3,10 +3,10 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.test.test_llinterp import gengraph, interpret, interpret_raises -FLOAT_PRECISION = 8 - class BaseRtypingTest(object): + FLOAT_PRECISION = 8 + def gengraph(self, func, argtypes=[], viewbefore='auto', policy=None, backendopt=False, config=None): return gengraph(func, argtypes, viewbefore, policy, type_system=self.type_system, @@ -23,7 +23,8 @@ def float_eq_approx(self, x, y): diff = abs(x-y) - return diff < 10**-FLOAT_PRECISION + error = diff/y + return error < 10**-self.FLOAT_PRECISION def is_of_type(self, x, type_): return type(x) is type_ Modified: pypy/dist/pypy/translator/cli/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_builtin.py (original) +++ pypy/dist/pypy/translator/cli/test/test_builtin.py Fri Dec 7 13:58:40 2007 @@ -14,7 +14,9 @@ class TestCliBuiltin(CliTest, BaseTestBuiltin): test_os_path_exists = skip_os test_os_isdir = skip_os + test_os_dup_oo = skip_os + def test_builtin_math_frexp(self): self._skip_powerpc("Mono math floating point problem") BaseTestBuiltin.test_builtin_math_frexp(self) Modified: pypy/dist/pypy/translator/jvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/jvm/test/runtest.py Fri Dec 7 13:58:40 2007 @@ -14,8 +14,6 @@ generate_source_for_function, JvmError, detect_missing_support_programs from pypy.translator.jvm.option import getoption -FLOAT_PRECISION = 5 - class StructTuple(tuple): def __getattr__(self, name): if name.startswith('item'): @@ -81,6 +79,9 @@ return res class JvmTest(BaseRtypingTest, OORtypeMixin): + + FLOAT_PRECISION = 7 + def __init__(self): self._func = None self._ann = None @@ -132,7 +133,7 @@ assert False, 'function did not raise any exception at all' def float_eq(self, x, y): - return round(x, FLOAT_PRECISION) == round(y, FLOAT_PRECISION) + return self.float_eq_approx(x, y) def is_of_type(self, x, type_): return True # we can't really test the type From arigo at codespeak.net Fri Dec 7 18:27:08 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Dec 2007 18:27:08 +0100 (CET) Subject: [pypy-svn] r49526 - in pypy/branch/pypy-interp-file: . module/_file module/_file/test Message-ID: <20071207172708.CA191168502@codespeak.net> Author: arigo Date: Fri Dec 7 18:27:06 2007 New Revision: 49526 Removed: pypy/branch/pypy-interp-file/module/_file/app_file.py Modified: pypy/branch/pypy-interp-file/conftest.py pypy/branch/pypy-interp-file/module/_file/interp_file.py pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Log: file.readinto(). With this, the interp-level version of the file type should be complete. Modified: pypy/branch/pypy-interp-file/conftest.py ============================================================================== --- pypy/branch/pypy-interp-file/conftest.py (original) +++ pypy/branch/pypy-interp-file/conftest.py Fri Dec 7 18:27:06 2007 @@ -126,6 +126,9 @@ def call_function(self, func, *args, **kwds): return func(*args, **kwds) + def call_method(self, obj, name, *args, **kwds): + return getattr(obj, name)(*args, **kwds) + def translation_test_so_skip_if_appdirect(): if option.runappdirect: py.test.skip("translation test, skipped for appdirect") Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/interp_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/interp_file.py Fri Dec 7 18:27:06 2007 @@ -363,6 +363,26 @@ return self.getrepr(self.space, info) file__repr__.unwrap_spec = ['self'] + def file_readinto(self, w_array): + """readinto() -> Undocumented. Don't use this; it may go away.""" + # completely inefficient and incomplete for now + space = self.space + w_len = space.appexec([space.wrap(self), w_array], """(self, a): + from array import array + if not isinstance(a, array): + raise TypeError('Can only read into array objects') + length = len(a) + data = self.read(length) + n = len(data) + if n < length: + data += a.tostring()[len(data):] + del a[:] + a.fromstring(data) + return n + """) + return w_len + file_readinto.unwrap_spec = ['self', W_Root] + # ____________________________________________________________ @@ -445,6 +465,7 @@ cls=W_File, doc="Support for 'print'."), __repr__ = interp2app(W_File.file__repr__), + readinto = interp2app(W_File.file_readinto), **dict([(name, interp2app(getattr(W_File, 'file_' + name))) for name in W_File._exposed_method_names]) ) Modified: pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py (original) +++ pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Fri Dec 7 18:27:06 2007 @@ -206,9 +206,12 @@ extra_args = () def setup_method(self, method): - from pypy.module._file.interp_file import W_File space = self.space - w_filetype = space.gettypeobject(W_File.typedef) + if hasattr(space, 'gettypeobject'): + from pypy.module._file.interp_file import W_File + w_filetype = space.gettypeobject(W_File.typedef) + else: + w_filetype = file # TinyObjSpace, for "py.test -A" self.w_file = space.call_function( w_filetype, space.wrap(self.expected_filename), @@ -261,10 +264,14 @@ extra_args = () def setup_method(self, method): - from pypy.module._file.interp_file import W_File space = self.space O_BINARY = getattr(os, "O_BINARY", 0) - w_filetype = space.gettypeobject(W_File.typedef) + if hasattr(space, 'gettypeobject'): + from pypy.module._file.interp_file import W_File + w_filetype = space.gettypeobject(W_File.typedef) + else: + w_filetype = os # TinyObjSpace, for "py.test -A" + # (CPython has no file.fdopen, only os.fdopen) fd = os.open(AppTestFile.expected_filename, os.O_RDONLY | O_BINARY) self.w_file = space.call_method( w_filetype, @@ -440,7 +447,7 @@ f.close() assert os.stat(fn).st_size == 3 - f = file(fn, 'wb', 100) + f = file(fn, 'wb', 1000) f.write('x') assert os.stat(fn).st_size == 0 f.write('\n') @@ -477,3 +484,17 @@ data = f.read(123) assert data == 'hel' f.close() + + def test_readinto(self): + from array import array + a = array('c') + a.fromstring('0123456789') + fn = self.temptestfile + f = open(fn, 'w+b') + f.write('foobar') + f.seek(0) + n = f.readinto(a) + f.close() + assert n == 6 + assert len(a) == 10 + assert a.tostring() == 'foobar6789' From arigo at codespeak.net Fri Dec 7 18:43:00 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Dec 2007 18:43:00 +0100 (CET) Subject: [pypy-svn] r49527 - pypy/branch/pypy-interp-file/module/_file Message-ID: <20071207174300.229631684D3@codespeak.net> Author: arigo Date: Fri Dec 7 18:42:59 2007 New Revision: 49527 Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py Log: Dead code. Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/interp_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/interp_file.py Fri Dec 7 18:42:59 2007 @@ -472,12 +472,6 @@ # ____________________________________________________________ -def fdopen_as_stream(space, fd, mode="r", buffering=-1): - is_mode_ok(space, mode) - return space.wrap(W_Stream( - space, streamio.fdopen_as_stream(fd, mode, buffering))) -fdopen_as_stream.unwrap_spec = [ObjSpace, int, str, int] - def wrap_list_of_str(space, lst): return space.newlist([space.wrap(s) for s in lst]) From arigo at codespeak.net Fri Dec 7 18:43:13 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Dec 2007 18:43:13 +0100 (CET) Subject: [pypy-svn] r49528 - pypy/branch/pypy-interp-file/module/marshal Message-ID: <20071207174313.C66A21684E7@codespeak.net> Author: arigo Date: Fri Dec 7 18:43:13 2007 New Revision: 49528 Modified: pypy/branch/pypy-interp-file/module/marshal/interp_marshal.py Log: Fix marshal on real file objects. Modified: pypy/branch/pypy-interp-file/module/marshal/interp_marshal.py ============================================================================== --- pypy/branch/pypy-interp-file/module/marshal/interp_marshal.py (original) +++ pypy/branch/pypy-interp-file/module/marshal/interp_marshal.py Fri Dec 7 18:43:13 2007 @@ -1,7 +1,8 @@ from pypy.interpreter.baseobjspace import ObjSpace from pypy.interpreter.error import OperationError from pypy.rlib.rarithmetic import intmask -#from pypy.module._file.interp_file import file2stream +from pypy.module._file.interp_file import W_File +from pypy.module._file.interp_stream import StreamErrors, wrap_streamerror import sys # Py_MARSHAL_VERSION = 2 @@ -15,9 +16,10 @@ def dump(space, w_data, w_f, w_version=Py_MARSHAL_VERSION): """Write the 'data' object into the open file 'f'.""" - w_stream = file2stream(space, w_f) - if w_stream is not None: - writer = StreamWriter(space, w_stream) + # special case real files for performance + file = space.interpclass_w(w_f) + if isinstance(file, W_File): + writer = DirectStreamWriter(space, file) else: writer = FileWriter(space, w_f) try: @@ -39,9 +41,9 @@ def load(space, w_f): """Read one value from the file 'f' and return it.""" # special case real files for performance - w_stream = file2stream(space, w_f) - if w_stream is not None: - reader = StreamReader(space, w_stream) + file = space.interpclass_w(w_f) + if isinstance(file, W_File): + reader = DirectStreamReader(space, file) else: reader = FileReader(space, w_f) try: @@ -114,27 +116,24 @@ class StreamReaderWriter(AbstractReaderWriter): - def __init__(self, space, w_stream): + def __init__(self, space, file): AbstractReaderWriter.__init__(self, space) - self.w_stream = w_stream - w_stream.descr_lock() + self.file = file + file.lock() def finished(self): - self.w_stream.descr_unlock() + self.file.unlock() -class StreamWriter(StreamReaderWriter): +class DirectStreamWriter(StreamReaderWriter): def write(self, data): - self.w_stream.do_write(data) + self.file.direct_write(data) -class StreamReader(StreamReaderWriter): +class DirectStreamReader(StreamReaderWriter): def read(self, n): - result = data = self.w_stream.do_read(n) - while len(result) < n: - if len(data) == 0: - self.raise_eof() - data = self.w_stream.do_read(n) - result += data - return result + data = self.file.direct_read(n) + if len(data) < n: + self.raise_eof() + return data MAX_MARSHAL_DEPTH = 5000 From arigo at codespeak.net Fri Dec 7 20:12:06 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Dec 2007 20:12:06 +0100 (CET) Subject: [pypy-svn] r49532 - in pypy/branch/pypy-interp-file/module/_file: . test Message-ID: <20071207191206.CAE7D1684D2@codespeak.net> Author: arigo Date: Fri Dec 7 20:12:04 2007 New Revision: 49532 Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Log: Files are weakrefable. Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/interp_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/interp_file.py Fri Dec 7 20:12:04 2007 @@ -6,7 +6,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import ObjSpace, W_Root, Arguments from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.interpreter.typedef import interp_attrproperty +from pypy.interpreter.typedef import interp_attrproperty, make_weakref_descr from pypy.interpreter.gateway import interp2app @@ -466,6 +466,7 @@ doc="Support for 'print'."), __repr__ = interp2app(W_File.file__repr__), readinto = interp2app(W_File.file_readinto), + __weakref__ = make_weakref_descr(W_File), **dict([(name, interp2app(getattr(W_File, 'file_' + name))) for name in W_File._exposed_method_names]) ) Modified: pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py (original) +++ pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Fri Dec 7 20:12:04 2007 @@ -498,3 +498,13 @@ assert n == 6 assert len(a) == 10 assert a.tostring() == 'foobar6789' + + def test_weakref(self): + """Files are weakrefable.""" + import weakref + fn = self.temptestfile + f = open(fn, 'wb') + ref = weakref.ref(f) + ref().write('hello') + assert f.tell() == 5 + f.close() From arigo at codespeak.net Fri Dec 7 20:21:41 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Dec 2007 20:21:41 +0100 (CET) Subject: [pypy-svn] r49533 - pypy/branch/pypy-interp-file/module/_file Message-ID: <20071207192141.445021684D3@codespeak.net> Author: arigo Date: Fri Dec 7 20:21:40 2007 New Revision: 49533 Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py Log: Not much point in testing this here. Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/interp_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/interp_file.py Fri Dec 7 20:21:40 2007 @@ -33,8 +33,7 @@ def __del__(self): # assume that the file and stream objects are only visible in the # thread that runs __del__, so no race condition should be possible - if self.stream is not None: - self.direct_close() + self.direct_close() def fdopenstream(self, stream, fd, mode, name): self.fd = fd From arigo at codespeak.net Fri Dec 7 20:27:20 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Dec 2007 20:27:20 +0100 (CET) Subject: [pypy-svn] r49534 - pypy/dist/lib-python/modified-2.4.1/test Message-ID: <20071207192720.63CA116843F@codespeak.net> Author: arigo Date: Fri Dec 7 20:27:19 2007 New Revision: 49534 Modified: pypy/dist/lib-python/modified-2.4.1/test/test_tempfile.py Log: Shallow failure creating confusion in non-all-working-modules builds. Modified: pypy/dist/lib-python/modified-2.4.1/test/test_tempfile.py ============================================================================== --- pypy/dist/lib-python/modified-2.4.1/test/test_tempfile.py (original) +++ pypy/dist/lib-python/modified-2.4.1/test/test_tempfile.py Fri Dec 7 20:27:19 2007 @@ -314,6 +314,11 @@ tester = '"%s"' % tester else: decorated = sys.executable + try: + import fcntl + except ImportError: + return # for interpreters without fcntl, on Unix platforms, + # we can't set the FD_CLOEXEC flag retval = os.spawnl(os.P_WAIT, sys.executable, decorated, tester, v, fd) file.close() From antocuni at codespeak.net Fri Dec 7 20:36:03 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 7 Dec 2007 20:36:03 +0100 (CET) Subject: [pypy-svn] r49536 - pypy/dist/pypy/rpython/test Message-ID: <20071207193603.AE2EA16843F@codespeak.net> Author: antocuni Date: Fri Dec 7 20:36:02 2007 New Revision: 49536 Modified: pypy/dist/pypy/rpython/test/tool.py Log: (antocuni, xoraxax) a better way to compare floats Modified: pypy/dist/pypy/rpython/test/tool.py ============================================================================== --- pypy/dist/pypy/rpython/test/tool.py (original) +++ pypy/dist/pypy/rpython/test/tool.py Fri Dec 7 20:36:02 2007 @@ -22,9 +22,16 @@ return x == y def float_eq_approx(self, x, y): - diff = abs(x-y) - error = diff/y - return error < 10**-self.FLOAT_PRECISION + maxError = 10**-self.FLOAT_PRECISION + if abs(x-y) < maxError: + return True + + if abs(y) > abs(x): + relativeError = abs((x - y) / y) + else: + relativeError = abs((x - y) / x) + + return relativeError < maxError def is_of_type(self, x, type_): return type(x) is type_ From cfbolz at codespeak.net Fri Dec 7 20:50:42 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 7 Dec 2007 20:50:42 +0100 (CET) Subject: [pypy-svn] r49537 - pypy/dist/pypy/rlib Message-ID: <20071207195042.2945016851C@codespeak.net> Author: cfbolz Date: Fri Dec 7 20:50:41 2007 New Revision: 49537 Modified: pypy/dist/pypy/rlib/rope.py Log: small simplification Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Fri Dec 7 20:50:41 2007 @@ -1033,21 +1033,11 @@ return self.node def nextnode(self): - below = self.node - while self.stack: - tookleft = self.tookleft.pop() - if tookleft: - node = self.stack[-1] - assert isinstance(node, BinaryConcatNode) - self.tookleft.append(False) - self.find_downward(node.right) - return self.node - self.stack.pop() - raise StopIteration + self.seekforward(0) def getnode(self): if self.index == self.node.length(): - return self.nextnode() + self.nextnode() return self.node def nextchar(self): From arigo at codespeak.net Fri Dec 7 21:16:59 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Dec 2007 21:16:59 +0100 (CET) Subject: [pypy-svn] r49538 - pypy/branch/pypy-interp-file/module/bz2 Message-ID: <20071207201659.86E1B168502@codespeak.net> Author: arigo Date: Fri Dec 7 21:16:58 2007 New Revision: 49538 Removed: pypy/branch/pypy-interp-file/module/bz2/app_bz2.py Modified: pypy/branch/pypy-interp-file/module/bz2/__init__.py pypy/branch/pypy-interp-file/module/bz2/interp_bz2.py Log: Trying to fix the bz2 module: * port BZ2File to interp-level. This is not as nice as it could be. * fix various bugs in ReadBZ2Filter that for some reason were not apparent before (probably differences in what stream methods get called in answer to what file-level operation). Modified: pypy/branch/pypy-interp-file/module/bz2/__init__.py ============================================================================== --- pypy/branch/pypy-interp-file/module/bz2/__init__.py (original) +++ pypy/branch/pypy-interp-file/module/bz2/__init__.py Fri Dec 7 21:16:58 2007 @@ -2,15 +2,18 @@ from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): + """The python bz2 module provides a comprehensive interface for +the bz2 compression library. It implements a complete file +interface, one shot (de)compression functions, and types for +sequential (de)compression.""" + interpleveldefs = { 'BZ2Compressor': 'interp_bz2.W_BZ2Compressor', 'BZ2Decompressor': 'interp_bz2.W_BZ2Decompressor', 'compress': 'interp_bz2.compress', 'decompress': 'interp_bz2.decompress', - '_open_file_as_stream': 'interp_bz2.open_file_as_stream' + 'BZ2File': 'interp_bz2.W_BZ2File', } appleveldefs = { - '__doc__': 'app_bz2.__doc__', - 'BZ2File': 'app_bz2.BZ2File', } Modified: pypy/branch/pypy-interp-file/module/bz2/interp_bz2.py ============================================================================== --- pypy/branch/pypy-interp-file/module/bz2/interp_bz2.py (original) +++ pypy/branch/pypy-interp-file/module/bz2/interp_bz2.py Fri Dec 7 21:16:58 2007 @@ -5,7 +5,7 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.typedef import interp_attrproperty -from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped, interp2app +from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped, interp2app, Arguments from pypy.rlib.streamio import Stream from pypy.translator.tool.cbuild import ExternalCompilationInfo import sys @@ -161,12 +161,83 @@ return current_size + BIGCHUNK return current_size + SMALLCHUNK -def open_file_as_stream(space, path, mode="r", buffering=-1, compresslevel=9): +# ____________________________________________________________ +# +# Make the BZ2File type by internally inheriting from W_File. +# XXX this depends on internal details of W_File to work properly. + +from pypy.module._file.interp_file import W_File + +class W_BZ2File(W_File): + + def direct___init__(self, name, mode='r', buffering=0, compresslevel=9): + self.direct_close() + # the stream should always be opened in binary mode + if "b" not in mode: + mode = mode + "b" + self.check_mode_ok(mode) + stream = open_bz2file_as_stream(self.space, name, mode, + buffering, compresslevel) + fd = stream.try_to_find_file_descriptor() + self.fdopenstream(stream, fd, mode, name) + + _exposed_method_names = [] + W_File._decl.im_func(locals(), "__init__", ['self', str, str, int, int], + """Opens a BZ2-compressed file.""") + + def bz2file__repr__(self): + if self.stream is None: + head = "closed" + else: + head = "open" + info = "%s bz2.BZ2File '%s', mode '%s'" % (head, self.name, self.mode) + return self.getrepr(self.space, info) + bz2file__repr__.unwrap_spec = ['self'] + +def descr_bz2file__new__(space, w_subtype, args): + bz2file = space.allocate_instance(W_BZ2File, w_subtype) + W_BZ2File.__init__(bz2file, space) + return space.wrap(bz2file) +descr_bz2file__new__.unwrap_spec = [ObjSpace, W_Root, Arguments] + +same_attributes_as_in_file = list(W_File._exposed_method_names) +same_attributes_as_in_file.remove('__init__') +same_attributes_as_in_file.extend([ + 'name', 'mode', 'encoding', 'closed', 'newlines', 'softspace', + '__weakref__']) + +extra_attrs = dict([(name, interp2app(getattr(W_BZ2File, 'file_' + name))) + for name in W_BZ2File._exposed_method_names]) +extra_attrs.update(dict([(name, W_File.typedef.rawdict[name]) + for name in same_attributes_as_in_file])) + +W_BZ2File.typedef = TypeDef( + "BZ2File", + __doc__ = """\ +BZ2File(name [, mode='r', buffering=0, compresslevel=9]) -> file object + +Open a bz2 file. The mode can be 'r' or 'w', for reading (default) or +writing. When opened for writing, the file will be created if it doesn't +exist, and truncated otherwise. If the buffering argument is given, 0 means +unbuffered, and larger numbers specify the buffer size. If compresslevel +is given, must be a number between 1 and 9. + +Add a 'U' to mode to open the file for input with universal newline +support. Any line ending in the input file will be seen as a '\\n' in +Python. Also, a file so opened gains the attribute 'newlines'; the value +for this attribute is one of None (no newline read yet), '\\r', '\\n', +'\\r\\n' or a tuple containing all the newline types seen. Universal +newlines are available only when reading.""", + __new__ = interp2app(descr_bz2file__new__), + __repr__ = interp2app(W_BZ2File.bz2file__repr__), + **extra_attrs) + +# ____________________________________________________________ + +def open_bz2file_as_stream(space, path, mode="r", buffering=-1, + compresslevel=9): from pypy.rlib.streamio import decode_mode, open_path_helper from pypy.rlib.streamio import construct_stream_tower - from pypy.module._file.interp_file import wrap_oserror_as_ioerror, W_Stream - from pypy.module._file.interp_file import is_mode_ok - is_mode_ok(space, mode) os_flags, universal, reading, writing, basemode = decode_mode(mode) if reading and writing: raise OperationError(space.w_ValueError, @@ -174,10 +245,7 @@ if basemode == "a": raise OperationError(space.w_ValueError, space.wrap("cannot append to bz2 file")) - try: - stream = open_path_helper(path, os_flags, False) - except OSError, exc: - raise wrap_oserror_as_ioerror(space, exc) + stream = open_path_helper(path, os_flags, False) if reading: bz2stream = ReadBZ2Filter(space, stream, compresslevel) else: @@ -185,8 +253,7 @@ bz2stream = WriteBZ2Filter(space, stream, compresslevel) stream = construct_stream_tower(bz2stream, buffering, universal, reading, writing) - return space.wrap(W_Stream(space, stream)) -open_file_as_stream.unwrap_spec = [ObjSpace, str, str, int, int] + return stream class ReadBZ2Filter(Stream): @@ -229,12 +296,18 @@ if not length: break else: - raise NotImplementedError + # first measure the length by reading everything left + while len(self.read(65536)) > 0: + pass + pos = self.readlength + offset + self.seek(pos, 0) def readall(self): w_result = self.decompressor.decompress(self.stream.readall()) result = self.space.str_w(w_result) self.readlength += len(result) + result = self.buffer + result + self.buffer = '' return result def read(self, n): @@ -244,8 +317,12 @@ while not self.buffer: if self.finished: return "" + moredata = self.stream.read(n) + if not moredata: + self.finished = True + return "" try: - w_read = self.decompressor.decompress(self.stream.read(n)) + w_read = self.decompressor.decompress(moredata) except OperationError, e: if e.match(self.space, self.space.w_EOFError): self.finished = True @@ -507,7 +584,9 @@ after the end of stream is found, EOFError will be raised. If any data was found after the end of stream, it'll be ignored and saved in unused_data attribute.""" - + + if data == '': + return self.space.wrap('') if not self.running: raise OperationError(self.space.w_EOFError, self.space.wrap("end of stream was already found")) From arigo at codespeak.net Fri Dec 7 21:28:22 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Dec 2007 21:28:22 +0100 (CET) Subject: [pypy-svn] r49539 - pypy/branch/pypy-interp-file/module/bz2 Message-ID: <20071207202822.DA55E168414@codespeak.net> Author: arigo Date: Fri Dec 7 21:28:20 2007 New Revision: 49539 Modified: pypy/branch/pypy-interp-file/module/bz2/interp_bz2.py Log: Ah. The docstring says 'buffering' defaults to 0, but the previous app-level version had a default of -1. The difference in performance for readline() ad readlines() is huuuuuuuge... Modified: pypy/branch/pypy-interp-file/module/bz2/interp_bz2.py ============================================================================== --- pypy/branch/pypy-interp-file/module/bz2/interp_bz2.py (original) +++ pypy/branch/pypy-interp-file/module/bz2/interp_bz2.py Fri Dec 7 21:28:20 2007 @@ -170,7 +170,7 @@ class W_BZ2File(W_File): - def direct___init__(self, name, mode='r', buffering=0, compresslevel=9): + def direct___init__(self, name, mode='r', buffering=-1, compresslevel=9): self.direct_close() # the stream should always be opened in binary mode if "b" not in mode: @@ -214,7 +214,7 @@ W_BZ2File.typedef = TypeDef( "BZ2File", __doc__ = """\ -BZ2File(name [, mode='r', buffering=0, compresslevel=9]) -> file object +BZ2File(name [, mode='r', buffering=-1, compresslevel=9]) -> file object Open a bz2 file. The mode can be 'r' or 'w', for reading (default) or writing. When opened for writing, the file will be created if it doesn't From arigo at codespeak.net Fri Dec 7 21:40:02 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Dec 2007 21:40:02 +0100 (CET) Subject: [pypy-svn] r49540 - pypy/branch/pypy-interp-file/module/bz2 Message-ID: <20071207204002.6DF9A16845E@codespeak.net> Author: arigo Date: Fri Dec 7 21:40:01 2007 New Revision: 49540 Modified: pypy/branch/pypy-interp-file/module/bz2/interp_bz2.py Log: hacking hacking... can't just use the name "__init__" again because the RTyper is confused about the two direct__init__() with a different signature, confusion caused by the fact that W_File.file__init__() would appear to contain an indirect call to one of the two versions of direct__init__(). Modified: pypy/branch/pypy-interp-file/module/bz2/interp_bz2.py ============================================================================== --- pypy/branch/pypy-interp-file/module/bz2/interp_bz2.py (original) +++ pypy/branch/pypy-interp-file/module/bz2/interp_bz2.py Fri Dec 7 21:40:01 2007 @@ -170,7 +170,8 @@ class W_BZ2File(W_File): - def direct___init__(self, name, mode='r', buffering=-1, compresslevel=9): + def direct_bz2__init__(self, name, mode='r', buffering=-1, + compresslevel=9): self.direct_close() # the stream should always be opened in binary mode if "b" not in mode: @@ -182,17 +183,22 @@ self.fdopenstream(stream, fd, mode, name) _exposed_method_names = [] - W_File._decl.im_func(locals(), "__init__", ['self', str, str, int, int], + W_File._decl.im_func(locals(), "bz2__init__", ['self', str, str, int, int], """Opens a BZ2-compressed file.""") + # XXX ^^^ hacking hacking... can't just use the name "__init__" again + # because the RTyper is confused about the two direct__init__() with + # a different signature, confusion caused by the fact that + # W_File.file__init__() would appear to contain an indirect call to + # one of the two versions of direct__init__(). - def bz2file__repr__(self): + def file_bz2__repr__(self): if self.stream is None: head = "closed" else: head = "open" info = "%s bz2.BZ2File '%s', mode '%s'" % (head, self.name, self.mode) return self.getrepr(self.space, info) - bz2file__repr__.unwrap_spec = ['self'] + file_bz2__repr__.unwrap_spec = ['self'] def descr_bz2file__new__(space, w_subtype, args): bz2file = space.allocate_instance(W_BZ2File, w_subtype) @@ -206,11 +212,6 @@ 'name', 'mode', 'encoding', 'closed', 'newlines', 'softspace', '__weakref__']) -extra_attrs = dict([(name, interp2app(getattr(W_BZ2File, 'file_' + name))) - for name in W_BZ2File._exposed_method_names]) -extra_attrs.update(dict([(name, W_File.typedef.rawdict[name]) - for name in same_attributes_as_in_file])) - W_BZ2File.typedef = TypeDef( "BZ2File", __doc__ = """\ @@ -229,8 +230,10 @@ '\\r\\n' or a tuple containing all the newline types seen. Universal newlines are available only when reading.""", __new__ = interp2app(descr_bz2file__new__), - __repr__ = interp2app(W_BZ2File.bz2file__repr__), - **extra_attrs) + __init__ = interp2app(W_BZ2File.file_bz2__init__), + __repr__ = interp2app(W_BZ2File.file_bz2__repr__), + **dict([(name, W_File.typedef.rawdict[name]) + for name in same_attributes_as_in_file])) # ____________________________________________________________ From arigo at codespeak.net Fri Dec 7 22:34:06 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 7 Dec 2007 22:34:06 +0100 (CET) Subject: [pypy-svn] r49542 - in pypy/branch/pypy-interp-file/module/_file: . test Message-ID: <20071207213406.15D0E1684D4@codespeak.net> Author: arigo Date: Fri Dec 7 22:34:05 2007 New Revision: 49542 Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Log: Two more checks for a closed file. Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/interp_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/interp_file.py Fri Dec 7 22:34:05 2007 @@ -93,6 +93,7 @@ stream.close() def direct_fileno(self): + self.getstream() # check if the file is still open return self.fd def direct_flush(self): @@ -200,6 +201,7 @@ direct_xreadlines = direct___iter__ def direct_isatty(self): + self.getstream() # check if the file is still open return os.isatty(self.fd) # ____________________________________________________________ Modified: pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py (original) +++ pypy/branch/pypy-interp-file/module/_file/test/test_file_extra.py Fri Dec 7 22:34:05 2007 @@ -508,3 +508,23 @@ ref().write('hello') assert f.tell() == 5 f.close() + + def test_ValueError(self): + fn = self.temptestfile + f = open(fn, 'wb') + f.close() + raises(ValueError, f.fileno) + raises(ValueError, f.flush) + raises(ValueError, f.isatty) + raises(ValueError, f.next) + raises(ValueError, f.read) + raises(ValueError, f.readline) + raises(ValueError, f.readlines) + raises(ValueError, f.seek, 0) + raises(ValueError, f.tell) + raises(ValueError, f.truncate) + raises(ValueError, f.write, "") + raises(ValueError, f.writelines, []) + raises(ValueError, iter, f) + raises(ValueError, f.xreadlines) + f.close() # accepted as a no-op From cfbolz at codespeak.net Sat Dec 8 15:27:44 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 8 Dec 2007 15:27:44 +0100 (CET) Subject: [pypy-svn] r49551 - in pypy/dist/pypy/rlib: . test Message-ID: <20071208142744.2F3C216855F@codespeak.net> Author: cfbolz Date: Sat Dec 8 15:27:42 2007 New Revision: 49551 Modified: pypy/dist/pypy/rlib/rope.py pypy/dist/pypy/rlib/test/test_rope.py Log: introduce ways to iterate backward too Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Sat Dec 8 15:27:42 2007 @@ -1005,6 +1005,17 @@ self.advance_index() return result +def make_seekable_method(resultgetter, backward=False): + if backward: + direction = -1 + else: + direction = 1 + def next(self): + node = self.getnode() + result = getattr(node, resultgetter)(self.index) + self.index += direction + return result + return next class SeekableItemIterator(object): def __init__(self, node): @@ -1032,31 +1043,19 @@ self.index = items return self.node - def nextnode(self): - self.seekforward(0) - def getnode(self): - if self.index == self.node.length(): - self.nextnode() + if self.index == self.nodelength: + self.seekforward(0) + if self.index == -1: + self.seekback(0) return self.node - def nextchar(self): - node = self.getnode() - result = node.getchar(self.index) - self.index += 1 - return result - - def nextunichar(self): - node = self.getnode() - result = node.getunichar(self.index) - self.index += 1 - return result - - def nextint(self): - node = self.getnode() - result = node.getint(self.index) - self.index += 1 - return result + nextchar = make_seekable_method("getchar") + nextunichar = make_seekable_method("getunichar") + nextint = make_seekable_method("getint") + lastchar = make_seekable_method("getchar", backward=True) + lastunichar = make_seekable_method("getunichar", backward=True) + lastint = make_seekable_method("getint", backward=True) def seekforward(self, numchars): if numchars < (self.nodelength - self.index): Modified: pypy/dist/pypy/rlib/test/test_rope.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rope.py (original) +++ pypy/dist/pypy/rlib/test/test_rope.py Sat Dec 8 15:27:42 2007 @@ -390,6 +390,17 @@ assert c2 == c py.test.raises(StopIteration, iter.nextchar) +def test_iterbackward(): + rope = BinaryConcatNode(BinaryConcatNode(LiteralStringNode("abc"), + LiteralStringNode("def")), + LiteralStringNode("ghi")) + iter = SeekableItemIterator(rope) + iter.seekforward(8) + for c in "abcdefghi"[::-1]: + c2 = iter.lastchar() + assert c2 == c + py.test.raises(StopIteration, iter.lastchar) + def test_find_int(): rope, st = make_random_string() rope = getslice_one(rope, 10, 100) From regmee at codespeak.net Sat Dec 8 22:06:37 2007 From: regmee at codespeak.net (regmee at codespeak.net) Date: Sat, 8 Dec 2007 22:06:37 +0100 (CET) Subject: [pypy-svn] r49564 - pypy/branch/clr-module-improvements/pypy/module/clr Message-ID: <20071208210637.F34EC168544@codespeak.net> Author: regmee Date: Sat Dec 8 22:06:36 2007 New Revision: 49564 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py pypy/branch/clr-module-improvements/pypy/module/clr/app_importer.py pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py Log: Removing the hardcoded Valid Namespaces used during import check. Introducing Valid Namespaces list for the check by scanning the Assemblies Modified: pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py Sat Dec 8 22:06:36 2007 @@ -14,6 +14,9 @@ '_CliObject_internal': 'interp_clr.W_CliObject', 'call_staticmethod': 'interp_clr.call_staticmethod', 'load_cli_class': 'interp_clr.load_cli_class', + 'load_valid_namespaces': 'interp_clr.load_valid_namespaces', + 'isDotNetType': 'interp_clr.isDotNetType', + 'load_assembly': 'interp_clr.load_assembly', } def setup_after_space_initialization(self): Modified: pypy/branch/clr-module-improvements/pypy/module/clr/app_importer.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/app_importer.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/app_importer.py Sat Dec 8 22:06:36 2007 @@ -6,14 +6,6 @@ import imp import sys -DotNetModuleList = ['System', - 'System.Collections', - 'System.Collections.ArrayList', - 'System.Collections.Stack', - 'System.Collections.Queue', - 'System.Math'] - - class loader(object): ''' This method returns the loaded module or raises an exception, ImportError @@ -46,26 +38,18 @@ ''' # If it is a call for a Class then return with the Class reference - code = 0 - if fullname == "System.Math": - code = 1 - import clr - sys.modules[fullname] = clr.load_cli_class('System','Math') - if fullname == "System.Collections.Queue": - code = 1 - import clr - sys.modules[fullname] = clr.load_cli_class('System.Collections','Queue') - if fullname == "System.Collections.Stack": - code = 1 - import clr - sys.modules[fullname] = clr.load_cli_class('System.Collections','Stack') - if fullname == "System.Collections.ArrayList": - code = 1 - import clr - sys.modules[fullname] = clr.load_cli_class('System.Collections','ArrayList') + import clr + if clr.isDotNetType(fullname): + ''' Task is to breakup System.Collections.ArrayList and call + clr.load_cli_class('System.Collections','ArrayList') + ''' + rindex = fullname.rfind('.') + if rindex != -1: + leftStr = fullname[:rindex] + rightStr= fullname[rindex+1:] + sys.modules[fullname] = clr.load_cli_class(leftStr, rightStr) - # if not a call for actual class assign an empty module for it. - if not code: + else: # if not a call for actual class (say for namespaces) assign an empty module mod = imp.new_module(fullname) mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) mod.__file__ = "<%s>" % self.__class__.__name__ @@ -92,11 +76,15 @@ been imported and added to sys.modules. ''' def __init__(self): + import clr + # this might not be the correct place to load the valid NameSpaces + self.ValidNameSpaces = clr.load_valid_namespaces() self.loader = loader() def find_module(self, fullname, path = None): - #print "( fullname = %s ) + ( path = %s )"%(fullname, path) - if fullname in DotNetModuleList: + # check for true NAMESPACE or .NET TYPE + import clr + if fullname in self.ValidNameSpaces or clr.isDotNetType(fullname): # fullname is a .Net Module if path != None: __path__ = path Modified: pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py Sat Dec 8 22:06:36 2007 @@ -184,6 +184,65 @@ return self.cache.get(fullname, None) CliClassCache = _CliClassCache() +def load_valid_namespaces(space): + """ + We use this function to play with reflection + and then return useful data to the app_importer module + + Parameters: + + Return: List of Valid .NET namespaces + """ + + listOfNamespaces = [] + currentDomain = System.AppDomain.get_CurrentDomain() + asEvidence = currentDomain.get_Evidence() + +# assembly1 = System.Reflection.Assembly.LoadFile("System.Xml.dll") +# assembly2 = System.Reflection.Assembly.LoadFile("mscorlib.dll") +# assembly3 = System.Reflection.Assembly.LoadFile("/home/amit/clrModImprove/pypy/module/clr/System.Windows.Forms.dll") + +# currentDomain.Load("System.Xml",asEvidence) +# currentDomain.Load("mscorlib",asEvidence) +# currentDomain.Load("System.Web",asEvidence) +# currentDomain.Load("System.Drawing",asEvidence) +# currentDomain.Load("System.Data",asEvidence) +# currentDomain.Load("System.Windows.Forms",asEvidence) + + assems = currentDomain.GetAssemblies() + for assembly in assems: + typesInAssembly = assembly.GetTypes(); + for type in typesInAssembly: + namespace = type.get_Namespace() + if namespace != None and namespace not in listOfNamespaces: + listOfNamespaces.append(namespace) + w_listOfNamespaces = wrap_list_of_strings(space, listOfNamespaces) + return w_listOfNamespaces + +def isDotNetType(space, nameFromImporter): + """ + determines if the string input is a DotNetType + + Return: + Boolean + """ + if System.Type.GetType(nameFromImporter) != None: + return space.wrap(True) + return space.wrap(False) + +isDotNetType.unwrap_spec = [ObjSpace, str] + +def load_assembly(space, assemblyName): + """ + Load the given .NET assembly into the PyPy interpreter + + Parameters: + + - assemblyName: the full name of the assembly + (e.g., ``System.Xml.dll``). + """ + pass + def load_cli_class(space, namespace, classname): """ Load the given .NET class into the PyPy interpreter and return a From pypy-svn at codespeak.net Sun Dec 9 18:20:57 2007 From: pypy-svn at codespeak.net (VIAGRA ® Official Site) Date: Sun, 9 Dec 2007 18:20:57 +0100 (CET) Subject: [pypy-svn] December 75% OFF Message-ID: <20071209112044.4586.qmail@ppp91-77-26-202.pppoe.mtu-net.ru> An HTML attachment was scrubbed... URL: From cfbolz at codespeak.net Sun Dec 9 20:43:41 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 9 Dec 2007 20:43:41 +0100 (CET) Subject: [pypy-svn] r49579 - pypy/dist/pypy/rlib Message-ID: <20071209194341.AFE371684D2@codespeak.net> Author: cfbolz Date: Sun Dec 9 20:43:40 2007 New Revision: 49579 Modified: pypy/dist/pypy/rlib/rope.py Log: some small optimizations. a lot of performance is lost for stack checks, I will need to do manual tail call opts in some places. Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Sun Dec 9 20:43:40 2007 @@ -290,6 +290,17 @@ id(self), len(self.u), repr(addinfo).replace('"', '').replace("\\", "\\\\"))) +def make_binary_get(getter): + def get(self, index): + while isinstance(self, BinaryConcatNode): + llen = self.left.length() + if index >= llen: + self = self.right + index -= llen + else: + self = self.left + return getattr(self, getter)(index) + return get class BinaryConcatNode(StringNode): def __init__(self, left, right): @@ -333,33 +344,10 @@ def depth(self): return self._depth - def getchar(self, index): - llen = self.left.length() - if index >= llen: - return self.right.getchar(index - llen) - else: - return self.left.getchar(index) - - def getunichar(self, index): - llen = self.left.length() - if index >= llen: - return self.right.getunichar(index - llen) - else: - return self.left.getunichar(index) - - def getint(self, index): - llen = self.left.length() - if index >= llen: - return self.right.getint(index - llen) - else: - return self.left.getint(index) - - def getrope(self, index): - llen = self.left.length() - if index >= llen: - return self.right.getrope(index - llen) - else: - return self.left.getrope(index) + getchar = make_binary_get("getchar") + getunichar = make_binary_get("getunichar") + getint = make_binary_get("getint") + getrope = make_binary_get("getrope") def can_contain_int(self, value): if self.is_bytestring() and value > 255: @@ -368,6 +356,17 @@ return False return (1 << (value & 0x1f)) & self.charbitmask + def getslice(self, start, stop): + if start == 0: + if stop == self.length(): + return self + return getslice_left(self, stop) + if stop == self.length(): + return getslice_right(self, start) + return concatenate( + getslice_right(self.left, start), + getslice_left(self.right, stop - self.left.length())) + def flatten_string(self): f = fringe(self) return "".join([node.flatten_string() for node in f]) @@ -448,18 +447,7 @@ def getslice_one(node, start, stop): start, stop, node = find_straddling(node, start, stop) - if isinstance(node, BinaryConcatNode): - if start == 0: - if stop == node.length(): - return node - return getslice_left(node, stop) - if stop == node.length(): - return getslice_right(node, start) - return concatenate( - getslice_right(node.left, start), - getslice_left(node.right, stop - node.left.length())) - else: - return node.getslice(start, stop) + return node.getslice(start, stop) def find_straddling(node, start, stop): while 1: @@ -546,13 +534,13 @@ return rebalance(nodelist, length) def rebalance(nodelist, sizehint=-1): - nodelist.reverse() if sizehint < 0: sizehint = 0 for node in nodelist: sizehint += node.length() if sizehint == 0: return LiteralStringNode.EMPTY + nodelist.reverse() # this code is based on the Fibonacci identity: # sum(fib(i) for i in range(n+1)) == fib(n+2) From cfbolz at codespeak.net Sun Dec 9 22:27:15 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 9 Dec 2007 22:27:15 +0100 (CET) Subject: [pypy-svn] r49587 - pypy/dist/pypy/objspace/std Message-ID: <20071209212715.819B1168506@codespeak.net> Author: cfbolz Date: Sun Dec 9 22:27:14 2007 New Revision: 49587 Modified: pypy/dist/pypy/objspace/std/ropeobject.py Log: kill some unnecessary code Modified: pypy/dist/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/ropeobject.py (original) +++ pypy/dist/pypy/objspace/std/ropeobject.py Sun Dec 9 22:27:14 2007 @@ -286,11 +286,8 @@ assert isinstance(w_s, W_RopeObject) node = w_s._node l.append(node) - selfnode = w_self._node - length = selfnode.length() - listlen_minus_one = len(list_w) - 1 try: - return W_RopeObject(rope.join(selfnode, l)) + return W_RopeObject(rope.join(self, l)) except OverflowError: raise OperationError(space.w_OverflowError, space.wrap("string too long")) From cfbolz at codespeak.net Sun Dec 9 22:47:38 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 9 Dec 2007 22:47:38 +0100 (CET) Subject: [pypy-svn] r49588 - pypy/dist/pypy/objspace/std Message-ID: <20071209214738.D0F30168511@codespeak.net> Author: cfbolz Date: Sun Dec 9 22:47:38 2007 New Revision: 49588 Modified: pypy/dist/pypy/objspace/std/ropeobject.py Log: more small optimizations Modified: pypy/dist/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/ropeobject.py (original) +++ pypy/dist/pypy/objspace/std/ropeobject.py Sun Dec 9 22:47:38 2007 @@ -531,7 +531,7 @@ def str_endswith__Rope_Tuple_ANY_ANY(space, w_self, w_suffixes, w_start, w_end): (self, _, start, end) = _convert_idx_params(space, w_self, - space.wrap(''), w_start, w_end) + W_RopeObject.EMPTY, w_start, w_end) for w_suffix in space.unpacktuple(w_suffixes): if space.is_true(space.isinstance(w_suffix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) @@ -549,7 +549,7 @@ return space.newbool(rope.startswith(self, prefix, start, end)) def str_startswith__Rope_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end): - (self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), + (self, _, start, end) = _convert_idx_params(space, w_self, W_RopeObject.EMPTY, w_start, w_end) for w_prefix in space.unpacktuple(w_prefixes): if space.is_true(space.isinstance(w_prefix, space.w_unicode)): @@ -741,7 +741,7 @@ space.w_TypeError, space.wrap("ord() expected a character, but string " "of length %d found"% (w_str._node.length(),))) - return space.wrap(ord(node.flatten_string()[0])) + return space.wrap(node.getint(0)) def getnewargs__Rope(space, w_str): return space.newtuple([W_RopeObject(w_str._node)]) @@ -853,7 +853,7 @@ raise OperationError(space.w_StopIteration, space.w_None) try: char = w_ropeiter.item_iter.nextchar() - w_item = space.wrap(char) + w_item = wrapchar(space, char) except StopIteration: w_ropeiter.node = None w_ropeiter.char_iter = None From cfbolz at codespeak.net Mon Dec 10 13:07:58 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 10 Dec 2007 13:07:58 +0100 (CET) Subject: [pypy-svn] r49590 - pypy/dist/pypy/rlib Message-ID: <20071210120758.65C2416840F@codespeak.net> Author: cfbolz Date: Mon Dec 10 13:07:57 2007 New Revision: 49590 Modified: pypy/dist/pypy/rlib/rope.py Log: be lazier in calculating all the rope extra info Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Mon Dec 10 13:07:57 2007 @@ -53,7 +53,6 @@ class StringNode(object): hash_cache = 0 - charbitmask = 0 def length(self): raise NotImplementedError("base class") @@ -69,6 +68,9 @@ def hash_part(self): raise NotImplementedError("base class") + def charbitmask(self): + raise NotImplementedError("base class") + def check_balanced(self): return True @@ -115,23 +117,28 @@ class LiteralStringNode(LiteralNode): - def __init__(self, s): + _is_ascii = False + _charbitmask = 0 + def __init__(self, s, charbitmask=0, is_ascii=False): assert isinstance(s, str) self.s = s - is_ascii = True - charbitmask = 0 - for c in s: - ordc = ord(c) - if ordc >= 128: - is_ascii = False - charbitmask |= 1 << (ordc & 0x1F) - self.charbitmask = charbitmask - self._is_ascii = is_ascii + if not s: + self._is_ascii = True + self._calculated = True + elif charbitmask: + self._charbitmask = charbitmask + self._is_ascii = is_ascii + self._calculated = True + else: + self._calculated = False + def length(self): return len(self.s) def is_ascii(self): + if not self._calculated: + self._calculate() return self._is_ascii def is_bytestring(self): @@ -154,6 +161,23 @@ h = self.hash_cache = x return h + def _calculate(self): + is_ascii = True + charbitmask = 0 + for c in self.s: + ordc = ord(c) + if ordc >= 128: + is_ascii = False + charbitmask |= intmask(1 << (ordc & 0x1F)) + self._is_ascii = is_ascii + self._charbitmask = charbitmask + self._calculated = True + + def charbitmask(self): + if not self._calculated: + self._calculate() + return self._charbitmask + def getchar(self, index): return self.s[index] @@ -171,7 +195,7 @@ return False if self.is_ascii() and value > 127: return False - return (1 << (value & 0x1f)) & self.charbitmask + return (1 << (value & 0x1f)) & self.charbitmask() def getslice(self, start, stop): assert 0 <= start <= stop @@ -203,6 +227,9 @@ yield ('"%s" [shape=box,label="length: %s\\n%s"];' % ( id(self), len(self.s), repr(addinfo).replace('"', '').replace("\\", "\\\\"))) + + def _freeze_(self): + self._calculate() LiteralStringNode.EMPTY = LiteralStringNode("") LiteralStringNode.PREBUILT = [LiteralStringNode(chr(i)) for i in range(256)] del i @@ -212,13 +239,7 @@ def __init__(self, u): assert isinstance(u, unicode) self.u = u - charbitmask = 0 - for c in u: - ordc = ord(c) - if ordc >= 128: - charbitmask |= 1 # be compatible with LiteralStringNode - charbitmask |= 1 << (ordc & 0x1F) - self.charbitmask = charbitmask + self._charbitmask = 0 def length(self): return len(self.u) @@ -243,6 +264,22 @@ h = self.hash_cache = x return h + def _calculate(self): + if len(self.u) == 0: + return + charbitmask = 0 + for c in self.u: + ordc = ord(c) + if ordc >= 128: + charbitmask |= 1 # be compatible with LiteralStringNode + charbitmask |= intmask(1 << (ordc & 0x1F)) + self._charbitmask = intmask(charbitmask) + + def charbitmask(self): + if not self._charbitmask: + self._calculate() + return self._charbitmask + def getunichar(self, index): return self.u[index] @@ -258,7 +295,7 @@ return LiteralUnicodeNode(unichr(ch)) def can_contain_int(self, value): - return (1 << (value & 0x1f)) & self.charbitmask + return (1 << (value & 0x1f)) & self.charbitmask() def getslice(self, start, stop): assert 0 <= start <= stop @@ -289,6 +326,8 @@ yield ('"%s" [shape=box,label="length: %s\\n%s"];' % ( id(self), len(self.u), repr(addinfo).replace('"', '').replace("\\", "\\\\"))) + def _freeze_(self): + self._calculate() def make_binary_get(getter): def get(self, index): @@ -303,46 +342,43 @@ return get class BinaryConcatNode(StringNode): - def __init__(self, left, right): + def __init__(self, left, right, balanced=False): self.left = left self.right = right try: self.len = ovfcheck(left.length() + right.length()) except OverflowError: raise - self._depth = max(left.depth(), right.depth()) + 1 - self.balanced = False - self._is_ascii = left.is_ascii() and right.is_ascii() - self._is_bytestring = left.is_bytestring() and right.is_bytestring() - self.charbitmask = left.charbitmask | right.charbitmask + self.balanced = balanced + self._calculated = False + self._depth = 0 def is_ascii(self): + if not self._calculated: + self._calculate() return self._is_ascii def is_bytestring(self): + if not self._calculated: + self._calculate() return self._is_bytestring def check_balanced(self): if self.balanced: return True - if not self.left.check_balanced() or not self.right.check_balanced(): - return False - left = self.left - right = self.right - llen = left.length() - rlen = right.length() - ldepth = left.depth() - rdepth = right.depth() - balanced = (find_fib_index(self.len // (NEW_NODE_WHEN_LENGTH / 2)) >= - self._depth) - self.balanced = balanced - return balanced + if not self._calculated: + self._calculate() + return self.balanced def length(self): return self.len def depth(self): - return self._depth + depth = self._depth + if not depth: + depth = self._depth = max(self.left.depth(), + self.right.depth()) + 1 + return depth getchar = make_binary_get("getchar") getunichar = make_binary_get("getunichar") @@ -354,7 +390,7 @@ return False if self.is_ascii() and value > 127: return False - return (1 << (value & 0x1f)) & self.charbitmask + return (1 << (value & 0x1f)) & self.charbitmask() def getslice(self, start, stop): if start == 0: @@ -385,7 +421,32 @@ h = self.hash_cache = x return h + def _calculate(self): + left = self.left + right = self.right + self._is_ascii = left.is_ascii() and right.is_ascii() + self._is_bytestring = left.is_bytestring() and right.is_bytestring() + self._charbitmask = left.charbitmask() | right.charbitmask() + # balance calculation + # XXX improve? + if self.balanced: + balanced = True + elif not left.check_balanced() or not right.check_balanced(): + balanced = False + else: + balanced = (find_fib_index(self.len // (NEW_NODE_WHEN_LENGTH / 2)) >= + self._depth) + self.balanced = balanced + self._calculated = True + + def charbitmask(self): + if not self._calculated: + self._calculate() + return self._charbitmask + def rebalance(self): + if self.balanced: + return self return rebalance([self], self.len) def dot(self, seen, toplevel=False): @@ -404,6 +465,8 @@ yield '"%s" -> "%s";' % (id(self), id(child)) for line in child.dot(seen): yield line + def _freeze_(self): + self._calculate() def concatenate(node1, node2): if node1.length() == 0: @@ -427,8 +490,8 @@ if slicelength == -1: # XXX for testing only slicelength = len(xrange(start, stop, step)) + start, stop, node = find_straddling(node, start, stop) if step != 1: - start, stop, node = find_straddling(node, start, stop) iter = SeekableItemIterator(node) iter.seekforward(start) if node.is_bytestring(): @@ -443,7 +506,7 @@ iter.seekforward(step - 1) result.append(iter.nextunichar()) return rope_from_unicharlist(result) - return getslice_one(node, start, stop) + return node.getslice(start, stop) def getslice_one(node, start, stop): start, stop, node = find_straddling(node, start, stop) @@ -589,7 +652,6 @@ if l[index] is not None: curr = BinaryConcatNode(l[index], curr) assert curr is not None - curr.check_balanced() return curr # __________________________________________________________________________ From arigo at codespeak.net Mon Dec 10 13:17:00 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 Dec 2007 13:17:00 +0100 (CET) Subject: [pypy-svn] r49591 - pypy/branch/pypy-interp-file/module/_file Message-ID: <20071210121700.21DC81684D5@codespeak.net> Author: arigo Date: Mon Dec 10 13:16:58 2007 New Revision: 49591 Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py Log: Kill dead code. Modified: pypy/branch/pypy-interp-file/module/_file/interp_file.py ============================================================================== --- pypy/branch/pypy-interp-file/module/_file/interp_file.py (original) +++ pypy/branch/pypy-interp-file/module/_file/interp_file.py Mon Dec 10 13:16:58 2007 @@ -215,10 +215,9 @@ raise wrap_streamerror(self.space, e) _exposed_method_names = [] - _exposed_classmethod_names = [] def _decl(class_scope, name, unwrap_spec, docstring, - as_classmethod=False, wrapresult="space.wrap(result)"): + wrapresult="space.wrap(result)"): # hack hack to build a wrapper around the direct_xxx methods. # The wrapper adds lock/unlock calls and a space.wrap() on # the result, conversion of stream errors to OperationErrors, @@ -256,11 +255,7 @@ """ % locals()) exec str(src) in globals(), class_scope class_scope['file_' + name].unwrap_spec = unwrap_spec - interp2app - if as_classmethod: - class_scope['_exposed_classmethod_names'].append(name) - else: - class_scope['_exposed_method_names'].append(name) + class_scope['_exposed_method_names'].append(name) _decl(locals(), "__init__", ['self', str, str, int], From cfbolz at codespeak.net Mon Dec 10 13:17:00 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 10 Dec 2007 13:17:00 +0100 (CET) Subject: [pypy-svn] r49592 - pypy/dist/pypy/rlib Message-ID: <20071210121700.E569C1684D8@codespeak.net> Author: cfbolz Date: Mon Dec 10 13:16:59 2007 New Revision: 49592 Modified: pypy/dist/pypy/rlib/rope.py Log: balance calculation shouldn't calculate the additional info Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Mon Dec 10 13:16:59 2007 @@ -350,7 +350,13 @@ except OverflowError: raise self.balanced = balanced + if balanced: + self.balance_known = True + else: + self.balance_known = False self._calculated = False + self._is_ascii = False + self._is_bytestring = False self._depth = 0 def is_ascii(self): @@ -364,10 +370,17 @@ return self._is_bytestring def check_balanced(self): - if self.balanced: - return True - if not self._calculated: - self._calculate() + if self.balance_known: + return self.balanced + # balance calculation + # XXX improve? + if not self.left.check_balanced() or not self.right.check_balanced(): + balanced = False + else: + balanced = (find_fib_index(self.len // (NEW_NODE_WHEN_LENGTH / 2)) >= + self._depth) + self.balanced = balanced + self.balance_known = True return self.balanced def length(self): @@ -427,16 +440,6 @@ self._is_ascii = left.is_ascii() and right.is_ascii() self._is_bytestring = left.is_bytestring() and right.is_bytestring() self._charbitmask = left.charbitmask() | right.charbitmask() - # balance calculation - # XXX improve? - if self.balanced: - balanced = True - elif not left.check_balanced() or not right.check_balanced(): - balanced = False - else: - balanced = (find_fib_index(self.len // (NEW_NODE_WHEN_LENGTH / 2)) >= - self._depth) - self.balanced = balanced self._calculated = True def charbitmask(self): From arigo at codespeak.net Mon Dec 10 13:17:45 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 Dec 2007 13:17:45 +0100 (CET) Subject: [pypy-svn] r49593 - pypy/branch/pypy-interp-file/module/bz2 Message-ID: <20071210121745.D65F31684D5@codespeak.net> Author: arigo Date: Mon Dec 10 13:17:45 2007 New Revision: 49593 Modified: pypy/branch/pypy-interp-file/module/bz2/interp_bz2.py Log: The ReadBZ2Filter acts like a read buffer itself, so we don't need to layer another one on top of it - at least if the peek() method is present. Modified: pypy/branch/pypy-interp-file/module/bz2/interp_bz2.py ============================================================================== --- pypy/branch/pypy-interp-file/module/bz2/interp_bz2.py (original) +++ pypy/branch/pypy-interp-file/module/bz2/interp_bz2.py Mon Dec 10 13:17:45 2007 @@ -250,7 +250,9 @@ space.wrap("cannot append to bz2 file")) stream = open_path_helper(path, os_flags, False) if reading: - bz2stream = ReadBZ2Filter(space, stream, compresslevel) + bz2stream = ReadBZ2Filter(space, stream, buffering) + buffering = 0 # by construction, the ReadBZ2Filter acts like + # a read buffer too - no need for another one else: assert writing bz2stream = WriteBZ2Filter(space, stream, compresslevel) @@ -263,13 +265,16 @@ """Standard I/O stream filter that decompresses the stream with bz2.""" - def __init__(self, space, stream, compresslevel): + def __init__(self, space, stream, buffering): self.space = space self.stream = stream self.decompressor = W_BZ2Decompressor(space) self.readlength = 0 self.buffer = "" self.finished = False + if buffering < 1024: + buffering = 1024 # minimum amount of compressed data read at once + self.buffering = buffering def close(self): self.stream.close() @@ -320,7 +325,7 @@ while not self.buffer: if self.finished: return "" - moredata = self.stream.read(n) + moredata = self.stream.read(max(self.buffering, n)) if not moredata: self.finished = True return "" @@ -341,6 +346,9 @@ self.readlength += len(result) return result + def peek(self): + return self.buffer + def try_to_find_file_descriptor(self): return self.stream.try_to_find_file_descriptor() From arigo at codespeak.net Mon Dec 10 14:41:25 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 Dec 2007 14:41:25 +0100 (CET) Subject: [pypy-svn] r49594 - in pypy: branch/pypy-interp-file dist/pypy dist/pypy/interpreter dist/pypy/module Message-ID: <20071210134125.DFDEF1684CC@codespeak.net> Author: arigo Date: Mon Dec 10 14:41:24 2007 New Revision: 49594 Added: pypy/dist/pypy/conftest.py - copied unchanged from r49593, pypy/branch/pypy-interp-file/conftest.py pypy/dist/pypy/interpreter/baseobjspace.py - copied unchanged from r49593, pypy/branch/pypy-interp-file/interpreter/baseobjspace.py pypy/dist/pypy/interpreter/module.py - copied unchanged from r49593, pypy/branch/pypy-interp-file/interpreter/module.py pypy/dist/pypy/interpreter/typedef.py - copied unchanged from r49593, pypy/branch/pypy-interp-file/interpreter/typedef.py pypy/dist/pypy/module/ - copied from r49593, pypy/branch/pypy-interp-file/module/ Removed: pypy/branch/pypy-interp-file/ Log: Move the 'file' class to interp-level and remove the direct app-level interface to streams. From fijal at codespeak.net Mon Dec 10 14:49:22 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 10 Dec 2007 14:49:22 +0100 (CET) Subject: [pypy-svn] r49595 - pypy/dist/pypy/rlib/parsing Message-ID: <20071210134922.8193F1684C9@codespeak.net> Author: fijal Date: Mon Dec 10 14:49:22 2007 New Revision: 49595 Added: pypy/dist/pypy/rlib/parsing/main.py (contents, props changed) Log: Add a convinient shortcut. Added: pypy/dist/pypy/rlib/parsing/main.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rlib/parsing/main.py Mon Dec 10 14:49:22 2007 @@ -0,0 +1,12 @@ + +from pypy.rlib.parsing.ebnfparse import parse_ebnf, make_parse_function +from pypy.rlib.parsing.parsing import ParseError, Rule +import py + +def make_parser_from_file(filename): + try: + t = py.path.local(filename).read(mode='U') + except ParserError, e: + print e.nice_error_message(filename=filename, source=t) + raise + return make_parse_function(regexs, rules, eof=True) From fijal at codespeak.net Mon Dec 10 15:01:25 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 10 Dec 2007 15:01:25 +0100 (CET) Subject: [pypy-svn] r49596 - pypy/dist/pypy/rlib/parsing Message-ID: <20071210140125.322A81684CF@codespeak.net> Author: fijal Date: Mon Dec 10 15:01:24 2007 New Revision: 49596 Modified: pypy/dist/pypy/rlib/parsing/main.py Log: Ooops, forgotten to check in this line. Modified: pypy/dist/pypy/rlib/parsing/main.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/main.py (original) +++ pypy/dist/pypy/rlib/parsing/main.py Mon Dec 10 15:01:24 2007 @@ -6,6 +6,7 @@ def make_parser_from_file(filename): try: t = py.path.local(filename).read(mode='U') + regexs, rules, ToAST = parse_ebnf(t) except ParserError, e: print e.nice_error_message(filename=filename, source=t) raise From fijal at codespeak.net Mon Dec 10 15:07:28 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 10 Dec 2007 15:07:28 +0100 (CET) Subject: [pypy-svn] r49597 - pypy/dist/pypy/rlib/parsing Message-ID: <20071210140728.5081C1684CE@codespeak.net> Author: fijal Date: Mon Dec 10 15:07:28 2007 New Revision: 49597 Modified: pypy/dist/pypy/rlib/parsing/main.py Log: typo Modified: pypy/dist/pypy/rlib/parsing/main.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/main.py (original) +++ pypy/dist/pypy/rlib/parsing/main.py Mon Dec 10 15:07:28 2007 @@ -7,7 +7,7 @@ try: t = py.path.local(filename).read(mode='U') regexs, rules, ToAST = parse_ebnf(t) - except ParserError, e: + except ParseError, e: print e.nice_error_message(filename=filename, source=t) raise return make_parse_function(regexs, rules, eof=True) From arigo at codespeak.net Mon Dec 10 15:18:43 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 Dec 2007 15:18:43 +0100 (CET) Subject: [pypy-svn] r49598 - pypy/dist/pypy/module/zlib/test Message-ID: <20071210141843.92EE91683DC@codespeak.net> Author: arigo Date: Mon Dec 10 15:18:43 2007 New Revision: 49598 Modified: pypy/dist/pypy/module/zlib/test/test_zlib.py Log: Skip for py.test -A. Modified: pypy/dist/pypy/module/zlib/test/test_zlib.py ============================================================================== --- pypy/dist/pypy/module/zlib/test/test_zlib.py (original) +++ pypy/dist/pypy/module/zlib/test/test_zlib.py Mon Dec 10 15:18:43 2007 @@ -3,7 +3,10 @@ Tests for the zlib module. """ -import zlib +try: + import zlib +except ImportError: + import py; py.test.skip("no zlib module on this host Python") from pypy.conftest import gettestobjspace From arigo at codespeak.net Mon Dec 10 15:22:04 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 Dec 2007 15:22:04 +0100 (CET) Subject: [pypy-svn] r49599 - pypy/dist/pypy/objspace/std/test Message-ID: <20071210142204.E4346168442@codespeak.net> Author: arigo Date: Mon Dec 10 15:22:04 2007 New Revision: 49599 Modified: pypy/dist/pypy/objspace/std/test/test_index.py Log: Support for py.test -A. Modified: pypy/dist/pypy/objspace/std/test/test_index.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_index.py (original) +++ pypy/dist/pypy/objspace/std/test/test_index.py Mon Dec 10 15:22:04 2007 @@ -260,13 +260,13 @@ def setup_method(self, method): SeqTestCase.setup_method(self, method) self.w_seq = self.space.wrap("this is a test") - self.w_const = self.space.w_str + self.w_const = self.space.appexec([], """(): return str""") class AppTest_UnicodeTestCase(SeqTestCase, StringTestCase): def setup_method(self, method): SeqTestCase.setup_method(self, method) self.w_seq = self.space.wrap(u"this is a test") - self.w_const = self.space.w_unicode + self.w_const = self.space.appexec([], """(): return unicode""") class AppTest_XRangeTestCase: From cfbolz at codespeak.net Mon Dec 10 15:48:44 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 10 Dec 2007 15:48:44 +0100 (CET) Subject: [pypy-svn] r49600 - pypy/dist/pypy/rlib Message-ID: <20071210144844.4F8851684D4@codespeak.net> Author: cfbolz Date: Mon Dec 10 15:48:43 2007 New Revision: 49600 Modified: pypy/dist/pypy/rlib/rope.py Log: don't rebalance recursively Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Mon Dec 10 15:48:43 2007 @@ -471,7 +471,7 @@ def _freeze_(self): self._calculate() -def concatenate(node1, node2): +def concatenate(node1, node2, rebalance=True): if node1.length() == 0: return node2 if node2.length() == 0: @@ -485,7 +485,7 @@ return BinaryConcatNode(node1.left, r.literal_concat(node2)) result = BinaryConcatNode(node1, node2) - if result.depth() > MAX_DEPTH: #XXX better check + if rebalance and result.depth() > MAX_DEPTH: #XXX better check return result.rebalance() return result @@ -637,7 +637,7 @@ # sweep all elements up to the preferred location for 'curr' while not (currlen < b and l[empty_up_to] is None): if l[empty_up_to] is not None: - curr = concatenate(l[empty_up_to], curr) + curr = concatenate(l[empty_up_to], curr, rebalance=False) l[empty_up_to] = None currlen = curr.length() else: From arigo at codespeak.net Mon Dec 10 16:14:06 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 Dec 2007 16:14:06 +0100 (CET) Subject: [pypy-svn] r49601 - in pypy/dist/pypy/objspace/std: . test Message-ID: <20071210151406.CB74F1684D6@codespeak.net> Author: arigo Date: Mon Dec 10 16:14:05 2007 New Revision: 49601 Modified: pypy/dist/pypy/objspace/std/test/test_shadowtracking.py pypy/dist/pypy/objspace/std/typetype.py Log: Test and fix for __bases__ assignment versus the method cache. Modified: pypy/dist/pypy/objspace/std/test/test_shadowtracking.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_shadowtracking.py (original) +++ pypy/dist/pypy/objspace/std/test/test_shadowtracking.py Mon Dec 10 16:14:05 2007 @@ -211,3 +211,22 @@ assert append_counter[0] >= 5 * len(names) for name, count in zip(names, names_counters): assert count[0] >= 5 + + def test_mutating_bases(self): + class C(object): + pass + class C2(object): + foo = 5 + class D(C): + pass + class E(D): + pass + d = D() + e = E() + D.__bases__ = (C2,) + assert e.foo == 5 + + class F(object): + foo = 3 + D.__bases__ = (C, F) + assert e.foo == 3 Modified: pypy/dist/pypy/objspace/std/typetype.py ============================================================================== --- pypy/dist/pypy/objspace/std/typetype.py (original) +++ pypy/dist/pypy/objspace/std/typetype.py Mon Dec 10 16:14:05 2007 @@ -172,9 +172,9 @@ raise OperationError(space.w_TypeError, space.wrap("__bases__ assignment: '%s' object layout differs from '%s'" % (w_type.getname(space, '?'), new_base.getname(space, '?')))) - if space.config.objspace.std.withtypeversion: - # it does not make sense to cache this type, it changes bases - w_type.version_tag = None + + # invalidate the version_tag of all the current subclasses + w_type.mutated() saved_bases = w_type.bases_w saved_base = w_type.w_bestbase From arigo at codespeak.net Mon Dec 10 16:45:08 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 Dec 2007 16:45:08 +0100 (CET) Subject: [pypy-svn] r49602 - pypy/dist/pypy/module/_weakref/test Message-ID: <20071210154508.0A4BB1684E1@codespeak.net> Author: arigo Date: Mon Dec 10 16:45:07 2007 New Revision: 49602 Modified: pypy/dist/pypy/module/_weakref/test/test_weakref.py Log: A test checking the del-weakref-id interactions. I suspect that this fails in py.test -A mode on a framework gc. Modified: pypy/dist/pypy/module/_weakref/test/test_weakref.py ============================================================================== --- pypy/dist/pypy/module/_weakref/test/test_weakref.py (original) +++ pypy/dist/pypy/module/_weakref/test/test_weakref.py Mon Dec 10 16:45:07 2007 @@ -268,6 +268,31 @@ a1 = w() assert a1 is None + def test_del_and_callback_and_id(self): + import gc, weakref + seen_del = [] + class A(object): + def __del__(self): + seen_del.append(id(self)) + seen_del.append(w1() is None) + seen_del.append(w2() is None) + seen_callback = [] + def callback(r): + seen_callback.append(r is w2) + seen_callback.append(w1() is None) + seen_callback.append(w2() is None) + a = A() + w1 = weakref.ref(a) + w2 = weakref.ref(a, callback) + aid = id(a) + del a + for i in range(5): + gc.collect() + if seen_del: + assert seen_del == [aid, True, True] + if seen_callback: + assert seen_callback == [True, True, True] + class AppTestProxy(object): def setup_class(cls): From arigo at codespeak.net Mon Dec 10 16:48:45 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 Dec 2007 16:48:45 +0100 (CET) Subject: [pypy-svn] r49603 - pypy/dist/pypy/module/_weakref Message-ID: <20071210154845.E5E941684E1@codespeak.net> Author: arigo Date: Mon Dec 10 16:48:45 2007 New Revision: 49603 Removed: pypy/dist/pypy/module/_weakref/app__weakref.py Log: Remove empty file. From arigo at codespeak.net Mon Dec 10 17:20:26 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 Dec 2007 17:20:26 +0100 (CET) Subject: [pypy-svn] r49604 - in pypy/dist/pypy: interpreter module/_weakref Message-ID: <20071210162026.2380C16847F@codespeak.net> Author: arigo Date: Mon Dec 10 17:20:24 2007 New Revision: 49604 Modified: pypy/dist/pypy/interpreter/typedef.py pypy/dist/pypy/module/_weakref/interp__weakref.py Log: Clear the app-level weakrefs before calling the app-level __del__. Modified: pypy/dist/pypy/interpreter/typedef.py ============================================================================== --- pypy/dist/pypy/interpreter/typedef.py (original) +++ pypy/dist/pypy/interpreter/typedef.py Mon Dec 10 17:20:24 2007 @@ -149,6 +149,15 @@ parent_destructor = getattr(cls, '__del__', None) class Proto(object): def __del__(self): + lifeline = self.getweakref() + if lifeline is not None: + # Clear all weakrefs to this object before we call + # the app-level __del__. We detach the lifeline + # first: if the app-level __del__ tries to use + # weakrefs again, they won't reuse the broken + # (already-cleared) ones from this lifeline. + self.setweakref(None) + lifeline.clear_all_weakrefs() try: self.space.userdel(self) except OperationError, e: Modified: pypy/dist/pypy/module/_weakref/interp__weakref.py ============================================================================== --- pypy/dist/pypy/module/_weakref/interp__weakref.py (original) +++ pypy/dist/pypy/module/_weakref/interp__weakref.py Mon Dec 10 17:20:24 2007 @@ -14,11 +14,28 @@ self.cached_proxy_index = -1 def __del__(self): + """This runs when the interp-level object goes away, and allows + its lifeline to go away. The purpose of this is to activate the + callbacks even if there is no __del__ method on the interp-level + W_Root subclass implementing the object. + """ for i in range(len(self.refs_weak) - 1, -1, -1): w_ref = self.refs_weak[i]() if w_ref is not None: w_ref.activate_callback() - + + def clear_all_weakrefs(self): + """Clear all weakrefs. This is called when an app-level object has + a __del__, just before the app-level __del__ method is called. + """ + for ref_w_ref in self.refs_weak: + w_ref = ref_w_ref() + if w_ref is not None: + w_ref.clear() + # Note that for no particular reason other than convenience, + # weakref callbacks are not invoked eagerly here. They are + # invoked by self.__del__() anyway. + def get_or_make_weakref(self, space, w_subtype, w_obj, w_callable): w_weakreftype = space.gettypeobject(W_Weakref.typedef) is_weakreftype = space.is_w(w_weakreftype, w_subtype) @@ -68,6 +85,17 @@ return w_ref return space.w_None +# ____________________________________________________________ + +class Dummy: + pass +dead_ref = weakref.ref(Dummy()) +for i in range(5): + if dead_ref() is not None: + import gc; gc.collect() +assert dead_ref() is None + + class W_WeakrefBase(Wrappable): def __init__(w_self, space, w_obj, w_callable): if space.is_w(w_callable, space.w_None): @@ -81,6 +109,9 @@ w_obj = self.w_obj_weak() return w_obj + def clear(self): + self.w_obj_weak = dead_ref + def activate_callback(w_self): if not w_self.w_callable is None: try: From cfbolz at codespeak.net Mon Dec 10 17:29:18 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 10 Dec 2007 17:29:18 +0100 (CET) Subject: [pypy-svn] r49605 - pypy/dist/pypy/rlib Message-ID: <20071210162918.2A9F41684C2@codespeak.net> Author: cfbolz Date: Mon Dec 10 17:29:17 2007 New Revision: 49605 Modified: pypy/dist/pypy/rlib/rope.py Log: not completely sure this is a good idea yet: put the additional info that some ropes have into its own object, which is not there for most rope nodes. saves some memory. Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Mon Dec 10 17:29:17 2007 @@ -51,8 +51,39 @@ return result +class GlobalRopeInfo(object): + """contains data that is "global" to the whole string, e.g. requires + an iteration over the whole string""" + def __init__(self): + self.charbitmask = 0 + self.hash = 0 + self.is_bytestring = False + self.is_ascii = False + + def combine(self, other, rightlength): + result = GlobalRopeInfo() + result.charbitmask = self.charbitmask | other.charbitmask + h1 = self.hash + h2 = other.hash + x = intmask(h2 + h1 * (masked_power(1000003, rightlength))) + x |= HIGHEST_BIT_SET + result.hash = x + result.is_ascii = self.is_ascii and other.is_ascii + result.is_bytestring = self.is_bytestring and right.is_bytestring() + return result + + class StringNode(object): - hash_cache = 0 + _additional_info = None + def additional_info(self): + addinfo = self._additional_info + if addinfo is None: + return self.compute_additional_info() + return addinfo + + def compute_additional_info(self): + raise NotImplementedError("base class") + def length(self): raise NotImplementedError("base class") @@ -107,6 +138,8 @@ def __add__(self, other): return concatenate(self, other) + def _freeze_(self): + self.additional_info() class LiteralNode(StringNode): def find_int(self, what, start, stop): @@ -117,29 +150,15 @@ class LiteralStringNode(LiteralNode): - _is_ascii = False - _charbitmask = 0 - def __init__(self, s, charbitmask=0, is_ascii=False): + def __init__(self, s): assert isinstance(s, str) self.s = s - if not s: - self._is_ascii = True - self._calculated = True - elif charbitmask: - self._charbitmask = charbitmask - self._is_ascii = is_ascii - self._calculated = True - else: - self._calculated = False - def length(self): return len(self.s) def is_ascii(self): - if not self._calculated: - self._calculate() - return self._is_ascii + return self.additional_info().is_ascii def is_bytestring(self): return True @@ -151,32 +170,30 @@ return self.s.decode('latin-1') def hash_part(self): - h = self.hash_cache - if not h: - x = 0 - for c in self.s: - x = (1000003*x) + ord(c) - x = intmask(x) - x |= HIGHEST_BIT_SET - h = self.hash_cache = x - return h + return self.additional_info().hash - def _calculate(self): + def compute_additional_info(self): + additional_info = GlobalRopeInfo() is_ascii = True charbitmask = 0 + partial_hash = 0 for c in self.s: ordc = ord(c) + partial_hash = (1000003*partial_hash) + ord(c) if ordc >= 128: is_ascii = False charbitmask |= intmask(1 << (ordc & 0x1F)) - self._is_ascii = is_ascii - self._charbitmask = charbitmask - self._calculated = True + partial_hash = intmask(partial_hash) + partial_hash |= HIGHEST_BIT_SET + + additional_info.hash = partial_hash + additional_info.is_ascii = is_ascii + additional_info.charbitmask = charbitmask + self._additional_info = additional_info + return additional_info def charbitmask(self): - if not self._calculated: - self._calculate() - return self._charbitmask + return self.additional_info().charbitmask def getchar(self, index): return self.s[index] @@ -227,9 +244,6 @@ yield ('"%s" [shape=box,label="length: %s\\n%s"];' % ( id(self), len(self.s), repr(addinfo).replace('"', '').replace("\\", "\\\\"))) - - def _freeze_(self): - self._calculate() LiteralStringNode.EMPTY = LiteralStringNode("") LiteralStringNode.PREBUILT = [LiteralStringNode(chr(i)) for i in range(256)] del i @@ -239,7 +253,6 @@ def __init__(self, u): assert isinstance(u, unicode) self.u = u - self._charbitmask = 0 def length(self): return len(self.u) @@ -254,31 +267,26 @@ return False def hash_part(self): - h = self.hash_cache - if not h: - x = 0 - for c in self.u: - x = (1000003*x) + ord(c) - x = intmask(x) - x |= HIGHEST_BIT_SET - h = self.hash_cache = x - return h + return self.additional_info().hash - def _calculate(self): - if len(self.u) == 0: - return + def compute_additional_info(self): + additional_info = GlobalRopeInfo() charbitmask = 0 + partial_hash = 0 for c in self.u: ordc = ord(c) - if ordc >= 128: - charbitmask |= 1 # be compatible with LiteralStringNode charbitmask |= intmask(1 << (ordc & 0x1F)) - self._charbitmask = intmask(charbitmask) + partial_hash = (1000003*partial_hash) + ordc + partial_hash = intmask(partial_hash) + partial_hash |= HIGHEST_BIT_SET + + additional_info.charbitmask = intmask(charbitmask) + additional_info.hash = partial_hash + self._additional_info = additional_info + return additional_info def charbitmask(self): - if not self._charbitmask: - self._calculate() - return self._charbitmask + return self.additional_info().charbitmask def getunichar(self, index): return self.u[index] @@ -326,8 +334,6 @@ yield ('"%s" [shape=box,label="length: %s\\n%s"];' % ( id(self), len(self.u), repr(addinfo).replace('"', '').replace("\\", "\\\\"))) - def _freeze_(self): - self._calculate() def make_binary_get(getter): def get(self, index): @@ -349,25 +355,19 @@ self.len = ovfcheck(left.length() + right.length()) except OverflowError: raise + self._depth = 0 + # XXX the balance should become part of the depth self.balanced = balanced if balanced: self.balance_known = True else: self.balance_known = False - self._calculated = False - self._is_ascii = False - self._is_bytestring = False - self._depth = 0 def is_ascii(self): - if not self._calculated: - self._calculate() - return self._is_ascii + return self.additional_info().is_ascii def is_bytestring(self): - if not self._calculated: - self._calculate() - return self._is_bytestring + return self.additional_info().is_bytestring def check_balanced(self): if self.balance_known: @@ -381,7 +381,7 @@ self._depth) self.balanced = balanced self.balance_known = True - return self.balanced + return balanced def length(self): return self.len @@ -425,27 +425,18 @@ return u"".join([node.flatten_unicode() for node in f]) def hash_part(self): - h = self.hash_cache - if not h: - h1 = self.left.hash_part() - h2 = self.right.hash_part() - x = intmask(h2 + h1 * (masked_power(1000003, self.right.length()))) - x |= HIGHEST_BIT_SET - h = self.hash_cache = x - return h - - def _calculate(self): - left = self.left - right = self.right - self._is_ascii = left.is_ascii() and right.is_ascii() - self._is_bytestring = left.is_bytestring() and right.is_bytestring() - self._charbitmask = left.charbitmask() | right.charbitmask() - self._calculated = True + return self.additional_info().hash + + def compute_additional_info(self): + leftaddinfo = self.left.additional_info() + rightaddinfo = self.right.additional_info() + additional_info = leftaddinfo.combine(rightaddinfo, + self.right.length()) + self._additional_info = additional_info + return additional_info def charbitmask(self): - if not self._calculated: - self._calculate() - return self._charbitmask + return self.additional_info().charbitmask def rebalance(self): if self.balanced: @@ -468,8 +459,7 @@ yield '"%s" -> "%s";' % (id(self), id(child)) for line in child.dot(seen): yield line - def _freeze_(self): - self._calculate() + def concatenate(node1, node2, rebalance=True): if node1.length() == 0: From arigo at codespeak.net Mon Dec 10 17:34:00 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 10 Dec 2007 17:34:00 +0100 (CET) Subject: [pypy-svn] r49606 - pypy/dist/pypy/interpreter Message-ID: <20071210163400.732921684C9@codespeak.net> Author: arigo Date: Mon Dec 10 17:33:59 2007 New Revision: 49606 Modified: pypy/dist/pypy/interpreter/typedef.py Log: Oups. Typo. Hard to test this code... Modified: pypy/dist/pypy/interpreter/typedef.py ============================================================================== --- pypy/dist/pypy/interpreter/typedef.py (original) +++ pypy/dist/pypy/interpreter/typedef.py Mon Dec 10 17:33:59 2007 @@ -156,7 +156,7 @@ # first: if the app-level __del__ tries to use # weakrefs again, they won't reuse the broken # (already-cleared) ones from this lifeline. - self.setweakref(None) + self.setweakref(self.space, None) lifeline.clear_all_weakrefs() try: self.space.userdel(self) From cfbolz at codespeak.net Mon Dec 10 19:07:30 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 10 Dec 2007 19:07:30 +0100 (CET) Subject: [pypy-svn] r49610 - pypy/dist/pypy/rlib Message-ID: <20071210180730.B5D7D1684CC@codespeak.net> Author: cfbolz Date: Mon Dec 10 19:07:30 2007 New Revision: 49610 Modified: pypy/dist/pypy/rlib/rope.py Log: simplify concatenation a bit Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Mon Dec 10 19:07:30 2007 @@ -135,6 +135,9 @@ def flatten_unicode(self): raise NotImplementedError("abstract base class") + def _concat(self, other): + raise NotImplementedError("abstract base class") + def __add__(self, other): return concatenate(self, other) @@ -145,8 +148,6 @@ def find_int(self, what, start, stop): raise NotImplementedError("abstract base class") - def literal_concat(self, other): - raise NotImplementedError("abstract base class") class LiteralStringNode(LiteralNode): @@ -224,7 +225,7 @@ return -1 return self.s.find(chr(what), start, stop) - def literal_concat(self, other): + def _concat(self, other): if (isinstance(other, LiteralStringNode) and len(other.s) + len(self.s) < NEW_NODE_WHEN_LENGTH): return LiteralStringNode(self.s + other.s) @@ -314,7 +315,7 @@ return -1 return self.u.find(unichr(what), start, stop) - def literal_concat(self, other): + def _concat(self, other): if (isinstance(other, LiteralUnicodeNode) and len(other.u) + len(self.u) < NEW_NODE_WHEN_LENGTH): return LiteralUnicodeNode(self.u + other.u) @@ -443,6 +444,15 @@ return self return rebalance([self], self.len) + + def _concat(self, other): + if isinstance(other, LiteralNode): + r = self.right + if isinstance(r, LiteralNode): + return BinaryConcatNode(self.left, + r._concat(other)) + return BinaryConcatNode(self, other) + def dot(self, seen, toplevel=False): if self in seen: return @@ -454,31 +464,24 @@ else: addition = "" yield '"%s" [shape=octagon,label="+\\ndepth=%s, length=%s"%s];' % ( - id(self), self._depth, self.len, addition) + id(self), self.depth(), self.len, addition) for child in [self.left, self.right]: yield '"%s" -> "%s";' % (id(self), id(child)) for line in child.dot(seen): yield line -def concatenate(node1, node2, rebalance=True): +def concatenate(node1, node2): if node1.length() == 0: return node2 if node2.length() == 0: return node1 - if isinstance(node2, LiteralNode): - if isinstance(node1, LiteralNode): - return node1.literal_concat(node2) - elif isinstance(node1, BinaryConcatNode): - r = node1.right - if isinstance(r, LiteralNode): - return BinaryConcatNode(node1.left, - r.literal_concat(node2)) - result = BinaryConcatNode(node1, node2) + result = node1._concat(node2) if rebalance and result.depth() > MAX_DEPTH: #XXX better check return result.rebalance() return result + def getslice(node, start, stop, step, slicelength=-1): if slicelength == -1: # XXX for testing only @@ -627,7 +630,7 @@ # sweep all elements up to the preferred location for 'curr' while not (currlen < b and l[empty_up_to] is None): if l[empty_up_to] is not None: - curr = concatenate(l[empty_up_to], curr, rebalance=False) + curr = l[empty_up_to]._concat(curr) l[empty_up_to] = None currlen = curr.length() else: From cfbolz at codespeak.net Mon Dec 10 19:33:30 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 10 Dec 2007 19:33:30 +0100 (CET) Subject: [pypy-svn] r49611 - pypy/branch/interplevel-oldstyle-classes Message-ID: <20071210183330.2A1B4168460@codespeak.net> Author: cfbolz Date: Mon Dec 10 19:33:29 2007 New Revision: 49611 Added: pypy/branch/interplevel-oldstyle-classes/ - copied from r49610, pypy/dist/ Log: a branch for the interplevelification of oldstyle classes From cfbolz at codespeak.net Mon Dec 10 20:18:40 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 10 Dec 2007 20:18:40 +0100 (CET) Subject: [pypy-svn] r49612 - pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__ Message-ID: <20071210191840.E081E16841D@codespeak.net> Author: cfbolz Date: Mon Dec 10 20:18:39 2007 New Revision: 49612 Removed: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/app_sets.py Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/__init__.py Log: sets are long on interplevel in the std obj space Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/__init__.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/__init__.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/__init__.py Mon Dec 10 20:18:39 2007 @@ -60,9 +60,6 @@ 'buffer' : 'app_buffer.buffer', 'reload' : 'app_misc.reload', - 'set' : 'app_sets.set', - 'frozenset' : 'app_sets.frozenset', - '__filestub' : 'app_file_stub.file', } From antocuni at codespeak.net Tue Dec 11 00:19:43 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 11 Dec 2007 00:19:43 +0100 (CET) Subject: [pypy-svn] r49615 - pypy/branch/clr-module-improvements/pypy/translator/cli Message-ID: <20071210231943.517D016845E@codespeak.net> Author: antocuni Date: Tue Dec 11 00:19:42 2007 New Revision: 49615 Modified: pypy/branch/clr-module-improvements/pypy/translator/cli/dotnet.py Log: fix for pythonnet 2.0 Modified: pypy/branch/clr-module-improvements/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/translator/cli/dotnet.py (original) +++ pypy/branch/clr-module-improvements/pypy/translator/cli/dotnet.py Tue Dec 11 00:19:42 2007 @@ -421,7 +421,7 @@ def _create_NativeException(cliClass): from pypy.translator.cli.query import getattr_ex TYPE = cliClass._INSTANCE - if PythonNet.__name__ == 'CLR': + if PythonNet.__name__ in ('CLR', 'clr'): # we are using pythonnet -- use the .NET class name = '%s.%s' % (TYPE._namespace, TYPE._classname) res = getattr_ex(PythonNet, name) From antocuni at codespeak.net Tue Dec 11 00:26:45 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 11 Dec 2007 00:26:45 +0100 (CET) Subject: [pypy-svn] r49616 - pypy/branch/clr-module-improvements/pypy/module/clr/test Message-ID: <20071210232645.E2EF6168462@codespeak.net> Author: antocuni Date: Tue Dec 11 00:26:44 2007 New Revision: 49616 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py Log: fix test Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py Tue Dec 11 00:26:44 2007 @@ -160,7 +160,7 @@ enum = x.GetEnumerator() assert enum.MoveNext() is False - def test_iteratrion(self): + def test_iteration(self): import clr # test iteration in ArrayList @@ -181,6 +181,7 @@ obj.Push(1) obj.Push(54) obj.Push(21) + sum = 0 for i in obj: sum += i assert sum == 1+54+21 From regmee at codespeak.net Tue Dec 11 00:48:47 2007 From: regmee at codespeak.net (regmee at codespeak.net) Date: Tue, 11 Dec 2007 00:48:47 +0100 (CET) Subject: [pypy-svn] r49617 - pypy/branch/clr-module-improvements/pypy/module/clr Message-ID: <20071210234847.ABA67168434@codespeak.net> Author: regmee Date: Tue Dec 11 00:48:47 2007 New Revision: 49617 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py Log: improving the Namespace scanning from assemblies Modified: pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py Tue Dec 11 00:48:47 2007 @@ -193,25 +193,20 @@ Return: List of Valid .NET namespaces """ - listOfNamespaces = [] - currentDomain = System.AppDomain.get_CurrentDomain() - asEvidence = currentDomain.get_Evidence() + assembliesToScan = [ "/usr/lib/mono/1.0/mscorlib.dll", + "/usr/lib/mono/1.0/System.dll", + "/usr/lib/mono/1.0/System.Web.dll", + "/usr/lib/mono/1.0/System.Data.dll", + "/usr/lib/mono/1.0/System.Xml.dll", + "/usr/lib/mono/1.0/System.Drawing.dll" + ] + assems = [] + for assemblyName in assembliesToScan: + assems.append(System.Reflection.Assembly.LoadFile(assemblyName)) -# assembly1 = System.Reflection.Assembly.LoadFile("System.Xml.dll") -# assembly2 = System.Reflection.Assembly.LoadFile("mscorlib.dll") -# assembly3 = System.Reflection.Assembly.LoadFile("/home/amit/clrModImprove/pypy/module/clr/System.Windows.Forms.dll") - -# currentDomain.Load("System.Xml",asEvidence) -# currentDomain.Load("mscorlib",asEvidence) -# currentDomain.Load("System.Web",asEvidence) -# currentDomain.Load("System.Drawing",asEvidence) -# currentDomain.Load("System.Data",asEvidence) -# currentDomain.Load("System.Windows.Forms",asEvidence) - - assems = currentDomain.GetAssemblies() - for assembly in assems: - typesInAssembly = assembly.GetTypes(); + for loadedAssembly in assems: + typesInAssembly = loadedAssembly.GetTypes() for type in typesInAssembly: namespace = type.get_Namespace() if namespace != None and namespace not in listOfNamespaces: @@ -241,6 +236,9 @@ - assemblyName: the full name of the assembly (e.g., ``System.Xml.dll``). """ + # listOfNamespaces = [] + # currentDomain = System.AppDomain.get_CurrentDomain() + # asEvidence = currentDomain.get_Evidence() pass def load_cli_class(space, namespace, classname): From cfbolz at codespeak.net Tue Dec 11 12:01:59 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 12:01:59 +0100 (CET) Subject: [pypy-svn] r49618 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071211110159.09FE9168453@codespeak.net> Author: cfbolz Date: Tue Dec 11 12:01:59 2007 New Revision: 49618 Added: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (contents, props changed) pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (contents, props changed) Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/__init__.py Log: bare-bone implementation of oldstyle classes on applevel Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/__init__.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/__init__.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/__init__.py Tue Dec 11 12:01:59 2007 @@ -79,6 +79,9 @@ # old-style classes dummy support '_classobj' : 'space.w_classobj', '_instance' : 'space.w_instance', + # new old-style classes implementation for testing + 'nclassobj' : 'interp_classobj.W_ClassObject', + 'ninstance' : 'interp_classobj.W_InstanceObject', # default __metaclass__ '__metaclass__' : '(space.w_type)', Added: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- (empty file) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Tue Dec 11 12:01:59 2007 @@ -0,0 +1,236 @@ +from pypy.interpreter.error import OperationError +from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped, applevel +from pypy.interpreter.gateway import interp2app, ObjSpace +from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.interpreter.argument import Arguments +from pypy.interpreter.baseobjspace import Wrappable +from pypy.rlib.rarithmetic import r_uint, intmask + + +def descr_classobj_new(space, w_subtype, w_name, w_bases, w_dict): + if w_bases is None: + w_bases = space.newtuple([]) + elif not space.is_true(space.isinstance(w_bases, space.w_tuple)): + raise_type_err(space, 'bases', 'tuple', w_bases) + + if not space.is_true(space.isinstance(w_dict, space.w_dict)): + raise_type_err(space, 'bases', 'tuple', w_bases) + + if not space.is_true(space.contains(w_dict, space.wrap("__doc__"))): + space.setitem(w_dict, space.wrap("__doc__"), space.w_None) + + if not space.is_true(space.contains(w_dict, space.wrap("__doc__"))): + space.setitem(w_dict, space.wrap("__doc__"), space.w_None) + + for w_base in space.unpackiterable(w_bases): + if not isinstance(w_base, W_ClassObject): + w_metaclass = space.call_function(space.w_type, w_base) + if space.is_true(space.callable(w_metaclass)): + return space.call_function(w_metaclass, w_name, + w_bases, w_dic) + raise OperationError(space.w_TypeError, + space.wrap("base must be class")) + + return W_ClassObject(space, w_name, w_bases, w_dict) + +class W_ClassObject(Wrappable): + def __init__(self, space, w_name, w_bases, w_dict): + self.w_name = w_name + self.name = space.str_w(w_name) + self.w_bases = w_bases + self.bases_w = space.unpackiterable(w_bases) + self.w_dict = w_dict + + def getdict(self): + return self.w_dict + + def fget_dict(space, self): + return self.w_dict + + def fset_dict(space, self, w_dict): + if not space.is_true(space.isinstance(w_dict, space.w_dict)): + raise OperationError( + space.w_TypeError, + space.wrap("__dict__ must be a dictionary object")) + self.w_dict = w_dict + + def fget_name(space, self): + return self.w_name + + def fset_name(space, self, w_newname): + if not space.is_true(space.isinstance(w_newname, space.w_str)): + raise OperationError( + space.w_TypeError, + space.wrap("__name__ must be a string object")) + self.w_name = w_newname + self.name = space.str_w(w_newname) + + def fget_bases(space, self): + return self.w_bases + + def fset_bases(space, self, w_bases): + if not space.is_true(space.isinstance(w_bases, space.w_tuple)): + raise OperationError( + space.w_TypeError, + space.wrap("__bases__ must be a tuple object")) + bases_w = space.unpackiterable(w_bases) + for w_base in bases_w: + if not isinstance(w_base, W_ClassObject): + raise OperationError(space.w_TypeError, + space.wrap("__bases__ items must be classes")) + self.w_bases = w_bases + self.bases_w = bases_w + + + def lookup(self, space, w_attr): + # returns w_value or interplevel None + try: + return space.getitem(self.w_dict, w_attr) + except OperationError, e: + if not e.match(space, space.w_KeyError): + raise + for base in self.bases_w: + w_result = base.lookup(space, w_attr) + if w_result is not None: + return w_result + return None + + def descr_getattr(self, space, w_attr): + import pdb; pdb.set_trace() + w_value = self.lookup(space, w_attr) + if w_value is not None: + raise OperationError( + space.w_AttributeError, + space.wrap("class %s has no attribute %s" % ( + self.name, space.str_w(space.str(w_attr))))) + + w_descr_get = space.lookup(w_value, '__get__') + if w_descr_get is None: + return w_value + return space.call_function(w_descr_get, w_value, space.w_None, self) + + # XXX missing: str, repr + + def descr_call(self, space, __args__): + w_inst = W_InstanceObject(space, self) + w_init = w_inst.getattr(space, space.wrap('__init__'), False) + if w_init is not None: + w_result = space.call(w_init, __args__) + if not space.is_w(w_result, space.w_None): + raise OperationError( + space.w_TypeError, + space.wrap("__init__() should return None")) + elif __args__.num_args() or __args__.num_kwds(): + raise OperationError( + space.w_TypeError, + space.wrap("this constructor takes no arguments")) + return w_inst + +W_ClassObject.typedef = TypeDef("classobj", + __new__ = interp2app(descr_classobj_new), + __dict__ = GetSetProperty(W_ClassObject.fget_dict, W_ClassObject.fset_dict), + __name__ = GetSetProperty(W_ClassObject.fget_name, W_ClassObject.fset_name), + __bases__ = GetSetProperty(W_ClassObject.fget_bases, + W_ClassObject.fset_bases), + __call__ = interp2app(W_ClassObject.descr_call, + unwrap_spec=['self', ObjSpace, Arguments]), + __getattr__ = interp2app(W_ClassObject.descr_getattr, + unwrap_spec=['self', ObjSpace, W_Root]), +) +W_ClassObject.typedef.acceptable_as_base_class = False + +class W_InstanceObject(Wrappable): + def __init__(self, space, w_class, w_dict=None): + if w_dict is None: + w_dict = space.newdict() + self.w_class = w_class + self.w_dict = w_dict + + def getdict(self): + return self.w_dict + + def fget_dict(space, self): + return self.w_dict + + def fset_dict(space, self, w_dict): + if not space.is_true(space.isinstance(w_dict, space.w_dict)): + raise OperationError( + space.w_TypeError, + space.wrap("__dict__ must be a dictionary object")) + self.w_dict = w_dict + + def fget_class(space, self): + return self.w_class + + def fset_class(space, self, w_class): + if not isinstance(w_class, W_ClassObject): + raise OperationError( + space.w_TypeError, + space.wrap("instance() first arg must be class")) + self.w_class = w_class + + def descr_new(space, w_type, w_class, w_dict=None): + # typ is not used at all + if not isinstance(w_class, W_ClassObject): + raise OperationError( + space.w_TypeError, + space.wrap("instance() first arg must be class")) + if w_dict is None: + w_dict = space.newdict() + elif not space.is_true(space.isinstance(w_dict, space.w_dict)): + raise TypeError("instance() second arg must be dictionary or None") + return W_InstanceObject(space, w_class, w_dict) + + def retrieve(self, space, w_attr, exc=True): + try: + return space.getitem(self.w_dict, w_attr) + except OperationError, e: + if not e.match(space, space.w_KeyError): + raise + if exc: + raise OperationError( + space.w_AttributeError, w_attr) + return None + + def getattr(self, space, w_name, exc=True): + try: + name = space.str_w(w_name) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + else: + if name == "__dict__": + return self.w_dict + elif name == "__class__": + return self.w_class + w_result = self.retrieve(space, w_name, False) + if w_result is not None: + return w_result + w_value = self.w_class.lookup(space, w_name) + if w_value is None: + if exc: + raise OperationError( + space.w_AttributeError, + space.wrap("%s instance has no attribute %s" % ( + space.str_w(self.w_class), space.str_w(space.str(name))))) + else: + return None + w_descr_get = space.lookup(w_value, '__get__') + if w_descr_get is None: + return w_value + return space.call_function(w_descr_get, w_value, self, self.w_class) + + def descr_getattr(self, space, w_attr): + #import pdb; pdb.set_trace() + return self.getattr(space, w_attr) + + +W_InstanceObject.typedef = TypeDef("instance", + __new__ = interp2app(W_InstanceObject.descr_new), + __dict__ = GetSetProperty(W_InstanceObject.fget_dict, + W_InstanceObject.fset_dict), + __class__ = GetSetProperty(W_InstanceObject.fget_class, + W_InstanceObject.fset_class), + __getattr__ = interp2app(W_InstanceObject.descr_getattr, + unwrap_spec=['self', ObjSpace, W_Root]), +) Added: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- (empty file) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Tue Dec 11 12:01:59 2007 @@ -0,0 +1,17 @@ + +class AppTestOldstyle(object): + def test_simple(self): + class A: + __metaclass__ = nclassobj + a = 1 + assert A.__name__ == 'A' + assert A.__bases__ == () + assert A.a == 1 + assert A.__dict__['a'] == 1 + a = A() + a.b = 2 + assert a.b == 2 + assert a.a == 1 + assert a.__class__ is A + assert a.__dict__ == {'b': 2} + From cfbolz at codespeak.net Tue Dec 11 12:08:12 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 12:08:12 +0100 (CET) Subject: [pypy-svn] r49619 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071211110812.64BA0168455@codespeak.net> Author: cfbolz Date: Tue Dec 11 12:08:11 2007 New Revision: 49619 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: test and fix for __init__. Of course I meant "interplevel" in the last checkin message. Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Tue Dec 11 12:08:11 2007 @@ -115,7 +115,7 @@ w_inst = W_InstanceObject(space, self) w_init = w_inst.getattr(space, space.wrap('__init__'), False) if w_init is not None: - w_result = space.call(w_init, __args__) + w_result = space.call_args(w_init, __args__) if not space.is_w(w_result, space.w_None): raise OperationError( space.w_TypeError, Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Tue Dec 11 12:08:11 2007 @@ -15,3 +15,12 @@ assert a.__class__ is A assert a.__dict__ == {'b': 2} + def test_init(self): + class A: + __metaclass__ = nclassobj + a = 1 + def __init__(self, a): + self.a = a + a = A(2) + assert a.a == 2 + From cfbolz at codespeak.net Tue Dec 11 12:10:09 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 12:10:09 +0100 (CET) Subject: [pypy-svn] r49620 - pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test Message-ID: <20071211111009.632FF168456@codespeak.net> Author: cfbolz Date: Tue Dec 11 12:10:09 2007 New Revision: 49620 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: test for type error when __init__ returns not None Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Tue Dec 11 12:10:09 2007 @@ -23,4 +23,9 @@ self.a = a a = A(2) assert a.a == 2 + class B: + __metaclass__ = nclassobj + def __init__(self, a): + return a + raises(TypeError, B, 2) From cfbolz at codespeak.net Tue Dec 11 12:12:08 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 12:12:08 +0100 (CET) Subject: [pypy-svn] r49621 - pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test Message-ID: <20071211111208.E77F116844C@codespeak.net> Author: cfbolz Date: Tue Dec 11 12:12:06 2007 New Revision: 49621 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: test for method Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Tue Dec 11 12:12:06 2007 @@ -29,3 +29,12 @@ return a raises(TypeError, B, 2) + + def test_method(self): + class A: + __metaclass__ = nclassobj + a = 1 + def f(self, a): + return self.a + a + a = A() + assert a.f(2) == 3 From cfbolz at codespeak.net Tue Dec 11 12:45:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 12:45:23 +0100 (CET) Subject: [pypy-svn] r49622 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071211114523.20D81168456@codespeak.net> Author: cfbolz Date: Tue Dec 11 12:45:22 2007 New Revision: 49622 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: test for inheritance and attributes Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Tue Dec 11 12:45:22 2007 @@ -96,9 +96,8 @@ return None def descr_getattr(self, space, w_attr): - import pdb; pdb.set_trace() w_value = self.lookup(space, w_attr) - if w_value is not None: + if w_value is None: raise OperationError( space.w_AttributeError, space.wrap("class %s has no attribute %s" % ( Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Tue Dec 11 12:45:22 2007 @@ -38,3 +38,23 @@ return self.a + a a = A() assert a.f(2) == 3 + + def test_inheritance(self): + class A: + __metaclass__ = nclassobj + a = 1 + b = 2 + class B(A): + a = 3 + c = 4 + assert B.__bases__ == (A, ) + assert B.a == 3 + assert B.b == 2 + assert B.c == 4 + b = B() + assert b.a == 3 + assert b.b == 2 + assert b.c == 4 + + + From fijal at codespeak.net Tue Dec 11 12:46:52 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 11 Dec 2007 12:46:52 +0100 (CET) Subject: [pypy-svn] r49623 - pypy/dist/pypy/objspace/std/test Message-ID: <20071211114652.5E976168459@codespeak.net> Author: fijal Date: Tue Dec 11 12:46:51 2007 New Revision: 49623 Modified: pypy/dist/pypy/objspace/std/test/test_proxy_internals.py Log: A failing test for proxy of user-created objects (right now is failing for some other reason) Modified: pypy/dist/pypy/objspace/std/test/test_proxy_internals.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_proxy_internals.py (original) +++ pypy/dist/pypy/objspace/std/test/test_proxy_internals.py Tue Dec 11 12:46:51 2007 @@ -122,6 +122,14 @@ lst = tproxy(list, f) assert get_tproxy_controller(lst) is f + def test_proxy_file(self): + from __pypy__ import tproxy + + def f(name, *args, **kwds): + pass + + t = tproxy(file, f) + class DONTAppTestProxyType(AppProxy): def test_filetype(self): f = self.get_proxy(file) From cfbolz at codespeak.net Tue Dec 11 12:48:57 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 12:48:57 +0100 (CET) Subject: [pypy-svn] r49624 - pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test Message-ID: <20071211114857.40D48168456@codespeak.net> Author: cfbolz Date: Tue Dec 11 12:48:56 2007 New Revision: 49624 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: some more passing tests Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Tue Dec 11 12:48:56 2007 @@ -38,15 +38,36 @@ return self.a + a a = A() assert a.f(2) == 3 + assert A.f(a, 2) == 3 + a.a = 5 + assert A.f(a, 2) == 7 def test_inheritance(self): class A: __metaclass__ = nclassobj a = 1 b = 2 + def af(self): + return 1 + def bf(self): + return 2 + assert A.a == 1 + assert A.b == 2 + a = A() + assert a.a == 1 + assert a.b == 2 + assert a.af() == 1 + assert a.bf() == 2 + assert A.af(a) == 1 + assert A.bf(a) == 2 + class B(A): a = 3 c = 4 + def af(self): + return 3 + def cf(self): + return 4 assert B.__bases__ == (A, ) assert B.a == 3 assert B.b == 2 @@ -55,6 +76,9 @@ assert b.a == 3 assert b.b == 2 assert b.c == 4 - - - + assert b.af() == 3 + assert b.bf() == 2 + assert b.cf() == 4 + assert B.af(b) == 3 + assert B.bf(b) == 2 + assert B.cf(b) == 4 From cfbolz at codespeak.net Tue Dec 11 13:09:27 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 13:09:27 +0100 (CET) Subject: [pypy-svn] r49625 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071211120927.F01D716842C@codespeak.net> Author: cfbolz Date: Tue Dec 11 13:09:27 2007 New Revision: 49625 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: this actually needs to be a __getattribute__, not a __getattr__ Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Tue Dec 11 13:09:27 2007 @@ -95,7 +95,19 @@ return w_result return None - def descr_getattr(self, space, w_attr): + def descr_getattribute(self, space, w_attr): + try: + name = space.str_w(w_attr) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + else: + if name == "__dict__": + return self.w_dict + elif name == "__name__": + return self.w_name + elif name == "__bases__": + return self.w_bases w_value = self.lookup(space, w_attr) if w_value is None: raise OperationError( @@ -133,7 +145,7 @@ W_ClassObject.fset_bases), __call__ = interp2app(W_ClassObject.descr_call, unwrap_spec=['self', ObjSpace, Arguments]), - __getattr__ = interp2app(W_ClassObject.descr_getattr, + __getattribute__ = interp2app(W_ClassObject.descr_getattribute, unwrap_spec=['self', ObjSpace, W_Root]), ) W_ClassObject.typedef.acceptable_as_base_class = False Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Tue Dec 11 13:09:27 2007 @@ -82,3 +82,16 @@ assert B.af(b) == 3 assert B.bf(b) == 2 assert B.cf(b) == 4 + + def test_inheritance_unbound_method(self): + class A: + __metaclass__ = nclassobj + def f(self): + return 1 + raises(TypeError, A.f, 1) + assert A.f(A()) == 1 + class B(A): + pass + raises(TypeError, B.f, 1) + raises(TypeError, B.f, A()) + assert B.f(B()) == 1 From cfbolz at codespeak.net Tue Dec 11 13:13:45 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 13:13:45 +0100 (CET) Subject: [pypy-svn] r49626 - pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test Message-ID: <20071211121345.48AA716842C@codespeak.net> Author: cfbolz Date: Tue Dec 11 13:13:43 2007 New Revision: 49626 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: test for mutating the special attributes of a class Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Tue Dec 11 13:13:43 2007 @@ -15,6 +15,24 @@ assert a.__class__ is A assert a.__dict__ == {'b': 2} + def test_mutate_class_special(self): + class A: + __metaclass__ = nclassobj + a = 1 + A.__name__ = 'B' + assert A.__name__ == 'B' + assert A.a == 1 + A.__dict__ = {'a': 5} + assert A.a == 5 + class B: + __metaclass__ = nclassobj + a = 17 + class C(A): + pass + assert C.a == 5 + C.__bases__ = (B, ) + assert C.a == 17 + def test_init(self): class A: __metaclass__ = nclassobj From cfbolz at codespeak.net Tue Dec 11 13:17:40 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 13:17:40 +0100 (CET) Subject: [pypy-svn] r49627 - pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test Message-ID: <20071211121740.DA0D7168443@codespeak.net> Author: cfbolz Date: Tue Dec 11 13:17:40 2007 New Revision: 49627 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: more passing tests Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Tue Dec 11 13:17:40 2007 @@ -27,11 +27,23 @@ class B: __metaclass__ = nclassobj a = 17 + b = 18 class C(A): - pass + c = 19 assert C.a == 5 + assert C.c == 19 C.__bases__ = (B, ) assert C.a == 17 + assert C.b == 18 + assert C.c == 19 + C.__bases__ = (B, A) + assert C.a == 17 + assert C.b == 18 + assert C.c == 19 + C.__bases__ = (A, B) + assert C.a == 5 + assert C.b == 18 + assert C.c == 19 def test_init(self): class A: From cfbolz at codespeak.net Tue Dec 11 13:21:22 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 13:21:22 +0100 (CET) Subject: [pypy-svn] r49628 - pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test Message-ID: <20071211122122.B6B4D168443@codespeak.net> Author: cfbolz Date: Tue Dec 11 13:21:22 2007 New Revision: 49628 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: even more passing tests Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Tue Dec 11 13:21:22 2007 @@ -45,6 +45,22 @@ assert C.b == 18 assert C.c == 19 + def test_mutate_instance_special(self): + class A: + __metaclass__ = nclassobj + a = 1 + class B: + __metaclass__ = nclassobj + a = 17 + b = 18 + a = A() + assert isinstance(a, A) + a.__class__ = B + assert isinstance(a, B) + assert a.a == 17 + assert a.b == 18 + + def test_init(self): class A: __metaclass__ = nclassobj From fijal at codespeak.net Tue Dec 11 16:37:54 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 11 Dec 2007 16:37:54 +0100 (CET) Subject: [pypy-svn] r49629 - pypy/dist/pypy/objspace/std/test Message-ID: <20071211153754.88B73168446@codespeak.net> Author: fijal Date: Tue Dec 11 16:37:53 2007 New Revision: 49629 Added: pypy/dist/pypy/objspace/std/test/test_proxy_usercreated.py (contents, props changed) Modified: pypy/dist/pypy/objspace/std/test/test_proxy_internals.py Log: Argh. Last checkin made little sense. Create new test and skip file test. Modified: pypy/dist/pypy/objspace/std/test/test_proxy_internals.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_proxy_internals.py (original) +++ pypy/dist/pypy/objspace/std/test/test_proxy_internals.py Tue Dec 11 16:37:53 2007 @@ -123,6 +123,7 @@ assert get_tproxy_controller(lst) is f def test_proxy_file(self): + skip("Not working") from __pypy__ import tproxy def f(name, *args, **kwds): Added: pypy/dist/pypy/objspace/std/test/test_proxy_usercreated.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/objspace/std/test/test_proxy_usercreated.py Tue Dec 11 16:37:53 2007 @@ -0,0 +1,23 @@ + +from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable +from pypy.objspace.std.test.test_proxy_internals import AppProxy +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import interp2app + +class W_Wrapped(Wrappable): + def new(space, w_type): + return space.wrap(W_Wrapped()) + +W_Wrapped.typedef = TypeDef( + 'Wrapped', + __new__ = interp2app(W_Wrapped.new.im_func) +) + +class AppTestProxyNewtype(AppProxy): + def setup_class(cls): + AppProxy.setup_class.im_func(cls) + cls.w_wrapped = cls.space.wrap(W_Wrapped()) + + def test_one(self): + x = type(self.wrapped)() + print x From arigo at codespeak.net Tue Dec 11 17:25:35 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Dec 2007 17:25:35 +0100 (CET) Subject: [pypy-svn] r49630 - pypy/dist/pypy/lang/smalltalk Message-ID: <20071211162535.8314E16843D@codespeak.net> Author: arigo Date: Tue Dec 11 17:25:33 2007 New Revision: 49630 Modified: pypy/dist/pypy/lang/smalltalk/model.py Log: Translation fix. Modified: pypy/dist/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/model.py (original) +++ pypy/dist/pypy/lang/smalltalk/model.py Tue Dec 11 17:25:33 2007 @@ -414,6 +414,7 @@ self.setchar(index0, chr(utility.unwrap_int(w_value))) def setchar(self, index0, character): + assert index0 >= 0 self.bytes = (self.bytes[:index0] + character + self.bytes[index0 + 1:]) From arigo at codespeak.net Tue Dec 11 17:25:43 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Dec 2007 17:25:43 +0100 (CET) Subject: [pypy-svn] r49631 - pypy/dist/pypy/rpython/memory Message-ID: <20071211162543.5D75016844B@codespeak.net> Author: arigo Date: Tue Dec 11 17:25:42 2007 New Revision: 49631 Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py Log: Comment fix. Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/gctypelayout.py Tue Dec 11 17:25:42 2007 @@ -26,7 +26,8 @@ assert self.can_add_new_types assert isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)) # Record the new type_id description as a small dict for now. - # It will be turned into a Struct("type_info") in finish() + # The framework gc transformer will turn it into a + # Struct("type_info") in flatten_table(). type_id = len(self.type_info_list) assert type_id & 0xffff == type_id # make sure it fits into 2 bytes info = {} From arigo at codespeak.net Tue Dec 11 17:44:48 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Dec 2007 17:44:48 +0100 (CET) Subject: [pypy-svn] r49632 - pypy/dist/pypy/translator/stackless Message-ID: <20071211164448.6A0E216842C@codespeak.net> Author: arigo Date: Tue Dec 11 17:44:48 2007 New Revision: 49632 Modified: pypy/dist/pypy/translator/stackless/transform.py Log: Oups! External function calls were thought to sometimes unwind the stack. That's wrong in general - I think they can never do that. This is a good performance boost for "--gc=framework --stackless". Modified: pypy/dist/pypy/translator/stackless/transform.py ============================================================================== --- pypy/dist/pypy/translator/stackless/transform.py (original) +++ pypy/dist/pypy/translator/stackless/transform.py Tue Dec 11 17:44:48 2007 @@ -260,6 +260,13 @@ return LL_OPERATIONS[op.opname].canunwindgc return False + def analyze_external_call(self, op): + # An external call cannot cause a stack unwind + # Note that this is essential to get good performance in framework GCs + # because there is a pseudo-external call to ROUND_UP_FOR_ALLOCATION + # in critical paths there. + return False + def vars_to_save(block): lastresult = block.operations[-1].result From arigo at codespeak.net Tue Dec 11 17:50:46 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Dec 2007 17:50:46 +0100 (CET) Subject: [pypy-svn] r49633 - in pypy/dist/pypy: rpython/tool translator/c translator/tool Message-ID: <20071211165046.2E21F168447@codespeak.net> Author: arigo Date: Tue Dec 11 17:50:46 2007 New Revision: 49633 Modified: pypy/dist/pypy/rpython/tool/rfficache.py pypy/dist/pypy/translator/c/genc.py pypy/dist/pypy/translator/tool/cbuild.py Log: Fix broken calls to build_executable() leading to a TypeError masked by a bare "except:". Remove the two bare "except:" while I'm at it. This means that some tests will run again - they were skipped by mistake. Modified: pypy/dist/pypy/rpython/tool/rfficache.py ============================================================================== --- pypy/dist/pypy/rpython/tool/rfficache.py (original) +++ pypy/dist/pypy/rpython/tool/rfficache.py Tue Dec 11 17:50:46 2007 @@ -5,8 +5,7 @@ import py import os import distutils -from pypy.translator.tool.cbuild import build_executable, \ - ExternalCompilationInfo +from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.tool.udir import udir from pypy.tool.autopath import pypydir from pypy.rlib import rarithmetic Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Tue Dec 11 17:50:46 2007 @@ -8,7 +8,7 @@ from pypy.translator.gensupp import uniquemodulename, NameManager from pypy.translator.tool.cbuild import so_ext, ExternalCompilationInfo from pypy.translator.tool.cbuild import compile_c_module -from pypy.translator.tool.cbuild import build_executable, CCompiler, ProfOpt +from pypy.translator.tool.cbuild import CCompiler, ProfOpt from pypy.translator.tool.cbuild import import_module_from_directory from pypy.translator.tool.cbuild import check_under_under_thread from pypy.rpython.lltypesystem import lltype Modified: pypy/dist/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/dist/pypy/translator/tool/cbuild.py (original) +++ pypy/dist/pypy/translator/tool/cbuild.py Tue Dec 11 17:50:46 2007 @@ -458,6 +458,7 @@ return str(compiler.outputfilename) def check_boehm_presence(): + import distutils.errors from pypy.tool.udir import udir try: cfile = udir.join('check_boehm.c') @@ -472,15 +473,18 @@ """) cfile.close() if sys.platform == 'win32': - build_executable([cfname], libraries=['gc_pypy'], noerr=True) + eci = ExternalCompilationInfo(libraries=['gc_pypy']) else: - build_executable([cfname], libraries=['gc'], noerr=True) - except: + eci = ExternalCompilationInfo(libraries=['gc']) + build_executable([cfname], eci, noerr=True) + except (distutils.errors.CompileError, + distutils.errors.LinkError): return False else: return True def check_under_under_thread(): + import distutils.errors from pypy.tool.udir import udir cfile = py.path.local(autopath.this_dir).join('__thread_test.c') fsource = cfile.open('r') @@ -491,12 +495,12 @@ fsource.write(source) fsource.close() try: - exe = build_executable([str(cfile)], + exe = build_executable([str(cfile)], ExternalCompilationInfo(), noerr=True) py.process.cmdexec(exe) - except (KeyboardInterrupt, SystemExit): - raise - except: + except (distutils.errors.CompileError, + distutils.errors.LinkError, + py.error.Error): return False else: return True From arigo at codespeak.net Tue Dec 11 17:56:25 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Dec 2007 17:56:25 +0100 (CET) Subject: [pypy-svn] r49634 - pypy/dist/pypy/objspace/std Message-ID: <20071211165625.66422168447@codespeak.net> Author: arigo Date: Tue Dec 11 17:56:24 2007 New Revision: 49634 Modified: pypy/dist/pypy/objspace/std/typeobject.py Log: Fix test_typeobject by moving the check for withtypeversion in mutated() instead of in each caller. Modified: pypy/dist/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/typeobject.py (original) +++ pypy/dist/pypy/objspace/std/typeobject.py Tue Dec 11 17:56:24 2007 @@ -221,7 +221,8 @@ def mutated(w_self): space = w_self.space - assert space.config.objspace.std.withtypeversion + if not space.config.objspace.std.withtypeversion: + return if w_self.version_tag is not None: w_self.version_tag = VersionTag() subclasses_w = w_self.get_subclasses() @@ -529,8 +530,7 @@ # Note. This is exactly the same thing as descroperation.descr__setattr__, # but it is needed at bootstrap to avoid a call to w_type.getdict() which # would un-lazify the whole type. - if space.config.objspace.std.withtypeversion: - w_type.mutated() + w_type.mutated() name = space.str_w(w_name) w_descr = space.lookup(w_type, name) if w_descr is not None: @@ -544,8 +544,7 @@ w_type.dict_w[name] = w_value def delattr__Type_ANY(space, w_type, w_name): - if space.config.objspace.std.withtypeversion: - w_type.mutated() + w_type.mutated() if w_type.lazyloaders: w_type._freeze_() # force un-lazification name = space.str_w(w_name) From arigo at codespeak.net Tue Dec 11 18:04:13 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Dec 2007 18:04:13 +0100 (CET) Subject: [pypy-svn] r49635 - pypy/dist/pypy/objspace/std/test Message-ID: <20071211170413.627DB168447@codespeak.net> Author: arigo Date: Tue Dec 11 18:04:12 2007 New Revision: 49635 Modified: pypy/dist/pypy/objspace/std/test/test_versionedtype.py Log: I *think* that this behavior is also correct: changing __bases__ now changes the version_tag instead of always resetting it to None. Modified: pypy/dist/pypy/objspace/std/test/test_versionedtype.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_versionedtype.py (original) +++ pypy/dist/pypy/objspace/std/test/test_versionedtype.py Tue Dec 11 18:04:12 2007 @@ -62,7 +62,7 @@ pass B.__bases__ = (D, ) """) - assert w_B.version_tag is None + assert w_B.version_tag is not btag def test_version_tag_of_builtin_types(self): space = self.space From arigo at codespeak.net Tue Dec 11 18:11:46 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Dec 2007 18:11:46 +0100 (CET) Subject: [pypy-svn] r49637 - pypy/dist/pypy/objspace/std Message-ID: <20071211171146.B9E6C16844B@codespeak.net> Author: arigo Date: Tue Dec 11 18:11:45 2007 New Revision: 49637 Modified: pypy/dist/pypy/objspace/std/typeobject.py Log: A comment. Modified: pypy/dist/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/typeobject.py (original) +++ pypy/dist/pypy/objspace/std/typeobject.py Tue Dec 11 18:11:45 2007 @@ -223,6 +223,13 @@ space = w_self.space if not space.config.objspace.std.withtypeversion: return + # Invariant: version_tag is None if and only if + # 'w_self.instancetypedef.hasdict' is True, which is the case + # for a built-in type that provides its instances with their own + # __dict__. If 'hasdict' is True for a type T then it is also + # True for all subtypes of T; so we don't need to look for + # version_tags to update in the subclasses of a type T whose + # version_tag is None. if w_self.version_tag is not None: w_self.version_tag = VersionTag() subclasses_w = w_self.get_subclasses() From fijal at codespeak.net Tue Dec 11 19:11:53 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 11 Dec 2007 19:11:53 +0100 (CET) Subject: [pypy-svn] r49639 - in pypy/dist/pypy: module/_file objspace/std objspace/std/test Message-ID: <20071211181153.A3321168446@codespeak.net> Author: fijal Date: Tue Dec 11 19:11:50 2007 New Revision: 49639 Modified: pypy/dist/pypy/module/_file/__init__.py pypy/dist/pypy/objspace/std/test/test_proxy_internals.py pypy/dist/pypy/objspace/std/test/test_proxy_usercreated.py pypy/dist/pypy/objspace/std/transparent.py Log: Fix tproxy for files. Seems to be a bit too hand-written to me, but we can attach a gateway magic (I hope) Modified: pypy/dist/pypy/module/_file/__init__.py ============================================================================== --- pypy/dist/pypy/module/_file/__init__.py (original) +++ pypy/dist/pypy/module/_file/__init__.py Tue Dec 11 19:11:50 2007 @@ -22,3 +22,8 @@ pass # key was removed in the meantime else: stream.flush() + + def setup_after_space_initialization(self): + from pypy.module._file.interp_file import W_File + from pypy.objspace.std.transparent import register_proxyable + register_proxyable(self.space, W_File) Modified: pypy/dist/pypy/objspace/std/test/test_proxy_internals.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_proxy_internals.py (original) +++ pypy/dist/pypy/objspace/std/test/test_proxy_internals.py Tue Dec 11 19:11:50 2007 @@ -123,7 +123,6 @@ assert get_tproxy_controller(lst) is f def test_proxy_file(self): - skip("Not working") from __pypy__ import tproxy def f(name, *args, **kwds): Modified: pypy/dist/pypy/objspace/std/test/test_proxy_usercreated.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_proxy_usercreated.py (original) +++ pypy/dist/pypy/objspace/std/test/test_proxy_usercreated.py Tue Dec 11 19:11:50 2007 @@ -3,21 +3,34 @@ from pypy.objspace.std.test.test_proxy_internals import AppProxy from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import interp2app +from pypy.objspace.std.transparent import register_proxyable class W_Wrapped(Wrappable): def new(space, w_type): return space.wrap(W_Wrapped()) + def name(self, space): + return space.wrap("wrapped") + name.unwrap_spec = ['self', ObjSpace] + W_Wrapped.typedef = TypeDef( 'Wrapped', - __new__ = interp2app(W_Wrapped.new.im_func) + __new__ = interp2app(W_Wrapped.new.im_func), + __name__ = interp2app(W_Wrapped.name), ) class AppTestProxyNewtype(AppProxy): def setup_class(cls): AppProxy.setup_class.im_func(cls) cls.w_wrapped = cls.space.wrap(W_Wrapped()) + register_proxyable(cls.space, W_Wrapped) def test_one(self): x = type(self.wrapped)() - print x + from __pypy__ import tproxy + + def f(name, *args, **kwds): + return getattr(x, name)(*args, **kwds) + + t = tproxy(type(x), f) + assert t.__name__ == x.__name__ Modified: pypy/dist/pypy/objspace/std/transparent.py ============================================================================== --- pypy/dist/pypy/objspace/std/transparent.py (original) +++ pypy/dist/pypy/objspace/std/transparent.py Tue Dec 11 19:11:50 2007 @@ -8,6 +8,8 @@ from pypy.objspace.std.proxyobject import * from pypy.objspace.std.typeobject import W_TypeObject +type_cache = {} + def proxy(space, w_type, w_controller): """tproxy(typ, controller) -> obj Return something that looks like it is of type typ. Its behaviour is @@ -37,9 +39,17 @@ return W_Transparent(space, w_type, w_controller) else: raise OperationError(space.w_TypeError, space.wrap("type expected as first argument")) - #return type_cache[w_type or w_type.w_bestbase] - raise OperationError(space.w_TypeError, space.wrap("Object type %s could not "\ - "be wrapped (YET)" % w_type.getname(space, "?"))) + try: + return type_cache[w_type or w_type.w_bestbase](space, w_type, w_controller) + except KeyError: + raise OperationError(space.w_TypeError, space.wrap("Object type %s could not "\ + "be wrapped (YET)" % w_type.getname(space, "?"))) + +def register_proxyable(space, cls): + tpdef = cls.typedef + class W_TransparentUserCreated(W_Transparent): + typedef = tpdef + type_cache[space.gettypeobject(tpdef)] = W_TransparentUserCreated def proxy_controller(space, w_object): """get_tproxy_controller(obj) -> controller From arigo at codespeak.net Tue Dec 11 19:12:33 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Dec 2007 19:12:33 +0100 (CET) Subject: [pypy-svn] r49640 - pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__ Message-ID: <20071211181233.9D0FB168446@codespeak.net> Author: arigo Date: Tue Dec 11 19:12:33 2007 New Revision: 49640 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Log: Some XXXs. Some of them are just reminders for features that cfbolz probably remembers are missing :-) Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Tue Dec 11 19:12:33 2007 @@ -8,6 +8,7 @@ def descr_classobj_new(space, w_subtype, w_name, w_bases, w_dict): + # XXX kill next two lines? no clue why they are there in _classobj either if w_bases is None: w_bases = space.newtuple([]) elif not space.is_true(space.isinstance(w_bases, space.w_tuple)): @@ -19,11 +20,11 @@ if not space.is_true(space.contains(w_dict, space.wrap("__doc__"))): space.setitem(w_dict, space.wrap("__doc__"), space.w_None) - if not space.is_true(space.contains(w_dict, space.wrap("__doc__"))): - space.setitem(w_dict, space.wrap("__doc__"), space.w_None) + # XXX missing: lengthy and obscure logic about "__module__" for w_base in space.unpackiterable(w_bases): if not isinstance(w_base, W_ClassObject): + # XXX use space.type(w_base) instead of the next line w_metaclass = space.call_function(space.w_type, w_base) if space.is_true(space.callable(w_metaclass)): return space.call_function(w_metaclass, w_name, @@ -35,8 +36,10 @@ class W_ClassObject(Wrappable): def __init__(self, space, w_name, w_bases, w_dict): + # XXX shouldn't store both w_name and name self.w_name = w_name self.name = space.str_w(w_name) + # XXX shouldn't store both w_bases and bases_w. Probably bases_w only self.w_bases = w_bases self.bases_w = space.unpackiterable(w_bases) self.w_dict = w_dict @@ -48,6 +51,8 @@ return self.w_dict def fset_dict(space, self, w_dict): + # XXX maybe also implement the setdict() method and move this + # logic over there if not space.is_true(space.isinstance(w_dict, space.w_dict)): raise OperationError( space.w_TypeError, @@ -81,10 +86,14 @@ self.w_bases = w_bases self.bases_w = bases_w + # XXX missing del descriptors for __name__, __bases__, __dict__ + def lookup(self, space, w_attr): # returns w_value or interplevel None try: + # XXX should use space.finditem(), which is more efficient + # with the stdobjspace return space.getitem(self.w_dict, w_attr) except OperationError, e: if not e.match(space, space.w_KeyError): @@ -164,6 +173,8 @@ return self.w_dict def fset_dict(space, self, w_dict): + # XXX maybe also implement the setdict() method and move this + # logic over there if not space.is_true(space.isinstance(w_dict, space.w_dict)): raise OperationError( space.w_TypeError, @@ -194,6 +205,7 @@ def retrieve(self, space, w_attr, exc=True): try: + # XXX use space.finditem() return space.getitem(self.w_dict, w_attr) except OperationError, e: if not e.match(space, space.w_KeyError): @@ -242,6 +254,7 @@ W_InstanceObject.fset_dict), __class__ = GetSetProperty(W_InstanceObject.fget_class, W_InstanceObject.fset_class), + # XXX there is no __getattr__ on this typedef in _classobj.py __getattr__ = interp2app(W_InstanceObject.descr_getattr, unwrap_spec=['self', ObjSpace, W_Root]), ) From arigo at codespeak.net Tue Dec 11 19:48:28 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Dec 2007 19:48:28 +0100 (CET) Subject: [pypy-svn] r49641 - pypy/dist/pypy/objspace/std Message-ID: <20071211184828.7D752168446@codespeak.net> Author: arigo Date: Tue Dec 11 19:48:26 2007 New Revision: 49641 Modified: pypy/dist/pypy/objspace/std/stringobject.py pypy/dist/pypy/objspace/std/stringtype.py pypy/dist/pypy/objspace/std/strjoinobject.py Log: Kill an obvious inefficiency. Modified: pypy/dist/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/stringobject.py (original) +++ pypy/dist/pypy/objspace/std/stringobject.py Tue Dec 11 19:48:26 2007 @@ -12,7 +12,7 @@ from pypy.objspace.std.tupleobject import W_TupleObject from pypy.objspace.std.stringtype import sliced, joined, wrapstr, wrapchar, \ - stringendswith, stringstartswith + stringendswith, stringstartswith, joined2 from pypy.objspace.std.formatting import mod_format @@ -827,7 +827,7 @@ def add__String_String(space, w_left, w_right): right = w_right._value left = w_left._value - return joined(space, [left, right]) + return joined2(space, left, right) def len__String(space, w_str): return space.wrap(len(w_str._value)) Modified: pypy/dist/pypy/objspace/std/stringtype.py ============================================================================== --- pypy/dist/pypy/objspace/std/stringtype.py (original) +++ pypy/dist/pypy/objspace/std/stringtype.py Tue Dec 11 19:48:26 2007 @@ -58,6 +58,14 @@ else: return wrapstr(space, "".join(strlist)) +def joined2(space, str1, str2): + assert not space.config.objspace.std.withrope + if space.config.objspace.std.withstrjoin: + from pypy.objspace.std.strjoinobject import W_StringJoinObject + return W_StringJoinObject([str1, str2]) + else: + return wrapstr(space, str1 + str2) + str_join = SMM('join', 2, doc='S.join(sequence) -> string\n\nReturn a string which is' ' the concatenation of the strings in the\nsequence. ' Modified: pypy/dist/pypy/objspace/std/strjoinobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/strjoinobject.py (original) +++ pypy/dist/pypy/objspace/std/strjoinobject.py Tue Dec 11 19:48:26 2007 @@ -2,7 +2,7 @@ from pypy.objspace.std.stringobject import W_StringObject from pypy.objspace.std.unicodeobject import delegate_String2Unicode -from pypy.objspace.std.stringtype import joined, wrapstr +from pypy.objspace.std.stringtype import wrapstr class W_StringJoinObject(W_Object): from pypy.objspace.std.stringtype import str_typedef as typedef From arigo at codespeak.net Tue Dec 11 19:59:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Dec 2007 19:59:24 +0100 (CET) Subject: [pypy-svn] r49642 - pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__ Message-ID: <20071211185924.3A1C3168459@codespeak.net> Author: arigo Date: Tue Dec 11 19:59:23 2007 New Revision: 49642 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Log: Comment. Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Tue Dec 11 19:59:23 2007 @@ -74,6 +74,9 @@ return self.w_bases def fset_bases(space, self, w_bases): + # XXX in theory, this misses a check against inheritence cycles + # although on pypy we don't get a segfault for infinite + # recursion anyway if not space.is_true(space.isinstance(w_bases, space.w_tuple)): raise OperationError( space.w_TypeError, From cfbolz at codespeak.net Tue Dec 11 20:17:38 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 20:17:38 +0100 (CET) Subject: [pypy-svn] r49643 - pypy/dist/pypy/rlib Message-ID: <20071211191738.9601516845F@codespeak.net> Author: cfbolz Date: Tue Dec 11 20:17:36 2007 New Revision: 49643 Modified: pypy/dist/pypy/rlib/rope.py Log: fix the ropes test failures Modified: pypy/dist/pypy/rlib/rope.py ============================================================================== --- pypy/dist/pypy/rlib/rope.py (original) +++ pypy/dist/pypy/rlib/rope.py Tue Dec 11 20:17:36 2007 @@ -69,7 +69,7 @@ x |= HIGHEST_BIT_SET result.hash = x result.is_ascii = self.is_ascii and other.is_ascii - result.is_bytestring = self.is_bytestring and right.is_bytestring() + result.is_bytestring = self.is_bytestring and other.is_bytestring return result @@ -190,6 +190,7 @@ additional_info.hash = partial_hash additional_info.is_ascii = is_ascii additional_info.charbitmask = charbitmask + additional_info.is_bytestring = True self._additional_info = additional_info return additional_info @@ -432,7 +433,7 @@ leftaddinfo = self.left.additional_info() rightaddinfo = self.right.additional_info() additional_info = leftaddinfo.combine(rightaddinfo, - self.right.length()) + self.right.length()) self._additional_info = additional_info return additional_info From cfbolz at codespeak.net Tue Dec 11 20:19:02 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 20:19:02 +0100 (CET) Subject: [pypy-svn] r49644 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071211191902.44911168462@codespeak.net> Author: cfbolz Date: Tue Dec 11 20:19:01 2007 New Revision: 49644 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: fix some of Armin's XXXs. Start implementing __special__ methods: __len__, __getitem__, __setitem__, __delitem__. Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Tue Dec 11 20:19:01 2007 @@ -24,8 +24,7 @@ for w_base in space.unpackiterable(w_bases): if not isinstance(w_base, W_ClassObject): - # XXX use space.type(w_base) instead of the next line - w_metaclass = space.call_function(space.w_type, w_base) + w_metaclass = space.type(w_base) if space.is_true(space.callable(w_metaclass)): return space.call_function(w_metaclass, w_name, w_bases, w_dic) @@ -36,11 +35,7 @@ class W_ClassObject(Wrappable): def __init__(self, space, w_name, w_bases, w_dict): - # XXX shouldn't store both w_name and name - self.w_name = w_name self.name = space.str_w(w_name) - # XXX shouldn't store both w_bases and bases_w. Probably bases_w only - self.w_bases = w_bases self.bases_w = space.unpackiterable(w_bases) self.w_dict = w_dict @@ -60,23 +55,22 @@ self.w_dict = w_dict def fget_name(space, self): - return self.w_name + return space.wrap(self.name) def fset_name(space, self, w_newname): if not space.is_true(space.isinstance(w_newname, space.w_str)): raise OperationError( space.w_TypeError, space.wrap("__name__ must be a string object")) - self.w_name = w_newname self.name = space.str_w(w_newname) def fget_bases(space, self): - return self.w_bases + return space.wrap(self.bases_w) def fset_bases(space, self, w_bases): # XXX in theory, this misses a check against inheritence cycles # although on pypy we don't get a segfault for infinite - # recursion anyway + # recursion anyway if not space.is_true(space.isinstance(w_bases, space.w_tuple)): raise OperationError( space.w_TypeError, @@ -86,7 +80,6 @@ if not isinstance(w_base, W_ClassObject): raise OperationError(space.w_TypeError, space.wrap("__bases__ items must be classes")) - self.w_bases = w_bases self.bases_w = bases_w # XXX missing del descriptors for __name__, __bases__, __dict__ @@ -117,9 +110,9 @@ if name == "__dict__": return self.w_dict elif name == "__name__": - return self.w_name + return space.wrap(self.name) elif name == "__bases__": - return self.w_bases + return space.newtuple(self.bases_w) w_value = self.lookup(space, w_attr) if w_value is None: raise OperationError( @@ -219,16 +212,11 @@ return None def getattr(self, space, w_name, exc=True): - try: - name = space.str_w(w_name) - except OperationError, e: - if not e.match(space, space.w_TypeError): - raise - else: - if name == "__dict__": - return self.w_dict - elif name == "__class__": - return self.w_class + name = space.str_w(w_name) + if name == "__dict__": + return self.w_dict + elif name == "__class__": + return self.w_class w_result = self.retrieve(space, w_name, False) if w_result is not None: return w_result @@ -238,7 +226,7 @@ raise OperationError( space.w_AttributeError, space.wrap("%s instance has no attribute %s" % ( - space.str_w(self.w_class), space.str_w(space.str(name))))) + self.w_class.name, name))) else: return None w_descr_get = space.lookup(w_value, '__get__') @@ -246,10 +234,36 @@ return w_value return space.call_function(w_descr_get, w_value, self, self.w_class) - def descr_getattr(self, space, w_attr): + def descr_getattribute(self, space, w_attr): #import pdb; pdb.set_trace() return self.getattr(space, w_attr) + def descr_len(self, space): + w_meth = self.getattr(space, space.wrap('__len__')) + w_result = space.call_function(w_meth) + if space.is_true(space.isinstance(ret, space.w_int)): + if space.is_true(space.le(w_result, space.wrap(0))): + raise OperationError( + space.w_ValueError, + space.wrap("__len__() should return >= 0")) + return w_result + else: + raise OperationError( + space.w_ValueError, + space.wrap("__len__() should return an int")) + + def descr_getitem(self, space, w_key): + w_meth = self.getattr(space, space.wrap('__getitem__')) + return space.call_function(w_meth, w_key) + + def descr_setitem(self, space, w_key, w_value): + w_meth = self.getattr(space, space.wrap('__setitem__')) + space.call_function(w_meth, w_key, w_value) + + def descr_delitem(self, space, w_key): + w_meth = self.getattr(space, space.wrap('__delitem__')) + space.call_function(w_meth, w_key) + W_InstanceObject.typedef = TypeDef("instance", __new__ = interp2app(W_InstanceObject.descr_new), @@ -257,7 +271,15 @@ W_InstanceObject.fset_dict), __class__ = GetSetProperty(W_InstanceObject.fget_class, W_InstanceObject.fset_class), - # XXX there is no __getattr__ on this typedef in _classobj.py - __getattr__ = interp2app(W_InstanceObject.descr_getattr, + __getattribute__ = interp2app(W_InstanceObject.descr_getattribute, + unwrap_spec=['self', ObjSpace, W_Root]), + __len__ = interp2app(W_InstanceObject.descr_len, + unwrap_spec=['self', ObjSpace]), + __getitem__ = interp2app(W_InstanceObject.descr_getitem, + unwrap_spec=['self', ObjSpace, W_Root]), + __setitem__ = interp2app(W_InstanceObject.descr_setitem, + unwrap_spec=['self', ObjSpace, W_Root, W_Root]), + __delitem__ = interp2app(W_InstanceObject.descr_delitem, unwrap_spec=['self', ObjSpace, W_Root]), ) + Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Tue Dec 11 20:19:01 2007 @@ -141,3 +141,14 @@ raises(TypeError, B.f, 1) raises(TypeError, B.f, A()) assert B.f(B()) == 1 + + def test_len_getsetdelitem(self): + class A: + __metaclass__ = nclassobj + a = A() + raises(AttributeError, len, a) + raises(AttributeError, "a[5]") + raises(AttributeError, "a[5] = 5") + raises(AttributeError, "del a[5]") + class A: + __metaclass__ = nclassobj From cfbolz at codespeak.net Tue Dec 11 20:25:33 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 20:25:33 +0100 (CET) Subject: [pypy-svn] r49645 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071211192533.CB4291684C2@codespeak.net> Author: cfbolz Date: Tue Dec 11 20:25:32 2007 New Revision: 49645 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: two more XXXs Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Tue Dec 11 20:25:32 2007 @@ -8,10 +8,7 @@ def descr_classobj_new(space, w_subtype, w_name, w_bases, w_dict): - # XXX kill next two lines? no clue why they are there in _classobj either - if w_bases is None: - w_bases = space.newtuple([]) - elif not space.is_true(space.isinstance(w_bases, space.w_tuple)): + if not space.is_true(space.isinstance(w_bases, space.w_tuple)): raise_type_err(space, 'bases', 'tuple', w_bases) if not space.is_true(space.isinstance(w_dict, space.w_dict)): @@ -54,6 +51,11 @@ space.wrap("__dict__ must be a dictionary object")) self.w_dict = w_dict + def fdel_dict(space, self): + raise OperationError( + space.w_TypeError, + space.wrap("__dict__ must be a dictionary object")) + def fget_name(space, self): return space.wrap(self.name) @@ -64,6 +66,12 @@ space.wrap("__name__ must be a string object")) self.name = space.str_w(w_newname) + def fdel_name(space, self): + raise OperationError( + space.w_TypeError, + space.wrap("__name__ must be a string object")) + + def fget_bases(space, self): return space.wrap(self.bases_w) @@ -82,8 +90,10 @@ space.wrap("__bases__ items must be classes")) self.bases_w = bases_w - # XXX missing del descriptors for __name__, __bases__, __dict__ - + def fdel_bases(space, self): + raise OperationError( + space.w_TypeError, + space.wrap("__bases__ must be a tuple object")) def lookup(self, space, w_attr): # returns w_value or interplevel None @@ -144,10 +154,13 @@ W_ClassObject.typedef = TypeDef("classobj", __new__ = interp2app(descr_classobj_new), - __dict__ = GetSetProperty(W_ClassObject.fget_dict, W_ClassObject.fset_dict), - __name__ = GetSetProperty(W_ClassObject.fget_name, W_ClassObject.fset_name), + __dict__ = GetSetProperty(W_ClassObject.fget_dict, W_ClassObject.fset_dict, + W_ClassObject.fdel_dict), + __name__ = GetSetProperty(W_ClassObject.fget_name, W_ClassObject.fset_name, + W_ClassObject.fdel_name), __bases__ = GetSetProperty(W_ClassObject.fget_bases, - W_ClassObject.fset_bases), + W_ClassObject.fset_bases, + W_ClassObject.fdel_bases), __call__ = interp2app(W_ClassObject.descr_call, unwrap_spec=['self', ObjSpace, Arguments]), __getattribute__ = interp2app(W_ClassObject.descr_getattribute, Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Tue Dec 11 20:25:32 2007 @@ -45,6 +45,15 @@ assert C.b == 18 assert C.c == 19 + def test_del_error_class_special(self): + class A: + __metaclass__ = nclassobj + a = 1 + raises(TypeError, "del A.__name__") + raises(TypeError, "del A.__dict__") + raises(TypeError, "del A.__bases__") + + def test_mutate_instance_special(self): class A: __metaclass__ = nclassobj From cfbolz at codespeak.net Tue Dec 11 20:28:26 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 20:28:26 +0100 (CET) Subject: [pypy-svn] r49646 - pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__ Message-ID: <20071211192826.665411683EE@codespeak.net> Author: cfbolz Date: Tue Dec 11 20:28:24 2007 New Revision: 49646 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Log: two more XXXs Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Tue Dec 11 20:28:24 2007 @@ -97,18 +97,14 @@ def lookup(self, space, w_attr): # returns w_value or interplevel None - try: - # XXX should use space.finditem(), which is more efficient - # with the stdobjspace - return space.getitem(self.w_dict, w_attr) - except OperationError, e: - if not e.match(space, space.w_KeyError): - raise - for base in self.bases_w: - w_result = base.lookup(space, w_attr) - if w_result is not None: - return w_result - return None + w_result = space.finditem(self.w_dict, w_attr) + if w_result is not None: + return w_result + for base in self.bases_w: + w_result = base.lookup(space, w_attr) + if w_result is not None: + return w_result + return None def descr_getattribute(self, space, w_attr): try: @@ -213,16 +209,13 @@ return W_InstanceObject(space, w_class, w_dict) def retrieve(self, space, w_attr, exc=True): - try: - # XXX use space.finditem() - return space.getitem(self.w_dict, w_attr) - except OperationError, e: - if not e.match(space, space.w_KeyError): - raise - if exc: - raise OperationError( - space.w_AttributeError, w_attr) - return None + w_result = space.finditem(self.w_dict, w_attr) + if w_result is not None: + return w_result + if exc: + raise OperationError( + space.w_AttributeError, w_attr) + return None def getattr(self, space, w_name, exc=True): name = space.str_w(w_name) From cfbolz at codespeak.net Tue Dec 11 20:32:59 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 20:32:59 +0100 (CET) Subject: [pypy-svn] r49647 - pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__ Message-ID: <20071211193259.4B98416845B@codespeak.net> Author: cfbolz Date: Tue Dec 11 20:32:58 2007 New Revision: 49647 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Log: an XXX and a typo Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Tue Dec 11 20:32:58 2007 @@ -39,18 +39,19 @@ def getdict(self): return self.w_dict - def fget_dict(space, self): - return self.w_dict - - def fset_dict(space, self, w_dict): - # XXX maybe also implement the setdict() method and move this - # logic over there + def setdict(self, space, w_dict): if not space.is_true(space.isinstance(w_dict, space.w_dict)): raise OperationError( space.w_TypeError, space.wrap("__dict__ must be a dictionary object")) self.w_dict = w_dict + def fget_dict(space, self): + return self.w_dict + + def fset_dict(space, self, w_dict): + self.setdict(self, space, w_dict) + def fdel_dict(space, self): raise OperationError( space.w_TypeError, @@ -76,7 +77,7 @@ return space.wrap(self.bases_w) def fset_bases(space, self, w_bases): - # XXX in theory, this misses a check against inheritence cycles + # XXX in theory, this misses a check against inheritance cycles # although on pypy we don't get a segfault for infinite # recursion anyway if not space.is_true(space.isinstance(w_bases, space.w_tuple)): From arigo at codespeak.net Tue Dec 11 20:33:48 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 11 Dec 2007 20:33:48 +0100 (CET) Subject: [pypy-svn] r49648 - pypy/dist/pypy/jit/tl/test Message-ID: <20071211193348.27A7516845B@codespeak.net> Author: arigo Date: Tue Dec 11 20:33:47 2007 New Revision: 49648 Modified: pypy/dist/pypy/jit/tl/test/test_tl.py Log: Fix completely shallow failures. The cause is that the new wrapping logic doesn't support default arguments any more. Modified: pypy/dist/pypy/jit/tl/test/test_tl.py ============================================================================== --- pypy/dist/pypy/jit/tl/test/test_tl.py (original) +++ pypy/dist/pypy/jit/tl/test/test_tl.py Tue Dec 11 20:33:47 2007 @@ -39,7 +39,7 @@ def test_tl_translatable(self): code = list2bytecode([PUSH,42, PUSH,100, ADD]) fn = self.getcompiled(self.interp, [str, int, int]) - assert self.interp(code) == fn(code) + assert self.interp(code, 0, 0) == fn(code, 0, 0) def test_swap(self): code = [PUSH,42, PUSH, 84] From cfbolz at codespeak.net Tue Dec 11 20:36:38 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 20:36:38 +0100 (CET) Subject: [pypy-svn] r49649 - pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__ Message-ID: <20071211193638.D0DD916845B@codespeak.net> Author: cfbolz Date: Tue Dec 11 20:36:38 2007 New Revision: 49649 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Log: deja vu: fix the same XXX in a different place and a different typo Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Tue Dec 11 20:36:38 2007 @@ -50,7 +50,7 @@ return self.w_dict def fset_dict(space, self, w_dict): - self.setdict(self, space, w_dict) + self.setdict(space, w_dict) def fdel_dict(space, self): raise OperationError( @@ -175,18 +175,19 @@ def getdict(self): return self.w_dict - def fget_dict(space, self): - return self.w_dict - - def fset_dict(space, self, w_dict): - # XXX maybe also implement the setdict() method and move this - # logic over there + def setdict(self, space, w_dict): if not space.is_true(space.isinstance(w_dict, space.w_dict)): raise OperationError( space.w_TypeError, space.wrap("__dict__ must be a dictionary object")) self.w_dict = w_dict + def fget_dict(space, self): + return self.w_dict + + def fset_dict(space, self, w_dict): + self.setdict(space, w_dict) + def fget_class(space, self): return self.w_class From cfbolz at codespeak.net Tue Dec 11 20:55:46 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 20:55:46 +0100 (CET) Subject: [pypy-svn] r49651 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071211195546.84DAC168466@codespeak.net> Author: cfbolz Date: Tue Dec 11 20:55:46 2007 New Revision: 49651 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: more tests & fixes for bugs found Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Tue Dec 11 20:55:46 2007 @@ -249,7 +249,7 @@ def descr_len(self, space): w_meth = self.getattr(space, space.wrap('__len__')) w_result = space.call_function(w_meth) - if space.is_true(space.isinstance(ret, space.w_int)): + if space.is_true(space.isinstance(w_result, space.w_int)): if space.is_true(space.le(w_result, space.wrap(0))): raise OperationError( space.w_ValueError, @@ -257,7 +257,7 @@ return w_result else: raise OperationError( - space.w_ValueError, + space.w_TypeError, space.wrap("__len__() should return an int")) def descr_getitem(self, space, w_key): Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Tue Dec 11 20:55:46 2007 @@ -161,3 +161,37 @@ raises(AttributeError, "del a[5]") class A: __metaclass__ = nclassobj + + class A: + __metaclass__ = nclassobj + def __init__(self): + self.list = [1, 2, 3, 4, 5] + def __len__(self): + return len(self.list) + def __getitem__(self, i): + return self.list[i] + def __setitem__(self, i, v): + self.list[i] = v + def __delitem__(self, i): + del self.list[i] + + a = A() + assert len(a) == 5 + del a[0] + assert len(a) == 4 + assert a[0] == 2 + a[0] = 5 + assert a[0] == 5 + + def test_len_errors(self): + class A: + __metaclass__ = nclassobj + def __len__(self): + return long(10) + raises(TypeError, len, A()) + class A: + __metaclass__ = nclassobj + def __len__(self): + return -1 + raises(ValueError, len, A()) + From cfbolz at codespeak.net Tue Dec 11 20:58:06 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 20:58:06 +0100 (CET) Subject: [pypy-svn] r49652 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071211195806.D83751684E1@codespeak.net> Author: cfbolz Date: Tue Dec 11 20:58:06 2007 New Revision: 49652 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: support for __call__ Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Tue Dec 11 20:58:06 2007 @@ -272,6 +272,9 @@ w_meth = self.getattr(space, space.wrap('__delitem__')) space.call_function(w_meth, w_key) + def descr_call(self, space, __args__): + w_meth = self.getattr(space, space.wrap('__call__')) + return space.call_args(w_meth, __args__) W_InstanceObject.typedef = TypeDef("instance", __new__ = interp2app(W_InstanceObject.descr_new), @@ -289,5 +292,7 @@ unwrap_spec=['self', ObjSpace, W_Root, W_Root]), __delitem__ = interp2app(W_InstanceObject.descr_delitem, unwrap_spec=['self', ObjSpace, W_Root]), + __call__ = interp2app(W_InstanceObject.descr_call, + unwrap_spec=['self', ObjSpace, Arguments]), ) Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Tue Dec 11 20:58:06 2007 @@ -195,3 +195,15 @@ return -1 raises(ValueError, len, A()) + def test_call(self): + class A: + __metaclass__ = nclassobj + a = A() + raises(AttributeError, a) + class A: + __metaclass__ = nclassobj + def __call__(self, a, b): + return a + b + a = A() + assert a(1, 2) == 3 + From cfbolz at codespeak.net Tue Dec 11 21:15:08 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 21:15:08 +0100 (CET) Subject: [pypy-svn] r49653 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071211201508.9C5DC1684E6@codespeak.net> Author: cfbolz Date: Tue Dec 11 21:15:06 2007 New Revision: 49653 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: bug in __len__. implement __nonzero__. Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Tue Dec 11 21:15:06 2007 @@ -250,15 +250,14 @@ w_meth = self.getattr(space, space.wrap('__len__')) w_result = space.call_function(w_meth) if space.is_true(space.isinstance(w_result, space.w_int)): - if space.is_true(space.le(w_result, space.wrap(0))): + if space.is_true(space.lt(w_result, space.wrap(0))): raise OperationError( space.w_ValueError, space.wrap("__len__() should return >= 0")) return w_result - else: - raise OperationError( - space.w_TypeError, - space.wrap("__len__() should return an int")) + raise OperationError( + space.w_TypeError, + space.wrap("__len__() should return an int")) def descr_getitem(self, space, w_key): w_meth = self.getattr(space, space.wrap('__getitem__')) @@ -276,6 +275,23 @@ w_meth = self.getattr(space, space.wrap('__call__')) return space.call_args(w_meth, __args__) + def descr_nonzero(self, space): + w_func = self.getattr(space, space.wrap('__nonzero__'), False) + if w_func is None: + w_func = self.getattr(space, space.wrap('__len__'), False) + if w_func is None: + return space.w_True + w_result = space.call_function(w_func) + if space.is_true(space.isinstance(w_result, space.w_int)): + if space.is_true(space.lt(w_result, space.wrap(0))): + raise OperationError( + space.w_ValueError, + space.wrap("__nonzero__() should return >= 0")) + return w_result + raise OperationError( + space.w_TypeError, + space.wrap("__nonzero__() should return an int")) + W_InstanceObject.typedef = TypeDef("instance", __new__ = interp2app(W_InstanceObject.descr_new), __dict__ = GetSetProperty(W_InstanceObject.fget_dict, @@ -294,5 +310,7 @@ unwrap_spec=['self', ObjSpace, W_Root]), __call__ = interp2app(W_InstanceObject.descr_call, unwrap_spec=['self', ObjSpace, Arguments]), + __nonzero__ = interp2app(W_InstanceObject.descr_nonzero, + unwrap_spec=['self', ObjSpace]), ) Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Tue Dec 11 21:15:06 2007 @@ -182,6 +182,15 @@ assert a[0] == 2 a[0] = 5 assert a[0] == 5 + assert a + assert bool(a) == True + del a[0] + del a[0] + del a[0] + del a[0] + assert len(a) == 0 + assert not a + assert bool(a) == False def test_len_errors(self): class A: @@ -207,3 +216,34 @@ a = A() assert a(1, 2) == 3 + def test_nonzero(self): + class A: + __metaclass__ = nclassobj + a = A() + assert a + assert bool(a) == True + class A: + __metaclass__ = nclassobj + def __init__(self, truth): + self.truth = truth + def __nonzero__(self): + return self.truth + a = A(1) + assert a + assert bool(a) == True + a = A(42) + assert a + assert bool(a) == True + a = A(True) + assert a + assert bool(a) == True + a = A(False) + assert not a + assert bool(a) == False + a = A(0) + assert not a + assert bool(a) == False + a = A(-1) + raises(ValueError, "assert a") + a = A("hello") + raises(TypeError, "assert a") From cfbolz at codespeak.net Tue Dec 11 22:49:31 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 22:49:31 +0100 (CET) Subject: [pypy-svn] r49655 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071211214931.A238A1684D0@codespeak.net> Author: cfbolz Date: Tue Dec 11 22:49:30 2007 New Revision: 49655 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: support for __str__ and __repr__ Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Tue Dec 11 22:49:30 2007 @@ -149,6 +149,17 @@ space.wrap("this constructor takes no arguments")) return w_inst + def get_module_string(self, space): + try: + w_mod = self.descr_getattribute(space, space.wrap("__module__")) + except OperationError, e: + if not e.match(space, space.w_AttributeError): + raise + return "?" + if space.is_true(space.isinstance(w_mod, space.w_str)): + return space.str_w(w_mod) + return "?" + W_ClassObject.typedef = TypeDef("classobj", __new__ = interp2app(descr_classobj_new), __dict__ = GetSetProperty(W_ClassObject.fget_dict, W_ClassObject.fset_dict, @@ -246,6 +257,20 @@ #import pdb; pdb.set_trace() return self.getattr(space, w_attr) + def descr_repr(self, space): + w_meth = self.getattr(space, space.wrap('__repr__'), False) + if w_meth is None: + w_class = self.w_class + mod = w_class.get_module_string(space) + return self.getrepr(space, "%s.%s instance" % (mod, w_class.name)) + return space.call_function(w_meth) + + def descr_str(self, space): + w_meth = self.getattr(space, space.wrap('__str__'), False) + if w_meth is None: + return self.descr_repr(space) + return space.call_function(w_meth) + def descr_len(self, space): w_meth = self.getattr(space, space.wrap('__len__')) w_result = space.call_function(w_meth) @@ -300,6 +325,10 @@ W_InstanceObject.fset_class), __getattribute__ = interp2app(W_InstanceObject.descr_getattribute, unwrap_spec=['self', ObjSpace, W_Root]), + __repr__ = interp2app(W_InstanceObject.descr_repr, + unwrap_spec=['self', ObjSpace]), + __str__ = interp2app(W_InstanceObject.descr_str, + unwrap_spec=['self', ObjSpace]), __len__ = interp2app(W_InstanceObject.descr_len, unwrap_spec=['self', ObjSpace]), __getitem__ = interp2app(W_InstanceObject.descr_getitem, Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Tue Dec 11 22:49:30 2007 @@ -247,3 +247,28 @@ raises(ValueError, "assert a") a = A("hello") raises(TypeError, "assert a") + + def test_repr(self): + class A: + __metaclass__ = nclassobj + a = A() + assert repr(a).startswith("<__builtin__.A instance at") + assert str(a).startswith("<__builtin__.A instance at") + A.__name__ = "Foo" + assert repr(a).startswith("<__builtin__.Foo instance at") + assert str(a).startswith("<__builtin__.Foo instance at") + class A: + __metaclass__ = nclassobj + def __repr__(self): + return "foo" + assert repr(A()) == "foo" + assert str(A()) == "foo" + + def test_str(self): + class A: + __metaclass__ = nclassobj + def __str__(self): + return "foo" + a = A() + assert repr(a).startswith("<__builtin__.A instance at") + assert str(a) == "foo" From fijal at codespeak.net Tue Dec 11 22:54:30 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 11 Dec 2007 22:54:30 +0100 (CET) Subject: [pypy-svn] r49656 - pypy/dist/pypy/objspace/std Message-ID: <20071211215430.B1D2C1684D0@codespeak.net> Author: fijal Date: Tue Dec 11 22:54:30 2007 New Revision: 49656 Modified: pypy/dist/pypy/objspace/std/transparent.py Log: A terrible hack to get rid of a problem that we cannot keep dictionary with classes as values. Modified: pypy/dist/pypy/objspace/std/transparent.py ============================================================================== --- pypy/dist/pypy/objspace/std/transparent.py (original) +++ pypy/dist/pypy/objspace/std/transparent.py Tue Dec 11 22:54:30 2007 @@ -7,8 +7,18 @@ from pypy.interpreter.error import OperationError from pypy.objspace.std.proxyobject import * from pypy.objspace.std.typeobject import W_TypeObject +from pypy.rlib.objectmodel import r_dict +from pypy.rlib.unroll import unrolling_iterable -type_cache = {} +class TypeCache(object): + def __init__(self): + self.cache = [] + + def _freeze_(self): + self.cache = unrolling_iterable(self.cache) + return True + +type_cache = TypeCache() def proxy(space, w_type, w_controller): """tproxy(typ, controller) -> obj @@ -40,7 +50,10 @@ else: raise OperationError(space.w_TypeError, space.wrap("type expected as first argument")) try: - return type_cache[w_type or w_type.w_bestbase](space, w_type, w_controller) + w_lookup = w_type or w_type.w_besttype + for k, v in type_cache.cache: + if w_lookup == k: + return v(space, w_type, w_controller) except KeyError: raise OperationError(space.w_TypeError, space.wrap("Object type %s could not "\ "be wrapped (YET)" % w_type.getname(space, "?"))) @@ -49,7 +62,7 @@ tpdef = cls.typedef class W_TransparentUserCreated(W_Transparent): typedef = tpdef - type_cache[space.gettypeobject(tpdef)] = W_TransparentUserCreated + type_cache.cache.append((space.gettypeobject(tpdef), W_TransparentUserCreated)) def proxy_controller(space, w_object): """get_tproxy_controller(obj) -> controller From cfbolz at codespeak.net Tue Dec 11 22:54:42 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 22:54:42 +0100 (CET) Subject: [pypy-svn] r49657 - pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test Message-ID: <20071211215442.0D7D91684D8@codespeak.net> Author: cfbolz Date: Tue Dec 11 22:54:42 2007 New Revision: 49657 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: some corner cases Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Tue Dec 11 22:54:42 2007 @@ -257,6 +257,15 @@ A.__name__ = "Foo" assert repr(a).startswith("<__builtin__.Foo instance at") assert str(a).startswith("<__builtin__.Foo instance at") + A.__module__ = "bar" + assert repr(a).startswith(" Author: fijal Date: Tue Dec 11 23:11:06 2007 New Revision: 49658 Modified: pypy/dist/pypy/lib/distributed/faker.py pypy/dist/pypy/lib/distributed/protocol.py Log: Few fixes. Looks like cowboy coding to me, we need some serious descriptor tests here. Modified: pypy/dist/pypy/lib/distributed/faker.py ============================================================================== --- pypy/dist/pypy/lib/distributed/faker.py (original) +++ pypy/dist/pypy/lib/distributed/faker.py Tue Dec 11 23:11:06 2007 @@ -28,7 +28,7 @@ from types import MethodType, FunctionType -def ignore(name): +def not_ignore(name): # we don't want to fake some default descriptors, because # they'll alter the way we set attributes l = ['__dict__', '__weakref__', '__class__', '__bases__', @@ -43,14 +43,13 @@ dict_w = {} for item in tp.__dict__.keys(): value = getattr(tp, item) - if ignore(item): + if not_ignore(item): # we've got shortcut for method if hasattr(value, '__get__') and not type(value) is MethodType: - name = type(value).__name__ if hasattr(value, '__set__'): - dict_w[item] = ('get', name) + dict_w[item] = ('get', item) else: - dict_w[item] = ('set', name) + dict_w[item] = ('set', item) else: dict_w[item] = protocol.wrap(value) bases_w = [protocol.wrap(i) for i in tp.__bases__ if i is not object] Modified: pypy/dist/pypy/lib/distributed/protocol.py ============================================================================== --- pypy/dist/pypy/lib/distributed/protocol.py (original) +++ pypy/dist/pypy/lib/distributed/protocol.py Tue Dec 11 23:11:06 2007 @@ -328,7 +328,12 @@ name, w_obj, w_type = data obj = protocol.unwrap(w_obj) type_ = protocol.unwrap(w_type) - send(('finished', protocol.wrap(getattr(type(obj), name).__get__(obj, type_)))) + if obj: + type__ = type(obj) + else: + type__ = type_ + send(('finished', protocol.wrap(getattr(type__, name).__get__(obj, type_)))) + elif command == 'desc_set': name, w_obj, w_value = data obj = protocol.unwrap(w_obj) From cfbolz at codespeak.net Tue Dec 11 23:25:32 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 11 Dec 2007 23:25:32 +0100 (CET) Subject: [pypy-svn] r49659 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071211222532.819491684D0@codespeak.net> Author: cfbolz Date: Tue Dec 11 23:25:31 2007 New Revision: 49659 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: str and repr support for the classes too Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Tue Dec 11 23:25:31 2007 @@ -132,8 +132,6 @@ return w_value return space.call_function(w_descr_get, w_value, space.w_None, self) - # XXX missing: str, repr - def descr_call(self, space, __args__): w_inst = W_InstanceObject(space, self) w_init = w_inst.getattr(space, space.wrap('__init__'), False) @@ -149,6 +147,17 @@ space.wrap("this constructor takes no arguments")) return w_inst + def descr_repr(self, space): + mod = self.get_module_string(space) + return self.getrepr(space, "class %s.%s" % (mod, self.name)) + + def descr_str(self, space): + mod = self.get_module_string(space) + if mod == "?": + return space.wrap(self.name) + else: + return space.wrap("%s.%s" % (mod, self.name)) + def get_module_string(self, space): try: w_mod = self.descr_getattribute(space, space.wrap("__module__")) @@ -169,6 +178,10 @@ __bases__ = GetSetProperty(W_ClassObject.fget_bases, W_ClassObject.fset_bases, W_ClassObject.fdel_bases), + __repr__ = interp2app(W_ClassObject.descr_repr, + unwrap_spec=['self', ObjSpace]), + __str__ = interp2app(W_ClassObject.descr_str, + unwrap_spec=['self', ObjSpace]), __call__ = interp2app(W_ClassObject.descr_call, unwrap_spec=['self', ObjSpace, Arguments]), __getattribute__ = interp2app(W_ClassObject.descr_getattribute, Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Tue Dec 11 23:25:31 2007 @@ -45,6 +45,32 @@ assert C.b == 18 assert C.c == 19 + def test_class_repr(self): + class A: + __metaclass__ = nclassobj + assert repr(A).startswith(" Author: fijal Date: Tue Dec 11 23:45:59 2007 New Revision: 49660 Modified: pypy/dist/pypy/lib/distributed/test/test_distributed.py pypy/dist/pypy/lib/distributed/test/test_socklayer.py Log: * Kill stupid assertion * Skip the test which is broken (I need to sit deeper on that) Modified: pypy/dist/pypy/lib/distributed/test/test_distributed.py ============================================================================== --- pypy/dist/pypy/lib/distributed/test/test_distributed.py (original) +++ pypy/dist/pypy/lib/distributed/test/test_distributed.py Tue Dec 11 23:45:59 2007 @@ -259,6 +259,7 @@ assert l def test_remote_file_access(self): + skip("Descriptor logic seems broken") protocol = self.test_env({'f':open}) xf = protocol.get_remote('f') data = xf('/etc/passwd').read() Modified: pypy/dist/pypy/lib/distributed/test/test_socklayer.py ============================================================================== --- pypy/dist/pypy/lib/distributed/test/test_socklayer.py (original) +++ pypy/dist/pypy/lib/distributed/test/test_socklayer.py Tue Dec 11 23:45:59 2007 @@ -28,6 +28,5 @@ def two(): rp = connect(('127.0.0.1', 21211), GreenSocket) assert rp.x.z == 3 - assert [i for i in dir(rp) if not i.startswith('__')] == ['x'] oneof(one, two) From pypy-svn at codespeak.net Wed Dec 12 03:39:38 2007 From: pypy-svn at codespeak.net (VIAGRA ® Official Site) Date: Wed, 12 Dec 2007 03:39:38 +0100 (CET) Subject: [pypy-svn] December 73% OFF Message-ID: <20071212163940.6748.qmail@mx-ll-58.147.38-21.tttmaxnet.com> An HTML attachment was scrubbed... URL: From cfbolz at codespeak.net Wed Dec 12 12:17:51 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 12 Dec 2007 12:17:51 +0100 (CET) Subject: [pypy-svn] r49667 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071212111751.C28081684EA@codespeak.net> Author: cfbolz Date: Wed Dec 12 12:17:48 2007 New Revision: 49667 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: add iter support Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Wed Dec 12 12:17:48 2007 @@ -309,6 +309,17 @@ w_meth = self.getattr(space, space.wrap('__delitem__')) space.call_function(w_meth, w_key) + def descr_iter(self, space): + w_meth = self.getattr(space, space.wrap('__iter__'), False) + if w_meth is not None: + return space.call_function(w_meth) + w_meth = self.getattr(space, space.wrap('__getitem__'), False) + if w_meth is None: + raise OperationError( + space.w_TypeError, + space.wrap("iteration over non-sequence")) + return space.newseqiter(self) + def descr_call(self, space, __args__): w_meth = self.getattr(space, space.wrap('__call__')) return space.call_args(w_meth, __args__) @@ -350,6 +361,8 @@ unwrap_spec=['self', ObjSpace, W_Root, W_Root]), __delitem__ = interp2app(W_InstanceObject.descr_delitem, unwrap_spec=['self', ObjSpace, W_Root]), + __iter__ = interp2app(W_InstanceObject.descr_iter, + unwrap_spec=['self', ObjSpace]), __call__ = interp2app(W_InstanceObject.descr_call, unwrap_spec=['self', ObjSpace, Arguments]), __nonzero__ = interp2app(W_InstanceObject.descr_nonzero, Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Wed Dec 12 12:17:48 2007 @@ -79,7 +79,6 @@ raises(TypeError, "del A.__dict__") raises(TypeError, "del A.__bases__") - def test_mutate_instance_special(self): class A: __metaclass__ = nclassobj @@ -307,3 +306,23 @@ a = A() assert repr(a).startswith("<__builtin__.A instance at") assert str(a) == "foo" + + def test_iter(self): + class A: + __metaclass__ = nclassobj + def __init__(self): + self.list = [1, 2, 3, 4, 5] + def __iter__(self): + return iter(self.list) + for i, element in enumerate(A()): + assert i + 1 == element + class A: + __metaclass__ = nclassobj + def __init__(self): + self.list = [1, 2, 3, 4, 5] + def __len__(self): + return len(self.list) + def __getitem__(self, i): + return self.list[i] + for i, element in enumerate(A()): + assert i + 1 == element From cfbolz at codespeak.net Wed Dec 12 12:28:16 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 12 Dec 2007 12:28:16 +0100 (CET) Subject: [pypy-svn] r49669 - pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__ Message-ID: <20071212112816.BD9AA168453@codespeak.net> Author: cfbolz Date: Wed Dec 12 12:28:16 2007 New Revision: 49669 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Log: add an XXX Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Wed Dec 12 12:28:16 2007 @@ -319,6 +319,8 @@ space.w_TypeError, space.wrap("iteration over non-sequence")) return space.newseqiter(self) + #XXX do I really need a next method? the old implementation had one, but I + # don't see the point def descr_call(self, space, __args__): w_meth = self.getattr(space, space.wrap('__call__')) From cfbolz at codespeak.net Wed Dec 12 13:06:09 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 12 Dec 2007 13:06:09 +0100 (CET) Subject: [pypy-svn] r49671 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071212120609.A0CC61684E2@codespeak.net> Author: cfbolz Date: Wed Dec 12 13:06:09 2007 New Revision: 49671 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: implement get/set/delattr Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Wed Dec 12 13:06:09 2007 @@ -200,26 +200,18 @@ return self.w_dict def setdict(self, space, w_dict): - if not space.is_true(space.isinstance(w_dict, space.w_dict)): + if (w_dict is None or + not space.is_true(space.isinstance(w_dict, space.w_dict))): raise OperationError( space.w_TypeError, space.wrap("__dict__ must be a dictionary object")) self.w_dict = w_dict - def fget_dict(space, self): - return self.w_dict - - def fset_dict(space, self, w_dict): - self.setdict(space, w_dict) - - def fget_class(space, self): - return self.w_class - - def fset_class(space, self, w_class): - if not isinstance(w_class, W_ClassObject): + def setclass(self, space, w_class): + if w_class is None or not isinstance(w_class, W_ClassObject): raise OperationError( space.w_TypeError, - space.wrap("instance() first arg must be class")) + space.wrap("__class__ must be set to a class")) self.w_class = w_class def descr_new(space, w_type, w_class, w_dict=None): @@ -268,7 +260,43 @@ def descr_getattribute(self, space, w_attr): #import pdb; pdb.set_trace() - return self.getattr(space, w_attr) + try: + return self.getattr(space, w_attr) + except OperationError, e: + if not e.match(space, space.w_AttributeError): + raise + w_meth = self.getattr(space, space.wrap('__getattr__'), False) + if w_meth is not None: + return space.call_function(w_meth, w_attr) + raise + + def descr_setattr(self, space, w_name, w_value): + name = space.str_w(w_name) + if name == '__dict__': + self.setdict(space, w_value) + elif name == '__class__': + self.setclass(space, w_value) + else: + w_meth = self.getattr(space, space.wrap('__setattr__'), False) + if w_meth is not None: + space.call_function(w_meth, w_name, w_value) + else: + self.setdictvalue(space, w_name, w_value) + + def descr_delattr(self, space, w_name): + name = space.str_w(w_name) + if name == '__dict__': + # use setdict to raise the error + self.setdict(space, None) + elif name == '__class__': + # use setclass to raise the error + self.setclass(space, None) + else: + w_meth = self.getattr(space, space.wrap('__delattr__'), False) + if w_meth is not None: + space.call_function(w_meth, w_name) + else: + self.deldictvalue(space, w_name) def descr_repr(self, space): w_meth = self.getattr(space, space.wrap('__repr__'), False) @@ -345,12 +373,12 @@ W_InstanceObject.typedef = TypeDef("instance", __new__ = interp2app(W_InstanceObject.descr_new), - __dict__ = GetSetProperty(W_InstanceObject.fget_dict, - W_InstanceObject.fset_dict), - __class__ = GetSetProperty(W_InstanceObject.fget_class, - W_InstanceObject.fset_class), __getattribute__ = interp2app(W_InstanceObject.descr_getattribute, unwrap_spec=['self', ObjSpace, W_Root]), + __setattr__ = interp2app(W_InstanceObject.descr_setattr, + unwrap_spec=['self', ObjSpace, W_Root, W_Root]), + __delattr__ = interp2app(W_InstanceObject.descr_delattr, + unwrap_spec=['self', ObjSpace, W_Root]), __repr__ = interp2app(W_InstanceObject.descr_repr, unwrap_spec=['self', ObjSpace]), __str__ = interp2app(W_InstanceObject.descr_str, Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Wed Dec 12 13:06:09 2007 @@ -326,3 +326,36 @@ return self.list[i] for i, element in enumerate(A()): assert i + 1 == element + + def test_getsetdelattr(self): + class A: + __metaclass__ = nclassobj + a = 1 + def __getattr__(self, attr): + return attr.upper() + a = A() + assert a.a == 1 + a.__dict__['b'] = 4 + assert a.b == 4 + assert a.c == "C" + class A: + __metaclass__ = nclassobj + a = 1 + def __setattr__(self, attr, value): + self.__dict__[attr.lower()] = value + a = A() + assert a.a == 1 + a.A = 2 + assert a.a == 2 + class A: + __metaclass__ = nclassobj + a = 1 + def __delattr__(self, attr): + del self.__dict__[attr.lower()] + a = A() + assert a.a == 1 + a.a = 2 + assert a.a == 2 + del a.A + assert a.a == 1 + From cfbolz at codespeak.net Wed Dec 12 13:12:38 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 12 Dec 2007 13:12:38 +0100 (CET) Subject: [pypy-svn] r49672 - pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test Message-ID: <20071212121238.EC7391684E2@codespeak.net> Author: cfbolz Date: Wed Dec 12 13:12:36 2007 New Revision: 49672 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: a test that you can override special attributes on the instance Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Wed Dec 12 13:12:36 2007 @@ -359,3 +359,15 @@ del a.A assert a.a == 1 + def test_instance_override(self): + class A: + __metaclass__ = nclassobj + def __str__(self): + return "foo" + def __str__(): + return "bar" + a = A() + assert str(a) == "foo" + a.__str__ = __str__ + assert str(a) == "bar" + From cfbolz at codespeak.net Wed Dec 12 13:16:42 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 12 Dec 2007 13:16:42 +0100 (CET) Subject: [pypy-svn] r49673 - pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__ Message-ID: <20071212121642.41B7A1684E2@codespeak.net> Author: cfbolz Date: Wed Dec 12 13:16:41 2007 New Revision: 49673 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Log: simplify a bit now that I know about space.finditem Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Wed Dec 12 13:16:41 2007 @@ -226,22 +226,13 @@ raise TypeError("instance() second arg must be dictionary or None") return W_InstanceObject(space, w_class, w_dict) - def retrieve(self, space, w_attr, exc=True): - w_result = space.finditem(self.w_dict, w_attr) - if w_result is not None: - return w_result - if exc: - raise OperationError( - space.w_AttributeError, w_attr) - return None - def getattr(self, space, w_name, exc=True): name = space.str_w(w_name) if name == "__dict__": return self.w_dict elif name == "__class__": return self.w_class - w_result = self.retrieve(space, w_name, False) + w_result = space.finditem(self.w_dict, w_name) if w_result is not None: return w_result w_value = self.w_class.lookup(space, w_name) From cfbolz at codespeak.net Wed Dec 12 13:39:39 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 12 Dec 2007 13:39:39 +0100 (CET) Subject: [pypy-svn] r49674 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071212123939.CDA431684E2@codespeak.net> Author: cfbolz Date: Wed Dec 12 13:39:39 2007 New Revision: 49674 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: add support for unary methods (neg pos abs invert int long float oct hex) Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Wed Dec 12 13:39:39 2007 @@ -1,3 +1,4 @@ +import new from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped, applevel from pypy.interpreter.gateway import interp2app, ObjSpace @@ -189,6 +190,13 @@ ) W_ClassObject.typedef.acceptable_as_base_class = False + +def make_unary_instance_method(name): + def unaryop(self, space): + w_meth = self.getattr(space, space.wrap(name), True) + return space.call_function(w_meth) + return unaryop + class W_InstanceObject(Wrappable): def __init__(self, space, w_class, w_dict=None): if w_dict is None: @@ -362,6 +370,18 @@ space.w_TypeError, space.wrap("__nonzero__() should return an int")) +rawdict = {} +for op in "neg pos abs invert int long float oct hex".split(): + specialname = "__%s__" % (op, ) + # fool the gateway logic by giving it a real unbound method + meth = new.instancemethod( + make_unary_instance_method(specialname), + None, + W_InstanceObject) + rawdict[specialname] = interp2app( + meth, + unwrap_spec=["self", ObjSpace]) + W_InstanceObject.typedef = TypeDef("instance", __new__ = interp2app(W_InstanceObject.descr_new), __getattribute__ = interp2app(W_InstanceObject.descr_getattribute, @@ -388,5 +408,6 @@ unwrap_spec=['self', ObjSpace, Arguments]), __nonzero__ = interp2app(W_InstanceObject.descr_nonzero, unwrap_spec=['self', ObjSpace]), + **rawdict ) Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Wed Dec 12 13:39:39 2007 @@ -371,3 +371,10 @@ a.__str__ = __str__ assert str(a) == "bar" + def test_unary_method(self): + class A: + __metaclass__ = nclassobj + def __pos__(self): + return -1 + a = A() + assert +a == -1 From cfbolz at codespeak.net Wed Dec 12 14:28:47 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 12 Dec 2007 14:28:47 +0100 (CET) Subject: [pypy-svn] r49675 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071212132847.9378B1684F1@codespeak.net> Author: cfbolz Date: Wed Dec 12 14:28:46 2007 New Revision: 49675 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: rich cmp implementation Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Wed Dec 12 14:28:46 2007 @@ -197,6 +197,15 @@ return space.call_function(w_meth) return unaryop +def make_richcmp_instance_method(name): + def richcmp(self, space, w_other): + w_meth = self.getattr(space, space.wrap(name), False) + if w_meth is None: + return space.w_NotImplemented + return space.call_function(w_meth, w_other) + return richcmp + + class W_InstanceObject(Wrappable): def __init__(self, space, w_class, w_dict=None): if w_dict is None: @@ -371,6 +380,8 @@ space.wrap("__nonzero__() should return an int")) rawdict = {} + +# unary operations for op in "neg pos abs invert int long float oct hex".split(): specialname = "__%s__" % (op, ) # fool the gateway logic by giving it a real unbound method @@ -382,6 +393,18 @@ meth, unwrap_spec=["self", ObjSpace]) +# rich comparison operations +for op in 'eq ne gt lt ge le'.split(): + specialname = "__%s__" % (op, ) + # fool the gateway logic by giving it a real unbound method + meth = new.instancemethod( + make_richcmp_instance_method(specialname), + None, + W_InstanceObject) + rawdict[specialname] = interp2app( + meth, + unwrap_spec=["self", ObjSpace, W_Root]) + W_InstanceObject.typedef = TypeDef("instance", __new__ = interp2app(W_InstanceObject.descr_new), __getattribute__ = interp2app(W_InstanceObject.descr_getattribute, Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Wed Dec 12 14:28:46 2007 @@ -378,3 +378,14 @@ return -1 a = A() assert +a == -1 + + def test_cmp(self): + class A: + __metaclass__ = nclassobj + def __lt__(self, other): + return True + a = A() + b = A() + assert a < b + assert b < a + assert a < 1 From cfbolz at codespeak.net Wed Dec 12 14:40:03 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 12 Dec 2007 14:40:03 +0100 (CET) Subject: [pypy-svn] r49676 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071212134003.355731684EE@codespeak.net> Author: cfbolz Date: Wed Dec 12 14:40:02 2007 New Revision: 49676 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: implement coerce Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Wed Dec 12 14:40:02 2007 @@ -197,7 +197,7 @@ return space.call_function(w_meth) return unaryop -def make_richcmp_instance_method(name): +def make_binary_returning_notimplemented_instance_method(name): def richcmp(self, space, w_other): w_meth = self.getattr(space, space.wrap(name), False) if w_meth is None: @@ -393,12 +393,13 @@ meth, unwrap_spec=["self", ObjSpace]) -# rich comparison operations -for op in 'eq ne gt lt ge le'.split(): +# binary operations that return NotImplemented if they fail +# e.g. rich comparisons and coerce +for op in 'eq ne gt lt ge le coerce'.split(): specialname = "__%s__" % (op, ) # fool the gateway logic by giving it a real unbound method meth = new.instancemethod( - make_richcmp_instance_method(specialname), + make_binary_returning_notimplemented_instance_method(specialname), None, W_InstanceObject) rawdict[specialname] = interp2app( Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Wed Dec 12 14:40:02 2007 @@ -389,3 +389,14 @@ assert a < b assert b < a assert a < 1 + + def test_coerce(self): + class B: + __metaclass__ = nclassobj + def __coerce__(self, other): + return other, self + b = B() + assert coerce(b, 1) == (1, b) + class B: + __metaclass__ = nclassobj + raises(TypeError, coerce, B(), []) From cfbolz at codespeak.net Wed Dec 12 17:00:16 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 12 Dec 2007 17:00:16 +0100 (CET) Subject: [pypy-svn] r49677 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071212160016.00B471684DA@codespeak.net> Author: cfbolz Date: Wed Dec 12 17:00:15 2007 New Revision: 49677 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: binary operations Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Wed Dec 12 17:00:15 2007 @@ -198,13 +198,52 @@ return unaryop def make_binary_returning_notimplemented_instance_method(name): - def richcmp(self, space, w_other): + def binaryop(self, space, w_other): w_meth = self.getattr(space, space.wrap(name), False) if w_meth is None: return space.w_NotImplemented return space.call_function(w_meth, w_other) - return richcmp + return binaryop +def make_binary_instance_method(name): + specialname = "__%s__" % (name, ) + rspecialname = "__r%s__" % (name, ) + objspacename = name + if name in ['and', 'or']: + objspacename = name + '_' + + def binaryop(self, space, w_other): + w_a, w_b = _coerce_helper(space, self, w_other) + if w_a is None: + w_a = self + w_b = w_other + if w_a is self: + w_meth = self.getattr(space, space.wrap(specialname), False) + if w_meth is None: + return space.w_NotImplemented + return space.call_function(w_meth, w_b) + else: + return getattr(space, objspacename)(w_a, w_b) + + def rbinaryop(self, space, w_other): + w_a, w_b = _coerce_helper(space, self, w_other) + if w_a is None or w_a is self: + w_meth = self.getattr(space, space.wrap(rspecialname), False) + if w_meth is None: + return space.w_NotImplemented + return space.call_function(w_meth, w_other) + else: + return getattr(space, objspacename)(w_b, w_a) + return binaryop, rbinaryop + +def _coerce_helper(space, w_self, w_other): + try: + w_tup = space.coerce(w_self, w_other) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + return (None, None) + return space.unpacktuple(w_tup, 2) class W_InstanceObject(Wrappable): def __init__(self, space, w_class, w_dict=None): @@ -406,6 +445,20 @@ meth, unwrap_spec=["self", ObjSpace, W_Root]) +for op in "or and xor lshift rshift add sub mul div mod divmod floordiv truediv".split(): + specialname = "__%s__" % (op, ) + rspecialname = "__r%s__" % (op, ) + func, rfunc = make_binary_instance_method(op) + # fool the gateway logic by giving it a real unbound method + meth = new.instancemethod(func, None, W_InstanceObject) + rawdict[specialname] = interp2app( + meth, + unwrap_spec=["self", ObjSpace, W_Root]) + rmeth = new.instancemethod(rfunc, None, W_InstanceObject) + rawdict[rspecialname] = interp2app( + rmeth, + unwrap_spec=["self", ObjSpace, W_Root]) + W_InstanceObject.typedef = TypeDef("instance", __new__ = interp2app(W_InstanceObject.descr_new), __getattribute__ = interp2app(W_InstanceObject.descr_getattribute, Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Wed Dec 12 17:00:15 2007 @@ -400,3 +400,38 @@ class B: __metaclass__ = nclassobj raises(TypeError, coerce, B(), []) + + def test_binaryop(self): + class A: + __metaclass__ = nclassobj + def __add__(self, other): + return 1 + other + a = A() + assert a + 1 == 2 + assert a + 1.1 == 2.1 + + def test_binaryop_coerces(self): + class A: + __metaclass__ = nclassobj + def __add__(self, other): + return 1 + other + def __coerce__(self, other): + return self, int(other) + + a = A() + assert a + 1 == 2 + assert a + 1.1 == 2 + + + def test_binaryop_calls_coerce_always(self): + l = [] + class A: + __metaclass__ = nclassobj + def __coerce__(self, other): + l.append(other) + + a = A() + raises(TypeError, "a + 1") + raises(TypeError, "a + 1.1") + assert l == [1, 1.1] + From cfbolz at codespeak.net Wed Dec 12 17:07:33 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 12 Dec 2007 17:07:33 +0100 (CET) Subject: [pypy-svn] r49678 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071212160733.43C341684DA@codespeak.net> Author: cfbolz Date: Wed Dec 12 17:07:32 2007 New Revision: 49678 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: zing. inplace methods Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Wed Dec 12 17:07:32 2007 @@ -433,8 +433,8 @@ unwrap_spec=["self", ObjSpace]) # binary operations that return NotImplemented if they fail -# e.g. rich comparisons and coerce -for op in 'eq ne gt lt ge le coerce'.split(): +# e.g. rich comparisons, coerce and inplace ops +for op in 'eq ne gt lt ge le coerce imod iand ipow itruediv ilshift ixor irshift ifloordiv idiv isub imul iadd ior'.split(): specialname = "__%s__" % (op, ) # fool the gateway logic by giving it a real unbound method meth = new.instancemethod( Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Wed Dec 12 17:07:32 2007 @@ -435,3 +435,18 @@ raises(TypeError, "a + 1.1") assert l == [1, 1.1] + def test_iadd(self): + class A: + __metaclass__ = nclassobj + def __init__(self): + self.l = [] + def __iadd__(self, other): + self.l.append(other) + return self + a1 = a = A() + a += 1 + assert a is a1 + a += 2 + assert a is a1 + assert a.l == [1, 2] + From cfbolz at codespeak.net Wed Dec 12 17:36:43 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 12 Dec 2007 17:36:43 +0100 (CET) Subject: [pypy-svn] r49679 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071212163643.A7F5616842C@codespeak.net> Author: cfbolz Date: Wed Dec 12 17:36:42 2007 New Revision: 49679 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: cmp Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Wed Dec 12 17:36:42 2007 @@ -418,6 +418,50 @@ space.w_TypeError, space.wrap("__nonzero__() should return an int")) + def descr_cmp(self, space, w_other): # do all the work here like CPython + w_a, w_b = _coerce_helper(space, self, w_other) + if w_a is None: + w_a = self + w_b = w_other + else: + if (not isinstance(w_a, W_InstanceObject) and + not isinstance(w_b, W_InstanceObject)): + return space.cmp(w_a, w_b) + if isinstance(w_a, W_InstanceObject): + w_func = w_a.getattr(space, space.wrap('__cmp__'), False) + if w_func is not None: + w_res = space.call_function(w_func, w_b) + try: + res = space.int_w(w_res) + except OperationError, e: + if e.match(space, space.w_TypeError): + raise OperationError( + space.w_TypeError, + space.wrap("__cmp__ must return int")) + raise + if res > 0: + return space.wrap(1) + if res < 0: + return space.wrap(-1) + return space.wrap(0) + if isinstance(w_b, W_InstanceObject): + w_func = w_b.getattr(space, space.wrap('__cmp__'), False) + if w_func is not None: + w_res = space.call_function(w_func, w_a) + try: + res = space.int_w(w_res) + except OperationError, e: + if e.match(space, space.w_TypeError): + raise OperationError( + space.w_TypeError, + space.wrap("__cmp__ must return int")) + raise + if res < 0: + return space.wrap(1) + if res > 0: + return space.wrap(-1) + return space.wrap(0) + return space.w_NotImplemented rawdict = {} # unary operations @@ -485,6 +529,8 @@ unwrap_spec=['self', ObjSpace, Arguments]), __nonzero__ = interp2app(W_InstanceObject.descr_nonzero, unwrap_spec=['self', ObjSpace]), + __cmp__ = interp2app(W_InstanceObject.descr_cmp, + unwrap_spec=['self', ObjSpace, W_Root]), **rawdict ) Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Wed Dec 12 17:36:42 2007 @@ -450,3 +450,31 @@ assert a is a1 assert a.l == [1, 2] + def test_cmp(self): + class A: + __metaclass__ = nclassobj + def __coerce__(self, other): + return (1, 2) + assert cmp(A(), 1) == -1 + class A: + def __cmp__(self, other): + return 1 + class B: + pass + + a = A() + b = B() + assert cmp(a, b) == 1 + assert cmp(b, a) == -1 + + class A: + def __cmp__(self, other): + return 1L + a = A() + assert cmp(a, b) == 1 + + class A: + def __cmp__(self, other): + return "hello?" + a = A() + raises(TypeError, cmp, a, b) From cfbolz at codespeak.net Wed Dec 12 18:21:59 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 12 Dec 2007 18:21:59 +0100 (CET) Subject: [pypy-svn] r49680 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071212172159.6AEB11684E6@codespeak.net> Author: cfbolz Date: Wed Dec 12 18:21:58 2007 New Revision: 49680 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: hash Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Wed Dec 12 18:21:58 2007 @@ -462,6 +462,25 @@ return space.wrap(-1) return space.wrap(0) return space.w_NotImplemented + + def descr_hash(self, space): + w_func = self.getattr(space, space.wrap('__hash__'), False) + if w_func is None: + w_eq = self.getattr(space, space.wrap('__eq__'), False) + w_cmp = self.getattr(space, space.wrap('__cmp__'), False) + if w_eq is not None or w_cmp is not None: + raise OperationError(space.w_TypeError, + space.wrap("unhashable instance")) + else: + return space.wrap(hash(self)) + w_ret = space.call_function(w_func) + if not space.is_true(space.isinstance(w_ret, space.w_int)): + raise OperationError( + space.w_TypeError, + space.wrap("__hash__ must return int")) + return w_ret + + rawdict = {} # unary operations @@ -531,6 +550,8 @@ unwrap_spec=['self', ObjSpace]), __cmp__ = interp2app(W_InstanceObject.descr_cmp, unwrap_spec=['self', ObjSpace, W_Root]), + __hash__ = interp2app(W_InstanceObject.descr_hash, + unwrap_spec=['self', ObjSpace]), **rawdict ) Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Wed Dec 12 18:21:58 2007 @@ -457,9 +457,11 @@ return (1, 2) assert cmp(A(), 1) == -1 class A: + __metaclass__ = nclassobj def __cmp__(self, other): return 1 class B: + __metaclass__ = nclassobj pass a = A() @@ -468,13 +470,44 @@ assert cmp(b, a) == -1 class A: + __metaclass__ = nclassobj def __cmp__(self, other): return 1L a = A() assert cmp(a, b) == 1 class A: + __metaclass__ = nclassobj def __cmp__(self, other): return "hello?" a = A() raises(TypeError, cmp, a, b) + + def test_hash(self): + class A: + __metaclass__ = nclassobj + pass + hash(A()) # does not crash + class A: + def __hash__(self): + return "hello?" + a = A() + raises(TypeError, hash, a) + class A: + __metaclass__ = nclassobj + def __hash__(self): + return 1 + a = A() + assert hash(a) == 1 + class A: + __metaclass__ = nclassobj + def __cmp__(self, other): + return 1 + a = A() + raises(TypeError, hash, a) + class A: + __metaclass__ = nclassobj + def __eq__(self, other): + return 1 + a = A() + raises(TypeError, hash, a) From cfbolz at codespeak.net Wed Dec 12 18:45:44 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 12 Dec 2007 18:45:44 +0100 (CET) Subject: [pypy-svn] r49681 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071212174544.C61871684EF@codespeak.net> Author: cfbolz Date: Wed Dec 12 18:45:43 2007 New Revision: 49681 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: index Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Wed Dec 12 18:45:43 2007 @@ -480,6 +480,14 @@ space.wrap("__hash__ must return int")) return w_ret + def descr_index(self, space): + w_func = self.getattr(space, space.wrap('__index__'), False) + if w_func is not None: + return space.call_function(w_func) + raise OperationError( + space.w_TypeError, + space.wrap("object cannot be interpreted as an index")) + rawdict = {} @@ -552,6 +560,8 @@ unwrap_spec=['self', ObjSpace, W_Root]), __hash__ = interp2app(W_InstanceObject.descr_hash, unwrap_spec=['self', ObjSpace]), + __index__ = interp2app(W_InstanceObject.descr_index, + unwrap_spec=['self', ObjSpace]), **rawdict ) Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Wed Dec 12 18:45:43 2007 @@ -511,3 +511,14 @@ return 1 a = A() raises(TypeError, hash, a) + + def test_index(self): + class A: + __metaclass__ = nclassobj + def __index__(self): + return 1 + l = [1, 2, 3] + assert l[A()] == 2 + class A: + __metaclass__ = nclassobj + raises(TypeError, "l[A()]") From arigo at codespeak.net Wed Dec 12 21:52:13 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 12 Dec 2007 21:52:13 +0100 (CET) Subject: [pypy-svn] r49682 - in pypy/branch/pypy-gc-traceopt: . jit/tl/test rpython/memory rpython/memory/gc rpython/memory/gctransform Message-ID: <20071212205213.E14E91684E1@codespeak.net> Author: arigo Date: Wed Dec 12 21:52:12 2007 New Revision: 49682 Added: pypy/branch/pypy-gc-traceopt/ - copied from r49641, pypy/dist/pypy/ pypy/branch/pypy-gc-traceopt/jit/tl/test/test_tl.py - copied unchanged from r49648, pypy/dist/pypy/jit/tl/test/test_tl.py Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/base.py pypy/branch/pypy-gc-traceopt/rpython/memory/gc/generation.py pypy/branch/pypy-gc-traceopt/rpython/memory/gc/semispace.py pypy/branch/pypy-gc-traceopt/rpython/memory/gctransform/framework.py pypy/branch/pypy-gc-traceopt/rpython/memory/gctypelayout.py pypy/branch/pypy-gc-traceopt/rpython/memory/gcwrapper.py Log: In-progress, hackish enough to go to a branch: try to speed up tracing for common object shapes. So far this crashes on the llinterp and can only run after translation to C. Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gc/base.py Wed Dec 12 21:52:12 2007 @@ -30,6 +30,10 @@ def setup(self): pass + def set_typeid_count(self, count): + from pypy.rpython.memory.gc import traceopt + traceopt.set_typeid_count(self, count) + def statistics(self, index): return -1 @@ -106,6 +110,35 @@ def x_become(self, target_addr, source_addr): raise RuntimeError("no support for x_become in the GC") + def trace(self, obj, callback): + from pypy.rpython.memory.gc import traceopt + typeid = self.get_type_id(obj) + if not traceopt.optimized_trace(self, obj, typeid, callback): + self._slow_trace(obj, typeid, callback) + trace._annspecialcase_ = 'specialize:arg(2)' + + def _slow_trace(self, obj, typeid, callback): + offsets = self.offsets_to_gc_pointers(typeid) + i = 0 + while i < len(offsets): + callback(self, obj + offsets[i]) + i += 1 + if self.is_varsize(typeid): + offset = self.varsize_offset_to_variable_part( + typeid) + length = (obj + self.varsize_offset_to_length(typeid)).signed[0] + offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) + itemlength = self.varsize_item_sizes(typeid) + i = 0 + while i < length: + item = obj + offset + itemlength * i + j = 0 + while j < len(offsets): + callback(self, item + offsets[j]) + j += 1 + i += 1 + _slow_trace._annspecialcase_ = 'specialize:arg(3)' + class MovingGCBase(GCBase): moving_gc = True Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gc/generation.py Wed Dec 12 21:52:12 2007 @@ -258,30 +258,11 @@ young objects it references out of the nursery. """ self.header(obj).tid |= GCFLAG_NO_YOUNG_PTRS - typeid = self.get_type_id(obj) - offsets = self.offsets_to_gc_pointers(typeid) - i = 0 - while i < len(offsets): - pointer = obj + offsets[i] - if self.is_in_nursery(pointer.address[0]): - pointer.address[0] = self.copy(pointer.address[0]) - i += 1 - if self.is_varsize(typeid): - offset = self.varsize_offset_to_variable_part( - typeid) - length = (obj + self.varsize_offset_to_length(typeid)).signed[0] - offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) - itemlength = self.varsize_item_sizes(typeid) - i = 0 - while i < length: - item = obj + offset + itemlength * i - j = 0 - while j < len(offsets): - pointer = item + offsets[j] - if self.is_in_nursery(pointer.address[0]): - pointer.address[0] = self.copy(pointer.address[0]) - j += 1 - i += 1 + self.trace(obj, GenerationGC._drag_out_of_nursery) + + def _drag_out_of_nursery(self, refaddr): + if self.is_in_nursery(refaddr.address[0]): + refaddr.address[0] = self.copy(refaddr.address[0]) def invalidate_young_weakrefs(self): # walk over the list of objects that contain weakrefs and are in the Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gc/semispace.py Wed Dec 12 21:52:12 2007 @@ -37,6 +37,7 @@ self.AddressLinkedList = AddressLinkedList def setup(self): + MovingGCBase.setup(self) self.tospace = llarena.arena_malloc(self.space_size, True) ll_assert(bool(self.tospace), "couldn't allocate tospace") self.top_of_space = self.tospace + self.space_size @@ -256,30 +257,11 @@ return newobj def trace_and_copy(self, obj): - typeid = self.get_type_id(obj) - offsets = self.offsets_to_gc_pointers(typeid) - i = 0 - while i < len(offsets): - pointer = obj + offsets[i] - if pointer.address[0] != NULL: - pointer.address[0] = self.copy(pointer.address[0]) - i += 1 - if self.is_varsize(typeid): - offset = self.varsize_offset_to_variable_part( - typeid) - length = (obj + self.varsize_offset_to_length(typeid)).signed[0] - offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) - itemlength = self.varsize_item_sizes(typeid) - i = 0 - while i < length: - item = obj + offset + itemlength * i - j = 0 - while j < len(offsets): - pointer = item + offsets[j] - if pointer.address[0] != NULL: - pointer.address[0] = self.copy(pointer.address[0]) - j += 1 - i += 1 + self.trace(obj, SemiSpaceGC._copy_ref) + + def _copy_ref(self, refaddr): + if refaddr.address[0] != NULL: + refaddr.address[0] = self.copy(refaddr.address[0]) def is_forwarded(self, obj): return self.header(obj).forw != NULL Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gctransform/framework.py Wed Dec 12 21:52:12 2007 @@ -202,6 +202,7 @@ q_varsize_offset_to_length, q_varsize_offsets_to_gcpointers_in_var_part, q_weakpointer_offset) + gcdata.gc.set_typeid_count(len(gcdata.type_info_table)) bk = self.translator.annotator.bookkeeper Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gctypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gctypelayout.py Wed Dec 12 21:52:12 2007 @@ -136,6 +136,10 @@ self.q_varsize_offsets_to_gcpointers_in_var_part, self.q_weakpointer_offset) + def count_typeids(self): + self.can_add_new_types = False + return len(self.type_info_list) + def consider_constant(self, TYPE, value, gc): if value is not lltype.top_container(value): return Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gcwrapper.py Wed Dec 12 21:52:12 2007 @@ -27,6 +27,7 @@ self.constantroots = layoutbuilder.addresses_of_static_ptrs self.constantrootsnongc = layoutbuilder.addresses_of_static_ptrs_in_nongc + self.gc.set_typeid_count(layoutbuilder.count_typeids()) def get_roots_from_llinterp(self, with_static=True): sizeofaddr = llmemory.sizeof(llmemory.Address) From arigo at codespeak.net Wed Dec 12 21:54:38 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 12 Dec 2007 21:54:38 +0100 (CET) Subject: [pypy-svn] r49683 - pypy/branch/pypy-gc-traceopt/rpython/memory/gc Message-ID: <20071212205438.8D6521684E1@codespeak.net> Author: arigo Date: Wed Dec 12 21:54:38 2007 New Revision: 49683 Added: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py (contents, props changed) Log: Forgot this file. Added: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py ============================================================================== --- (empty file) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py Wed Dec 12 21:54:38 2007 @@ -0,0 +1,92 @@ +from pypy.rlib.unroll import unrolling_iterable +from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.memory.gctypelayout import offsets_to_gc_pointers + +# ____________________________________________________________ + +TRACEOPT_INDEXES = lltype.Array(lltype.Char, hints={'nolength': True}) + +def set_typeid_count(gc, count): + print 'SET_TYPEID_COUNT', count + gc.traceopt_index = lltype.malloc(TRACEOPT_INDEXES, count, flavor='raw') + i = 0 + while i < count: + gc.traceopt_index[i] = chr(0) + i += 1 + +def enumerate_lltypes(): + for n in range(32): + fields = [] + for i in range(5): + if n & (1< Author: arigo Date: Wed Dec 12 22:00:54 2007 New Revision: 49684 Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py Log: This doesn't just work. Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py Wed Dec 12 22:00:54 2007 @@ -36,7 +36,7 @@ for offset in unroll_offsets: callback(gc, obj + offset) tracer._annspecialcase_ = 'specialize:arg(2)' - tracer._inline_ = True + #xxx force inline tracer def compatible_shape(gc, typeid): if gc.is_varsize(typeid): @@ -50,7 +50,7 @@ return False i += 1 return True - compatible_shape._inline_ = True + #xxx force inline compatible_shape return compatible_shape, tracer From regmee at codespeak.net Wed Dec 12 22:27:47 2007 From: regmee at codespeak.net (regmee at codespeak.net) Date: Wed, 12 Dec 2007 22:27:47 +0100 (CET) Subject: [pypy-svn] r49685 - pypy/branch/clr-module-improvements/pypy/module/clr Message-ID: <20071212212747.1F2041684E8@codespeak.net> Author: regmee Date: Wed Dec 12 22:27:45 2007 New Revision: 49685 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py Log: assembly file loading with clr.load_assembly() and namespaces scaning from currently loaded assemblies Modified: pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py Wed Dec 12 22:27:45 2007 @@ -17,6 +17,7 @@ 'load_valid_namespaces': 'interp_clr.load_valid_namespaces', 'isDotNetType': 'interp_clr.isDotNetType', 'load_assembly': 'interp_clr.load_assembly', + 'list_of_loadedAssemblies': 'interp_clr.list_of_loadedAssemblies', } def setup_after_space_initialization(self): Modified: pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py Wed Dec 12 22:27:45 2007 @@ -194,17 +194,8 @@ Return: List of Valid .NET namespaces """ listOfNamespaces = [] - assembliesToScan = [ "/usr/lib/mono/1.0/mscorlib.dll", - "/usr/lib/mono/1.0/System.dll", - "/usr/lib/mono/1.0/System.Web.dll", - "/usr/lib/mono/1.0/System.Data.dll", - "/usr/lib/mono/1.0/System.Xml.dll", - "/usr/lib/mono/1.0/System.Drawing.dll" - ] - assems = [] - for assemblyName in assembliesToScan: - assems.append(System.Reflection.Assembly.LoadFile(assemblyName)) - + currentDomain = System.AppDomain.get_CurrentDomain() + assems = currentDomain.GetAssemblies() for loadedAssembly in assems: typesInAssembly = loadedAssembly.GetTypes() for type in typesInAssembly: @@ -227,19 +218,56 @@ isDotNetType.unwrap_spec = [ObjSpace, str] -def load_assembly(space, assemblyName): +def list_of_loadedAssemblies(space): + """ + return a List of currently loaded .NET assembliesy + + return: + list eg: ['System.Xml','System.Data','mscorlib'] + """ + loadedAssemblies = [] + currentDomain = System.AppDomain.get_CurrentDomain() + currentLoadedAssemblies = currentDomain.GetAssemblies() + + for lAssembly in currentLoadedAssemblies: + strName = lAssembly.get_FullName() + rindexComma = strName.find(',') + if rindexComma != -1: + loadedAssemblies.append(strName[:rindexComma]) + return space.wrap(loadedAssemblies) +list_of_loadedAssemblies.unwrap_spec = [ObjSpace] + +def load_assembly(space, assemblyPath): """ Load the given .NET assembly into the PyPy interpreter Parameters: - - assemblyName: the full name of the assembly - (e.g., ``System.Xml.dll``). + - assemblyPath: the full path of the assembly + (e.g., /usr/lib/mono/2.0/System.Data.dll). """ - # listOfNamespaces = [] - # currentDomain = System.AppDomain.get_CurrentDomain() - # asEvidence = currentDomain.get_Evidence() - pass + assemblyToLoad = "" + loadedAssemblies = space.unwrap(list_of_loadedAssemblies(space)) + + # split the name to pull out "System.Data.dll" + # then check if it has already been loaded. + rindexSlash = assemblyPath.rfind('/') + if rindexSlash != -1: + strAfterSlash = assemblyPath[rindexSlash +1 : ] + rindexDot = strAfterSlash.rfind('.') + if rindexDot != -1: + assemblyToLoad = strAfterSlash[: rindexDot] + + if assemblyToLoad in loadedAssemblies: + print " won't reload loaded assembly " + pass + else: + try: + System.Reflection.Assembly.LoadFile(assemblyPath) + print "Loaded %s"%assemblyPath + except: + print " can not load the assembly " +load_assembly.unwrap_spec = [ObjSpace, str] def load_cli_class(space, namespace, classname): """ From misto at codespeak.net Wed Dec 12 23:01:31 2007 From: misto at codespeak.net (misto at codespeak.net) Date: Wed, 12 Dec 2007 23:01:31 +0100 (CET) Subject: [pypy-svn] r49687 - in pypy/branch/newdotviewer: . test Message-ID: <20071212220131.07D991684E1@codespeak.net> Author: misto Date: Wed Dec 12 23:01:29 2007 New Revision: 49687 Added: pypy/branch/newdotviewer/ - copied from r49685, pypy/dist/dotviewer/ Modified: pypy/branch/newdotviewer/conftest.py pypy/branch/newdotviewer/drawgraph.py pypy/branch/newdotviewer/graphclient.py pypy/branch/newdotviewer/graphparse.py pypy/branch/newdotviewer/msgstruct.py pypy/branch/newdotviewer/test/__init__.py pypy/branch/newdotviewer/test/test_msgstruct.py pypy/branch/newdotviewer/test/test_translator.py Log: dotviewer refactorization Modified: pypy/branch/newdotviewer/conftest.py ============================================================================== --- pypy/dist/dotviewer/conftest.py (original) +++ pypy/branch/newdotviewer/conftest.py Wed Dec 12 23:01:29 2007 @@ -2,6 +2,6 @@ Option = py.test.config.Option option = py.test.config.addoptions("dotviewer options", - Option('--pygame', action="store_true", dest="pygame", default=False, + Option('--pygame', action="store_true", dest="pygame", default=True, help="allow interactive tests using Pygame"), ) Modified: pypy/branch/newdotviewer/drawgraph.py ============================================================================== --- pypy/dist/dotviewer/drawgraph.py (original) +++ pypy/branch/newdotviewer/drawgraph.py Wed Dec 12 23:01:29 2007 @@ -260,10 +260,10 @@ else: return (color, None) - def setscale(self, scale): + def setscale(self, scale, graph): scale = max(min(scale, self.SCALEMAX), self.SCALEMIN) self.scale = float(scale) - w, h = self.graphlayout.boundingbox + w, h = graph.boundingbox self.margin = int(self.MARGIN * scale) self.width = int(w * scale) + (2 * self.margin) self.height = int(h * scale) + (2 * self.margin) Modified: pypy/branch/newdotviewer/graphclient.py ============================================================================== --- pypy/dist/dotviewer/graphclient.py (original) +++ pypy/branch/newdotviewer/graphclient.py Wed Dec 12 23:01:29 2007 @@ -83,6 +83,8 @@ try: io.sendmsg(*msg) except IOError, ioerror: + import sys + print >>sys.stderr, "Error: sending ", msg , str(ioerror) break # wait for MSG_OK or MSG_ERROR try: Modified: pypy/branch/newdotviewer/graphparse.py ============================================================================== --- pypy/dist/dotviewer/graphparse.py (original) +++ pypy/branch/newdotviewer/graphparse.py Wed Dec 12 23:01:29 2007 @@ -45,7 +45,7 @@ else: cmdline = 'neato -Tplain' #print >> sys.stderr, '* running:', cmdline - child_in, child_out = os.popen2(cmdline, 'r') + child_in, child_out = os.popen2 (cmdline) try: import thread except ImportError: Modified: pypy/branch/newdotviewer/msgstruct.py ============================================================================== --- pypy/dist/dotviewer/msgstruct.py (original) +++ pypy/branch/newdotviewer/msgstruct.py Wed Dec 12 23:01:29 2007 @@ -128,3 +128,5 @@ def close(self): self.s.close() + + Modified: pypy/branch/newdotviewer/test/__init__.py ============================================================================== --- pypy/dist/dotviewer/test/__init__.py (original) +++ pypy/branch/newdotviewer/test/__init__.py Wed Dec 12 23:01:29 2007 @@ -0,0 +1,47 @@ +ONE_NODE_GRAPH = r''' +digraph prova { +A-> B +} +''' + + +BIG_GRAPH = r'''digraph _generated__graph { +subgraph _generated__ { +_generated__ [shape="box", label="generated", color="black", fillcolor="#a5e6f0", style="filled", width="0.75"]; +edge [label="startblock", style="dashed", color="black", dir="forward", weight="5"]; +_generated__ -> _generated____1 +_generated____1 [shape="box", label="generated__1\ninputargs: v2720\n\n", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v2720", style="solid", color="black", dir="forward", weight="5"]; +_generated____1 -> _generated____2 +_generated____2 [shape="octagon", label="generated__2\ninputargs: v2721\n\nv2722 = int_gt(v2721, (2))\l\lexitswitch: v2722", color="red", fillcolor="white", style="filled", width="0.75"]; +edge [label="False: v2721", style="dotted", color="red", dir="forward", weight="5"]; +_generated____2 -> _generated____3 +edge [label="True: v2721", style="dotted", color="red", dir="forward", weight="5"]; +_generated____2 -> _generated____4 +_generated____3 [shape="octagon", label="generated__3\ninputargs: v2723\n\nv2724 = int_gt(v2723, (0))\l\lexitswitch: v2724", color="red", fillcolor="white", style="filled", width="0.75"]; +edge [label="False: (22) v2723", style="dotted", color="red", dir="forward", weight="5"]; +_generated____3 -> _generated____5 +edge [label="True: v2723 v2724", style="dotted", color="red", dir="forward", weight="5"]; +_generated____3 -> _generated____6 +_generated____5 [shape="box", label="generated__5\ninputargs: v2727 v2725\n\nv2726 = int_sub(v2725, (1))\lv2728 = int_add(v2727, (1))\lv2729 = int_add(v2728, v2726)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v2729", style="solid", color="black", dir="forward", weight="5"]; +_generated____5 -> _generated____7 +_generated____7 [shape="box", label="generated__7\ninputargs: v2730\n\nreturn v2730", color="black", fillcolor="green", style="filled", width="0.75"]; +_generated____6 [shape="box", label="generated__6\ninputargs: v2732 v2733\n\nv2731 = same_as((17))\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v2731 v2732", style="solid", color="black", dir="forward", weight="5"]; +_generated____6 -> _generated____5 +_generated____4 [shape="box", label="generated__4\ninputargs: v2734\n\nv2735 = int_sub(v2734, (1))\lv2736 = int_add((55), v2735)\l", color="black", fillcolor="white", style="filled", width="0.75"]; +edge [label="v2736", style="solid", color="black", dir="forward", weight="5"]; +_generated____4 -> _generated____7 +} +} +''' + + +SOURCE1_TO_MSG_LIST = [('i', -998470283), ('[', 'graph_id', '1.000', '6.944', '16.278'), ('n', '_generated__', '3.750', '16.028', '1.149', '0.500', 'generated', 'filled', 'box', 'black', '#a5e6f0'), ('n', '_generated____1', '3.750', '14.556', '1.826', '0.875', 'generated__1\ninputargs: v2720\n\n', 'filled', 'box', 'black', 'white'), ('n', '_generated____2', '3.750', '12.069', '3.738', '2.509', 'generated__2\ninputargs: v2721\n\nv2722 = int_gt(v2721, (2))\\l\\lexitswitch: v2722', 'filled', 'octagon', 'red', 'white'), ('n', '_generated____3', '2.014', '8.764', '3.738', '2.509', 'generated__3\ninputargs: v2723\n\nv2724 = int_gt(v2723, (0))\\l\\lexitswitch: v2724', 'filled', 'octagon', 'red', 'white'), ('n', '_generated____4', '5.417', '6.542', '3.066', '1.417', 'generated__4\ninputargs: v2734\n\nv2735 = int_sub(v2734, (1))\\lv2736 = int_add((55), v2735)\\l', 'filled', 'box', 'black', 'white'), ('n', '_generated____5', '3.403', '2.792', '3.253', '1.688', 'generated__5\ninputargs: v2727 v2725\n\nv2726 = int_sub(v2725, (1))\\lv2728 = int_add(v2727, (1))\\lv2729 = int_add(v2728, v2726)\\l', 'filled', 'box', 'black', 'white'), ('n', '_generated____6', '3.597', '5.000', '2.472', '1.146', 'generated__6\ninputargs: v2732 v2733\n\nv2731 = same_as((17))\\l', 'filled', 'box', 'black', 'white'), ('n', '_generated____7', '4.708', '0.583', '1.826', '1.146', 'generated__7\ninputargs: v2730\n\nreturn v2730', 'filled', 'box', 'black', 'green'), ('e', '_generated__', '_generated____1', '4', '3.750', '15.778', '3.750', '15.597', '3.750', '15.361', '3.750', '15.139', 'startblock', '4.222', '15.389', 'dashed', 'black'), ('e', '_generated____1', '_generated____2', '4', '3.750', '14.111', '3.750', '13.931', '3.750', '13.708', '3.750', '13.472', 'v2720', '4.042', '13.722', 'solid', 'black'), ('e', '_generated____2', '_generated____3', '4', '3.097', '10.819', '2.986', '10.597', '2.861', '10.375', '2.750', '10.153', 'False: v2721', '3.514', '10.417', 'dotted', 'red'), ('e', '_generated____2', '_generated____4', '4', '4.125', '10.806', '4.444', '9.764', '4.889', '8.306', '5.167', '7.389', 'True: v2721', '4.833', '10.417', 'dotted', 'red'), ('e', '_generated____3', '_generated____5', '7', '1.111', '7.583', '0.542', '6.681', '0.000', '5.417', '0.569', '4.417', '0.819', '3.986', '1.222', '3.667', '1.653', '3.431', 'False: (22) v2723', '1.403', '5.000', 'dotted', 'red'), ('e', '_generated____3', '_generated____6', '7', '1.694', '7.500', '1.639', '6.958', '1.653', '6.319', '1.958', '5.833', '2.028', '5.708', '2.125', '5.597', '2.250', '5.514', 'True: v2723 v2724', '2.861', '6.542', 'dotted', 'red'), ('e', '_generated____4', '_generated____7', '7', '5.431', '5.833', '5.444', '4.917', '5.403', '3.292', '5.153', '1.944', '5.111', '1.722', '5.042', '1.486', '4.972', '1.292', 'v2736', '5.667', '4.028', 'solid', 'black'), ('e', '_generated____5', '_generated____7', '4', '3.903', '1.944', '4.028', '1.722', '4.167', '1.486', '4.306', '1.278', 'v2729', '4.444', '1.556', 'solid', 'black'), ('e', '_generated____6', '_generated____5', '4', '3.542', '4.431', '3.528', '4.236', '3.514', '4.000', '3.486', '3.778', 'v2731 v2732', '4.125', '4.028', 'solid', 'black'), (']',)] + + +def compare ( alist , blist): + for a,b in zip(alist,blist): + assert a == b + \ No newline at end of file Modified: pypy/branch/newdotviewer/test/test_msgstruct.py ============================================================================== --- pypy/dist/dotviewer/test/test_msgstruct.py (original) +++ pypy/branch/newdotviewer/test/test_msgstruct.py Wed Dec 12 23:01:29 2007 @@ -14,3 +14,5 @@ assert decodemessage(encoded[:-1]) == (None, encoded[:-1]) assert decodemessage(encoded) == (args, '') assert decodemessage(encoded + 'FooBar') == (args, 'FooBar') + + Modified: pypy/branch/newdotviewer/test/test_translator.py ============================================================================== --- pypy/dist/dotviewer/test/test_translator.py (original) +++ pypy/branch/newdotviewer/test/test_translator.py Wed Dec 12 23:01:29 2007 @@ -20,8 +20,37 @@ return len(divisors) == 2 -def test_annotated(): +def xtest_annotated(): from pypy.translator.interactive import Translation t = Translation(is_prime) t.annotate([int]) - t.viewcg() + #t.viewcg() + +def test_new_annotated(): + from pypy.translator.interactive import Translation + t = Translation(is_prime) + t.annotate([int]) + process (show_flowgraph(t.context)) + process (show_callgraph(t.context)) + +def show_flowgraph (test): + """Shows the control flow graph with annotations if computed. + Requires 'dot' and pygame.""" + from pypy.translator.tool.graphpage import FlowGraphPage + flowpage = FlowGraphPage(test) + page = flowpage.content() + return page + +def process (page): + from dotviewer import display + from dotviewer.test.test_graphparse import graph_from_dot_buffer + graph = graph_from_dot_buffer (page.source) + dis = display.MyDisplay ((800,600)) + dis.display_graph (graph) + dis.driver.loop () + +def show_callgraph (test): + """Shows the whole call graph and the class hierarchy, based on + the computed annotations.""" + from pypy.translator.tool.graphpage import TranslatorPage + return TranslatorPage (test).content() From misto at codespeak.net Wed Dec 12 23:07:04 2007 From: misto at codespeak.net (misto at codespeak.net) Date: Wed, 12 Dec 2007 23:07:04 +0100 (CET) Subject: [pypy-svn] r49688 - in pypy/branch/newdotviewer: . test Message-ID: <20071212220704.F33301684E1@codespeak.net> Author: misto Date: Wed Dec 12 23:07:04 2007 New Revision: 49688 Added: pypy/branch/newdotviewer/display.py pypy/branch/newdotviewer/drivers.py pypy/branch/newdotviewer/protocol.py pypy/branch/newdotviewer/render.py pypy/branch/newdotviewer/test/test_graphclient.py pypy/branch/newdotviewer/test/test_graphparse.py pypy/branch/newdotviewer/test/test_service.py Modified: pypy/branch/newdotviewer/test/test_translator.py Log: added test files and temp refactoring modules Added: pypy/branch/newdotviewer/display.py ============================================================================== --- (empty file) +++ pypy/branch/newdotviewer/display.py Wed Dec 12 23:07:04 2007 @@ -0,0 +1,203 @@ +from graphdisplay import GraphDisplay +from drivers import PyGameDriver +from render import MyGraphRenderer + +import pygame # TODO BAD BAD + + +class View: + SCALEMAX = 100 + SCALEMIN = 1 + + def __init__ (self, display): + self.display = display + self.margin = 0.6 + self.setscale (75) + self.setoffset (0, 0) + + def calculate_zoom_to_fit (self, view): + width, height = self.display.size + + return min(float(width) / view.width, + float(height) / view.height, + float(view.SCALEMAX) / view.scale) + + def setoffset (self, offsetx, offsety): + "Set the (x,y) origin of the rectangle where the graph will be rendered." + self.ofsx = offsetx + self.ofsy = offsety + + #ofsx = property (lambda self, off: off - self.margin ) + #ofsy = property (lambda self, off: off - self.margin ) + + def setscale (self, scale): + scale = max(min(scale, self.SCALEMAX), self.SCALEMIN) + self.scale = float(scale) + self.bboxh = self.display.size[1] + + #margin = property (lambda self: int(self.MARGIN * self.scale)) + width = property (lambda self: int(self.display.size[0] * self.scale) + (2 * self.margin)) + height = property (lambda self: int(self.display.size[1] * self.scale) + (2 * self.margin)) + + def map (self, x, y): + "" + mx = (x * self.scale) + 2*self.margin*self.scale + my = (self.display.size[1] - y* self.scale) - 2 * self.margin*self.scale + return int(mx), int(my) + + def shiftoffset(self, dx, dy): + self.ofsx += dx + self.ofsy += dy + + def reoffset(self, swidth, sheight): + offsetx = noffsetx = self.ofsx + offsety = noffsety = self.ofsy + width = self.width + height = self.height + + # if it fits, center it, otherwise clamp + if width <= swidth: + noffsetx = (width - swidth) // 2 + else: + noffsetx = min(max(0, offsetx), width - swidth) + + if height <= sheight: + noffsety = (height - sheight) // 2 + else: + noffsety = min(max(0, offsety), height - sheight) + + self.ofsx = noffsetx + self.ofsy = noffsety + + def shiftscale (self, factor, fix=None): + if fix is None: + fixx, fixy = self.display.size + fixx //= 2 + fixy //= 2 + else: + fixx, fixy = fix + x, y = self.revmap(fixx, fixy) + w, h = self.display.size + + self.setscale(self.scale * factor, w, h) + newx, newy = self.map(x, y) + self.shiftoffset(newx - fixx, newy - fixy) + + def calculate_scale_to_fit (self, (width,height)): + maxw, maxh = self.display.size + scale = min(float(maxw) / width, + float(maxh) / height) + self.setscale (scale) + + def revmap(self, px, py): + return ((px + (self.ofsx - self.margin)) / self.scale, + self.bboxh - (py + (self.ofsy - self.margin)) / self.scale) + + def getboundingbox(self): + "Get the rectangle where the graph will be rendered." + return (-self.ofsx, -self.ofsy, self.width, self.height) + + def visible(self, x1, y1, x2, y2): + """Is any part of the box visible (i.e. within the bounding box)? + + We have to perform clipping ourselves because with big graphs the + coordinates may sometimes become longs and cause OverflowErrors + within pygame. + """ + return x1 < self.width and x2 > 0 and y1 < self.height and y2 > 0 + + def visible_node (self, x, y, w, h): + + + x, y = self.map (x, y) + nw2 = int(w * self.scale)//2 + nh2 = int(h * self.scale)//2 + + return True + + return x-nw2 < w and x+nw2 > 0 and y-nh2 < h and y+nh2 > 0 + + def visible_edge (self, x1, y1, x2, y2): + return True + w, h = self.width, self.height + x1, y1 = self.map(x1, y1) + x2, y2 = self.map(x2, y2) + return x1 < w and y1 < h and x2 > 0 and y2 > 0 + +class MyDisplay (GraphDisplay): + + def __init__(self, size): + self.driver = PyGameDriver() + self.viewers_history = [] + self.forward_viewers_history = [] + self.highlight_word = None + self.highlight_obj = None + self.viewer = None + self.method_cache = {} + self.key_cache = {} + self.ascii_key_cache = {} + self.status_bar_height = 0 + self.searchstr = None + self.searchpos = 0 + self.searchresults = [] + self.size = size + self.driver.resize (size) + self.initialize_keys() + self.font = pygame.font.Font (self.STATUSBARFONT, 16) + + def updated_viewer(self, view,keep_highlight=False): + width , height = self.size + view.reoffset(width, height) + if not keep_highlight: + self.sethighlight() + self.update_status_bar() + self.must_redraw = True + + def redraw_now(self): + self.status_bar_height = 0 + pygame.display.flip() + self.must_redraw = False + + def display_graph (self, graph): + rend = MyGraphRenderer (self.driver) + view = View (self) + view.calculate_scale_to_fit(graph.boundingbox) + rend.render (graph, view) + self.redraw_now () + + def process_event (self, event): + graph = event.get ('layout', None) + if graph: + self.display_graph (graph) + self.driver.update() + + def controller_process_event(self, event): + method = self.method_cache.get(event.type, KeyError) + if method is KeyError: + method = getattr(self, 'process_%s' % (pygame.event.event_name(event.type),), None) + self.method_cache[method] = method + if method is not None: + method(event) + + def notifymousepos(self, pos): + pass + + def notifyclick (self, pos): + pass + + def zoom (self, amount): + pass + + def loop (self): + self.dragging = self.click_origin = self.click_time = None + try: + EventQueue = [] + while True: + if not EventQueue: + EventQueue.append(pygame.event.wait()) + EventQueue.extend(pygame.event.get()) + + self.controller_process_event(EventQueue.pop(0)) + + except StopIteration: + pass \ No newline at end of file Added: pypy/branch/newdotviewer/drivers.py ============================================================================== --- (empty file) +++ pypy/branch/newdotviewer/drivers.py Wed Dec 12 23:07:04 2007 @@ -0,0 +1,132 @@ +import pygame + +COLOR = { + 'black': (0,0,0), + 'white': (255,255,255), + 'red': (255,0,0), + 'green': (0,255,0), + 'blue': (0,0,255), + 'yellow': (255,255,0), + } +import os +this_dir = os.path.dirname(os.path.abspath(__file__)) +FONT = os.path.join(this_dir, 'cyrvetic.ttf') +FIXEDFONT = os.path.join(this_dir, 'VeraMoBd.ttf') + + +# TODO +class FakeImg: + fakeimg = True + def get_size(self): + return (0 , 0) + +class FakeFont: + fakefont = True + def render (self, *args): + return FakeImg() + +class PyGameDriver: + + def __init__ (self): + self.FONTCACHE = {} + self.size = 0,0 + + def resize(self, (w,h)): + self.width = w + self.height = h + self.size = w,h + pygame.display.init() + pygame.font.init() + self.screen = pygame.display.set_mode((w, h), pygame.HWSURFACE|pygame.RESIZABLE, 32) + + def combine(self, color1, color2, alpha): + r1, g1, b1 = color1 + r2, g2, b2 = color2 + beta = 1.0 - alpha + return (int(r1 * alpha + r2 * beta), + int(g1 * alpha + g2 * beta), + int(b1 * alpha + b2 * beta)) + + def highlight_color (self,color): + if color == (0, 0, 0): # black becomes magenta + return (255, 0, 255) + elif color == (255, 255, 255): # white becomes yellow + return (255, 255, 0) + intensity = sum(color) + if intensity > 191 * 3: + return self.combine(color, (128, 192, 0), 0.2) + else: + return self.combine(color, (255, 255, 0), 0.2) + + def parse_webcolor (self, name): + return (int(name[1:3],16), int(name[3:5],16), int(name[5:7],16)) + + def getcolor (self, name, default): + if name in COLOR: + return COLOR[name] + elif name.startswith('#') and len(name) == 7: + rval = COLOR[name] = self.parse_webcolor (name) + return rval + else: + return default + + def get_canvas_size (self): + return self.width, self.height + + def process_event (self, event, display): + def display_async_quit(): + pygame.event.post(pygame.event.Event(QUIT)) + + def display_async_cmd(**kwds): + pygame.event.post(pygame.event.Event(USEREVENT, **kwds)) + + def getfont (self, size, fixedfont=False): + selected = None + if size in self.FONTCACHE: + selected = self.FONTCACHE[size] + elif size < 5: + self.FONTCACHE[size] = None + else: + if fixedfont: + filename = FIXEDFONT + else: + filename = FONT + selected = self.FONTCACHE[size] = pygame.font.Font(filename, size) + return selected or FakeFont() + + def fill (self, *args): + self.screen.fill(*args) + + def draw_lines (self, *args): + pygame.draw.lines (self.screen, *args) + + def draw_rect (self, *args): + pygame.draw.rect(self.screen, *args) + + def draw_polygon (self, *args): + pygame.draw.polygon (self.screen, *args) + + def update (self): + pygame.display.flip() + + def draw_ellipse(self, *args): + pygame.draw.ellipse(self.screen, *args) + + def blit_image (self, *args): + img = args[0] + if getattr(img,'fakeimg', False): + return + self.screen.blit (*args) + + def render_font (self, font, *args): + img = None + try: + try: + img = font.render (*args) + except pygame.error, e: + # Try *with* anti-aliasing to work around a bug in SDL + img = font.render (*args) + except pygame.error: + del parts[i] # Text has zero width + return img + Added: pypy/branch/newdotviewer/protocol.py ============================================================================== --- (empty file) +++ pypy/branch/newdotviewer/protocol.py Wed Dec 12 23:07:04 2007 @@ -0,0 +1,169 @@ +import msgstruct +import sys + +class FakeMessageTransport: + def __init__ (self): + self.source=None + self.sentmessages = [] + self.closed = False + + def recvmsg (self): + while self.source: + return self.source.pop (0) + raise EOFError() + + def sendmsg (self, *args): + self.sentmessages.append (args) + + def close (self): + self.closed = True + +class Session: + def __init__ (self, transport=FakeMessageTransport()): + self.history = [] + self.pagecache = {} + self.save_tmp_file = False + self.protocol = MessageProtocol(transport) + + def show (self, page): + self.history.append (page) + self.reload (page) + + def getpage (self, graph_id): + page = self.history[graph_id] + result = None + try: + result = self.pagecache[page] + except KeyError: + result = page.content() + self.pagecache.clear() # a cache of a single entry should be enough + self.pagecache[page] = result + return result + + def reload (self,page): + if self.save_tmp_file: + f = open(save_tmp_file, 'w') + f.write(page.source) + f.close() + self.protocol.send_page (page) + + + def send_error(self, e): + try: + errmsg = str(e) + if errmsg: + errmsg = '%s: %s' % (e.__class__.__name__, errmsg) + else: + errmsg = '%s' % (e.__class__.__name__,) + self.transport.sendmsg(msgstruct.CMSG_SAY, errmsg) + except Exception: + pass + +#from dotviewer import msgstruct +class MessageProtocol: + INIT = []#(msgstruct.CMSG_INIT, msgstruct.MAGIC)] + + def __init__ (self, transport): + self.transport = transport + self.send_messages = self.first_message #dynamic state pattern + + def first_message (self, message): + self._send_messages ((MessageProtocol.INIT)) + self.send_messages = self._send_messages + + def _send_messages (self, messages): + ioerror = None + for msg in messages: + try: + self.transport.sendmsg (*msg) + except IOError, ioerror: + print >>sys.stderr, "Error: sending ", msg , str(ioerror) + break + return #XXX + # wait for MSG_OK or MSG_ERROR + try: + while True: + msg = self.transport.recvmsg() + if msg[0] == msgstruct.MSG_OK: + break + except EOFError: + ioerror = IOError("connexion unexpectedly closed " + "(graphserver crash?)") + if ioerror is not None: + raise ioerror + + def send_page (self, page): + import graphparse + messages = graphparse.parse_dot(0, page.content(), page.links,getattr(page, 'fixedfont', False)) + self.send_messages (messages) + +class MessageDrivenPainter: + + def __init__ (self,service): + self.service = service + + def process_all (self, messages): + try: + for msg in messages: + self.process_message (msg) + except EOFError: + from drawgraph import display_async_quit + display_async_quit() + + def generate_layout (self, messages): + try: + for msg in messages: + self.process_message (msg) + except EOFError, errmsg: + print "Cannot process ", msg + return self.newlayout + + def process_message (self, msg): + fn = self.MESSAGES.get(msg[0]) + if fn: + fn(self, *msg[1:]) + else: + self.log("unknown message code %r" % (msg[0],)) + + def log(self, info): + print >> sys.stderr, info + + def cmsg_start_graph(self, graph_id, scale, width, height, *rest): + from drawgraph import GraphLayout + self.newlayout = GraphLayout(float(scale), float(width), float(height)) + + def cmsg_add_node(self, *args): + self.newlayout.add_node(*args) + + def cmsg_add_edge(self, *args): + self.newlayout.add_edge(*args) + + def cmsg_add_link(self, word, *info): + if len(info) == 1: + info = info[0] + elif len(info) >= 4: + info = (info[0], info[1:4]) + self.newlayout.links[word] = info + + def cmsg_fixed_font(self, *rest): + self.newlayout.fixedfont = True + + def cmsg_stop_graph(self, *rest): + self.service.post_event (layout=self.newlayout) + + def cmsg_missing_link(self, *rest): + self.setlayout(None) + + def cmsg_say(self, errmsg, *rest): + self.service.post_event (say=errmsg) + + MESSAGES = { + msgstruct.CMSG_START_GRAPH: cmsg_start_graph, + msgstruct.CMSG_ADD_NODE: cmsg_add_node, + msgstruct.CMSG_ADD_EDGE: cmsg_add_edge, + msgstruct.CMSG_ADD_LINK: cmsg_add_link, + msgstruct.CMSG_FIXED_FONT: cmsg_fixed_font, + msgstruct.CMSG_STOP_GRAPH: cmsg_stop_graph, + msgstruct.CMSG_MISSING_LINK:cmsg_missing_link, + msgstruct.CMSG_SAY: cmsg_say, + } \ No newline at end of file Added: pypy/branch/newdotviewer/render.py ============================================================================== --- (empty file) +++ pypy/branch/newdotviewer/render.py Wed Dec 12 23:07:04 2007 @@ -0,0 +1,346 @@ +import drawgraph + + + +class BoundingBox: #TODO this should be view ... + def __init__(self, node, xcenter, ycenter, view): + + self.xcenter, self.ycenter = xcenter,ycenter + self.boxwidth = int(node.w * view.scale) #TODO + self.boxheight = int(node.h * view.scale) + + self.wmax = self.boxwidth + self.hmax = self.boxheight + + def finalize (self): + self.xleft = self.xcenter - self.wmax//2 + self.xright = self.xcenter + self.wmax//2 + self.ytop = self.ycenter - self.hmax//2 + self.x = self.xcenter-self.boxwidth//2 + self.y = self.ycenter-self.boxheight//2 + +class Graph: + def __init__(self): + self.fixedfont = True + self.nodes = [] + self.edges = [] + + def __equals__ (self, other): + #fixedfont TODO + return other.nodes == self.nodes and other.edges == self.edges + +class MyGraphRenderer (drawgraph.GraphRenderer): + def __init__ (self, driver, scale=75): + self.driver = driver + self.textzones = [] + self.highlightwords = [] + + def setscale(self, scale, graph): + scale = max(min(scale, self.SCALEMAX), self.SCALEMIN) + self.scale = float(scale) + w, h = graph.boundingbox + self.margin = int(self.MARGIN * scale) + self.width = int(w * scale) + (2 * self.margin) + self.height = int(h * scale) + (2 * self.margin) + self.bboxh = h + size = int(15 * (scale-10) / 75) + self.font = self.driver.getfont (size, graph) + + def computevisible (self, graph, view): + visible = Graph() + + for node in graph.nodes.values(): + if view.visible_node (node.x,node.y, node.w, node.h): + visible.nodes.append(node) + + for edge in graph.edges: + x1, y1, x2, y2 = edge.limits() + if view.visible_edge (x1, y1, x2, y2): + visible.edges.append (edge) + return visible + + def draw_background (self, view): + bbox = view.getboundingbox() + + ox, oy, width, height = bbox + dpy_width, dpy_height = self.driver.size + # some versions of the SDL misinterpret widely out-of-range values, + # so clamp them + if ox < 0: + width += ox + ox = 0 + if oy < 0: + height += oy + oy = 0 + if width > dpy_width: + width = dpy_width + if height > dpy_height: + height = dpy_height + + self.driver.fill((224, 255, 224), (ox, oy, width, height)) + + # gray off-bkgnd areas + gray = (128, 128, 128) + if ox > 0: + self.driver.fill(gray, (0, 0, ox, dpy_height)) + if oy > 0: + self.driver.fill(gray, (0, 0, dpy_width, oy)) + w = dpy_width - (ox + width) + if w > 0: + self.driver.fill(gray, (dpy_width-w, 0, w, dpy_height)) + h = dpy_height - (oy + height) + if h > 0: + self.driver.fill(gray, (0, dpy_height-h, dpy_width, h)) + + + def render (self, graph, view): + + # self.driver.setscale (scale,graph) the view has the scale TODO + # self.driver.setoffset (0,0) the view knows the offset + + visible = self.computevisible (graph, view) + self.draw_background (view) + + # draw the graph and record the position of texts + del self.textzones[:] + for cmd in self.draw_commands (visible, view): + cmd() + + def draw_image (self, img, bbox): + w, h = img.get_size() + img.draw (bbox.xcenter-w//2, bbox.ytop+bbox.y) #XXXX WTF ??? + + def if_font_none (self, commands, lines, graph, bgcolor, bbox): + if lines: + raw_line = lines[0].replace('\\l','').replace('\r','') + if raw_line: + for size in (12, 10, 8, 6, 4): + font = self.driver.getfont (size, graph.fixedfont) + img = MyTextSnippet(self.driver, self, font, raw_line, (0, 0, 0), bgcolor) + w, h = img.get_size() + if (w >= bbox.boxwidth or h >= bbox.boxheight): + continue + else: + if w > bbox.wmax: bbox.wmax = w + def cmd(bbox, img=img): + self.draw_image (img, bbox) + commands.append (cmd) + bbox.hmax += h + break + + def draw_node_commands (self, node, graph, bbox, view): + xcenter, ycenter = view.map(node.x, node.y) + + boxwidth = int(node.w * view.scale) #TODO + boxheight = int(node.h * view.scale) + + fgcolor = self.driver.getcolor(node.color, (0,0,0)) + bgcolor = self.driver.getcolor(node.fillcolor, (255,255,255)) + if node.highlight: + fgcolor = self.driver.highlight_color(fgcolor) + bgcolor = self.driver.highlight_color(bgcolor) + + text = node.label + lines = text.replace('\\l','\\l\n').replace('\r','\r\n').split('\n') + # ignore a final newline + if not lines[-1]: + del lines[-1] + + commands = [] + bkgndcommands = [] + + self.font = None + if self.font is None: + self.if_font_none(commands, lines, graph, bgcolor, bbox) + else: + for line in lines: + raw_line = line.replace('\\l','').replace('\r','') or ' ' + img = MyTextSnippet(self, raw_line, (0, 0, 0), bgcolor) + w, h = img.get_size() + if w > bbox.wmax: bbox.wmax = w + if raw_line.strip(): + if line.endswith('\\l'): + def cmd(bbox,img=img): + img.draw(view.xleft, view.ytop+view.y) + elif line.endswith('\r'): + def cmd(bbox, img=img): + img.draw(view.xright-view.w, view.ytop+view.y) + else: + def cmd(bbox, img=img): + img.draw(view.xcenter-view.w//2, view.ytop+y) + commands.append(cmd) + bbox.hmax += h + #hmax += 8 + + # we know the bounding box only now; setting these variables will + # have an effect on the values seen inside the cmd() functions above + + self.draw_node_shape (node, bkgndcommands, commands, bbox, boxheight, boxwidth, bgcolor, fgcolor) + return bkgndcommands, commands + + + def draw_node_shape (self, node, bkgndcommands, commands, view, boxheight, boxwidth, bgcolor, fgcolor): + if node.shape == 'box': + def cmd (view): + rect = (view.x-1, view.y-1, boxwidth+2, boxheight+2) + self.driver.fill(bgcolor, rect) + bkgndcommands.append (cmd) + def cmd(view): + #pygame.draw.rect(self.screen, fgcolor, rect, 1) + rect = (view.x-1, view.y-1, boxwidth+2, boxheight+2) + self.driver.draw_rect (fgcolor, rect, 1) + commands.append(cmd) + elif node.shape == 'ellipse': + def cmd(view): + rect = (view.x-1, view.y-1, boxwidth+2, boxheight+2) + self.driver.draw_ellipse(bgcolor, rect, 0) + bkgndcommands.append(cmd) + def cmd(view): + rect = (view.x-1, view.y-1, boxwidth+2, boxheight+2) + self.driver.draw_ellipse(fgcolor, rect, 1) + commands.append(cmd) + elif node.shape == 'octagon': + import math + def cmd(view): + step = 1-math.sqrt(2)/2 + points = [(int(view.x+boxwidth*fx), int(view.y+boxheight*fy)) + for fx, fy in [(step,0), (1-step,0), + (1,step), (1,1-step), + (1-step,1), (step,1), + (0,1-step), (0,step)]] + self.driver.draw_polygon(bgcolor, points, 0) + bkgndcommands.append(cmd) + def cmd(view): + step = 1-math.sqrt(2)/2 + points = [(int(view.x+boxwidth*fx), int(view.y+boxheight*fy)) + for fx, fy in [(step,0), (1-step,0), + (1,step), (1,1-step), + (1-step,1), (step,1), + (0,1-step), (0,step)]] + self.driver.draw_polygon(fgcolor, points, 1) + commands.append(cmd) + + + + def draw_edge_body (self, points, fgcolor): + #pygame.draw.lines(self.screen, fgcolor, False, points) + self.driver.draw_lines (fgcolor, False, points) + + def draw_edge (self, edge, edgebodycmd, edgeheadcmd, view): + fgcolor = self.driver.getcolor(edge.color, (0,0,0)) + if edge.highlight: + fgcolor = self.driver.highlight_color(fgcolor) + points = [view.map(*xy) for xy in edge.bezierpoints()] + + edgebodycmd.append (Command (self.draw_edge_body, points, fgcolor)) + + points = [view.map(*xy) for xy in edge.arrowhead()] + if points: + def drawedgehead(points=points, fgcolor=fgcolor): + self.driver.draw_polygon(fgcolor, points, 0) + edgeheadcmd.append(drawedgehead) + + if edge.label: + x, y = view.map(edge.xl, edge.yl) + font = self.driver.getfont (10) #TODO style + img = MyTextSnippet (self.driver, self, font, edge.label, (0, 0, 0)) + w, h = img.get_size() + if view.visible(x-w//2, y-h//2, x+w//2, y+h//2): + def drawedgelabel(img=img, x1=x-w//2, y1=y-h//2): + img.draw(x1, y1) + edgeheadcmd.append(drawedgelabel) + + def draw_commands(self, graph, view): + nodebkgndcmd = [] + nodecmd = [] + + for node in graph.nodes: + xcenter,ycenter = view.map (node.x, node.y) + bbox = BoundingBox (node, xcenter,ycenter, view) + + cmd1, cmd2 = self.draw_node_commands(node, graph, bbox,view) + bbox.finalize() + nodebkgndcmd += [ Command (c, bbox) for c in cmd1 ] + nodecmd += [ Command (c, bbox) for c in cmd2 ] + + edgebodycmd = [] + edgeheadcmd = [] + for edge in graph.edges: + self.draw_edge(edge, edgebodycmd, edgeheadcmd, view) + + return edgebodycmd + nodebkgndcmd + edgeheadcmd + nodecmd + +class Command: + def __init__ (self, fun, *args): + self.fun = fun + self.args = args + def __call__ (self): + self.fun (*self.args) + +from drawgraph import TextSnippet + +class MyTextSnippet (TextSnippet): + + def __init__(self, driver, renderer, font, text, fgcolor, bgcolor=None): + self.renderer = renderer + self.driver = driver + self.imgs = [] + self.parts = [] + import re + re_nonword=re.compile(r'([^0-9a-zA-Z_.]+)') + + parts = self.parts + for word in re_nonword.split(text): + if not word: + continue + if word in renderer.highlightwords: + fg, bg = renderer.wordcolor(word) + bg = bg or bgcolor + else: + fg, bg = fgcolor, bgcolor + parts.append((word, fg, bg)) + # consolidate sequences of words with the same color + for i in range(len(parts)-2, -1, -1): + if parts[i][1:] == parts[i+1][1:]: + word, fg, bg = parts[i] + parts[i] = word + parts[i+1][0], fg, bg + del parts[i+1] + # delete None backgrounds + for i in range(len(parts)): + if parts[i][2] is None: + parts[i] = parts[i][:2] + + + # render parts + i = 0 + while i < len(parts): + part = parts[i] + word = part[0] + img = self.driver.render_font (font, word, False, *part[1:]) + if img: + self.imgs.append(img) + i += 1 + + try: + + try: + img = font.render(word, False, *part[1:]) + except pygame.error, e: + # Try *with* anti-aliasing to work around a bug in SDL + img = font.render(word, True, *part[1:]) + + except pygame.error: + del parts[i] # Text has zero width + else: + self.imgs.append(img) + i += 1 + + + def draw (self, x, y): + for part, img in zip(self.parts, self.imgs): + word = part[0] + self.renderer.driver.blit_image (img, (x, y)) + + w, h = img.get_size() + self.renderer.textzones.append((x, y, w, h, word)) + x += w \ No newline at end of file Added: pypy/branch/newdotviewer/test/test_graphclient.py ============================================================================== --- (empty file) +++ pypy/branch/newdotviewer/test/test_graphclient.py Wed Dec 12 23:07:04 2007 @@ -0,0 +1,267 @@ +from dotviewer import graphclient +from dotviewer import protocol +from dotviewer import graphpage + +import Queue + +class MyPage (graphpage.GraphPage): + + def __init__(self, source): + self.source = source + + def content(self): + return self.source + +import threading +import Queue + +from dotviewer.graphdisplay import GraphDisplay + +class FakeDisplay: + "I am responsable to handle video state" + def __init__ (self, size= (100, 100)): + self.size = 100, 100 + +from dotviewer.display import MyDisplay + +class FakeImg: + fakeimg = True + def get_size(self): + return (0 , 0) + +class FakeFont: + fakefont = True + def render (self, *args): + return FakeImg() + +class FakeDriver: + + def highlight_color (self,color): + return None + + def getcolor (self, name, default): + return None + + def __init__(self): + self.events = [] + self.actions = [] + self.size = 100,100 + + def process_event (self, event, display): + self.events.append( event) + if event.get('layout', None): + self.set_layout (event.get('layout')) + + def get_canvas_size (self): + return 1000,1000 # WARNING this could affect a lot + + + def getfont (self, size, fixedfont=False): + return FakeFont() + + def render_font (self, font, *args): + return font.render (*args) + + def fill (self, *args): + self.actions.append (( "Filled ", args )) + + def draw_lines (self, *args): + self.actions.append (("drawing lines",args)) + + def draw_rect (self, *args): + self.actions.append (( "drawing rect",args)) + + def draw_polygon (self, *args): + self.actions.append (( "drawing rect",args)) + + def draw_ellipsis(self, *args): + self.actions.append (( "drawing rect",args)) + + def blit_image (self, *args): + self.actions.append (( "blitting image",args)) + +class DisplayService (threading.Thread): + def __init__ (self, display, **kwds): + threading.Thread.__init__(self,**kwds) + self.cmdqueue = Queue.Queue() + self.display = display + self.alive = False + + def post_event (self, **event): + self.cmdqueue.put (dict(**event)) + + def stop (self): + self.alive = False + + def run (self): + self.dragging = self.click_origin = self.click_time = None + self.alive = True + try: + + while self.alive: +# if self.must_redraw and not EventQueue: +# self.redraw_now() + +# if self.cmdqueue: +# wait_for_events() + try: + event = self.cmdqueue.get(block=True,timeout=0.5) + self.display.process_event (event) + + except Queue.Empty: + pass + except StopIteration: + pass + + + +from dotviewer import drawgraph + +from test_graphparse import graph_from_dot_buffer + + + + +#class Frame: +# def __init__(self): +# self.x = 0 +# self.y = 0 +# self.width = 100 +# self.height = 100 +# +# def at (self, x, y): +# self.x = x +# self.y = y +# +# def point (self, x,y): +# scale = 1.5 +# return scale + (x * scale), (self.height - (y * scale)) + + +def test_view_width_must_be_scaled (): + display = FakeDisplay() + display.size = 1, 1 + v = View (display) + v.margin = 0 + v.setscale (100) + assert v.width == 100 + assert v.height == 100 + +def test_scale_commands_doesnot_stacks (): + display = FakeDisplay((1,1)) + v = View (display) + v.margin = 0 + v.setscale (1) + + assert v.getboundingbox() == (0,0,100,100) + + v.setscale (10) + v.setscale (10) + assert v.getboundingbox() == (0,0,1000,1000) + +def test_map (): + display = FakeDisplay() + display.size = 100, 100 + v = View (display) + v.margin = 0 + v.setscale (1.0) + assert v.map (0,0) == (0, 100) + +def test_boundingbox(): + display = FakeDisplay() + display.size = 100, 100 + v = View (display) + v.margin = 0 + v.setscale (1.0) + assert v.getboundingbox() == (0,0, 100, 100) + +def test_shift_offset (): + display = FakeDisplay() + display.size = 100, 100 + v = View (display) + v.MARGIN = 0 + + v.setoffset(0, 0) + x,y,w,h = v.getboundingbox() + assert x == 0 + assert y == 0 + + v.shiftoffset (10, 10) + x,y,w,h = v.getboundingbox() + assert x == -10 + assert y == -10 + + assert h,w == display.size + +def test_simple_graph_must_be_visible (): + graph = graph_from_dot_buffer (test.ONE_NODE_GRAPH) + assert len(graph.nodes) == 2 + display = FakeDisplay () + view = View (display) + driver = FakeDriver() + + view.owidth, view.oheight = graph.boundingbox + view.calculate_scale_to_fit (graph.boundingbox) + + x,y,w,h = view.getboundingbox() + rw , rh = graph.boundingbox + assert w,h == int (rw, rh) + + g = MyGraphRenderer (driver) + + for name,node in graph.nodes.iteritems(): + assert view.visible_node (node.x, node.y, node.w, node.h) + + visible = g.computevisible (graph, view) + assert visible.nodes + assert visible.edges + + assert len(visible.nodes) == len(graph.nodes.values()) + + assert len(visible.edges) == len(graph.edges) + +from dotviewer.render import MyGraphRenderer +from dotviewer.display import View + +def test_graph_renderer (): + graph = graph_from_dot_buffer (test.BIG_GRAPH) + driver = FakeDriver() + display = FakeDisplay () + g = MyGraphRenderer (driver) + view = View (display) + g.render (graph, view) + + +def test_view_fit_to_bbox (): + display = FakeDisplay((100,100)) + view = View (display) + + scale = view.calculate_scale_to_fit ((10,10)) + + assert view.scale == 10 + +def test_graph_renderer_with_pygame (): + #PYGAME + size = 800,600 + display = MyDisplay (size) + + graph = graph_from_dot_buffer (test.ONE_NODE_GRAPH) + display.display_graph (graph) + display.driver.update() + + graph = graph_from_dot_buffer (test.BIG_GRAPH) + display.display_graph (graph) + display.loop() + + +from dotviewer import graphclient +from dotviewer import graphpage +from dotviewer import test + +import sys +import os +this_dir = os.path.dirname(os.path.abspath(__file__)) +GRAPHSERVER = os.path.join(this_dir, 'graphserver.py') + + + \ No newline at end of file Added: pypy/branch/newdotviewer/test/test_graphparse.py ============================================================================== --- (empty file) +++ pypy/branch/newdotviewer/test/test_graphparse.py Wed Dec 12 23:07:04 2007 @@ -0,0 +1,350 @@ +from dotviewer import graphparse +import os, sys, re + +from dotviewer import msgstruct as msgx + +from dotviewer.drawgraph import GraphLayout + +class GraphGenerator: + + def __init__ (self): + self._current = None + + def new_graph (self,graph_id, header): + name, scale, w, h = header + self._current = GraphLayout (float(scale), float(w),float(h)) + + def new_node (self, node): + self._current.add_node (*node[1:]) + + def new_edge (self, line): + self._current.add_edge (*line[1:]) + + def end (self): + pass + + def get_graph (self): + return self._current + +class GraphParseEventGenerator: + def __init__ (self): + self._current = None + + def new_graph (self,graph_id, header): + self._current = [] + item = (msgx.CMSG_START_GRAPH, graph_id) + tuple(header[1:]) + self._current.append (item) + return item + + def new_node (self, line): + item = (msgx.CMSG_ADD_NODE,) + tuple(line[1:]) + self._current.append (item) + return item + + def new_edge (self, line): + item = (msgx.CMSG_ADD_EDGE,) + tuple(line[1:]) + self._current.append (item) + return item + + def new_link (self, word, statusbartext, color=(255,255,255)): + item = (msgx.CMSG_ADD_LINK, word, + statusbartext, color[0], color[1], color[2]) + self._current.append (item) + return item + + def fixed_font (self): + item = (msgx.CMSG_FIXED_FONT,) + self._current.append (item) + return item + + def end(self): + item = (msgx.CMSG_STOP_GRAPH,) + self._current.append(item) + return item + + def get_graph (self): + return self._current + +re_nonword = re.compile(r'([^0-9a-zA-Z_.]+)') +re_plain = re.compile(r'graph [-0-9.]+ [-0-9.]+ [-0-9.]+$', re.MULTILINE) +re_digraph = re.compile(r'\b(graph|digraph)\b', re.IGNORECASE) + +def guess_type(content): + # try to see whether it is a directed graph or not, + # or already a .plain file + # XXX not a perfect heursitic + if re_plain.match(content): + return 'plain' # already a .plain file + # look for the word 'graph' or 'digraph' followed by a '{'. + bracepos = None + lastfound = '' + for match in re_digraph.finditer(content): + position = match.start() + if bracepos is None: + bracepos = content.find('{', position) + if bracepos < 0: + break + elif position > bracepos: + break + lastfound = match.group() + if lastfound.lower() == 'digraph': + return 'dot' + if lastfound.lower() == 'graph': + return 'neato' + print >> sys.stderr, "Warning: could not guess file type, using 'dot'" + return 'unknown' + +class DotTool: + cmdline = 'dot -Tplain' + + def convert (self,content): + child_in, child_out = os.popen2 (self.cmdline) + def bkgndwrite(f, data): + f.write(data) + f.close() + try: + import thread + except ImportError: + bkgndwrite(child_in, content) + else: + thread.start_new_thread(bkgndwrite, (child_in, content)) + plaincontent = child_out.read() + child_out.close() + + return plaincontent + +def convert_using_tool (content, contenttype): + if contenttype == 'plain': + return content + + if contenttype != 'neato': + cmdline = 'dot -Tplain' + else: + cmdline = 'neato -Tplain' + return DotTool().convert (content) + +def convert_using_codespeak (content, contenttype): + import urllib + request = urllib.urlencode({'dot': content}) + url = 'http://codespeak.net/pypy/convertdot.cgi' + print >> sys.stderr, '* posting:', url + g = urllib.urlopen(url, data=request) + result = [] + while True: + data = g.read(16384) + if not data: + break + result.append(data) + g.close() + plaincontent = ''.join(result) + # very simple-minded way to give a somewhat better error message + if plaincontent.startswith(' i + 2: + texts.append(line[i]) + if line[0] == 'stop': + break + +def extract_header (lines): + header = splitline(lines.pop(0)) + if header[0] != 'graph': + raise PlainParseError("should start with 'graph'") + return header + +def normalize_lines (plaincontent): + lines = plaincontent.splitlines(True) + + for i in xrange(len(lines)-2, -1, -1): + if lines[i].endswith('\\\n'): # line ending in '\' + lines[i] = lines[i][:-2] + lines[i+1] + del lines[i+1] + return lines + +def generate_links (text): + # only include the links that really appear in the graph + seen = {} + for text in texts: + for word in re_nonword.split(text): + if word and word in links and word not in seen: + t = links[word] + if isinstance(t, tuple): + statusbartext, color = t + else: + statusbartext = t + color = None + generator.new_link (word, statusbartext, color) + seen[word] = True + +def parse_plain(graph_id, plaincontent, links={}, fixedfont=False, generator=GraphParseEventGenerator()): + lines = normalize_lines (plaincontent) + generator.new_graph (graph_id, extract_header(lines)) + + texts = [] + parse_body(lines, generator,texts) + + if links: + generate_links(text) + + +class PlainParseError(Exception): + pass + +def my_parse_dot ( graph_id, content, links={}, fixedfont = False): + contenttype = guess_type(content) + message_list = None + + try: + plaincontent = dot2plain(content, contenttype, use_codespeak=False) + except PlainParseError: + # failed, retry via codespeak + plaincontent = dot2plain(content, contenttype, use_codespeak=True) + + generator = GraphParseEventGenerator() + parse_plain(graph_id, plaincontent, links, fixedfont, generator) + if fixedfont: + generator.fixed_font () + generator.end() + return generator.get_graph () + +class DotParseError(Exception): + pass +class DotReader: + def __init__ (self): + self.tool = DotTool() + self.format = re.compile(r'[^\s"]\S*|["]["]|["].*?[^\\]["]') + + def select_tool (self, buffer): + return self.tool + + def read_words (self, line): + result = [] + for word in self.format.findall (line): + if word.startswith('"'): + word = eval(word) + result.append(word) + return result + + def traverse (self, buffer, visitor): + tool = self.select_tool (buffer) + converted = tool.convert (buffer) + + lines = self.collapse_lines (converted) + header = self.read_words(lines.pop(0)) + + if header[0] != 'graph': + raise DotParseError("should start with 'graph'") + + visitor.new_graph (0, header) + text = self.parse_body (lines, visitor) + self.generate_links (text) + res = visitor.end() + + def generate_links (self,texts, links={}): + # only include the links that really appear in the graph + seen = {} + for text in texts: + for word in re_nonword.split(text): + if word and word in links and word not in seen: + t = links[word] + if isinstance(t, tuple): + statusbartext, color = t + else: + statusbartext = t + color = None + generator.new_link (word, statusbartext, color) + seen[word] = True + + def parse_node (self, words): + pass + + def parse_body (self, lines, generator): + texts = [] + for line in lines: + line = self.read_words (line) + if line[0] == 'node': + if len(line) != 11: + raise PlainParseError("bad 'node'") + generator.new_node (line) + texts.append(line[6]) + if line[0] == 'edge': + generator.new_edge (line) + i = 4 + 2 * int(line[3]) + if len(line) > i + 2: + texts.append(line[i]) + if line[0] == 'stop': + break + return texts + + def collapse_lines (self, buffer): + lines = buffer.splitlines(True) + for i in xrange(len(lines)-2, -1, -1): + if lines[i].endswith('\\\n'): # line ending in '\' + lines[i] = lines[i][:-2] + lines[i+1] + del lines[i+1] + return lines + +def graph_from_dot_buffer (dotbuffer): + reader = DotReader () + visitor = GraphGenerator() + reader.traverse (dotbuffer, visitor) + return visitor.get_graph() + +from dotviewer import test +def test_graph_dot_buffer (): + graph = graph_from_dot_buffer (test.ONE_NODE_GRAPH) + assert len(graph.nodes) == 2 + +def test_graph_dot_buffer (): + graph = graph_from_dot_buffer ('digraph {') + assert len(graph.nodes) == 0 + +def xxtest_graphparse (): + g = graphparse.parse_plain ("id", graphparse.dot2plain(test.BIG_GRAPH, '')) + result = [ m for m in g] + assert result == my_parse_dot ("id", test.BIG_GRAPH) + + g = graphparse.parse_plain ("id", graphparse.dot2plain(test.BIG_GRAPH, ''), fixedfont=False) + result = [ m for m in g] + my_result = my_parse_dot("id", test.BIG_GRAPH,fixedfont=False) + assert result == my_result + +def xtest_remote_codespeak (): + g1 = graphparse.parse_dot("id", graphparse.dot2plain(SOURCE1, '', use_codespeak=True)) + g2 = graphparse.parse_dot("id", dot2plain(SOURCE1, '', use_codespeak=True)) + + for a,b in zip(g1,g2): + assert a == b + + \ No newline at end of file Added: pypy/branch/newdotviewer/test/test_service.py ============================================================================== --- (empty file) +++ pypy/branch/newdotviewer/test/test_service.py Wed Dec 12 23:07:04 2007 @@ -0,0 +1,13 @@ +from test_graphclient import DisplayService, graph_from_dot_buffer + +from dotviewer import protocol, test +from dotviewer.display import MyDisplay +from drivers import PyGameDriver + +def test_display_service (): + page = graph_from_dot_buffer(test.BIG_GRAPH) + width, height = 800, 600 + display = MyDisplay ((width, height)) + service = DisplayService (display) + service.start() + service.post_event (layout= page) Modified: pypy/branch/newdotviewer/test/test_translator.py ============================================================================== --- pypy/branch/newdotviewer/test/test_translator.py (original) +++ pypy/branch/newdotviewer/test/test_translator.py Wed Dec 12 23:07:04 2007 @@ -47,7 +47,7 @@ graph = graph_from_dot_buffer (page.source) dis = display.MyDisplay ((800,600)) dis.display_graph (graph) - dis.driver.loop () + dis.loop () def show_callgraph (test): """Shows the whole call graph and the class hierarchy, based on From cfbolz at codespeak.net Wed Dec 12 23:30:15 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 12 Dec 2007 23:30:15 +0100 (CET) Subject: [pypy-svn] r49691 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071212223015.AA02E1684DE@codespeak.net> Author: cfbolz Date: Wed Dec 12 23:30:15 2007 New Revision: 49691 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: add contains Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Wed Dec 12 23:30:15 2007 @@ -488,6 +488,21 @@ space.w_TypeError, space.wrap("object cannot be interpreted as an index")) + def descr_contains(self, space, w_obj): + w_func = self.getattr(space, space.wrap('__contains__'), False) + if w_func is not None: + return space.wrap(space.is_true(space.call_function(w_func, w_obj))) + # now do it ourselves + w_iter = space.iter(self) + while 1: + try: + w_x = space.next(w_iter) + except OperationError, e: + if e.match(space, space.w_StopIteration): + return space.w_False + if space.eq_w(w_x, w_obj): + return space.w_True + rawdict = {} @@ -562,6 +577,8 @@ unwrap_spec=['self', ObjSpace]), __index__ = interp2app(W_InstanceObject.descr_index, unwrap_spec=['self', ObjSpace]), + __contains__ = interp2app(W_InstanceObject.descr_contains, + unwrap_spec=['self', ObjSpace, W_Root]), **rawdict ) Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Wed Dec 12 23:30:15 2007 @@ -522,3 +522,36 @@ class A: __metaclass__ = nclassobj raises(TypeError, "l[A()]") + + def test_contains(self): + class A: + __metaclass__ = nclassobj + def __contains__(self, other): + return True + a = A() + assert 1 in a + assert None in a + class A: + __metaclass__ = nclassobj + a = A() + raises(TypeError, "1 in a") + class A: + __metaclass__ = nclassobj + def __init__(self): + self.list = [1, 2, 3, 4, 5] + def __iter__(self): + return iter(self.list) + a = A() + for i in range(1, 6): + assert i in a + class A: + __metaclass__ = nclassobj + def __init__(self): + self.list = [1, 2, 3, 4, 5] + def __len__(self): + return len(self.list) + def __getitem__(self, i): + return self.list[i] + a = A() + for i in range(1, 6): + assert i in a From arigo at codespeak.net Thu Dec 13 11:45:35 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 11:45:35 +0100 (CET) Subject: [pypy-svn] r49697 - in pypy/dist/pypy/annotation: . test Message-ID: <20071213104535.8EB4C168461@codespeak.net> Author: arigo Date: Thu Dec 13 11:45:34 2007 New Revision: 49697 Modified: pypy/dist/pypy/annotation/specialize.py pypy/dist/pypy/annotation/test/test_annrpython.py Log: Support for "specialize:arg" receiving a bound method of an instance. This is a kind-of-natural extension of the idea of specializing when receiving a function. Modified: pypy/dist/pypy/annotation/specialize.py ============================================================================== --- pypy/dist/pypy/annotation/specialize.py (original) +++ pypy/dist/pypy/annotation/specialize.py Thu Dec 13 11:45:34 2007 @@ -421,7 +421,19 @@ return constgraphbuilder def specialize_argvalue(funcdesc, args_s, *argindices): - key = tuple([args_s[i].const for i in argindices]) + from pypy.annotation.model import SomePBC + key = [] + for i in argindices: + s = args_s[i] + if s.is_constant(): + key.append(s.const) + elif isinstance(s, SomePBC) and len(s.descriptions) == 1: + # for test_specialize_arg_bound_method + key.append(s.descriptions.keys()[0]) + else: + raise Exception("specialize:arg(%d): argument not constant: %r" + % (i, s)) + key = tuple(key) return funcdesc.cachedgraph(key) def specialize_argtype(funcdesc, args_s, *argindices): Modified: pypy/dist/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/test/test_annrpython.py (original) +++ pypy/dist/pypy/annotation/test/test_annrpython.py Thu Dec 13 11:45:34 2007 @@ -1099,6 +1099,32 @@ assert graph1 in a.translator.graphs assert graph2 in a.translator.graphs + def test_specialize_arg_bound_method(self): + class GC(object): + def trace(self, callback, arg): + return callback(arg) + trace._annspecialcase_ = "specialize:arg(1)" + + def callback1(self, arg): + self.x = arg + return "hello" + + def callback2(self, arg): + self.y = arg + return 6 + + def f(): + gc = GC() + s1 = gc.trace(gc.callback1, None) + n2 = gc.trace(gc.callback2, 7) + return (s1, n2, gc.x, gc.y) + a = self.RPythonAnnotator() + s = a.build_types(f, []) + assert s.items[0].const == "hello" + assert s.items[1].const == 6 + assert s.items[2].const == None + assert s.items[3].const == 7 + def test_assert_list_doesnt_lose_info(self): class T(object): pass From arigo at codespeak.net Thu Dec 13 11:52:20 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 11:52:20 +0100 (CET) Subject: [pypy-svn] r49699 - pypy/dist/pypy/annotation/test Message-ID: <20071213105220.BC577168461@codespeak.net> Author: arigo Date: Thu Dec 13 11:52:20 2007 New Revision: 49699 Modified: pypy/dist/pypy/annotation/test/test_annrpython.py Log: Harder and harder... (I want to use this to write a trace() function in GCBase) Modified: pypy/dist/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/test/test_annrpython.py (original) +++ pypy/dist/pypy/annotation/test/test_annrpython.py Thu Dec 13 11:52:20 2007 @@ -1101,29 +1101,31 @@ def test_specialize_arg_bound_method(self): class GC(object): - def trace(self, callback, arg): - return callback(arg) + def trace(self, callback, *args): + return callback(*args) trace._annspecialcase_ = "specialize:arg(1)" - def callback1(self, arg): - self.x = arg + def callback1(self, arg1): + self.x = arg1 return "hello" - def callback2(self, arg): - self.y = arg + def callback2(self, arg2, arg3): + self.y = arg2 + self.z = arg3 return 6 def f(): gc = GC() - s1 = gc.trace(gc.callback1, None) - n2 = gc.trace(gc.callback2, 7) - return (s1, n2, gc.x, gc.y) + s1 = gc.trace(gc.callback1, "foo") + n2 = gc.trace(gc.callback2, 7, 2) + return (s1, n2, gc.x, gc.y, gc.z) a = self.RPythonAnnotator() s = a.build_types(f, []) assert s.items[0].const == "hello" assert s.items[1].const == 6 - assert s.items[2].const == None + assert s.items[2].const == "foo" assert s.items[3].const == 7 + assert s.items[4].const == 2 def test_assert_list_doesnt_lose_info(self): class T(object): From arigo at codespeak.net Thu Dec 13 13:07:27 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 13:07:27 +0100 (CET) Subject: [pypy-svn] r49701 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20071213120727.092C6168487@codespeak.net> Author: arigo Date: Thu Dec 13 13:07:27 2007 New Revision: 49701 Modified: pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/gc/marksweep.py pypy/dist/pypy/rpython/memory/gc/semispace.py Log: Remove some code duplication by moving the tracing logic in the base class. Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Thu Dec 13 13:07:27 2007 @@ -106,6 +106,34 @@ def x_become(self, target_addr, source_addr): raise RuntimeError("no support for x_become in the GC") + def trace(self, obj, callback, arg): + """Enumerate the locations inside the given obj that can contain + GC pointers. For each such location, callback(pointer, arg) is + called, where 'pointer' is an address inside the object. + Typically, 'callback' is a bound method and 'arg' can be None. + """ + typeid = self.get_type_id(obj) + offsets = self.offsets_to_gc_pointers(typeid) + i = 0 + while i < len(offsets): + callback(obj + offsets[i], arg) + i += 1 + if self.is_varsize(typeid): + offset = self.varsize_offset_to_variable_part( + typeid) + length = (obj + self.varsize_offset_to_length(typeid)).signed[0] + offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) + itemlength = self.varsize_item_sizes(typeid) + i = 0 + while i < length: + item = obj + offset + itemlength * i + j = 0 + while j < len(offsets): + callback(item + offsets[j], arg) + j += 1 + i += 1 + trace._annspecialcase_ = 'specialize:arg(2)' + class MovingGCBase(GCBase): moving_gc = True Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Thu Dec 13 13:07:27 2007 @@ -258,30 +258,11 @@ young objects it references out of the nursery. """ self.header(obj).tid |= GCFLAG_NO_YOUNG_PTRS - typeid = self.get_type_id(obj) - offsets = self.offsets_to_gc_pointers(typeid) - i = 0 - while i < len(offsets): - pointer = obj + offsets[i] - if self.is_in_nursery(pointer.address[0]): - pointer.address[0] = self.copy(pointer.address[0]) - i += 1 - if self.is_varsize(typeid): - offset = self.varsize_offset_to_variable_part( - typeid) - length = (obj + self.varsize_offset_to_length(typeid)).signed[0] - offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) - itemlength = self.varsize_item_sizes(typeid) - i = 0 - while i < length: - item = obj + offset + itemlength * i - j = 0 - while j < len(offsets): - pointer = item + offsets[j] - if self.is_in_nursery(pointer.address[0]): - pointer.address[0] = self.copy(pointer.address[0]) - j += 1 - i += 1 + self.trace(obj, self._trace_drag_out, None) + + def _trace_drag_out(self, pointer, ignored): + if self.is_in_nursery(pointer.address[0]): + pointer.address[0] = self.copy(pointer.address[0]) def invalidate_young_weakrefs(self): # walk over the list of objects that contain weakrefs and are in the Modified: pypy/dist/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/dist/pypy/rpython/memory/gc/marksweep.py Thu Dec 13 13:07:27 2007 @@ -458,33 +458,18 @@ STAT_BYTES_MALLOCED = 1 STATISTICS_NUMBERS = 2 - def add_reachable_to_stack(self, obj, objects): + def get_type_id(self, obj): size_gc_header = self.gcheaderbuilder.size_gc_header gc_info = obj - size_gc_header hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - typeid = hdr.typeid >> 1 - offsets = self.offsets_to_gc_pointers(typeid) - i = 0 - while i < len(offsets): - pointer = obj + offsets[i] - objects.append(pointer.address[0]) - i += 1 - if self.is_varsize(typeid): - offset = self.varsize_offset_to_variable_part( - typeid) - length = (obj + self.varsize_offset_to_length(typeid)).signed[0] - obj += offset - offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) - itemlength = self.varsize_item_sizes(typeid) - i = 0 - while i < length: - item = obj + itemlength * i - j = 0 - while j < len(offsets): - pointer = item + offsets[j] - objects.append(pointer.address[0]) - j += 1 - i += 1 + return hdr.typeid >> 1 + + def add_reachable_to_stack(self, obj, objects): + self.trace(obj, self._add_reachable, objects) + + def _add_reachable(pointer, objects): + objects.append(pointer.address[0]) + _add_reachable = staticmethod(_add_reachable) def statistics(self, index): # no memory allocation here! @@ -675,48 +660,18 @@ # reinstall the pool that was current at the beginning of x_clone() clonedata.pool = self.x_swap_pool(curpool) - def add_reachable_to_stack2(self, obj, objects, target_addr, source_addr): + def add_reachable_to_stack2(self, obj, objects): size_gc_header = self.gcheaderbuilder.size_gc_header gc_info = obj - size_gc_header hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) if hdr.typeid & 1: return - typeid = hdr.typeid >> 1 - offsets = self.offsets_to_gc_pointers(typeid) - i = 0 - while i < len(offsets): - pointer = obj + offsets[i] - # ------------------------------------------------- - # begin difference from collect - if pointer.address[0] == target_addr: - pointer.address[0] = source_addr - # end difference from collect - # ------------------------------------------------- - objects.append(pointer.address[0]) - i += 1 - if self.is_varsize(typeid): - offset = self.varsize_offset_to_variable_part( - typeid) - length = (obj + self.varsize_offset_to_length(typeid)).signed[0] - obj += offset - offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) - itemlength = self.varsize_item_sizes(typeid) - i = 0 - while i < length: - item = obj + itemlength * i - j = 0 - while j < len(offsets): - pointer = item + offsets[j] - # ------------------------------------------------- - # begin difference from collect - if pointer.address[0] == target_addr: - pointer.address[0] = source_addr - ## end difference from collect - # ------------------------------------------------- - objects.append(pointer.address[0]) - j += 1 - i += 1 + self.trace(obj, self._add_reachable_and_rename, objects) + def _add_reachable_and_rename(self, pointer, objects): + if pointer.address[0] == self.x_become_target_addr: + pointer.address[0] = self.x_become_source_addr + objects.append(pointer.address[0]) def x_become(self, target_addr, source_addr): # 1. mark from the roots, and also the objects that objects-with-del @@ -754,6 +709,9 @@ curr_heap_size = 0 freed_size = 0 + self.x_become_target_addr = target_addr + self.x_become_source_addr = source_addr + # mark objects reachable by objects with a finalizer, but not those # themselves. add their size to curr_heap_size, since they always # survive the collection @@ -763,7 +721,7 @@ typeid = hdr.typeid >> 1 gc_info = llmemory.cast_ptr_to_adr(hdr) obj = gc_info + size_gc_header - self.add_reachable_to_stack2(obj, objects, target_addr, source_addr) + self.add_reachable_to_stack2(obj, objects) addr = llmemory.cast_ptr_to_adr(hdr) size = self.fixed_size(typeid) if self.is_varsize(typeid): @@ -777,7 +735,7 @@ # stack until the stack is empty while objects.non_empty(): #mark curr = objects.pop() - self.add_reachable_to_stack2(curr, objects, target_addr, source_addr) + self.add_reachable_to_stack2(curr, objects) gc_info = curr - size_gc_header hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) if hdr.typeid & 1: Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Thu Dec 13 13:07:27 2007 @@ -256,30 +256,11 @@ return newobj def trace_and_copy(self, obj): - typeid = self.get_type_id(obj) - offsets = self.offsets_to_gc_pointers(typeid) - i = 0 - while i < len(offsets): - pointer = obj + offsets[i] - if pointer.address[0] != NULL: - pointer.address[0] = self.copy(pointer.address[0]) - i += 1 - if self.is_varsize(typeid): - offset = self.varsize_offset_to_variable_part( - typeid) - length = (obj + self.varsize_offset_to_length(typeid)).signed[0] - offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) - itemlength = self.varsize_item_sizes(typeid) - i = 0 - while i < length: - item = obj + offset + itemlength * i - j = 0 - while j < len(offsets): - pointer = item + offsets[j] - if pointer.address[0] != NULL: - pointer.address[0] = self.copy(pointer.address[0]) - j += 1 - i += 1 + self.trace(obj, self._trace_copy, None) + + def _trace_copy(self, pointer, ignored): + if pointer.address[0] != NULL: + pointer.address[0] = self.copy(pointer.address[0]) def is_forwarded(self, obj): return self.header(obj).forw != NULL From arigo at codespeak.net Thu Dec 13 13:21:11 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 13:21:11 +0100 (CET) Subject: [pypy-svn] r49703 - in pypy/dist/pypy/rpython/memory: . gc gctransform Message-ID: <20071213122111.78DBE168487@codespeak.net> Author: arigo Date: Thu Dec 13 13:21:10 2007 New Revision: 49703 Modified: pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gctypelayout.py Log: Add a query function has_gcptr_in_varsize(). Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Thu Dec 13 13:21:10 2007 @@ -7,7 +7,8 @@ needs_write_barrier = False needs_zero_gc_pointers = True - def set_query_functions(self, is_varsize, getfinalizer, + def set_query_functions(self, is_varsize, has_gcptr_in_varsize, + getfinalizer, offsets_to_gc_pointers, fixed_size, varsize_item_sizes, varsize_offset_to_variable_part, @@ -16,6 +17,7 @@ weakpointer_offset): self.getfinalizer = getfinalizer self.is_varsize = is_varsize + self.has_gcptr_in_varsize = has_gcptr_in_varsize self.offsets_to_gc_pointers = offsets_to_gc_pointers self.fixed_size = fixed_size self.varsize_item_sizes = varsize_item_sizes @@ -118,7 +120,7 @@ while i < len(offsets): callback(obj + offsets[i], arg) i += 1 - if self.is_varsize(typeid): + if self.has_gcptr_in_varsize(typeid): offset = self.varsize_offset_to_variable_part( typeid) length = (obj + self.varsize_offset_to_length(typeid)).signed[0] Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Thu Dec 13 13:21:10 2007 @@ -116,7 +116,7 @@ # types of the GC information tables OFFSETS_TO_GC_PTR = lltype.Array(lltype.Signed) TYPE_INFO = lltype.Struct("type_info", - ("isvarsize", lltype.Bool), + ("isvarsize", lltype.Signed), ("finalizer", self.FINALIZERTYPE), ("fixedsize", lltype.Signed), ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), @@ -130,7 +130,11 @@ def q_is_varsize(typeid): ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].isvarsize + return gcdata.type_info_table[typeid].isvarsize != 0 + + def q_has_gcptr_in_varsize(typeid): + ll_assert(typeid > 0, "invalid type_id") + return gcdata.type_info_table[typeid].isvarsize > 1 def q_finalizer(typeid): ll_assert(typeid > 0, "invalid type_id") @@ -194,6 +198,7 @@ gcdata.gc.setup() gcdata.gc.set_query_functions( q_is_varsize, + q_has_gcptr_in_varsize, q_finalizer, q_offsets_to_gc_pointers, q_fixed_size, Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/gctypelayout.py Thu Dec 13 13:21:10 2007 @@ -38,7 +38,7 @@ info["finalizer"] = self.make_finalizer_funcptr_for_type(TYPE) info["weakptrofs"] = weakpointer_offset(TYPE) if not TYPE._is_varsize(): - info["isvarsize"] = False + info["isvarsize"] = 0 info["fixedsize"] = llarena.round_up_for_allocation( llmemory.sizeof(TYPE)) info["ofstolength"] = -1 @@ -47,7 +47,6 @@ # varsize ones, the GC must anyway compute the size at run-time # and round up that result. else: - info["isvarsize"] = True info["fixedsize"] = llmemory.sizeof(TYPE, 0) if isinstance(TYPE, lltype.Struct): ARRAY = TYPE._flds[TYPE._arrayfld] @@ -69,11 +68,13 @@ assert isinstance(ARRAY, lltype.Array) if ARRAY.OF != lltype.Void: offsets = offsets_to_gc_pointers(ARRAY.OF) - info["varofstoptrs"] = self.offsets2table(offsets, ARRAY.OF) - info["varitemsize"] = llmemory.sizeof(ARRAY.OF) else: - info["varofstoptrs"] = self.offsets2table((), lltype.Void) - info["varitemsize"] = llmemory.sizeof(ARRAY.OF) + offsets = () + info["varofstoptrs"] = self.offsets2table(offsets, ARRAY.OF) + info["varitemsize"] = llmemory.sizeof(ARRAY.OF) + info["isvarsize"] = 1 + (len(offsets) > 0) + # isvarsize is set to 1 if there are no gc ptrs in the + # varsize part, and to 2 if there are. return type_id def offsets2table(self, offsets, TYPE): @@ -91,7 +92,11 @@ def q_is_varsize(self, typeid): assert typeid > 0 - return self.type_info_list[typeid]["isvarsize"] + return self.type_info_list[typeid]["isvarsize"] != 0 + + def q_has_gcptr_in_varsize(self, typeid): + assert typeid > 0 + return self.type_info_list[typeid]["isvarsize"] > 1 def q_finalizer(self, typeid): assert typeid > 0 @@ -127,6 +132,7 @@ def get_query_functions(self): return (self.q_is_varsize, + self.q_has_gcptr_in_varsize, self.q_finalizer, self.q_offsets_to_gc_pointers, self.q_fixed_size, From arigo at codespeak.net Thu Dec 13 13:23:10 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 13:23:10 +0100 (CET) Subject: [pypy-svn] r49704 - pypy/branch/pypy-gc-traceopt/rpython/memory/gc Message-ID: <20071213122310.B12E0168450@codespeak.net> Author: arigo Date: Thu Dec 13 13:23:09 2007 New Revision: 49704 Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py Log: Clearer field names. Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py Thu Dec 13 13:23:09 2007 @@ -19,10 +19,9 @@ fields = [] for i in range(5): if n & (1< Author: arigo Date: Thu Dec 13 13:23:23 2007 New Revision: 49705 Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/base.py pypy/branch/pypy-gc-traceopt/rpython/memory/gctransform/framework.py pypy/branch/pypy-gc-traceopt/rpython/memory/gctypelayout.py Log: Merge of the trunk's r49703. Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/memory/gc/base.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gc/base.py Thu Dec 13 13:23:23 2007 @@ -7,7 +7,8 @@ needs_write_barrier = False needs_zero_gc_pointers = True - def set_query_functions(self, is_varsize, getfinalizer, + def set_query_functions(self, is_varsize, has_gcptr_in_varsize, + getfinalizer, offsets_to_gc_pointers, fixed_size, varsize_item_sizes, varsize_offset_to_variable_part, @@ -16,6 +17,7 @@ weakpointer_offset): self.getfinalizer = getfinalizer self.is_varsize = is_varsize + self.has_gcptr_in_varsize = has_gcptr_in_varsize self.offsets_to_gc_pointers = offsets_to_gc_pointers self.fixed_size = fixed_size self.varsize_item_sizes = varsize_item_sizes @@ -123,7 +125,7 @@ while i < len(offsets): callback(self, obj + offsets[i]) i += 1 - if self.is_varsize(typeid): + if self.has_gcptr_in_varsize(typeid): offset = self.varsize_offset_to_variable_part( typeid) length = (obj + self.varsize_offset_to_length(typeid)).signed[0] Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gctransform/framework.py Thu Dec 13 13:23:23 2007 @@ -116,7 +116,7 @@ # types of the GC information tables OFFSETS_TO_GC_PTR = lltype.Array(lltype.Signed) TYPE_INFO = lltype.Struct("type_info", - ("isvarsize", lltype.Bool), + ("isvarsize", lltype.Signed), ("finalizer", self.FINALIZERTYPE), ("fixedsize", lltype.Signed), ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), @@ -130,7 +130,11 @@ def q_is_varsize(typeid): ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].isvarsize + return gcdata.type_info_table[typeid].isvarsize != 0 + + def q_has_gcptr_in_varsize(typeid): + ll_assert(typeid > 0, "invalid type_id") + return gcdata.type_info_table[typeid].isvarsize > 1 def q_finalizer(typeid): ll_assert(typeid > 0, "invalid type_id") @@ -194,6 +198,7 @@ gcdata.gc.setup() gcdata.gc.set_query_functions( q_is_varsize, + q_has_gcptr_in_varsize, q_finalizer, q_offsets_to_gc_pointers, q_fixed_size, Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/memory/gctypelayout.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gctypelayout.py Thu Dec 13 13:23:23 2007 @@ -38,7 +38,7 @@ info["finalizer"] = self.make_finalizer_funcptr_for_type(TYPE) info["weakptrofs"] = weakpointer_offset(TYPE) if not TYPE._is_varsize(): - info["isvarsize"] = False + info["isvarsize"] = 0 info["fixedsize"] = llarena.round_up_for_allocation( llmemory.sizeof(TYPE)) info["ofstolength"] = -1 @@ -47,7 +47,6 @@ # varsize ones, the GC must anyway compute the size at run-time # and round up that result. else: - info["isvarsize"] = True info["fixedsize"] = llmemory.sizeof(TYPE, 0) if isinstance(TYPE, lltype.Struct): ARRAY = TYPE._flds[TYPE._arrayfld] @@ -69,11 +68,13 @@ assert isinstance(ARRAY, lltype.Array) if ARRAY.OF != lltype.Void: offsets = offsets_to_gc_pointers(ARRAY.OF) - info["varofstoptrs"] = self.offsets2table(offsets, ARRAY.OF) - info["varitemsize"] = llmemory.sizeof(ARRAY.OF) else: - info["varofstoptrs"] = self.offsets2table((), lltype.Void) - info["varitemsize"] = llmemory.sizeof(ARRAY.OF) + offsets = () + info["varofstoptrs"] = self.offsets2table(offsets, ARRAY.OF) + info["varitemsize"] = llmemory.sizeof(ARRAY.OF) + info["isvarsize"] = 1 + (len(offsets) > 0) + # isvarsize is set to 1 if there are no gc ptrs in the + # varsize part, and to 2 if there are. return type_id def offsets2table(self, offsets, TYPE): @@ -91,7 +92,11 @@ def q_is_varsize(self, typeid): assert typeid > 0 - return self.type_info_list[typeid]["isvarsize"] + return self.type_info_list[typeid]["isvarsize"] != 0 + + def q_has_gcptr_in_varsize(self, typeid): + assert typeid > 0 + return self.type_info_list[typeid]["isvarsize"] > 1 def q_finalizer(self, typeid): assert typeid > 0 @@ -127,6 +132,7 @@ def get_query_functions(self): return (self.q_is_varsize, + self.q_has_gcptr_in_varsize, self.q_finalizer, self.q_offsets_to_gc_pointers, self.q_fixed_size, From arigo at codespeak.net Thu Dec 13 13:30:01 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 13:30:01 +0100 (CET) Subject: [pypy-svn] r49706 - pypy/branch/pypy-gc-traceopt/rpython/memory/gc Message-ID: <20071213123001.60542168450@codespeak.net> Author: arigo Date: Thu Dec 13 13:30:01 2007 New Revision: 49706 Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/base.py pypy/branch/pypy-gc-traceopt/rpython/memory/gc/generation.py pypy/branch/pypy-gc-traceopt/rpython/memory/gc/semispace.py pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py Log: Intermediate checkin (this doesn't run). Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/memory/gc/base.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gc/base.py Thu Dec 13 13:30:01 2007 @@ -1,7 +1,8 @@ from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rlib.debug import ll_assert +from pypy.rpython.memory.gc import traceopt -class GCBase(object): +class GCBase(traceopt.TraceOptGCBase): _alloc_flavor_ = "raw" moving_gc = False needs_write_barrier = False @@ -32,10 +33,6 @@ def setup(self): pass - def set_typeid_count(self, count): - from pypy.rpython.memory.gc import traceopt - traceopt.set_typeid_count(self, count) - def statistics(self, index): return -1 @@ -112,18 +109,11 @@ def x_become(self, target_addr, source_addr): raise RuntimeError("no support for x_become in the GC") - def trace(self, obj, callback): - from pypy.rpython.memory.gc import traceopt - typeid = self.get_type_id(obj) - if not traceopt.optimized_trace(self, obj, typeid, callback): - self._slow_trace(obj, typeid, callback) - trace._annspecialcase_ = 'specialize:arg(2)' - - def _slow_trace(self, obj, typeid, callback): + def _slow_trace(self, obj, typeid, callback, arg): offsets = self.offsets_to_gc_pointers(typeid) i = 0 while i < len(offsets): - callback(self, obj + offsets[i]) + callback(obj + offsets[i], arg) i += 1 if self.has_gcptr_in_varsize(typeid): offset = self.varsize_offset_to_variable_part( @@ -136,7 +126,7 @@ item = obj + offset + itemlength * i j = 0 while j < len(offsets): - callback(self, item + offsets[j]) + callback(item + offsets[j], arg) j += 1 i += 1 _slow_trace._annspecialcase_ = 'specialize:arg(3)' Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/memory/gc/generation.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gc/generation.py Thu Dec 13 13:30:01 2007 @@ -258,11 +258,11 @@ young objects it references out of the nursery. """ self.header(obj).tid |= GCFLAG_NO_YOUNG_PTRS - self.trace(obj, GenerationGC._drag_out_of_nursery) + self.trace(obj, self._trace_drag_out, None) - def _drag_out_of_nursery(self, refaddr): - if self.is_in_nursery(refaddr.address[0]): - refaddr.address[0] = self.copy(refaddr.address[0]) + def _trace_drag_out(self, pointer, ignored): + if self.is_in_nursery(pointer.address[0]): + pointer.address[0] = self.copy(pointer.address[0]) def invalidate_young_weakrefs(self): # walk over the list of objects that contain weakrefs and are in the Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/memory/gc/semispace.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gc/semispace.py Thu Dec 13 13:30:01 2007 @@ -37,7 +37,6 @@ self.AddressLinkedList = AddressLinkedList def setup(self): - MovingGCBase.setup(self) self.tospace = llarena.arena_malloc(self.space_size, True) ll_assert(bool(self.tospace), "couldn't allocate tospace") self.top_of_space = self.tospace + self.space_size @@ -257,11 +256,11 @@ return newobj def trace_and_copy(self, obj): - self.trace(obj, SemiSpaceGC._copy_ref) + self.trace(obj, self._trace_copy, None) - def _copy_ref(self, refaddr): - if refaddr.address[0] != NULL: - refaddr.address[0] = self.copy(refaddr.address[0]) + def _trace_copy(self, pointer, ignored): + if pointer.address[0] != NULL: + pointer.address[0] = self.copy(pointer.address[0]) def is_forwarded(self, obj): return self.header(obj).forw != NULL Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py Thu Dec 13 13:30:01 2007 @@ -2,18 +2,34 @@ from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.memory.gctypelayout import offsets_to_gc_pointers + +class TraceOptGCBase(object): + + def set_typeid_count(gc, count): + print 'SET_TYPEID_COUNT', count + self.traceopt_index = lltype.malloc(TRACEOPT_INDEXES, count, + flavor='raw') + i = 0 + while i < count: + self.traceopt_index[i] = chr(0) + i += 1 + + def trace(self, obj, callback, arg): + typeid = self.get_type_id(obj) + shapeindex = ord(gc.traceopt_index[typeid]) + for idx, tracer in unroll_common_shape_tracers: + if shapeindex == idx: + tracer(gc, obj, callback, arg) + return True + if shapeindex == 0: + find_common_shape(gc, typeid) + self._slow_trace(obj, typeid, callback, arg) + trace._annspecialcase_ = 'specialize:arg(2)' + # ____________________________________________________________ TRACEOPT_INDEXES = lltype.Array(lltype.Char, hints={'nolength': True}) -def set_typeid_count(gc, count): - print 'SET_TYPEID_COUNT', count - gc.traceopt_index = lltype.malloc(TRACEOPT_INDEXES, count, flavor='raw') - i = 0 - while i < count: - gc.traceopt_index[i] = chr(0) - i += 1 - def enumerate_lltypes(): for n in range(32): fields = [] @@ -70,17 +86,6 @@ unroll_common_shape_checkers= unrolling_iterable(common_shape_checkers.items()) unroll_common_shape_tracers = unrolling_iterable(common_shape_tracers.items()) -def optimized_trace(gc, obj, typeid, callback): - shapeindex = ord(gc.traceopt_index[typeid]) - for idx, tracer in unroll_common_shape_tracers: - if shapeindex == idx: - tracer(gc, obj, callback) - return True - if shapeindex == 0: - find_common_shape(gc, typeid) - return False -optimized_trace._annspecialcase_ = 'specialize:arg(3)' - def find_common_shape(gc, typeid): shapeindex = 255 for idx, checker in unroll_common_shape_checkers: From arigo at codespeak.net Thu Dec 13 13:37:28 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 13:37:28 +0100 (CET) Subject: [pypy-svn] r49707 - in pypy/dist/pypy/rpython/memory: . gctransform Message-ID: <20071213123728.A7D64168487@codespeak.net> Author: arigo Date: Thu Dec 13 13:37:27 2007 New Revision: 49707 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gctypelayout.py Log: It seems to be a bit better to not be clever here. Separate Bool fields are nicer for gcc. Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Thu Dec 13 13:37:27 2007 @@ -116,7 +116,8 @@ # types of the GC information tables OFFSETS_TO_GC_PTR = lltype.Array(lltype.Signed) TYPE_INFO = lltype.Struct("type_info", - ("isvarsize", lltype.Signed), + ("isvarsize", lltype.Bool), + ("gcptrinvarsize", lltype.Bool), ("finalizer", self.FINALIZERTYPE), ("fixedsize", lltype.Signed), ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), @@ -130,11 +131,11 @@ def q_is_varsize(typeid): ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].isvarsize != 0 + return gcdata.type_info_table[typeid].isvarsize def q_has_gcptr_in_varsize(typeid): ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].isvarsize > 1 + return gcdata.type_info_table[typeid].gcptrinvarsize def q_finalizer(typeid): ll_assert(typeid > 0, "invalid type_id") Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/gctypelayout.py Thu Dec 13 13:37:27 2007 @@ -38,7 +38,8 @@ info["finalizer"] = self.make_finalizer_funcptr_for_type(TYPE) info["weakptrofs"] = weakpointer_offset(TYPE) if not TYPE._is_varsize(): - info["isvarsize"] = 0 + info["isvarsize"] = False + info["gcptrinvarsize"] = False info["fixedsize"] = llarena.round_up_for_allocation( llmemory.sizeof(TYPE)) info["ofstolength"] = -1 @@ -47,6 +48,7 @@ # varsize ones, the GC must anyway compute the size at run-time # and round up that result. else: + info["isvarsize"] = True info["fixedsize"] = llmemory.sizeof(TYPE, 0) if isinstance(TYPE, lltype.Struct): ARRAY = TYPE._flds[TYPE._arrayfld] @@ -72,9 +74,7 @@ offsets = () info["varofstoptrs"] = self.offsets2table(offsets, ARRAY.OF) info["varitemsize"] = llmemory.sizeof(ARRAY.OF) - info["isvarsize"] = 1 + (len(offsets) > 0) - # isvarsize is set to 1 if there are no gc ptrs in the - # varsize part, and to 2 if there are. + info["gcptrinvarsize"] = len(offsets) > 0 return type_id def offsets2table(self, offsets, TYPE): @@ -92,11 +92,11 @@ def q_is_varsize(self, typeid): assert typeid > 0 - return self.type_info_list[typeid]["isvarsize"] != 0 + return self.type_info_list[typeid]["isvarsize"] def q_has_gcptr_in_varsize(self, typeid): assert typeid > 0 - return self.type_info_list[typeid]["isvarsize"] > 1 + return self.type_info_list[typeid]["gcptrinvarsize"] def q_finalizer(self, typeid): assert typeid > 0 From arigo at codespeak.net Thu Dec 13 13:38:28 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 13:38:28 +0100 (CET) Subject: [pypy-svn] r49708 - in pypy/branch/pypy-gc-traceopt/rpython/memory: . gctransform Message-ID: <20071213123828.A4D88168453@codespeak.net> Author: arigo Date: Thu Dec 13 13:38:28 2007 New Revision: 49708 Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gctransform/framework.py pypy/branch/pypy-gc-traceopt/rpython/memory/gctypelayout.py Log: Merge of r49707 from the trunk. Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gctransform/framework.py Thu Dec 13 13:38:28 2007 @@ -116,7 +116,8 @@ # types of the GC information tables OFFSETS_TO_GC_PTR = lltype.Array(lltype.Signed) TYPE_INFO = lltype.Struct("type_info", - ("isvarsize", lltype.Signed), + ("isvarsize", lltype.Bool), + ("gcptrinvarsize", lltype.Bool), ("finalizer", self.FINALIZERTYPE), ("fixedsize", lltype.Signed), ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), @@ -130,11 +131,11 @@ def q_is_varsize(typeid): ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].isvarsize != 0 + return gcdata.type_info_table[typeid].isvarsize def q_has_gcptr_in_varsize(typeid): ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].isvarsize > 1 + return gcdata.type_info_table[typeid].gcptrinvarsize def q_finalizer(typeid): ll_assert(typeid > 0, "invalid type_id") Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/memory/gctypelayout.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gctypelayout.py Thu Dec 13 13:38:28 2007 @@ -38,7 +38,8 @@ info["finalizer"] = self.make_finalizer_funcptr_for_type(TYPE) info["weakptrofs"] = weakpointer_offset(TYPE) if not TYPE._is_varsize(): - info["isvarsize"] = 0 + info["isvarsize"] = False + info["gcptrinvarsize"] = False info["fixedsize"] = llarena.round_up_for_allocation( llmemory.sizeof(TYPE)) info["ofstolength"] = -1 @@ -47,6 +48,7 @@ # varsize ones, the GC must anyway compute the size at run-time # and round up that result. else: + info["isvarsize"] = True info["fixedsize"] = llmemory.sizeof(TYPE, 0) if isinstance(TYPE, lltype.Struct): ARRAY = TYPE._flds[TYPE._arrayfld] @@ -72,9 +74,7 @@ offsets = () info["varofstoptrs"] = self.offsets2table(offsets, ARRAY.OF) info["varitemsize"] = llmemory.sizeof(ARRAY.OF) - info["isvarsize"] = 1 + (len(offsets) > 0) - # isvarsize is set to 1 if there are no gc ptrs in the - # varsize part, and to 2 if there are. + info["gcptrinvarsize"] = len(offsets) > 0 return type_id def offsets2table(self, offsets, TYPE): @@ -92,11 +92,11 @@ def q_is_varsize(self, typeid): assert typeid > 0 - return self.type_info_list[typeid]["isvarsize"] != 0 + return self.type_info_list[typeid]["isvarsize"] def q_has_gcptr_in_varsize(self, typeid): assert typeid > 0 - return self.type_info_list[typeid]["isvarsize"] > 1 + return self.type_info_list[typeid]["gcptrinvarsize"] def q_finalizer(self, typeid): assert typeid > 0 From arigo at codespeak.net Thu Dec 13 13:40:54 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 13:40:54 +0100 (CET) Subject: [pypy-svn] r49709 - in pypy/branch/pypy-gc-traceopt/annotation: . test Message-ID: <20071213124054.87CEA168453@codespeak.net> Author: arigo Date: Thu Dec 13 13:40:54 2007 New Revision: 49709 Modified: pypy/branch/pypy-gc-traceopt/annotation/specialize.py pypy/branch/pypy-gc-traceopt/annotation/test/test_annrpython.py Log: Merge of r49697 from the trunk. Modified: pypy/branch/pypy-gc-traceopt/annotation/specialize.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/annotation/specialize.py (original) +++ pypy/branch/pypy-gc-traceopt/annotation/specialize.py Thu Dec 13 13:40:54 2007 @@ -421,7 +421,19 @@ return constgraphbuilder def specialize_argvalue(funcdesc, args_s, *argindices): - key = tuple([args_s[i].const for i in argindices]) + from pypy.annotation.model import SomePBC + key = [] + for i in argindices: + s = args_s[i] + if s.is_constant(): + key.append(s.const) + elif isinstance(s, SomePBC) and len(s.descriptions) == 1: + # for test_specialize_arg_bound_method + key.append(s.descriptions.keys()[0]) + else: + raise Exception("specialize:arg(%d): argument not constant: %r" + % (i, s)) + key = tuple(key) return funcdesc.cachedgraph(key) def specialize_argtype(funcdesc, args_s, *argindices): Modified: pypy/branch/pypy-gc-traceopt/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/annotation/test/test_annrpython.py (original) +++ pypy/branch/pypy-gc-traceopt/annotation/test/test_annrpython.py Thu Dec 13 13:40:54 2007 @@ -1099,6 +1099,32 @@ assert graph1 in a.translator.graphs assert graph2 in a.translator.graphs + def test_specialize_arg_bound_method(self): + class GC(object): + def trace(self, callback, arg): + return callback(arg) + trace._annspecialcase_ = "specialize:arg(1)" + + def callback1(self, arg): + self.x = arg + return "hello" + + def callback2(self, arg): + self.y = arg + return 6 + + def f(): + gc = GC() + s1 = gc.trace(gc.callback1, None) + n2 = gc.trace(gc.callback2, 7) + return (s1, n2, gc.x, gc.y) + a = self.RPythonAnnotator() + s = a.build_types(f, []) + assert s.items[0].const == "hello" + assert s.items[1].const == 6 + assert s.items[2].const == None + assert s.items[3].const == 7 + def test_assert_list_doesnt_lose_info(self): class T(object): pass From arigo at codespeak.net Thu Dec 13 13:51:52 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 13:51:52 +0100 (CET) Subject: [pypy-svn] r49711 - pypy/branch/pypy-gc-traceopt/rpython/memory/gc Message-ID: <20071213125152.B838C16840C@codespeak.net> Author: arigo Date: Thu Dec 13 13:51:52 2007 New Revision: 49711 Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py Log: Some of test_newgc passes again. Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py Thu Dec 13 13:51:52 2007 @@ -1,12 +1,14 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.memory.gctypelayout import offsets_to_gc_pointers +from pypy.rpython.lltypesystem.lloperation import llop class TraceOptGCBase(object): + _alloc_flavor_ = "raw" - def set_typeid_count(gc, count): - print 'SET_TYPEID_COUNT', count + def set_typeid_count(self, count): + llop.debug_print(lltype.Void, 'SET_TYPEID_COUNT', count) self.traceopt_index = lltype.malloc(TRACEOPT_INDEXES, count, flavor='raw') i = 0 @@ -16,16 +18,34 @@ def trace(self, obj, callback, arg): typeid = self.get_type_id(obj) - shapeindex = ord(gc.traceopt_index[typeid]) + # first see if we have an optimized tracer for this typeid + # (this becomes a single switch) + shapeindex = ord(self.traceopt_index[typeid]) for idx, tracer in unroll_common_shape_tracers: if shapeindex == idx: - tracer(gc, obj, callback, arg) - return True + tracer(obj, callback, arg) # yes + return + # none found. If the shapeindex is still 0, check at run-time + # if the offsets for the typeid match one of the offsets for + # the available optimized tracers. if shapeindex == 0: - find_common_shape(gc, typeid) + self.find_common_shape(typeid) self._slow_trace(obj, typeid, callback, arg) trace._annspecialcase_ = 'specialize:arg(2)' + def find_common_shape(self, typeid): + # see if the offsets of the typeid match one of the optimized + # tracers. This is a bit slow but it's done only once per typeid + # per execution of the program. + shapeindex = 255 + for idx, checker in list_common_shape_checkers: + if checker(self, typeid): + shapeindex = idx + break + llop.debug_print(lltype.Void, "find_common_shape:", typeid, + "is shape", shapeindex) + self.traceopt_index[typeid] = chr(shapeindex) + # ____________________________________________________________ TRACEOPT_INDEXES = lltype.Array(lltype.Char, hints={'nolength': True}) @@ -47,14 +67,14 @@ offsets = offsets_to_gc_pointers(TYPE) unroll_offsets = unrolling_iterable(offsets) - def tracer(gc, obj, callback): + def tracer(obj, callback, arg): for offset in unroll_offsets: - callback(gc, obj + offset) - tracer._annspecialcase_ = 'specialize:arg(2)' + callback(obj + offset, arg) + tracer._annspecialcase_ = 'specialize:arg(1)' #xxx force inline tracer def compatible_shape(gc, typeid): - if gc.is_varsize(typeid): + if gc.has_gcptr_in_varsize(typeid): return False cur_offsets = gc.offsets_to_gc_pointers(typeid) if len(cur_offsets) != len(offsets): @@ -65,7 +85,6 @@ return False i += 1 return True - #xxx force inline compatible_shape return compatible_shape, tracer @@ -83,14 +102,5 @@ return checkers, tracers common_shape_checkers, common_shape_tracers = get_common_shape_fns() -unroll_common_shape_checkers= unrolling_iterable(common_shape_checkers.items()) +list_common_shape_checkers = common_shape_checkers.items() unroll_common_shape_tracers = unrolling_iterable(common_shape_tracers.items()) - -def find_common_shape(gc, typeid): - shapeindex = 255 - for idx, checker in unroll_common_shape_checkers: - if checker(gc, typeid): - shapeindex = idx - break - print "find_common_shape:", typeid, "is shape", shapeindex - gc.traceopt_index[typeid] = chr(shapeindex) From arigo at codespeak.net Thu Dec 13 14:11:15 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 14:11:15 +0100 (CET) Subject: [pypy-svn] r49713 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20071213131115.3EA431684D1@codespeak.net> Author: arigo Date: Thu Dec 13 14:11:14 2007 New Revision: 49713 Modified: pypy/dist/pypy/rpython/memory/gc/base.py Log: Avoid this multiplication in here. Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Thu Dec 13 14:11:14 2007 @@ -121,19 +121,18 @@ callback(obj + offsets[i], arg) i += 1 if self.has_gcptr_in_varsize(typeid): - offset = self.varsize_offset_to_variable_part( - typeid) + item = obj + self.varsize_offset_to_variable_part(typeid) length = (obj + self.varsize_offset_to_length(typeid)).signed[0] offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) itemlength = self.varsize_item_sizes(typeid) i = 0 while i < length: - item = obj + offset + itemlength * i j = 0 while j < len(offsets): callback(item + offsets[j], arg) j += 1 i += 1 + item += itemlength trace._annspecialcase_ = 'specialize:arg(2)' From arigo at codespeak.net Thu Dec 13 14:12:47 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 14:12:47 +0100 (CET) Subject: [pypy-svn] r49714 - pypy/branch/pypy-gc-traceopt/rpython/memory/gc Message-ID: <20071213131247.F234C1684D0@codespeak.net> Author: arigo Date: Thu Dec 13 14:12:47 2007 New Revision: 49714 Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/base.py Log: Merge of r49713 from the trunk. Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/memory/gc/base.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gc/base.py Thu Dec 13 14:12:47 2007 @@ -116,19 +116,18 @@ callback(obj + offsets[i], arg) i += 1 if self.has_gcptr_in_varsize(typeid): - offset = self.varsize_offset_to_variable_part( - typeid) + item = obj + self.varsize_offset_to_variable_part(typeid) length = (obj + self.varsize_offset_to_length(typeid)).signed[0] offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) itemlength = self.varsize_item_sizes(typeid) i = 0 while i < length: - item = obj + offset + itemlength * i j = 0 while j < len(offsets): callback(item + offsets[j], arg) j += 1 i += 1 + item += itemlength _slow_trace._annspecialcase_ = 'specialize:arg(3)' From arigo at codespeak.net Thu Dec 13 14:20:49 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 14:20:49 +0100 (CET) Subject: [pypy-svn] r49715 - pypy/branch/pypy-gc-traceopt/rpython/memory/gc Message-ID: <20071213132049.43E4A1684D0@codespeak.net> Author: arigo Date: Thu Dec 13 14:20:48 2007 New Revision: 49715 Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/base.py pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py Log: Support for GcArrays in the list of common shapes. Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/memory/gc/base.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gc/base.py Thu Dec 13 14:20:48 2007 @@ -120,14 +120,13 @@ length = (obj + self.varsize_offset_to_length(typeid)).signed[0] offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) itemlength = self.varsize_item_sizes(typeid) - i = 0 - while i < length: + while length > 0: j = 0 while j < len(offsets): callback(item + offsets[j], arg) j += 1 - i += 1 item += itemlength + length -= 1 _slow_trace._annspecialcase_ = 'specialize:arg(3)' Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py Thu Dec 13 14:20:48 2007 @@ -59,11 +59,16 @@ else: fields.append(('n%d' % i, llmemory.Address)) yield lltype.GcStruct('dummy', *fields) - #yield lltype.GcArray(llmemory.GCREF) + yield lltype.GcArray(llmemory.GCREF) def build_common_shape_fn(TYPE): + if TYPE._is_varsize(): + return build_common_shape_fn_varsize(TYPE) + else: + return build_common_shape_fn_nonvarsize(TYPE) + +def build_common_shape_fn_nonvarsize(TYPE): assert isinstance(TYPE, lltype.GcStruct) - assert not TYPE._is_varsize() offsets = offsets_to_gc_pointers(TYPE) unroll_offsets = unrolling_iterable(offsets) @@ -88,6 +93,48 @@ return compatible_shape, tracer +def build_common_shape_fn_varsize(TYPE): + assert isinstance(TYPE, lltype.GcArray) # for now, no varsized GcStruct + ARRAY = TYPE + assert ARRAY.OF != lltype.Void # an empty GcStruct is compatible + # with an array of Voids, gc-wise + offsets = offsets_to_gc_pointers(ARRAY.OF) + assert len(offsets) > 0 # an array without pointers doesn't need an + # optimized version - it's like an empty GcStruct + ofstolength = llmemory.ArrayLengthOffset(ARRAY) + ofstovar = llmemory.itemoffsetof(ARRAY, 0) + itemlength = llmemory.sizeof(ARRAY.OF) + unroll_offsets = unrolling_iterable(offsets) + + def tracer(obj, callback, arg): + item = obj + ofstovar + length = (obj + ofstolength).signed[0] + while length > 0: + for offset in unroll_offsets: + callback(item + offset, arg) + item += itemlength + length -= 1 + tracer._annspecialcase_ = 'specialize:arg(1)' + + def compatible_shape(gc, typeid): + if not (gc.has_gcptr_in_varsize(typeid) and + len(gc.offsets_to_gc_pointers(typeid)) == 0 and + gc.varsize_offset_to_variable_part(typeid) == ofstovar and + gc.varsize_offset_to_length(typeid) == ofstolength and + gc.varsize_item_sizes(typeid) != itemlength): + return False + cur_offsets = gc.varsize_offsets_to_gcpointers_in_var_part(typeid) + if len(cur_offsets) != len(offsets): + return False + i = 0 + while i < len(offsets): + if cur_offsets[i] != offsets[i]: + return False + i += 1 + return True + + return compatible_shape, tracer + def get_common_shape_fns(): # index 0: not decided yet # index 255: reserved for the unoptimized case From arigo at codespeak.net Thu Dec 13 14:30:58 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 14:30:58 +0100 (CET) Subject: [pypy-svn] r49716 - pypy/branch/pypy-gc-traceopt/rpython/memory/gc Message-ID: <20071213133058.1C5B1168453@codespeak.net> Author: arigo Date: Thu Dec 13 14:30:56 2007 New Revision: 49716 Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py Log: Typo. Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py Thu Dec 13 14:30:56 2007 @@ -121,7 +121,7 @@ len(gc.offsets_to_gc_pointers(typeid)) == 0 and gc.varsize_offset_to_variable_part(typeid) == ofstovar and gc.varsize_offset_to_length(typeid) == ofstolength and - gc.varsize_item_sizes(typeid) != itemlength): + gc.varsize_item_sizes(typeid) == itemlength): return False cur_offsets = gc.varsize_offsets_to_gcpointers_in_var_part(typeid) if len(cur_offsets) != len(offsets): From arigo at codespeak.net Thu Dec 13 15:21:52 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 15:21:52 +0100 (CET) Subject: [pypy-svn] r49717 - pypy/dist/pypy/rpython/ootypesystem/test Message-ID: <20071213142152.55FD11684E2@codespeak.net> Author: arigo Date: Thu Dec 13 15:21:51 2007 New Revision: 49717 Modified: pypy/dist/pypy/rpython/ootypesystem/test/test_oortype.py Log: Skip this test for now. Modified: pypy/dist/pypy/rpython/ootypesystem/test/test_oortype.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/test/test_oortype.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/test/test_oortype.py Thu Dec 13 15:21:51 2007 @@ -193,6 +193,7 @@ assert rtyper.exceptiondata.is_exception_instance(INST) def test_string_annotation(): + py.test.skip("XXX interpret() does special magic about String arguments") def oof(lst): return lst.ll_strlen() From arigo at codespeak.net Thu Dec 13 16:02:49 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 16:02:49 +0100 (CET) Subject: [pypy-svn] r49718 - pypy/dist/pypy/module/_file Message-ID: <20071213150249.BC1A91684ED@codespeak.net> Author: arigo Date: Thu Dec 13 16:02:47 2007 New Revision: 49718 Modified: pypy/dist/pypy/module/_file/interp_stream.py Log: An extra assert. Modified: pypy/dist/pypy/module/_file/interp_stream.py ============================================================================== --- pypy/dist/pypy/module/_file/interp_stream.py (original) +++ pypy/dist/pypy/module/_file/interp_stream.py Thu Dec 13 16:02:47 2007 @@ -72,6 +72,7 @@ if self.slockowner is me: return False # already acquired by the current thread self.slock.acquire(True) + assert self.slockowner is None self.slockowner = me return True From arigo at codespeak.net Thu Dec 13 16:03:28 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 16:03:28 +0100 (CET) Subject: [pypy-svn] r49719 - pypy/dist/pypy/objspace Message-ID: <20071213150328.08DC71684ED@codespeak.net> Author: arigo Date: Thu Dec 13 16:03:28 2007 New Revision: 49719 Modified: pypy/dist/pypy/objspace/trace.py Log: Always return the same result from getexecutioncontext(). The _file module, at least, uses that result as an identifier for the current thread. Modified: pypy/dist/pypy/objspace/trace.py ============================================================================== --- pypy/dist/pypy/objspace/trace.py (original) +++ pypy/dist/pypy/objspace/trace.py Thu Dec 13 16:03:28 2007 @@ -165,7 +165,7 @@ def __getattribute__(self, name): obj = super(Trace, self).__getattribute__(name) - if name in ["_result", "_in_cache", + if name in ["_result", "_in_cache", "_ect_cache", "_tracing", "_config_options"]: return obj @@ -189,6 +189,7 @@ def settrace(self): self._result = TraceResult(self, **self._config_options) + self._ect_cache = {} self._tracing = True def unsettrace(self): @@ -200,8 +201,13 @@ def getexecutioncontext(self): ec = super(Trace, self).getexecutioncontext() if not self._in_cache: - assert not isinstance(ec, ExecutionContextTracer) - ec = ExecutionContextTracer(self._result, ec) + try: + ect = self._ect_cache[ec] + except KeyError: + assert not isinstance(ec, ExecutionContextTracer) + ect = ExecutionContextTracer(self._result, ec) + self._ect_cache[ec] = ect + return ect return ec # XXX Rename @@ -210,7 +216,8 @@ space.__class__ = space.__oldclass__ del space.__oldclass__ - for k in ["_result", "_in_cache", "_config_options", "_operations"]: + for k in ["_result", "_in_cache", "_ect_cache", + "_config_options", "_operations"]: if hasattr(self, k): delattr(self, k) @@ -221,6 +228,7 @@ from pypy.tool.traceconfig import config space._tracing = False space._result = None + space._ect_cache = {} space._in_cache = 0 space._config_options = config From arigo at codespeak.net Thu Dec 13 16:28:21 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 16:28:21 +0100 (CET) Subject: [pypy-svn] r49720 - pypy/extradoc/pypy.org Message-ID: <20071213152821.237011684D1@codespeak.net> Author: arigo Date: Thu Dec 13 16:28:19 2007 New Revision: 49720 Modified: pypy/extradoc/pypy.org/index.txt Log: Redirect more clearly to the main PyPy page. Modified: pypy/extradoc/pypy.org/index.txt ============================================================================== --- pypy/extradoc/pypy.org/index.txt (original) +++ pypy/extradoc/pypy.org/index.txt Thu Dec 13 16:28:19 2007 @@ -1,3 +1,8 @@ +Welcome! + *This is the European Union project page for PyPy.* + *Please visit the main page at* http://codespeak.net/pypy/. + + PyPy EU project title (contract number: 004779) ------------------------------------------------ From arigo at codespeak.net Thu Dec 13 16:34:04 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 16:34:04 +0100 (CET) Subject: [pypy-svn] r49721 - in pypy/dist/pypy: rpython rpython/lltypesystem rpython/ootypesystem translator/cli translator/jvm Message-ID: <20071213153404.611971684D8@codespeak.net> Author: arigo Date: Thu Dec 13 16:34:03 2007 New Revision: 49721 Modified: pypy/dist/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/rpbc.py pypy/dist/pypy/rpython/ootypesystem/rdict.py pypy/dist/pypy/rpython/rexternalobj.py pypy/dist/pypy/rpython/rmodel.py pypy/dist/pypy/rpython/rpbc.py pypy/dist/pypy/rpython/rtyper.py pypy/dist/pypy/translator/cli/comparer.py pypy/dist/pypy/translator/jvm/metavm.py Log: Kill HalfConcreteWrapper. Yay! It was still (ab)used by the ootypesystem implementation of r_dict. I think I fixed the cli and jvm backends too. Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Thu Dec 13 16:34:03 2007 @@ -1215,12 +1215,12 @@ def wrap_callable(llinterpreter, fn, obj, method_name): if method_name is None: - # fn is a HalfConcreteWrapper wrapping a StaticMethod + # fn is a StaticMethod if obj is not None: self_arg = [obj] else: self_arg = [] - func_graph = fn.concretize().value.graph + func_graph = fn.graph else: # obj is an instance, we want to call 'method_name' on it assert fn is None Modified: pypy/dist/pypy/rpython/lltypesystem/rpbc.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rpbc.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rpbc.py Thu Dec 13 16:34:03 2007 @@ -7,7 +7,7 @@ from pypy.rpython.lltypesystem.lltype import \ typeOf, Void, ForwardReference, Struct, Bool, Char, \ Ptr, malloc, nullptr, Array, Signed, FuncType -from pypy.rpython.rmodel import Repr, TyperError, inputconst, inputdesc, HalfConcreteWrapper +from pypy.rpython.rmodel import Repr, TyperError, inputconst, inputdesc from pypy.rpython.rpbc import samesig,\ commonbase, allattributenames, adjust_shape, \ AbstractClassesPBCRepr, AbstractMethodsPBCRepr, OverriddenFunctionPBCRepr, \ @@ -261,8 +261,7 @@ class __extend__(pairtype(SmallFunctionSetPBCRepr, FunctionsPBCRepr)): def convert_from_to((r_set, r_ptr), v, llops): if r_ptr.lowleveltype is Void: - wrapper = HalfConcreteWrapper(r_ptr.get_unique_llfn) - return inputconst(Void, wrapper) + return inputconst(Void, None) else: assert v.concretetype is Char v_int = llops.genop('cast_char_to_int', [v], Modified: pypy/dist/pypy/rpython/ootypesystem/rdict.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rdict.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rdict.py Thu Dec 13 16:34:03 2007 @@ -160,7 +160,7 @@ r_func, nimplicitarg = r_func.get_r_implfunc() else: obj = None - callable = r_func.convert_const(fn) + callable = r_func.get_unique_llfn().value func_name, interp_fn = llinterp.wrap_callable(interp, callable, obj, None) return ootype.static_meth(TYPE, func_name, _callable=interp_fn) @@ -232,7 +232,7 @@ def _get_call_vars(hop, r_func, arg, params_annotation): if isinstance(r_func, AbstractFunctionsPBCRepr): - v_fn = hop.inputarg(r_func, arg=arg) + v_fn = r_func.get_unique_llfn() v_obj = hop.inputconst(ootype.Void, None) c_method_name = hop.inputconst(ootype.Void, None) elif isinstance(r_func, AbstractMethodsPBCRepr): @@ -243,7 +243,7 @@ c_method_name = hop.inputconst(ootype.Void, methodname) elif isinstance(r_func, MethodOfFrozenPBCRepr): r_impl, nimplicitarg = r_func.get_r_implfunc() - v_fn = hop.inputconst(r_impl, r_func.funcdesc.pyobj) + v_fn = r_impl.get_unique_llfn() v_obj = hop.inputarg(r_func, arg=arg) c_method_name = hop.inputconst(ootype.Void, None) Modified: pypy/dist/pypy/rpython/rexternalobj.py ============================================================================== --- pypy/dist/pypy/rpython/rexternalobj.py (original) +++ pypy/dist/pypy/rpython/rexternalobj.py Thu Dec 13 16:34:03 2007 @@ -1,7 +1,7 @@ from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype -from pypy.rpython.rmodel import Repr, HalfConcreteWrapper +from pypy.rpython.rmodel import Repr from pypy.rpython import rbuiltin from pypy.objspace.flow.model import Constant, Variable from pypy.rpython import extregistry Modified: pypy/dist/pypy/rpython/rmodel.py ============================================================================== --- pypy/dist/pypy/rpython/rmodel.py (original) +++ pypy/dist/pypy/rpython/rmodel.py Thu Dec 13 16:34:03 2007 @@ -421,13 +421,6 @@ else: return '%s_%s' % (prefix, name) -class HalfConcreteWrapper: - # see rtyper.gendirectcall() - def __init__(self, callback): - self.concretize = callback # should produce a concrete const - def _freeze_(self): - return True - # __________ utilities __________ PyObjPtr = Ptr(PyObject) Modified: pypy/dist/pypy/rpython/rpbc.py ============================================================================== --- pypy/dist/pypy/rpython/rpbc.py (original) +++ pypy/dist/pypy/rpython/rpbc.py Thu Dec 13 16:34:03 2007 @@ -7,7 +7,7 @@ from pypy.rpython.lltypesystem.lltype import \ typeOf, Void, Bool, nullptr, frozendict, Ptr, Struct, malloc from pypy.rpython.error import TyperError -from pypy.rpython.rmodel import Repr, inputconst, HalfConcreteWrapper, CanBeNull, \ +from pypy.rpython.rmodel import Repr, inputconst, CanBeNull, \ mangle, inputdesc, warning, impossible_repr from pypy.rpython import rclass from pypy.rpython import robject @@ -225,7 +225,7 @@ except KeyError: pass if self.lowleveltype is Void: - result = HalfConcreteWrapper(self.get_unique_llfn) + result = None else: llfns = {} found_anything = False @@ -255,7 +255,7 @@ if isinstance(value, types.MethodType) and value.im_self is None: value = value.im_func # unbound method -> bare function if self.lowleveltype is Void: - return HalfConcreteWrapper(self.get_unique_llfn) + return None if value is None: null = self.rtyper.type_system.null_callable(self.lowleveltype) return null @@ -346,8 +346,7 @@ if r_fpbc1.lowleveltype is Void: return inputconst(r_fpbc2, r_fpbc1.s_pbc.const) if r_fpbc2.lowleveltype is Void: - wrapper = HalfConcreteWrapper(r_fpbc2.get_unique_llfn) - return inputconst(Void, wrapper) + return inputconst(Void, None) return NotImplemented class OverriddenFunctionPBCRepr(Repr): Modified: pypy/dist/pypy/rpython/rtyper.py ============================================================================== --- pypy/dist/pypy/rpython/rtyper.py (original) +++ pypy/dist/pypy/rpython/rtyper.py Thu Dec 13 16:34:03 2007 @@ -27,7 +27,7 @@ from pypy.translator.unsimplify import insert_empty_block from pypy.rpython.error import TyperError from pypy.rpython.rmodel import Repr, inputconst, BrokenReprTyperError -from pypy.rpython.rmodel import warning, HalfConcreteWrapper +from pypy.rpython.rmodel import warning from pypy.rpython.annlowlevel import annotate_lowlevel_helper, LowLevelAnnotatorPolicy from pypy.rpython.typesystem import LowLevelTypeSystem,\ ObjectOrientedTypeSystem @@ -884,12 +884,6 @@ raise TyperError("non-constant variable of type Void") if not isinstance(s_value, annmodel.SomePBC): raise TyperError("non-PBC Void argument: %r", (s_value,)) - if isinstance(s_value.const, HalfConcreteWrapper): - # Modify args_v so that 'v' gets the concrete value - # returned by the wrapper - wrapper = s_value.const - v = wrapper.concretize() - s_value = annmodel.lltype_to_annotation(v.concretetype) args_s.append(s_value) else: args_s.append(annmodel.lltype_to_annotation(v.concretetype)) Modified: pypy/dist/pypy/translator/cli/comparer.py ============================================================================== --- pypy/dist/pypy/translator/cli/comparer.py (original) +++ pypy/dist/pypy/translator/cli/comparer.py Thu Dec 13 16:34:03 2007 @@ -50,10 +50,9 @@ self._call_function(fn_args.graph, len(arglist)) else: fn, obj, method_name = fn_args - # fn is a HalfConcreteWrapper - sm = fn.value.concretize().value + # fn is a Constant(StaticMethod) if method_name.value is None: - self._call_function(sm.graph, len(arglist)) + self._call_function(fn.value.graph, len(arglist)) else: assert False, 'XXX' Modified: pypy/dist/pypy/translator/jvm/metavm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/metavm.py (original) +++ pypy/dist/pypy/translator/jvm/metavm.py Thu Dec 13 16:34:03 2007 @@ -110,7 +110,7 @@ # Standalone function: find the delegate class and # instantiate it. assert method_name.value is None - smimpl = fn.value.concretize().value # ootype._static_meth + smimpl = fn.value # ootype._static_meth db.record_delegate(smimpl._TYPE) # _TYPE is a StaticMethod ty = db.record_delegate_standalone_func_impl(smimpl.graph) gen.new_with_jtype(ty) From arigo at codespeak.net Thu Dec 13 16:39:47 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 16:39:47 +0100 (CET) Subject: [pypy-svn] r49722 - pypy/branch/pypy-gc-traceopt/rpython/memory/gc Message-ID: <20071213153947.8888B1684D9@codespeak.net> Author: arigo Date: Thu Dec 13 16:39:47 2007 New Revision: 49722 Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py Log: A comment. Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py Thu Dec 13 16:39:47 2007 @@ -51,6 +51,9 @@ TRACEOPT_INDEXES = lltype.Array(lltype.Char, hints={'nolength': True}) def enumerate_lltypes(): + """Enumerates some GcStruct and GcArray with 'common' shapes. + These are the shapes for which a specialized tracer is generated. + """ for n in range(32): fields = [] for i in range(5): From arigo at codespeak.net Thu Dec 13 16:41:14 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 16:41:14 +0100 (CET) Subject: [pypy-svn] r49723 - in pypy/branch/pypy-gc-traceopt: rpython rpython/lltypesystem rpython/ootypesystem translator/cli translator/jvm Message-ID: <20071213154114.E51491684E8@codespeak.net> Author: arigo Date: Thu Dec 13 16:41:14 2007 New Revision: 49723 Modified: pypy/branch/pypy-gc-traceopt/rpython/llinterp.py pypy/branch/pypy-gc-traceopt/rpython/lltypesystem/rpbc.py pypy/branch/pypy-gc-traceopt/rpython/ootypesystem/rdict.py pypy/branch/pypy-gc-traceopt/rpython/rexternalobj.py pypy/branch/pypy-gc-traceopt/rpython/rmodel.py pypy/branch/pypy-gc-traceopt/rpython/rpbc.py pypy/branch/pypy-gc-traceopt/rpython/rtyper.py pypy/branch/pypy-gc-traceopt/translator/cli/comparer.py pypy/branch/pypy-gc-traceopt/translator/jvm/metavm.py Log: Merge r49721 from the trunk. Modified: pypy/branch/pypy-gc-traceopt/rpython/llinterp.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/llinterp.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/llinterp.py Thu Dec 13 16:41:14 2007 @@ -1215,12 +1215,12 @@ def wrap_callable(llinterpreter, fn, obj, method_name): if method_name is None: - # fn is a HalfConcreteWrapper wrapping a StaticMethod + # fn is a StaticMethod if obj is not None: self_arg = [obj] else: self_arg = [] - func_graph = fn.concretize().value.graph + func_graph = fn.graph else: # obj is an instance, we want to call 'method_name' on it assert fn is None Modified: pypy/branch/pypy-gc-traceopt/rpython/lltypesystem/rpbc.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/lltypesystem/rpbc.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/lltypesystem/rpbc.py Thu Dec 13 16:41:14 2007 @@ -7,7 +7,7 @@ from pypy.rpython.lltypesystem.lltype import \ typeOf, Void, ForwardReference, Struct, Bool, Char, \ Ptr, malloc, nullptr, Array, Signed, FuncType -from pypy.rpython.rmodel import Repr, TyperError, inputconst, inputdesc, HalfConcreteWrapper +from pypy.rpython.rmodel import Repr, TyperError, inputconst, inputdesc from pypy.rpython.rpbc import samesig,\ commonbase, allattributenames, adjust_shape, \ AbstractClassesPBCRepr, AbstractMethodsPBCRepr, OverriddenFunctionPBCRepr, \ @@ -261,8 +261,7 @@ class __extend__(pairtype(SmallFunctionSetPBCRepr, FunctionsPBCRepr)): def convert_from_to((r_set, r_ptr), v, llops): if r_ptr.lowleveltype is Void: - wrapper = HalfConcreteWrapper(r_ptr.get_unique_llfn) - return inputconst(Void, wrapper) + return inputconst(Void, None) else: assert v.concretetype is Char v_int = llops.genop('cast_char_to_int', [v], Modified: pypy/branch/pypy-gc-traceopt/rpython/ootypesystem/rdict.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/ootypesystem/rdict.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/ootypesystem/rdict.py Thu Dec 13 16:41:14 2007 @@ -160,7 +160,7 @@ r_func, nimplicitarg = r_func.get_r_implfunc() else: obj = None - callable = r_func.convert_const(fn) + callable = r_func.get_unique_llfn().value func_name, interp_fn = llinterp.wrap_callable(interp, callable, obj, None) return ootype.static_meth(TYPE, func_name, _callable=interp_fn) @@ -232,7 +232,7 @@ def _get_call_vars(hop, r_func, arg, params_annotation): if isinstance(r_func, AbstractFunctionsPBCRepr): - v_fn = hop.inputarg(r_func, arg=arg) + v_fn = r_func.get_unique_llfn() v_obj = hop.inputconst(ootype.Void, None) c_method_name = hop.inputconst(ootype.Void, None) elif isinstance(r_func, AbstractMethodsPBCRepr): @@ -243,7 +243,7 @@ c_method_name = hop.inputconst(ootype.Void, methodname) elif isinstance(r_func, MethodOfFrozenPBCRepr): r_impl, nimplicitarg = r_func.get_r_implfunc() - v_fn = hop.inputconst(r_impl, r_func.funcdesc.pyobj) + v_fn = r_impl.get_unique_llfn() v_obj = hop.inputarg(r_func, arg=arg) c_method_name = hop.inputconst(ootype.Void, None) Modified: pypy/branch/pypy-gc-traceopt/rpython/rexternalobj.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/rexternalobj.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/rexternalobj.py Thu Dec 13 16:41:14 2007 @@ -1,7 +1,7 @@ from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype -from pypy.rpython.rmodel import Repr, HalfConcreteWrapper +from pypy.rpython.rmodel import Repr from pypy.rpython import rbuiltin from pypy.objspace.flow.model import Constant, Variable from pypy.rpython import extregistry Modified: pypy/branch/pypy-gc-traceopt/rpython/rmodel.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/rmodel.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/rmodel.py Thu Dec 13 16:41:14 2007 @@ -421,13 +421,6 @@ else: return '%s_%s' % (prefix, name) -class HalfConcreteWrapper: - # see rtyper.gendirectcall() - def __init__(self, callback): - self.concretize = callback # should produce a concrete const - def _freeze_(self): - return True - # __________ utilities __________ PyObjPtr = Ptr(PyObject) Modified: pypy/branch/pypy-gc-traceopt/rpython/rpbc.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/rpbc.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/rpbc.py Thu Dec 13 16:41:14 2007 @@ -7,7 +7,7 @@ from pypy.rpython.lltypesystem.lltype import \ typeOf, Void, Bool, nullptr, frozendict, Ptr, Struct, malloc from pypy.rpython.error import TyperError -from pypy.rpython.rmodel import Repr, inputconst, HalfConcreteWrapper, CanBeNull, \ +from pypy.rpython.rmodel import Repr, inputconst, CanBeNull, \ mangle, inputdesc, warning, impossible_repr from pypy.rpython import rclass from pypy.rpython import robject @@ -225,7 +225,7 @@ except KeyError: pass if self.lowleveltype is Void: - result = HalfConcreteWrapper(self.get_unique_llfn) + result = None else: llfns = {} found_anything = False @@ -255,7 +255,7 @@ if isinstance(value, types.MethodType) and value.im_self is None: value = value.im_func # unbound method -> bare function if self.lowleveltype is Void: - return HalfConcreteWrapper(self.get_unique_llfn) + return None if value is None: null = self.rtyper.type_system.null_callable(self.lowleveltype) return null @@ -346,8 +346,7 @@ if r_fpbc1.lowleveltype is Void: return inputconst(r_fpbc2, r_fpbc1.s_pbc.const) if r_fpbc2.lowleveltype is Void: - wrapper = HalfConcreteWrapper(r_fpbc2.get_unique_llfn) - return inputconst(Void, wrapper) + return inputconst(Void, None) return NotImplemented class OverriddenFunctionPBCRepr(Repr): Modified: pypy/branch/pypy-gc-traceopt/rpython/rtyper.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/rtyper.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/rtyper.py Thu Dec 13 16:41:14 2007 @@ -27,7 +27,7 @@ from pypy.translator.unsimplify import insert_empty_block from pypy.rpython.error import TyperError from pypy.rpython.rmodel import Repr, inputconst, BrokenReprTyperError -from pypy.rpython.rmodel import warning, HalfConcreteWrapper +from pypy.rpython.rmodel import warning from pypy.rpython.annlowlevel import annotate_lowlevel_helper, LowLevelAnnotatorPolicy from pypy.rpython.typesystem import LowLevelTypeSystem,\ ObjectOrientedTypeSystem @@ -884,12 +884,6 @@ raise TyperError("non-constant variable of type Void") if not isinstance(s_value, annmodel.SomePBC): raise TyperError("non-PBC Void argument: %r", (s_value,)) - if isinstance(s_value.const, HalfConcreteWrapper): - # Modify args_v so that 'v' gets the concrete value - # returned by the wrapper - wrapper = s_value.const - v = wrapper.concretize() - s_value = annmodel.lltype_to_annotation(v.concretetype) args_s.append(s_value) else: args_s.append(annmodel.lltype_to_annotation(v.concretetype)) Modified: pypy/branch/pypy-gc-traceopt/translator/cli/comparer.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/translator/cli/comparer.py (original) +++ pypy/branch/pypy-gc-traceopt/translator/cli/comparer.py Thu Dec 13 16:41:14 2007 @@ -50,10 +50,9 @@ self._call_function(fn_args.graph, len(arglist)) else: fn, obj, method_name = fn_args - # fn is a HalfConcreteWrapper - sm = fn.value.concretize().value + # fn is a Constant(StaticMethod) if method_name.value is None: - self._call_function(sm.graph, len(arglist)) + self._call_function(fn.value.graph, len(arglist)) else: assert False, 'XXX' Modified: pypy/branch/pypy-gc-traceopt/translator/jvm/metavm.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/translator/jvm/metavm.py (original) +++ pypy/branch/pypy-gc-traceopt/translator/jvm/metavm.py Thu Dec 13 16:41:14 2007 @@ -110,7 +110,7 @@ # Standalone function: find the delegate class and # instantiate it. assert method_name.value is None - smimpl = fn.value.concretize().value # ootype._static_meth + smimpl = fn.value # ootype._static_meth db.record_delegate(smimpl._TYPE) # _TYPE is a StaticMethod ty = db.record_delegate_standalone_func_impl(smimpl.graph) gen.new_with_jtype(ty) From fijal at codespeak.net Thu Dec 13 16:42:37 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 13 Dec 2007 16:42:37 +0100 (CET) Subject: [pypy-svn] r49724 - pypy/dist/pypy/module/fcntl Message-ID: <20071213154237.370591684E8@codespeak.net> Author: fijal Date: Thu Dec 13 16:42:36 2007 New Revision: 49724 Modified: pypy/dist/pypy/module/fcntl/interp_fcntl.py Log: Add rffi.setintfield everywhere Modified: pypy/dist/pypy/module/fcntl/interp_fcntl.py ============================================================================== --- pypy/dist/pypy/module/fcntl/interp_fcntl.py (original) +++ pypy/dist/pypy/module/fcntl/interp_fcntl.py Thu Dec 13 16:42:36 2007 @@ -151,7 +151,9 @@ space.wrap(_get_error_msg())) else: l = _check_flock_op(space, op) - l.c_l_whence = l.c_l_start = l.c_l_len = 0 + rffi.setintfield(l, 'c_l_whence', 0) + rffi.setintfield(l, 'c_l_start', 0) + rffi.setintfield(l, 'c_l_len', 0) op = [F_SETLKW, F_SETLK][op & LOCK_NB] fcntl_flock(fd, op, l) lltype.free(l, flavor='raw') @@ -185,12 +187,15 @@ fd = _conv_descriptor(space, w_fd) l = _check_flock_op(space, op) - l.c_l_start = l.c_l_len = 0 - if start: - l.c_l_start = int(start) + rffi.setintfield(l, 'c_l_start', int(start)) + else: + rffi.setintfield(l, 'c_l_start', 0) if len: - l.c_l_len = int(length) + rffi.setintfield(l, 'c_l_len', int(length)) + else: + rffi.setintfield(l, 'c_l_len', 0) + l.c_l_whence = rffi.cast(rffi.SHORT, whence) try: From cfbolz at codespeak.net Thu Dec 13 16:49:09 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 13 Dec 2007 16:49:09 +0100 (CET) Subject: [pypy-svn] r49726 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071213154909.CC3961684E8@codespeak.net> Author: cfbolz Date: Thu Dec 13 16:49:09 2007 New Revision: 49726 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: implement pow Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Thu Dec 13 16:49:09 2007 @@ -504,6 +504,47 @@ return space.w_True + def descr_pow(self, space, w_other, w_modulo=None): + if space.is_w(w_modulo, space.w_None): + w_a, w_b = _coerce_helper(space, self, w_other) + if w_a is None: + w_a = self + w_b = w_other + if w_a is self: + w_func = self.getattr(space, space.wrap('__pow__'), False) + if w_func is not None: + return space.call_function(w_func, w_other) + return space.w_NotImplemented + else: + return space.pow(w_a, w_b, space.w_None) + else: + # CPython also doesn't try coercion in this case + w_func = self.getattr(space, space.wrap('__pow__'), False) + if w_func is not None: + return space.call_function(w_func, w_other, w_modulo) + return space.w_NotImplemented + + def descr_rpow(self, space, w_other, w_modulo=None): + if space.is_w(w_modulo, space.w_None): + w_a, w_b = _coerce_helper(space, self, w_other) + if w_a is None: + w_a = self + w_b = w_other + if w_a is self: + w_func = self.getattr(space, space.wrap('__rpow__'), False) + if w_func is not None: + return space.call_function(w_func, w_other) + return space.w_NotImplemented + else: + return space.pow(w_b, w_a, space.w_None) + else: + # CPython also doesn't try coercion in this case + w_func = self.getattr(space, space.wrap('__rpow__'), False) + if w_func is not None: + return space.call_function(w_func, w_other, w_modulo) + return space.w_NotImplemented + + rawdict = {} # unary operations @@ -579,6 +620,10 @@ unwrap_spec=['self', ObjSpace]), __contains__ = interp2app(W_InstanceObject.descr_contains, unwrap_spec=['self', ObjSpace, W_Root]), + __pow__ = interp2app(W_InstanceObject.descr_pow, + unwrap_spec=['self', ObjSpace, W_Root, W_Root]), + __rpow__ = interp2app(W_InstanceObject.descr_rpow, + unwrap_spec=['self', ObjSpace, W_Root, W_Root]), **rawdict ) Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Thu Dec 13 16:49:09 2007 @@ -555,3 +555,27 @@ a = A() for i in range(1, 6): assert i in a + + def test_pow(self): + class A: + __metaclass__ = nclassobj + def __pow__(self, other, mod=None): + if mod is None: + return 2 ** other + return mod ** other + a = A() + assert a ** 4 == 16 + assert pow(a, 4) == 16 + assert pow(a, 4, 5) == 625 + raises(TypeError, "4 ** a") + class A: + __metaclass__ = nclassobj + def __rpow__(self, other, mod=None): + if mod is None: + return 2 ** other + return mod ** other + a = A() + assert 4 ** a == 16 + assert pow(4, a) == 16 + raises(TypeError, "a ** 4") + assert pow(4, a, 5) == 625 From cfbolz at codespeak.net Thu Dec 13 17:17:55 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 13 Dec 2007 17:17:55 +0100 (CET) Subject: [pypy-svn] r49727 - pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test Message-ID: <20071213161755.4C5531684F3@codespeak.net> Author: cfbolz Date: Thu Dec 13 17:17:54 2007 New Revision: 49727 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: test slicing too Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Thu Dec 13 17:17:54 2007 @@ -216,6 +216,12 @@ assert len(a) == 0 assert not a assert bool(a) == False + a = A() + assert a[1:3] == [2, 3] + a[1:3] = [1, 2, 3] + assert a.list == [1, 1, 2, 3, 4, 5] + del a[1:4] + assert a.list == [1, 4, 5] def test_len_errors(self): class A: From cfbolz at codespeak.net Thu Dec 13 17:26:40 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 13 Dec 2007 17:26:40 +0100 (CET) Subject: [pypy-svn] r49728 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071213162640.C34111684F3@codespeak.net> Author: cfbolz Date: Thu Dec 13 17:26:39 2007 New Revision: 49728 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: implement get/set/delslice. puh Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Thu Dec 13 17:26:39 2007 @@ -397,6 +397,28 @@ #XXX do I really need a next method? the old implementation had one, but I # don't see the point + def descr_getslice(self, space, w_i, w_j): + w_meth = self.getattr(space, space.wrap('__getslice__'), False) + if w_meth is not None: + return space.call_function(w_meth, w_i, w_j) + else: + return space.getitem(self, space.newslice(w_i, w_j, space.w_None)) + + def descr_setslice(self, space, w_i, w_j, w_sequence): + w_meth = self.getattr(space, space.wrap('__setslice__'), False) + if w_meth is not None: + space.call_function(w_meth, w_i, w_j, w_sequence) + else: + space.setitem(self, space.newslice(w_i, w_j, space.w_None), + w_sequence) + + def descr_delslice(self, space, w_i, w_j): + w_meth = self.getattr(space, space.wrap('__delslice__'), False) + if w_meth is not None: + space.call_function(w_meth, w_i, w_j) + else: + return space.delitem(self, space.newslice(w_i, w_j, space.w_None)) + def descr_call(self, space, __args__): w_meth = self.getattr(space, space.wrap('__call__')) return space.call_args(w_meth, __args__) @@ -608,6 +630,13 @@ unwrap_spec=['self', ObjSpace, W_Root]), __iter__ = interp2app(W_InstanceObject.descr_iter, unwrap_spec=['self', ObjSpace]), + __getslice__ = interp2app(W_InstanceObject.descr_getslice, + unwrap_spec=['self', ObjSpace, W_Root, W_Root]), + __setslice__ = interp2app(W_InstanceObject.descr_setslice, + unwrap_spec=['self', ObjSpace, W_Root, + W_Root, W_Root]), + __delslice__ = interp2app(W_InstanceObject.descr_delslice, + unwrap_spec=['self', ObjSpace, W_Root, W_Root]), __call__ = interp2app(W_InstanceObject.descr_call, unwrap_spec=['self', ObjSpace, Arguments]), __nonzero__ = interp2app(W_InstanceObject.descr_nonzero, Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Thu Dec 13 17:26:39 2007 @@ -585,3 +585,21 @@ assert pow(4, a) == 16 raises(TypeError, "a ** 4") assert pow(4, a, 5) == 625 + + def test_getsetdelslice(self): + + class A: + __metaclass__ = nclassobj + def __getslice__(self, i, j): + return i + j + def __setslice__(self, i, j, seq): + self.last = (i, j, seq) + def __delslice__(self, i, j): + self.last = (i, j, None) + a = A() + assert a[1:3] == 4 + a[1:3] = [1, 2, 3] + assert a.last == (1, 3, [1, 2, 3]) + del a[1:4] + assert a.last == (1, 4, None) + From arigo at codespeak.net Thu Dec 13 17:39:53 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 17:39:53 +0100 (CET) Subject: [pypy-svn] r49729 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20071213163953.A7AA01684F0@codespeak.net> Author: arigo Date: Thu Dec 13 17:39:53 2007 New Revision: 49729 Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py Log: Experimental support to access GcArrays of pointers in a general way while still making fakeaddress happy. Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py Thu Dec 13 17:39:53 2007 @@ -61,7 +61,8 @@ assert index.startswith('item') # itemN => N index = int(index[4:]) return parent.getitem(index + self.repeat)._as_ptr() - elif isinstance(A, lltype.FixedSizeArray) and A.OF == self.TYPE: + elif (isinstance(A, lltype.FixedSizeArray) and + array_item_type_match(A.OF, self.TYPE)): # for array of primitives or pointers return lltype.direct_ptradd(firstitemptr, self.repeat) else: @@ -191,7 +192,7 @@ return '< ArrayItemsOffset %r >' % (self.TYPE,) def ref(self, arrayptr): - assert lltype.typeOf(arrayptr).TO == self.TYPE + assert array_type_match(lltype.typeOf(arrayptr).TO, self.TYPE) if isinstance(self.TYPE.OF, lltype.ContainerType): # XXX this doesn't support empty arrays o = arrayptr._obj.getitem(0) @@ -228,7 +229,7 @@ return '< ArrayLengthOffset %r >' % (self.TYPE,) def ref(self, arrayptr): - assert lltype.typeOf(arrayptr).TO == self.TYPE + assert array_type_match(lltype.typeOf(arrayptr).TO, self.TYPE) return lltype._arraylenref._makeptr(arrayptr._obj, arrayptr._solid) @@ -405,6 +406,21 @@ # GCREF is similar to Address but it is GC-aware GCREF = lltype.Ptr(lltype.GcOpaqueType('GCREF')) +# A placeholder for any type that is a GcArray of pointers. +# This can be used in the symbolic offsets above to access such arrays +# in a generic way. +GCARRAY_OF_PTR = lltype.GcArray(GCREF, hints={'placeholder': True}) +gcarrayofptr_lengthoffset = ArrayLengthOffset(GCARRAY_OF_PTR) +gcarrayofptr_itemsoffset = ArrayItemsOffset(GCARRAY_OF_PTR) +gcarrayofptr_singleitemoffset = ItemOffset(GCARRAY_OF_PTR.OF) +def array_type_match(A1, A2): + return A1 == A2 or (A2 == GCARRAY_OF_PTR and + isinstance(A1, lltype.GcArray) and + isinstance(A1.OF, lltype.Ptr) and + not A1._hints.get('nolength')) +def array_item_type_match(T1, T2): + return T1 == T2 or (T2 == GCREF and isinstance(T1, lltype.Ptr)) + class _fakeaccessor(object): def __init__(self, addr): Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py Thu Dec 13 17:39:53 2007 @@ -524,3 +524,21 @@ assert weakref_deref(lltype.Ptr(S), w) == lltype.nullptr(S) assert weakref_deref(lltype.Ptr(S1), w) == lltype.nullptr(S1) +def test_generic_gcarray_of_ptr(): + S1 = lltype.GcStruct('S1', ('x', lltype.Signed)) + A1 = lltype.GcArray(lltype.Ptr(S1)) + A2 = lltype.GcArray(lltype.Ptr(A1)) + a2 = lltype.malloc(A2, 3) + a2[1] = lltype.malloc(A1, 4) + a2[1][2] = lltype.malloc(S1) + a2[1][2].x = -33 + + adr = cast_ptr_to_adr(a2) + assert (adr + gcarrayofptr_lengthoffset).signed[0] == 3 + adr += gcarrayofptr_itemsoffset + adr += gcarrayofptr_singleitemoffset + adr = adr.address[0] # => a2[1] + assert (adr + gcarrayofptr_lengthoffset).signed[0] == 4 + adr += gcarrayofptr_itemsoffset + 2 * gcarrayofptr_singleitemoffset + adr = adr.address[0] # => s2[1][2] + assert (adr + FieldOffset(S1, 'x')).signed[0] == -33 From arigo at codespeak.net Thu Dec 13 17:47:40 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 17:47:40 +0100 (CET) Subject: [pypy-svn] r49730 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20071213164740.D3CA0168453@codespeak.net> Author: arigo Date: Thu Dec 13 17:47:40 2007 New Revision: 49730 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py Log: Close your open files please :-) Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Thu Dec 13 17:47:40 2007 @@ -534,6 +534,7 @@ all.sort() for typeid, TYPE in all: f.write("%s %s\n" % (typeid, TYPE)) + f.close() def transform_graph(self, graph): if self.write_barrier_ptr: From arigo at codespeak.net Thu Dec 13 17:48:57 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 17:48:57 +0100 (CET) Subject: [pypy-svn] r49731 - in pypy/dist/pypy/rpython/memory: . gc Message-ID: <20071213164857.85B46168453@codespeak.net> Author: arigo Date: Thu Dec 13 17:48:57 2007 New Revision: 49731 Modified: pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/gctypelayout.py Log: Experimental: special-case GcArray(gcptr) in trace(). So far, optimizing this case was the most important measured win in the pypy-gc-traceopt branch. Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Thu Dec 13 17:48:57 2007 @@ -7,6 +7,8 @@ needs_write_barrier = False needs_zero_gc_pointers = True + TYPEID_OF_GCARRAY_OF_GCPTR = 1 + def set_query_functions(self, is_varsize, has_gcptr_in_varsize, getfinalizer, offsets_to_gc_pointers, @@ -115,6 +117,15 @@ Typically, 'callback' is a bound method and 'arg' can be None. """ typeid = self.get_type_id(obj) + if typeid == GCBase.TYPEID_OF_GCARRAY_OF_GCPTR: + # a performance shortcut for GcArray(gcptr) + length = (obj + llmemory.gcarrayofptr_lengthoffset).signed[0] + item = obj + llmemory.gcarrayofptr_itemsoffset + while length > 0: + callback(item, arg) + item += llmemory.gcarrayofptr_singleitemoffset + length -= 1 + return offsets = self.offsets_to_gc_pointers(typeid) i = 0 while i < len(offsets): @@ -125,14 +136,13 @@ length = (obj + self.varsize_offset_to_length(typeid)).signed[0] offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) itemlength = self.varsize_item_sizes(typeid) - i = 0 - while i < length: + while length > 0: j = 0 while j < len(offsets): callback(item + offsets[j], arg) j += 1 - i += 1 item += itemlength + length -= 1 trace._annspecialcase_ = 'specialize:arg(2)' Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/gctypelayout.py Thu Dec 13 17:48:57 2007 @@ -1,4 +1,5 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena +from pypy.rpython.memory.gc.base import GCBase class TypeLayoutBuilder(object): @@ -25,6 +26,22 @@ except KeyError: assert self.can_add_new_types assert isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)) + + if TYPE != llmemory.GCARRAY_OF_PTR: + # first, force GcArray(gcptr) to have typeid 1, for the + # performance-related special-casing done by GCBase.trace(). + if llmemory.GCARRAY_OF_PTR not in self.id_of_type: + typeid = self.get_type_id(llmemory.GCARRAY_OF_PTR) + assert typeid == GCBase.TYPEID_OF_GCARRAY_OF_GCPTR + # all types that are of the shape GcArray(gcptr) get a + # typeid of 1. + if (isinstance(TYPE, lltype.GcArray) + and isinstance(TYPE.OF, lltype.Ptr) + and TYPE.OF.TO._gckind == 'gc'): + type_id = GCBase.TYPEID_OF_GCARRAY_OF_GCPTR + self.id_of_type[TYPE] = type_id + return type_id + # Record the new type_id description as a small dict for now. # The framework gc transformer will turn it into a # Struct("type_info") in flatten_table(). From arigo at codespeak.net Thu Dec 13 18:28:20 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 18:28:20 +0100 (CET) Subject: [pypy-svn] r49733 - pypy/branch/pypy-gc-traceopt/rpython/memory/gc Message-ID: <20071213172820.BAF851684F8@codespeak.net> Author: arigo Date: Thu Dec 13 18:28:19 2007 New Revision: 49733 Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py Log: Count the number of traces with almost no overhead. Must be enabled by setting LOG_COUNT_TRACES to a file name. Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py Thu Dec 13 18:28:19 2007 @@ -4,6 +4,10 @@ from pypy.rpython.lltypesystem.lloperation import llop +#LOG_COUNT_TRACES = "traces.log" +LOG_COUNT_TRACES = None + + class TraceOptGCBase(object): _alloc_flavor_ = "raw" @@ -15,9 +19,13 @@ while i < count: self.traceopt_index[i] = chr(0) i += 1 + if LOG_COUNT_TRACES: + self.logt = setup_log_count_traces(count) def trace(self, obj, callback, arg): typeid = self.get_type_id(obj) + if LOG_COUNT_TRACES: + log_count_trace(self.logt, typeid) # first see if we have an optimized tracer for this typeid # (this becomes a single switch) shapeindex = ord(self.traceopt_index[typeid]) @@ -154,3 +162,42 @@ common_shape_checkers, common_shape_tracers = get_common_shape_fns() list_common_shape_checkers = common_shape_checkers.items() unroll_common_shape_tracers = unrolling_iterable(common_shape_tracers.items()) + +# ____________________________________________________________ + +if LOG_COUNT_TRACES: + import os + from pypy.rpython.lltypesystem import rffi + from pypy.rlib import rmmap + + def setup_log_count_traces(max): + # XXX no error checks + fd = os.open(LOG_COUNT_TRACES, os.O_RDWR|os.O_CREAT|os.O_TRUNC, 0666) + length = max * rffi.sizeof(rffi.ULONGLONG) + os.write(fd, '\x00' * length) + os.lseek(fd, 0, 0) + return rmmap.mmap(fd, length) + + def log_count_trace(map, typeid): + p = rffi.cast(rffi.ULONGLONGP, map.data) + p[typeid] += 1 + +# display the content of a log file when run as __main__ +if __name__ == '__main__': + import sys, struct + if len(sys.argv) == 1: + filename = LOG_COUNT_TRACES + else: + filename = sys.argv[1] + data = open(filename, 'rb').read() + results = [] + sz = struct.calcsize("Q") + for i in range(0, len(data), sz): + count, = struct.unpack("Q", data[i:i+sz]) + results.append((count, i)) + results.sort() + results.reverse() + N = 20 + print '%d top results: (typeid, number of traces)' % (N,) + for count, i in results[:N]: + print '%5d %15d' % (i, count) From arigo at codespeak.net Thu Dec 13 18:34:37 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 18:34:37 +0100 (CET) Subject: [pypy-svn] r49734 - pypy/branch/pypy-gc-traceopt/rpython/memory/gc Message-ID: <20071213173437.BB6991684F8@codespeak.net> Author: arigo Date: Thu Dec 13 18:34:37 2007 New Revision: 49734 Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py Log: Bug. Modified: pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/memory/gc/traceopt.py Thu Dec 13 18:34:37 2007 @@ -194,7 +194,7 @@ sz = struct.calcsize("Q") for i in range(0, len(data), sz): count, = struct.unpack("Q", data[i:i+sz]) - results.append((count, i)) + results.append((count, i // sz)) results.sort() results.reverse() N = 20 From arigo at codespeak.net Thu Dec 13 19:11:10 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 19:11:10 +0100 (CET) Subject: [pypy-svn] r49735 - pypy/dist/pypy/rpython Message-ID: <20071213181110.F12A816844A@codespeak.net> Author: arigo Date: Thu Dec 13 19:11:10 2007 New Revision: 49735 Modified: pypy/dist/pypy/rpython/rpbc.py Log: Overlooked this while killing the HalfConcreteWrapper. The idea is that the various convert_const() methods should now always return None if the lowleveltype is Void. Modified: pypy/dist/pypy/rpython/rpbc.py ============================================================================== --- pypy/dist/pypy/rpython/rpbc.py (original) +++ pypy/dist/pypy/rpython/rpbc.py Thu Dec 13 19:11:10 2007 @@ -666,7 +666,7 @@ if desc not in self.s_pbc.descriptions: raise TyperError("%r not in %r" % (cls, self)) if self.lowleveltype is Void: - return desc.pyobj + return None return rclass.get_type_repr(self.rtyper).convert_desc(desc) def convert_const(self, cls): From arigo at codespeak.net Thu Dec 13 19:11:42 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 13 Dec 2007 19:11:42 +0100 (CET) Subject: [pypy-svn] r49736 - pypy/branch/pypy-gc-traceopt/rpython Message-ID: <20071213181142.7329816844A@codespeak.net> Author: arigo Date: Thu Dec 13 19:11:42 2007 New Revision: 49736 Modified: pypy/branch/pypy-gc-traceopt/rpython/rpbc.py Log: Merge of r49735 from the trunk. Modified: pypy/branch/pypy-gc-traceopt/rpython/rpbc.py ============================================================================== --- pypy/branch/pypy-gc-traceopt/rpython/rpbc.py (original) +++ pypy/branch/pypy-gc-traceopt/rpython/rpbc.py Thu Dec 13 19:11:42 2007 @@ -666,7 +666,7 @@ if desc not in self.s_pbc.descriptions: raise TyperError("%r not in %r" % (cls, self)) if self.lowleveltype is Void: - return desc.pyobj + return None return rclass.get_type_repr(self.rtyper).convert_desc(desc) def convert_const(self, cls): From fijal at codespeak.net Thu Dec 13 20:17:45 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 13 Dec 2007 20:17:45 +0100 (CET) Subject: [pypy-svn] r49737 - pypy/dist/pypy/module/crypt Message-ID: <20071213191745.B420E168510@codespeak.net> Author: fijal Date: Thu Dec 13 20:17:44 2007 New Revision: 49737 Modified: pypy/dist/pypy/module/crypt/interp_crypt.py Log: (mwh) we don't need crypt on top of OS X Modified: pypy/dist/pypy/module/crypt/interp_crypt.py ============================================================================== --- pypy/dist/pypy/module/crypt/interp_crypt.py (original) +++ pypy/dist/pypy/module/crypt/interp_crypt.py Thu Dec 13 20:17:44 2007 @@ -2,8 +2,12 @@ from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.rpython.lltypesystem import rffi, lltype from pypy.translator.tool.cbuild import ExternalCompilationInfo +import sys -eci = ExternalCompilationInfo(libraries=['crypt']) +if sys.platform.startswith('darwin'): + eci = ExternalCompilationInfo() +else: + eci = ExternalCompilationInfo(libraries=['crypt']) c_crypt = rffi.llexternal('crypt', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP, compilation_info=eci, threadsafe=False) From fijal at codespeak.net Thu Dec 13 20:25:19 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 13 Dec 2007 20:25:19 +0100 (CET) Subject: [pypy-svn] r49738 - pypy/dist/pypy/translator/goal Message-ID: <20071213192519.BF451168511@codespeak.net> Author: fijal Date: Thu Dec 13 20:25:17 2007 New Revision: 49738 Modified: pypy/dist/pypy/translator/goal/bench-cronjob.py Log: Join --faassen and --allworkingmodules into one job. Modified: pypy/dist/pypy/translator/goal/bench-cronjob.py ============================================================================== --- pypy/dist/pypy/translator/goal/bench-cronjob.py (original) +++ pypy/dist/pypy/translator/goal/bench-cronjob.py Thu Dec 13 20:25:17 2007 @@ -182,10 +182,9 @@ backends = [backend.strip() for backend in """ c c--stackless--_faassen - c--_faassen + c--_faassen--_allworkingmodules c--thread c--_objspace=taint - c--_allworkingmodules c--gc=marksweep--_faassen c--gc=semispace--_faassen c--gc=generation--_faassen From fijal at codespeak.net Thu Dec 13 20:31:48 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 13 Dec 2007 20:31:48 +0100 (CET) Subject: [pypy-svn] r49739 - pypy/dist/pypy/translator/goal Message-ID: <20071213193148.C9D2E168405@codespeak.net> Author: fijal Date: Thu Dec 13 20:31:48 2007 New Revision: 49739 Modified: pypy/dist/pypy/translator/goal/bench-cronjob.py Log: Strike taint Modified: pypy/dist/pypy/translator/goal/bench-cronjob.py ============================================================================== --- pypy/dist/pypy/translator/goal/bench-cronjob.py (original) +++ pypy/dist/pypy/translator/goal/bench-cronjob.py Thu Dec 13 20:31:48 2007 @@ -184,7 +184,6 @@ c--stackless--_faassen c--_faassen--_allworkingmodules c--thread - c--_objspace=taint c--gc=marksweep--_faassen c--gc=semispace--_faassen c--gc=generation--_faassen From cfbolz at codespeak.net Thu Dec 13 21:11:59 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 13 Dec 2007 21:11:59 +0100 (CET) Subject: [pypy-svn] r49740 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071213201159.154681684F6@codespeak.net> Author: cfbolz Date: Thu Dec 13 21:11:58 2007 New Revision: 49740 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: fix a bug found by the flow object space Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Thu Dec 13 21:11:58 2007 @@ -522,6 +522,7 @@ except OperationError, e: if e.match(space, space.w_StopIteration): return space.w_False + raise if space.eq_w(w_x, w_obj): return space.w_True Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Thu Dec 13 21:11:58 2007 @@ -603,3 +603,9 @@ del a[1:4] assert a.last == (1, 4, None) + def test_contains_bug(self): + class A: + __metaclass__ = nclassobj + def __iter__(self): + return self + raises(TypeError, "1 in A()") From cfbolz at codespeak.net Thu Dec 13 21:32:11 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 13 Dec 2007 21:32:11 +0100 (CET) Subject: [pypy-svn] r49741 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071213203211.637FB168519@codespeak.net> Author: cfbolz Date: Thu Dec 13 21:32:10 2007 New Revision: 49741 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: another bug Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Thu Dec 13 21:32:10 2007 @@ -25,7 +25,7 @@ w_metaclass = space.type(w_base) if space.is_true(space.callable(w_metaclass)): return space.call_function(w_metaclass, w_name, - w_bases, w_dic) + w_bases, w_dict) raise OperationError(space.w_TypeError, space.wrap("base must be class")) Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Thu Dec 13 21:32:10 2007 @@ -609,3 +609,6 @@ def __iter__(self): return self raises(TypeError, "1 in A()") + + def test_class_instantiation_bug(self): + raises(TypeError, "class A(1, 2): __metaclass__ = nclassobj") From regmee at codespeak.net Thu Dec 13 21:52:29 2007 From: regmee at codespeak.net (regmee at codespeak.net) Date: Thu, 13 Dec 2007 21:52:29 +0100 (CET) Subject: [pypy-svn] r49742 - pypy/branch/clr-module-improvements/pypy/module/clr Message-ID: <20071213205229.5A7B3168518@codespeak.net> Author: regmee Date: Thu Dec 13 21:52:28 2007 New Revision: 49742 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py Log: bug fix to enable support for generic Dictionary class Modified: pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py Thu Dec 13 21:52:28 2007 @@ -161,8 +161,9 @@ is_static = get_meth.get_IsStatic() if b_prop.get_CanWrite(): set_meth = b_prop.GetSetMethod() - set_name = set_meth.get_Name() - is_static = set_meth.get_IsStatic() + if set_meth: + set_name = set_meth.get_Name() + is_static = set_meth.get_IsStatic() b_indexparams = b_prop.GetIndexParameters() if len(b_indexparams) == 0: properties.append((b_prop.get_Name(), get_name, set_name, is_static)) From cfbolz at codespeak.net Thu Dec 13 21:55:44 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 13 Dec 2007 21:55:44 +0100 (CET) Subject: [pypy-svn] r49743 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071213205544.BED65168451@codespeak.net> Author: cfbolz Date: Thu Dec 13 21:55:43 2007 New Revision: 49743 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: a missing helper function. tests that hit the problem Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Thu Dec 13 21:55:43 2007 @@ -8,6 +8,13 @@ from pypy.rlib.rarithmetic import r_uint, intmask +def raise_type_err(space, argument, expected, w_obj): + type_name = space.type(w_obj).getname(space, '?') + w_error = space.wrap("argument %s must be %s, not %s" % ( + argument, expected, type_name)) + raise OperationError(space.w_TypeError, + w_error) + def descr_classobj_new(space, w_subtype, w_name, w_bases, w_dict): if not space.is_true(space.isinstance(w_bases, space.w_tuple)): raise_type_err(space, 'bases', 'tuple', w_bases) Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Thu Dec 13 21:55:43 2007 @@ -612,3 +612,7 @@ def test_class_instantiation_bug(self): raises(TypeError, "class A(1, 2): __metaclass__ = nclassobj") + raises(TypeError, "nclassobj(1, (), {})") + raises(TypeError, "nclassobj('abc', 1, {})") + raises(TypeError, "nclassobj('abc', (1, ), {})") + raises(TypeError, "nclassobj('abc', (), 3)") From regmee at codespeak.net Thu Dec 13 22:21:07 2007 From: regmee at codespeak.net (regmee at codespeak.net) Date: Thu, 13 Dec 2007 22:21:07 +0100 (CET) Subject: [pypy-svn] r49744 - pypy/branch/clr-module-improvements/pypy/module/clr/test Message-ID: <20071213212107.548B5168504@codespeak.net> Author: regmee Date: Thu Dec 13 22:21:06 2007 New Revision: 49744 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py Log: some Test cases for Generic classes Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py Thu Dec 13 22:21:06 2007 @@ -186,3 +186,22 @@ sum += i assert sum == 1+54+21 + def test_generic_class(self): + import clr + ListInt = clr.load_cli_class("System.Collections.Generic", "List`1[System.Int32]") + x = ListInt() + x.Add(42) + x.Add(4) + x.Add(4) + sum = 0 + for i in x: + sum += i + assert sum == 42+4+4 + + genDictIntStr = clr.load_cli_class("System.Collections.Generic","Dictionary`2[System.Int32,System.String]") + x = genDictIntStr() + x[1] = "test" + x[2] = "rest" + assert x[1] == "test" + assert x[2] == "rest" + From cfbolz at codespeak.net Thu Dec 13 22:44:01 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 13 Dec 2007 22:44:01 +0100 (CET) Subject: [pypy-svn] r49745 - pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__ Message-ID: <20071213214401.3F0F9168506@codespeak.net> Author: cfbolz Date: Thu Dec 13 22:43:59 2007 New Revision: 49745 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Log: fix annotation problem Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Thu Dec 13 22:43:59 2007 @@ -249,7 +249,7 @@ except OperationError, e: if not e.match(space, space.w_TypeError): raise - return (None, None) + return [None, None] return space.unpacktuple(w_tup, 2) class W_InstanceObject(Wrappable): From cfbolz at codespeak.net Thu Dec 13 22:58:13 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 13 Dec 2007 22:58:13 +0100 (CET) Subject: [pypy-svn] r49746 - pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__ Message-ID: <20071213215813.89D3E168514@codespeak.net> Author: cfbolz Date: Thu Dec 13 22:58:13 2007 New Revision: 49746 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Log: translation fixes. not perfectly efficient, but not too bad either Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Thu Dec 13 22:58:13 2007 @@ -27,6 +27,7 @@ # XXX missing: lengthy and obscure logic about "__module__" + bases = [] for w_base in space.unpackiterable(w_bases): if not isinstance(w_base, W_ClassObject): w_metaclass = space.type(w_base) @@ -35,13 +36,14 @@ w_bases, w_dict) raise OperationError(space.w_TypeError, space.wrap("base must be class")) + bases.append(w_base) - return W_ClassObject(space, w_name, w_bases, w_dict) + return W_ClassObject(space, w_name, bases, w_dict) class W_ClassObject(Wrappable): - def __init__(self, space, w_name, w_bases, w_dict): + def __init__(self, space, w_name, bases, w_dict): self.name = space.str_w(w_name) - self.bases_w = space.unpackiterable(w_bases) + self.bases_w = bases self.w_dict = w_dict def getdict(self): @@ -82,7 +84,7 @@ def fget_bases(space, self): - return space.wrap(self.bases_w) + return space.newtuple(self.bases_w) def fset_bases(space, self, w_bases): # XXX in theory, this misses a check against inheritance cycles @@ -93,11 +95,13 @@ space.w_TypeError, space.wrap("__bases__ must be a tuple object")) bases_w = space.unpackiterable(w_bases) + bases = [] for w_base in bases_w: if not isinstance(w_base, W_ClassObject): raise OperationError(space.w_TypeError, space.wrap("__bases__ items must be classes")) - self.bases_w = bases_w + bases.append(w_base) + self.bases_w = bases def fdel_bases(space, self): raise OperationError( From cfbolz at codespeak.net Thu Dec 13 23:27:43 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 13 Dec 2007 23:27:43 +0100 (CET) Subject: [pypy-svn] r49747 - pypy/branch/interplevel-oldstyle-classes/pypy/module/_random Message-ID: <20071213222743.DFAD4168519@codespeak.net> Author: cfbolz Date: Thu Dec 13 23:27:42 2007 New Revision: 49747 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/_random/interp_random.py Log: give the Random class a sensible name Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/_random/interp_random.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/_random/interp_random.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/_random/interp_random.py Thu Dec 13 23:27:42 2007 @@ -110,7 +110,7 @@ getrandbits.unwrap_spec = ['self', ObjSpace, int] -W_Random.typedef = TypeDef("W_Random", +W_Random.typedef = TypeDef("Random", __new__ = interp2app(descr_new__), random = interp2app(W_Random.random), seed = interp2app(W_Random.seed), From cfbolz at codespeak.net Thu Dec 13 23:35:59 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 13 Dec 2007 23:35:59 +0100 (CET) Subject: [pypy-svn] r49748 - pypy/branch/interplevel-oldstyle-classes/pypy/interpreter Message-ID: <20071213223559.2C07E168501@codespeak.net> Author: cfbolz Date: Thu Dec 13 23:35:59 2007 New Revision: 49748 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/interpreter/gateway.py Log: typo in a comment Modified: pypy/branch/interplevel-oldstyle-classes/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/interpreter/gateway.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/interpreter/gateway.py Thu Dec 13 23:35:59 2007 @@ -328,7 +328,7 @@ def visit__object(self, typ): if typ not in (int, str, float, unicode, r_longlong): - assert False, "unsupported basic type in uwnrap_spec" + assert False, "unsupported basic type in unwrap_spec" self.unwrap.append("space.%s_w(%s)" % (typ.__name__, self.nextarg())) From cfbolz at codespeak.net Fri Dec 14 00:02:21 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 14 Dec 2007 00:02:21 +0100 (CET) Subject: [pypy-svn] r49751 - in pypy/branch/interplevel-oldstyle-classes/pypy: lib module/__builtin__ module/__builtin__/test objspace/std Message-ID: <20071213230221.A753816851B@codespeak.net> Author: cfbolz Date: Fri Dec 14 00:02:21 2007 New Revision: 49751 Removed: pypy/branch/interplevel-oldstyle-classes/pypy/lib/_classobj.py Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/__init__.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/objspace/std/objspace.py Log: make the new interplevel version of oldstyle classes the default and kill the old one Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/__init__.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/__init__.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/__init__.py Fri Dec 14 00:02:21 2007 @@ -77,11 +77,8 @@ 'open' : 'state.get(space).w_file', # old-style classes dummy support - '_classobj' : 'space.w_classobj', - '_instance' : 'space.w_instance', - # new old-style classes implementation for testing - 'nclassobj' : 'interp_classobj.W_ClassObject', - 'ninstance' : 'interp_classobj.W_InstanceObject', + '_classobj' : 'interp_classobj.W_ClassObject', + '_instance' : 'interp_classobj.W_InstanceObject', # default __metaclass__ '__metaclass__' : '(space.w_type)', Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Fri Dec 14 00:02:21 2007 @@ -1,8 +1,11 @@ class AppTestOldstyle(object): + def setup_class(cls): + from pypy.conftest import gettestobjspace + cls.space = gettestobjspace(**{"objspace.std.oldstyle": True}) + def test_simple(self): class A: - __metaclass__ = nclassobj a = 1 assert A.__name__ == 'A' assert A.__bases__ == () @@ -17,7 +20,6 @@ def test_mutate_class_special(self): class A: - __metaclass__ = nclassobj a = 1 A.__name__ = 'B' assert A.__name__ == 'B' @@ -25,7 +27,6 @@ A.__dict__ = {'a': 5} assert A.a == 5 class B: - __metaclass__ = nclassobj a = 17 b = 18 class C(A): @@ -47,7 +48,7 @@ def test_class_repr(self): class A: - __metaclass__ = nclassobj + pass assert repr(A).startswith(" Author: cfbolz Date: Fri Dec 14 00:08:16 2007 New Revision: 49752 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Log: more annotation stuff Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Fri Dec 14 00:08:16 2007 @@ -27,8 +27,8 @@ # XXX missing: lengthy and obscure logic about "__module__" - bases = [] - for w_base in space.unpackiterable(w_bases): + bases_w = space.unpackiterable(w_bases) + for w_base in bases_w: if not isinstance(w_base, W_ClassObject): w_metaclass = space.type(w_base) if space.is_true(space.callable(w_metaclass)): @@ -36,9 +36,8 @@ w_bases, w_dict) raise OperationError(space.w_TypeError, space.wrap("base must be class")) - bases.append(w_base) - return W_ClassObject(space, w_name, bases, w_dict) + return W_ClassObject(space, w_name, bases_w, w_dict) class W_ClassObject(Wrappable): def __init__(self, space, w_name, bases, w_dict): @@ -95,13 +94,11 @@ space.w_TypeError, space.wrap("__bases__ must be a tuple object")) bases_w = space.unpackiterable(w_bases) - bases = [] for w_base in bases_w: if not isinstance(w_base, W_ClassObject): raise OperationError(space.w_TypeError, space.wrap("__bases__ items must be classes")) - bases.append(w_base) - self.bases_w = bases + self.bases_w = bases_w def fdel_bases(space, self): raise OperationError( @@ -114,6 +111,8 @@ if w_result is not None: return w_result for base in self.bases_w: + # XXX fix annotation of bases_w to be a list of W_ClassObjects + assert isinstance(base, W_ClassObject) w_result = base.lookup(space, w_attr) if w_result is not None: return w_result @@ -260,6 +259,7 @@ def __init__(self, space, w_class, w_dict=None): if w_dict is None: w_dict = space.newdict() + assert isinstance(w_class, W_ClassObject) self.w_class = w_class self.w_dict = w_dict From regmee at codespeak.net Fri Dec 14 00:10:55 2007 From: regmee at codespeak.net (regmee at codespeak.net) Date: Fri, 14 Dec 2007 00:10:55 +0100 (CET) Subject: [pypy-svn] r49753 - pypy/branch/clr-module-improvements/pypy/module/clr/test Message-ID: <20071213231055.82E1616851D@codespeak.net> Author: regmee Date: Fri Dec 14 00:10:55 2007 New Revision: 49753 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py Log: improving the test case for Generic Classes Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py Fri Dec 14 00:10:55 2007 @@ -197,6 +197,7 @@ for i in x: sum += i assert sum == 42+4+4 + raises(TypeError, x.Add, "test") genDictIntStr = clr.load_cli_class("System.Collections.Generic","Dictionary`2[System.Int32,System.String]") x = genDictIntStr() @@ -204,4 +205,7 @@ x[2] = "rest" assert x[1] == "test" assert x[2] == "rest" + raises(TypeError, x.__setitem__, 3, 3) + raises(TypeError, x.__setitem__, 4, 4.453) + raises(TypeError, x.__setitem__, "test", 3) From cfbolz at codespeak.net Fri Dec 14 00:17:21 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 14 Dec 2007 00:17:21 +0100 (CET) Subject: [pypy-svn] r49754 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071213231721.3579216851E@codespeak.net> Author: cfbolz Date: Fri Dec 14 00:17:20 2007 New Revision: 49754 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: make instance.__new__ work Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Fri Dec 14 00:17:20 2007 @@ -255,6 +255,18 @@ return [None, None] return space.unpacktuple(w_tup, 2) +def descr_instance_new(space, w_type, w_class, w_dict=None): + # w_type is not used at all + if not isinstance(w_class, W_ClassObject): + raise OperationError( + space.w_TypeError, + space.wrap("instance() first arg must be class")) + if w_dict is None: + w_dict = space.newdict() + elif not space.is_true(space.isinstance(w_dict, space.w_dict)): + raise TypeError("instance() second arg must be dictionary or None") + return W_InstanceObject(space, w_class, w_dict) + class W_InstanceObject(Wrappable): def __init__(self, space, w_class, w_dict=None): if w_dict is None: @@ -281,17 +293,6 @@ space.wrap("__class__ must be set to a class")) self.w_class = w_class - def descr_new(space, w_type, w_class, w_dict=None): - # typ is not used at all - if not isinstance(w_class, W_ClassObject): - raise OperationError( - space.w_TypeError, - space.wrap("instance() first arg must be class")) - if w_dict is None: - w_dict = space.newdict() - elif not space.is_true(space.isinstance(w_dict, space.w_dict)): - raise TypeError("instance() second arg must be dictionary or None") - return W_InstanceObject(space, w_class, w_dict) def getattr(self, space, w_name, exc=True): name = space.str_w(w_name) @@ -621,7 +622,7 @@ unwrap_spec=["self", ObjSpace, W_Root]) W_InstanceObject.typedef = TypeDef("instance", - __new__ = interp2app(W_InstanceObject.descr_new), + __new__ = interp2app(descr_instance_new), __getattribute__ = interp2app(W_InstanceObject.descr_getattribute, unwrap_spec=['self', ObjSpace, W_Root]), __setattr__ = interp2app(W_InstanceObject.descr_setattr, Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Fri Dec 14 00:17:20 2007 @@ -569,3 +569,11 @@ raises(TypeError, "_classobj('abc', 1, {})") raises(TypeError, "_classobj('abc', (1, ), {})") raises(TypeError, "_classobj('abc', (), 3)") + + def test_instance_new(self): + class A: + b = 1 + a = A() + a = type(a).__new__(type(a), A, {'c': 2}) + assert a.b == 1 + assert a.c == 2 From cfbolz at codespeak.net Fri Dec 14 01:06:58 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 14 Dec 2007 01:06:58 +0100 (CET) Subject: [pypy-svn] r49755 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071214000658.6CAD81684D9@codespeak.net> Author: cfbolz Date: Fri Dec 14 01:06:57 2007 New Revision: 49755 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: support for __del__. not quite elegant, but works. Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Fri Dec 14 01:06:57 2007 @@ -44,7 +44,7 @@ self.name = space.str_w(w_name) self.bases_w = bases self.w_dict = w_dict - + def getdict(self): return self.w_dict @@ -144,7 +144,10 @@ return space.call_function(w_descr_get, w_value, space.w_None, self) def descr_call(self, space, __args__): - w_inst = W_InstanceObject(space, self) + if self.lookup(space, space.wrap('__del__')) is not None: + w_inst = W_InstanceObjectWithDel(space, self) + else: + w_inst = W_InstanceObject(space, self) w_init = w_inst.getattr(space, space.wrap('__init__'), False) if w_init is not None: w_result = space.call_args(w_init, __args__) @@ -274,6 +277,7 @@ assert isinstance(w_class, W_ClassObject) self.w_class = w_class self.w_dict = w_dict + self.space = space def getdict(self): return self.w_dict @@ -669,3 +673,28 @@ **rawdict ) +class W_InstanceObjectWithDel(W_InstanceObject): + # XXX this is code duplication from pypy.interpreter.typedef + # find a way to prevent this. + def __del__(self): + + lifeline = self.getweakref() + if lifeline is not None: + # Clear all weakrefs to this object before we call + # the app-level __del__. We detach the lifeline + # first: if the app-level __del__ tries to use + # weakrefs again, they won't reuse the broken + # (already-cleared) ones from this lifeline. + self.setweakref(self.space, None) + lifeline.clear_all_weakrefs() + try: + self.descr_del() + except OperationError, e: + e.write_unraisable(self.space, 'method __del__ of ', self) + e.clear(self.space) # break up reference cycles + + def descr_del(self): + space = self.space + w_func = self.getattr(space, space.wrap('__del__'), False) + if w_func is not None: + space.call_function(w_func) Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Fri Dec 14 01:06:57 2007 @@ -577,3 +577,24 @@ a = type(a).__new__(type(a), A, {'c': 2}) assert a.b == 1 assert a.c == 2 + + def test_del(self): + import gc + l = [] + class A: + def __del__(self): + l.append(1) + a = A() + a = None + gc.collect() + gc.collect() + gc.collect() + assert l == [1] + class B(A): + pass + b = B() + b = None + gc.collect() + gc.collect() + gc.collect() + assert l == [1, 1] From cfbolz at codespeak.net Fri Dec 14 01:34:51 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 14 Dec 2007 01:34:51 +0100 (CET) Subject: [pypy-svn] r49756 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071214003451.94AC4168509@codespeak.net> Author: cfbolz Date: Fri Dec 14 01:34:50 2007 New Revision: 49756 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: test and fix for a problem that CPython's test_class revealed Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Fri Dec 14 01:34:50 2007 @@ -319,7 +319,13 @@ w_descr_get = space.lookup(w_value, '__get__') if w_descr_get is None: return w_value - return space.call_function(w_descr_get, w_value, self, self.w_class) + try: + return space.call_function(w_descr_get, w_value, self, self.w_class) + except OperationError, e: + if exc or not e.match(space, space.w_AttributeError): + raise + return None + def descr_getattribute(self, space, w_attr): #import pdb; pdb.set_trace() Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Fri Dec 14 01:34:50 2007 @@ -598,3 +598,13 @@ gc.collect() gc.collect() assert l == [1, 1] + + def test_catch_attributeerror_of_descriptor(self): + def booh(self): + raise AttributeError, "booh" + + class E: + __eq__ = property(booh) + + # does not crash + E() == E() From arigo at codespeak.net Fri Dec 14 11:41:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Dec 2007 11:41:24 +0100 (CET) Subject: [pypy-svn] r49769 - pypy/dist/pypy/translator/c/test Message-ID: <20071214104124.6D0BF16852B@codespeak.net> Author: arigo Date: Fri Dec 14 11:41:24 2007 New Revision: 49769 Modified: pypy/dist/pypy/translator/c/test/test_boehm.py Log: Fix test. (It was skipped by mistake until two days ago) Modified: pypy/dist/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_boehm.py (original) +++ pypy/dist/pypy/translator/c/test/test_boehm.py Fri Dec 14 11:41:24 2007 @@ -98,8 +98,9 @@ assert 0 < res <= 84 def test_id_is_weak(self): - # test that id(obj) does not keep obj alive + # test that compute_unique_id(obj) does not keep obj alive from pypy.rpython.lltypesystem.lloperation import llop + from pypy.rlib.objectmodel import compute_unique_id class State: pass s = State() @@ -113,11 +114,11 @@ pass def run_once(): a = A() - ida = id(a) + ida = compute_unique_id(a) b = B() - idb = id(b) + idb = compute_unique_id(b) c = C() - idc = id(c) + idc = compute_unique_id(c) llop.gc__collect(lltype.Void) llop.gc__collect(lltype.Void) llop.gc__collect(lltype.Void) From pypy-svn at codespeak.net Fri Dec 14 12:03:07 2007 From: pypy-svn at codespeak.net (VIAGRA ® Official Site) Date: Fri, 14 Dec 2007 12:03:07 +0100 (CET) Subject: [pypy-svn] Lovers package at discount price! Message-ID: <20071214050307.17910.qmail@mail.voron-auto.ru> An HTML attachment was scrubbed... URL: From cfbolz at codespeak.net Fri Dec 14 12:04:43 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 14 Dec 2007 12:04:43 +0100 (CET) Subject: [pypy-svn] r49770 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071214110443.C854516853D@codespeak.net> Author: cfbolz Date: Fri Dec 14 12:04:43 2007 New Revision: 49770 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: fix a bug found by test_new Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Fri Dec 14 12:04:43 2007 @@ -267,7 +267,9 @@ if w_dict is None: w_dict = space.newdict() elif not space.is_true(space.isinstance(w_dict, space.w_dict)): - raise TypeError("instance() second arg must be dictionary or None") + raise OperationError( + space.w_TypeError, + space.wrap("instance() second arg must be dictionary or None")) return W_InstanceObject(space, w_class, w_dict) class W_InstanceObject(Wrappable): Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Fri Dec 14 12:04:43 2007 @@ -577,6 +577,7 @@ a = type(a).__new__(type(a), A, {'c': 2}) assert a.b == 1 assert a.c == 2 + raises(TypeError, type(a).__new__, type(a), A, 1) def test_del(self): import gc @@ -608,3 +609,19 @@ # does not crash E() == E() + + def test_multiple_inheritance_more(self): + l = [] + class A: # classic class + def save(self): + l.append("A") + class B(A): + pass + class C(A): + def save(self): + l.append("C") + class D(B, C): + pass + + D().save() + assert l == ['A'] From antocuni at codespeak.net Fri Dec 14 12:14:52 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 14 Dec 2007 12:14:52 +0100 (CET) Subject: [pypy-svn] r49771 - pypy/branch/clr-module-improvements/pypy/module/clr/test Message-ID: <20071214111452.8B88A16851D@codespeak.net> Author: antocuni Date: Fri Dec 14 12:14:52 2007 New Revision: 49771 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py Log: split one big test into smaller ones Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py Fri Dec 14 12:14:52 2007 @@ -162,8 +162,6 @@ def test_iteration(self): import clr - - # test iteration in ArrayList ArrayList = clr.load_cli_class('System.Collections', 'ArrayList') x = ArrayList() x.Add(1) @@ -175,7 +173,8 @@ sum += i assert sum == 1+2+3+4 - # test iteration in Stack + def test_iteration_stack(self): + import clr Stack = clr.load_cli_class('System.Collections', 'Stack') obj = Stack() obj.Push(1) @@ -186,7 +185,7 @@ sum += i assert sum == 1+54+21 - def test_generic_class(self): + def test_load_generic_class(self): import clr ListInt = clr.load_cli_class("System.Collections.Generic", "List`1[System.Int32]") x = ListInt() @@ -197,8 +196,15 @@ for i in x: sum += i assert sum == 42+4+4 + + def test_generic_class_typeerror(self): + import clr + ListInt = clr.load_cli_class("System.Collections.Generic", "List`1[System.Int32]") + x = ListInt() raises(TypeError, x.Add, "test") + def test_generic_dict(self): + import clr genDictIntStr = clr.load_cli_class("System.Collections.Generic","Dictionary`2[System.Int32,System.String]") x = genDictIntStr() x[1] = "test" From fijal at codespeak.net Fri Dec 14 12:15:50 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 14 Dec 2007 12:15:50 +0100 (CET) Subject: [pypy-svn] r49772 - pypy/dist/pypy/doc Message-ID: <20071214111550.C959E16853A@codespeak.net> Author: fijal Date: Fri Dec 14 12:15:50 2007 New Revision: 49772 Modified: pypy/dist/pypy/doc/prolog-interpreter.txt Log: Typo Modified: pypy/dist/pypy/doc/prolog-interpreter.txt ============================================================================== --- pypy/dist/pypy/doc/prolog-interpreter.txt (original) +++ pypy/dist/pypy/doc/prolog-interpreter.txt Fri Dec 14 12:15:50 2007 @@ -25,7 +25,7 @@ .. _`Learn Prolog Now!`: http://www.coli.uni-saarland.de/~kris/learn-prolog-now/ .. _`A Prolog interpreter in Python`: http://codespeak.net/pypy/extradoc/paper/prolog-in-python.pdf -Example useage +Example usage ============== First some simple examples that show simple unification:: From cfbolz at codespeak.net Fri Dec 14 12:36:12 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 14 Dec 2007 12:36:12 +0100 (CET) Subject: [pypy-svn] r49773 - in pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__: . test Message-ID: <20071214113612.2C38316853C@codespeak.net> Author: cfbolz Date: Fri Dec 14 12:36:11 2007 New Revision: 49773 Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Log: allow weakrefability Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/interp_classobj.py Fri Dec 14 12:36:11 2007 @@ -2,7 +2,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped, applevel from pypy.interpreter.gateway import interp2app, ObjSpace -from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.interpreter.typedef import TypeDef, GetSetProperty, make_weakref_descr from pypy.interpreter.argument import Arguments from pypy.interpreter.baseobjspace import Wrappable from pypy.rlib.rarithmetic import r_uint, intmask @@ -678,6 +678,7 @@ unwrap_spec=['self', ObjSpace, W_Root, W_Root]), __rpow__ = interp2app(W_InstanceObject.descr_rpow, unwrap_spec=['self', ObjSpace, W_Root, W_Root]), + __weakref__ = make_weakref_descr(W_InstanceObject), **rawdict ) Modified: pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/test/test_classobj.py Fri Dec 14 12:36:11 2007 @@ -625,3 +625,16 @@ D().save() assert l == ['A'] + + def test_weakref(self): + import weakref, gc + class A: + pass + a = A() + ref = weakref.ref(a) + assert ref() is a + a = None + gc.collect() + gc.collect() + gc.collect() + assert ref() is None From fijal at codespeak.net Fri Dec 14 13:13:47 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 14 Dec 2007 13:13:47 +0100 (CET) Subject: [pypy-svn] r49774 - pypy/branch/lazy-write-barrier Message-ID: <20071214121347.2DAB3168525@codespeak.net> Author: fijal Date: Fri Dec 14 13:13:46 2007 New Revision: 49774 Removed: pypy/branch/lazy-write-barrier/ Log: Remove the branch (in order to branch again) From fijal at codespeak.net Fri Dec 14 13:14:11 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 14 Dec 2007 13:14:11 +0100 (CET) Subject: [pypy-svn] r49775 - pypy/branch/lazy-write-barrier Message-ID: <20071214121411.87AEC168524@codespeak.net> Author: fijal Date: Fri Dec 14 13:14:11 2007 New Revision: 49775 Added: pypy/branch/lazy-write-barrier/ - copied from r49774, pypy/dist/ Log: Create a branch for lazy write barrier From fijal at codespeak.net Fri Dec 14 13:15:09 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 14 Dec 2007 13:15:09 +0100 (CET) Subject: [pypy-svn] r49776 - pypy/branch/lazy-write-barrier Message-ID: <20071214121509.145CB168524@codespeak.net> Author: fijal Date: Fri Dec 14 13:15:08 2007 New Revision: 49776 Removed: pypy/branch/lazy-write-barrier/ Log: Remove branch once again From fijal at codespeak.net Fri Dec 14 13:15:57 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 14 Dec 2007 13:15:57 +0100 (CET) Subject: [pypy-svn] r49777 - pypy/branch/lazy-write-barrier Message-ID: <20071214121557.95FF4168525@codespeak.net> Author: fijal Date: Fri Dec 14 13:15:57 2007 New Revision: 49777 Added: pypy/branch/lazy-write-barrier/ - copied from r49724, pypy/dist/ Log: Create a branch from 49724 to experiment with lazy write barrier From arigo at codespeak.net Fri Dec 14 13:16:41 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Dec 2007 13:16:41 +0100 (CET) Subject: [pypy-svn] r49778 - in pypy/dist/pypy/rpython/memory: . gc Message-ID: <20071214121641.05C7C168524@codespeak.net> Author: arigo Date: Fri Dec 14 13:16:41 2007 New Revision: 49778 Modified: pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/gctypelayout.py Log: Revert r49731, which causes many test failures. Will try again. Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Fri Dec 14 13:16:41 2007 @@ -7,8 +7,6 @@ needs_write_barrier = False needs_zero_gc_pointers = True - TYPEID_OF_GCARRAY_OF_GCPTR = 1 - def set_query_functions(self, is_varsize, has_gcptr_in_varsize, getfinalizer, offsets_to_gc_pointers, @@ -117,15 +115,6 @@ Typically, 'callback' is a bound method and 'arg' can be None. """ typeid = self.get_type_id(obj) - if typeid == GCBase.TYPEID_OF_GCARRAY_OF_GCPTR: - # a performance shortcut for GcArray(gcptr) - length = (obj + llmemory.gcarrayofptr_lengthoffset).signed[0] - item = obj + llmemory.gcarrayofptr_itemsoffset - while length > 0: - callback(item, arg) - item += llmemory.gcarrayofptr_singleitemoffset - length -= 1 - return offsets = self.offsets_to_gc_pointers(typeid) i = 0 while i < len(offsets): @@ -136,13 +125,14 @@ length = (obj + self.varsize_offset_to_length(typeid)).signed[0] offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) itemlength = self.varsize_item_sizes(typeid) - while length > 0: + i = 0 + while i < length: j = 0 while j < len(offsets): callback(item + offsets[j], arg) j += 1 + i += 1 item += itemlength - length -= 1 trace._annspecialcase_ = 'specialize:arg(2)' Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/gctypelayout.py Fri Dec 14 13:16:41 2007 @@ -1,5 +1,4 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena -from pypy.rpython.memory.gc.base import GCBase class TypeLayoutBuilder(object): @@ -26,22 +25,6 @@ except KeyError: assert self.can_add_new_types assert isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)) - - if TYPE != llmemory.GCARRAY_OF_PTR: - # first, force GcArray(gcptr) to have typeid 1, for the - # performance-related special-casing done by GCBase.trace(). - if llmemory.GCARRAY_OF_PTR not in self.id_of_type: - typeid = self.get_type_id(llmemory.GCARRAY_OF_PTR) - assert typeid == GCBase.TYPEID_OF_GCARRAY_OF_GCPTR - # all types that are of the shape GcArray(gcptr) get a - # typeid of 1. - if (isinstance(TYPE, lltype.GcArray) - and isinstance(TYPE.OF, lltype.Ptr) - and TYPE.OF.TO._gckind == 'gc'): - type_id = GCBase.TYPEID_OF_GCARRAY_OF_GCPTR - self.id_of_type[TYPE] = type_id - return type_id - # Record the new type_id description as a small dict for now. # The framework gc transformer will turn it into a # Struct("type_info") in flatten_table(). From arigo at codespeak.net Fri Dec 14 13:25:01 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Dec 2007 13:25:01 +0100 (CET) Subject: [pypy-svn] r49779 - in pypy/dist/pypy/rpython/memory: . gc gctransform Message-ID: <20071214122501.B439A168525@codespeak.net> Author: arigo Date: Fri Dec 14 13:24:59 2007 New Revision: 49779 Modified: pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gctypelayout.py Log: Reintroduce r49731 in a variant that looks marginally less efficient, but which works on top of the llinterp and is more extensible. Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Fri Dec 14 13:24:59 2007 @@ -8,6 +8,7 @@ needs_zero_gc_pointers = True def set_query_functions(self, is_varsize, has_gcptr_in_varsize, + is_gcarrayofgcptr, getfinalizer, offsets_to_gc_pointers, fixed_size, varsize_item_sizes, @@ -18,6 +19,7 @@ self.getfinalizer = getfinalizer self.is_varsize = is_varsize self.has_gcptr_in_varsize = has_gcptr_in_varsize + self.is_gcarrayofgcptr = is_gcarrayofgcptr self.offsets_to_gc_pointers = offsets_to_gc_pointers self.fixed_size = fixed_size self.varsize_item_sizes = varsize_item_sizes @@ -115,6 +117,15 @@ Typically, 'callback' is a bound method and 'arg' can be None. """ typeid = self.get_type_id(obj) + if self.is_gcarrayofgcptr(typeid): + # a performance shortcut for GcArray(gcptr) + length = (obj + llmemory.gcarrayofptr_lengthoffset).signed[0] + item = obj + llmemory.gcarrayofptr_itemsoffset + while length > 0: + callback(item, arg) + item += llmemory.gcarrayofptr_singleitemoffset + length -= 1 + return offsets = self.offsets_to_gc_pointers(typeid) i = 0 while i < len(offsets): @@ -125,14 +136,13 @@ length = (obj + self.varsize_offset_to_length(typeid)).signed[0] offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) itemlength = self.varsize_item_sizes(typeid) - i = 0 - while i < length: + while length > 0: j = 0 while j < len(offsets): callback(item + offsets[j], arg) j += 1 - i += 1 item += itemlength + length -= 1 trace._annspecialcase_ = 'specialize:arg(2)' Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Fri Dec 14 13:24:59 2007 @@ -118,6 +118,7 @@ TYPE_INFO = lltype.Struct("type_info", ("isvarsize", lltype.Bool), ("gcptrinvarsize", lltype.Bool), + ("gcarrayofgcptr", lltype.Bool), ("finalizer", self.FINALIZERTYPE), ("fixedsize", lltype.Signed), ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), @@ -137,6 +138,10 @@ ll_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].gcptrinvarsize + def q_is_gcarrayofgcptr(typeid): + ll_assert(typeid > 0, "invalid type_id") + return gcdata.type_info_table[typeid].gcarrayofgcptr + def q_finalizer(typeid): ll_assert(typeid > 0, "invalid type_id") return gcdata.type_info_table[typeid].finalizer @@ -200,6 +205,7 @@ gcdata.gc.set_query_functions( q_is_varsize, q_has_gcptr_in_varsize, + q_is_gcarrayofgcptr, q_finalizer, q_offsets_to_gc_pointers, q_fixed_size, Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/gctypelayout.py Fri Dec 14 13:24:59 2007 @@ -75,6 +75,11 @@ info["varofstoptrs"] = self.offsets2table(offsets, ARRAY.OF) info["varitemsize"] = llmemory.sizeof(ARRAY.OF) info["gcptrinvarsize"] = len(offsets) > 0 + # if the type is of the shape GcArray(gcptr) then we record, + # for now, a flag in the 'info'. XXX could use a bit in typeid + info["gcarrayofgcptr"] = (isinstance(TYPE, lltype.GcArray) + and isinstance(TYPE.OF, lltype.Ptr) + and TYPE.OF.TO._gckind == 'gc') return type_id def offsets2table(self, offsets, TYPE): @@ -98,6 +103,10 @@ assert typeid > 0 return self.type_info_list[typeid]["gcptrinvarsize"] + def q_is_gcarrayofgcptr(self, typeid): + assert typeid > 0 + return self.type_info_list[typeid]["gcarrayofgcptr"] + def q_finalizer(self, typeid): assert typeid > 0 return self.type_info_list[typeid]["finalizer"] @@ -133,6 +142,7 @@ def get_query_functions(self): return (self.q_is_varsize, self.q_has_gcptr_in_varsize, + self.q_is_gcarrayofgcptr, self.q_finalizer, self.q_offsets_to_gc_pointers, self.q_fixed_size, From fijal at codespeak.net Fri Dec 14 13:55:46 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 14 Dec 2007 13:55:46 +0100 (CET) Subject: [pypy-svn] r49780 - in pypy/branch/lazy-write-barrier/pypy/rpython/memory: . gc gctransform test Message-ID: <20071214125546.AAC5416851D@codespeak.net> Author: fijal Date: Fri Dec 14 13:55:46 2007 New Revision: 49780 Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/marksweep.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/test/test_transformed_gc.py Log: inprogress checkin of lazy write barrier approach Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py Fri Dec 14 13:55:46 2007 @@ -13,6 +13,10 @@ # pointer to a young object. GCFLAG_NO_YOUNG_PTRS = 2 << GCFLAGSHIFT +# The following flag is set for static roots which are not on the list +# of static roots yet, but will appear with write barrier +GCFLAG_NEVER_SET = 3 << GCFLAGSHIFT + DEBUG_PRINT = False class GenerationGC(SemiSpaceGC): @@ -287,6 +291,15 @@ def write_barrier(self, oldvalue, newvalue, addr_struct): if self.header(addr_struct).tid & GCFLAG_NO_YOUNG_PTRS: self.remember_young_pointer(addr_struct, newvalue) + if self.header(addr_struct).tid & GCFLAG_NEVER_SET: + self.move_to_static_roots(addr_struct) + + def append_to_static_roots(self, pointer, arg): + self.get_roots.append_static_root(pointer) + + def move_to_static_roots(self, addr_struct): + self.header(addr_struct).tid &= ~GCFLAG_NEVER_SET + self.trace(addr_struct, self.append_to_static_roots, None) def remember_young_pointer(self, addr_struct, addr): ll_assert(not self.is_in_nursery(addr_struct), Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/marksweep.py Fri Dec 14 13:55:46 2007 @@ -483,8 +483,9 @@ hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) hdr.typeid = typeid << 1 - def init_gc_object_immortal(self, addr, typeid): + def init_gc_object_immortal(self, addr, typeid, flags=0): # prebuilt gc structures always have the mark bit set + # ignore flags hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) hdr.typeid = (typeid << 1) | 1 Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py Fri Dec 14 13:55:46 2007 @@ -403,6 +403,7 @@ gcdata = self.gcdata sizeofaddr = llmemory.sizeof(llmemory.Address) rootstacksize = sizeofaddr * self.root_stack_depth + TP = lltype.Ptr(lltype.Array(llmemory.Address)) class StackRootIterator: _alloc_flavor_ = 'raw' @@ -421,6 +422,11 @@ gcdata.root_stack_top = top + n*sizeofaddr return top incr_stack = staticmethod(incr_stack) + + def append_static_root(adr): + gcdata.static_root_end.address[0] = adr + gcdata.static_root_end += sizeofaddr + append_static_root = staticmethod(append_static_root) def decr_stack(n): top = gcdata.root_stack_top - n*sizeofaddr @@ -509,8 +515,11 @@ self.layoutbuilder.addresses_of_static_ptrs + self.layoutbuilder.addresses_of_static_ptrs_in_nongc) log.info("found %s static roots" % (len(addresses_of_static_ptrs), )) + additional_ptrs = self.layoutbuilder.additional_roots_sources + log.info("additional %d potential static roots" % additional_ptrs) ll_static_roots_inside = lltype.malloc(lltype.Array(llmemory.Address), - len(addresses_of_static_ptrs), + len(addresses_of_static_ptrs) + + additional_ptrs, immortal=True) for i in range(len(addresses_of_static_ptrs)): ll_static_roots_inside[i] = addresses_of_static_ptrs[i] Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py Fri Dec 14 13:55:46 2007 @@ -17,6 +17,8 @@ self.addresses_of_static_ptrs = [] # this lists contains pointers in raw Structs and Arrays self.addresses_of_static_ptrs_in_nongc = [] + # dictionary address -> list of addresses + self.additional_roots_sources = 0 self.finalizer_funcptrs = {} def get_type_id(self, TYPE): @@ -149,11 +151,22 @@ return self.seen_roots[id(value)] = True + # XXX hack, a lot of gengc details here + from pypy.rpython.memory.gc.generation import GenerationGC + from pypy.rpython.memory.gc.generation import GCFLAG_NEVER_SET + if isinstance(gc, GenerationGC): + gen_gc = True + else: + gen_gc = False + if isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)): typeid = self.get_type_id(TYPE) hdr = gc.gcheaderbuilder.new_header(value) adr = llmemory.cast_ptr_to_adr(hdr) - gc.init_gc_object_immortal(adr, typeid) + flags = 0 + if gen_gc: + flags = GCFLAG_NEVER_SET + gc.init_gc_object_immortal(adr, typeid, flags=flags) # The following collects the addresses of all the fields that have # a GC Pointer type, inside the current prebuilt object. All such @@ -161,7 +174,17 @@ # they could be changed later to point to GC heap objects. adr = llmemory.cast_ptr_to_adr(value._as_ptr()) if TYPE._gckind == "gc": - appendto = self.addresses_of_static_ptrs + if gen_gc: + # check if have any + gen = mutable_gc_pointers_inside(value, adr) + try: + gen.next() + except StopIteration: + return + self.additional_roots_sources += 1 + return + else: + appendto = self.addresses_of_static_ptrs else: appendto = self.addresses_of_static_ptrs_in_nongc for a in mutable_gc_pointers_inside(value, adr): Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/test/test_transformed_gc.py Fri Dec 14 13:55:46 2007 @@ -37,7 +37,8 @@ gcpolicy = None stacklessgc = False - def runner(self, f, nbargs=0, statistics=False, **extraconfigopts): + def runner(self, f, nbargs=0, statistics=False, transformer=False, + **extraconfigopts): if nbargs == 2: def entrypoint(args): x = args[0] @@ -84,6 +85,8 @@ def statistics(index): return llinterp.eval_graph(statisticsgraph, [ll_gc, index]) return run, statistics + elif transformer: + return run, db.gctransformer else: return run @@ -862,6 +865,32 @@ run = self.runner(f, nbargs=0) run([]) + def test_immutable_to_old_promotion(self): + T_CHILD = lltype.Ptr(lltype.GcStruct('Child', ('field', lltype.Signed))) + T_PARENT = lltype.Ptr(lltype.GcStruct('Parent', ('sub', T_CHILD))) + child = lltype.malloc(T_CHILD.TO) + child2 = lltype.malloc(T_CHILD.TO) + parent = lltype.malloc(T_PARENT.TO) + parent2 = lltype.malloc(T_PARENT.TO) + parent.sub = child + child.field = 3 + parent2.sub = child2 + child2.field = 8 + + T_ALL = lltype.Ptr(lltype.GcArray(T_PARENT)) + all = lltype.malloc(T_ALL.TO, 2) + all[0] = parent + all[1] = parent2 + + def f(x, y): + res = all[x] + #all[x] = lltype.nullptr(T_PARENT.TO) + return res.sub.field + + run, transformer = self.runner(f, nbargs=2, transformer=True) + run([1, 4]) + assert len(transformer.layoutbuilder.addresses_of_static_ptrs) == 0 + assert transformer.layoutbuilder.additional_roots_sources == 5 class TestGenerationalNoFullCollectGC(GCTest): # test that nursery is doing its job and that no full collection From fijal at codespeak.net Fri Dec 14 14:05:24 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 14 Dec 2007 14:05:24 +0100 (CET) Subject: [pypy-svn] r49781 - pypy/branch/lazy-write-barrier/pypy/rpython/memory Message-ID: <20071214130524.F315A16852B@codespeak.net> Author: fijal Date: Fri Dec 14 14:05:19 2007 New Revision: 49781 Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py Log: Fix calculation Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py Fri Dec 14 14:05:19 2007 @@ -175,13 +175,8 @@ adr = llmemory.cast_ptr_to_adr(value._as_ptr()) if TYPE._gckind == "gc": if gen_gc: - # check if have any - gen = mutable_gc_pointers_inside(value, adr) - try: - gen.next() - except StopIteration: - return - self.additional_roots_sources += 1 + for a in mutable_gc_pointers_inside(value, adr): + self.additional_roots_sources += 1 return else: appendto = self.addresses_of_static_ptrs From arigo at codespeak.net Fri Dec 14 14:21:25 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Dec 2007 14:21:25 +0100 (CET) Subject: [pypy-svn] r49782 - in pypy: branch/interplevel-oldstyle-classes dist/pypy/interpreter dist/pypy/lib dist/pypy/module/__builtin__ dist/pypy/module/_random dist/pypy/objspace/std Message-ID: <20071214132125.D4E0216852C@codespeak.net> Author: arigo Date: Fri Dec 14 14:21:24 2007 New Revision: 49782 Added: pypy/dist/pypy/interpreter/ - copied from r49781, pypy/branch/interplevel-oldstyle-classes/pypy/interpreter/ pypy/dist/pypy/module/__builtin__/ - copied from r49781, pypy/branch/interplevel-oldstyle-classes/pypy/module/__builtin__/ pypy/dist/pypy/module/_random/ - copied from r49781, pypy/branch/interplevel-oldstyle-classes/pypy/module/_random/ pypy/dist/pypy/objspace/std/objspace.py - copied unchanged from r49781, pypy/branch/interplevel-oldstyle-classes/pypy/objspace/std/objspace.py Removed: pypy/branch/interplevel-oldstyle-classes/ pypy/dist/pypy/lib/_classobj.py Log: Merge of the interplevel-oldstyle-classes branch by cfbolz: move old-style classes to interp-level. From arigo at codespeak.net Fri Dec 14 15:03:23 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Dec 2007 15:03:23 +0100 (CET) Subject: [pypy-svn] r49783 - in pypy/dist/pypy/rpython/memory: . gctransform Message-ID: <20071214140323.CB01916853F@codespeak.net> Author: arigo Date: Fri Dec 14 15:03:23 2007 New Revision: 49783 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gctypelayout.py pypy/dist/pypy/rpython/memory/gcwrapper.py Log: Move the GCData class from gctransform/framework.py to gctypelayout.py and use it for the gcwrapper case too. This reduces some code duplication and makes the details of the TYPE_INFO table encoding local to gctypelayout.py. Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Fri Dec 14 15:03:23 2007 @@ -91,8 +91,6 @@ print "found %s initializing stores in %s" % (len(result), graph.name) return result -ADDRESS_VOID_FUNC = lltype.FuncType([llmemory.Address], lltype.Void) - class FrameworkGCTransformer(GCTransformer): use_stackless = False root_stack_depth = 163840 @@ -111,80 +109,18 @@ # for regular translation: pick the GC from the config GCClass, GC_PARAMS = choose_gc_from_config(translator.config) - self.FINALIZERTYPE = lltype.Ptr(ADDRESS_VOID_FUNC) - class GCData(object): - # types of the GC information tables - OFFSETS_TO_GC_PTR = lltype.Array(lltype.Signed) - TYPE_INFO = lltype.Struct("type_info", - ("isvarsize", lltype.Bool), - ("gcptrinvarsize", lltype.Bool), - ("gcarrayofgcptr", lltype.Bool), - ("finalizer", self.FINALIZERTYPE), - ("fixedsize", lltype.Signed), - ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), - ("varitemsize", lltype.Signed), - ("ofstovar", lltype.Signed), - ("ofstolength", lltype.Signed), - ("varofstoptrs",lltype.Ptr(OFFSETS_TO_GC_PTR)), - ("weakptrofs", lltype.Signed), - ) - TYPE_INFO_TABLE = lltype.Array(TYPE_INFO) - - def q_is_varsize(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].isvarsize - - def q_has_gcptr_in_varsize(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].gcptrinvarsize - - def q_is_gcarrayofgcptr(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].gcarrayofgcptr - - def q_finalizer(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].finalizer - - def q_offsets_to_gc_pointers(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].ofstoptrs - - def q_fixed_size(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].fixedsize - - def q_varsize_item_sizes(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].varitemsize - - def q_varsize_offset_to_variable_part(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].ofstovar - - def q_varsize_offset_to_length(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].ofstolength - - def q_varsize_offsets_to_gcpointers_in_var_part(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].varofstoptrs - - def q_weakpointer_offset(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].weakptrofs - self.layoutbuilder = TransformerLayoutBuilder(self) self.get_type_id = self.layoutbuilder.get_type_id - gcdata = GCData() # set up dummy a table, to be overwritten with the real one in finish() - gcdata.type_info_table = lltype.malloc(GCData.TYPE_INFO_TABLE, 0, - immortal=True) + type_info_table = lltype.malloc(gctypelayout.GCData.TYPE_INFO_TABLE, 0, + immortal=True) + gcdata = gctypelayout.GCData(type_info_table) + # initialize the following two fields with a random non-NULL address, # to make the annotator happy. The fields are patched in finish() - # to point to a real array (not 'static_roots', another one). - a_random_address = llmemory.cast_ptr_to_adr(gcdata.type_info_table) + # to point to a real array (not 'type_info_table', another one). + a_random_address = llmemory.cast_ptr_to_adr(type_info_table) gcdata.static_root_start = a_random_address # patched in finish() gcdata.static_root_nongcstart = a_random_address # patched in finish() gcdata.static_root_end = a_random_address # patched in finish() @@ -202,30 +138,16 @@ # run-time initialization code StackRootIterator.setup_root_stack() gcdata.gc.setup() - gcdata.gc.set_query_functions( - q_is_varsize, - q_has_gcptr_in_varsize, - q_is_gcarrayofgcptr, - q_finalizer, - q_offsets_to_gc_pointers, - q_fixed_size, - q_varsize_item_sizes, - q_varsize_offset_to_variable_part, - q_varsize_offset_to_length, - q_varsize_offsets_to_gcpointers_in_var_part, - q_weakpointer_offset) + gcdata.set_query_functions(gcdata.gc) bk = self.translator.annotator.bookkeeper # the point of this little dance is to not annotate # self.gcdata.type_info_table as a constant. - data_classdef = bk.getuniqueclassdef(GCData) + data_classdef = bk.getuniqueclassdef(gctypelayout.GCData) data_classdef.generalize_attr( 'type_info_table', - annmodel.SomePtr(lltype.Ptr(GCData.TYPE_INFO_TABLE))) - data_classdef.generalize_attr( - 'static_roots', - annmodel.SomePtr(lltype.Ptr(lltype.Array(llmemory.Address)))) + annmodel.SomePtr(lltype.Ptr(gctypelayout.GCData.TYPE_INFO_TABLE))) data_classdef.generalize_attr( 'static_root_start', annmodel.SomeAddress()) @@ -571,7 +493,7 @@ c_type_id = rmodel.inputconst(lltype.Signed, type_id) info = self.layoutbuilder.type_info_list[type_id] - c_size = rmodel.inputconst(lltype.Signed, info["fixedsize"]) + c_size = rmodel.inputconst(lltype.Signed, info.fixedsize) has_finalizer = bool(self.finalizer_funcptr_for_type(TYPE)) c_has_finalizer = rmodel.inputconst(lltype.Bool, has_finalizer) @@ -590,8 +512,8 @@ c_has_finalizer, rmodel.inputconst(lltype.Bool, False)] else: v_length = op.args[-1] - c_ofstolength = rmodel.inputconst(lltype.Signed, info['ofstolength']) - c_varitemsize = rmodel.inputconst(lltype.Signed, info['varitemsize']) + c_ofstolength = rmodel.inputconst(lltype.Signed, info.ofstolength) + c_varitemsize = rmodel.inputconst(lltype.Signed, info.varitemsize) malloc_ptr = self.malloc_varsize_clear_ptr ## if op.opname.startswith('zero'): ## malloc_ptr = self.malloc_varsize_clear_ptr @@ -622,7 +544,7 @@ c_type_id = rmodel.inputconst(lltype.Signed, type_id) info = self.layoutbuilder.type_info_list[type_id] - c_size = rmodel.inputconst(lltype.Signed, info["fixedsize"]) + c_size = rmodel.inputconst(lltype.Signed, info.fixedsize) has_finalizer = bool(self.finalizer_funcptr_for_type(TYPE)) assert not has_finalizer @@ -633,8 +555,8 @@ args = [self.c_const_gc, v_coallocator, c_type_id, c_size] else: v_length = op.args[-1] - c_ofstolength = rmodel.inputconst(lltype.Signed, info['ofstolength']) - c_varitemsize = rmodel.inputconst(lltype.Signed, info['varitemsize']) + c_ofstolength = rmodel.inputconst(lltype.Signed, info.ofstolength) + c_varitemsize = rmodel.inputconst(lltype.Signed, info.varitemsize) malloc_ptr = self.coalloc_varsize_clear_ptr args = [self.c_const_gc, v_coallocator, c_type_id, v_length, c_size, c_varitemsize, c_ofstolength] @@ -696,7 +618,7 @@ c_type_id = rmodel.inputconst(lltype.Signed, type_id) info = self.layoutbuilder.type_info_list[type_id] - c_size = rmodel.inputconst(lltype.Signed, info["fixedsize"]) + c_size = rmodel.inputconst(lltype.Signed, info.fixedsize) malloc_ptr = self.malloc_fixedsize_ptr c_has_finalizer = rmodel.inputconst(lltype.Bool, False) c_has_weakptr = c_can_collect = rmodel.inputconst(lltype.Bool, True) @@ -854,31 +776,9 @@ [llmemory.Address], lltype.Void) else: - fptr = lltype.nullptr(ADDRESS_VOID_FUNC) + fptr = lltype.nullptr(gctypelayout.GCData.FINALIZERTYPE.TO) return fptr - def offsets2table(self, offsets, TYPE): - try: - return self.offsettable_cache[TYPE] - except KeyError: - gcdata = self.transformer.gcdata - cachedarray = lltype.malloc(gcdata.OFFSETS_TO_GC_PTR, - len(offsets), immortal=True) - for i, value in enumerate(offsets): - cachedarray[i] = value - self.offsettable_cache[TYPE] = cachedarray - return cachedarray - - def flatten_table(self): - self.can_add_new_types = False - table = lltype.malloc(self.transformer.gcdata.TYPE_INFO_TABLE, - len(self.type_info_list), immortal=True) - for tableentry, newcontent in zip(table, self.type_info_list): - for key, value in newcontent.items(): - setattr(tableentry, key, value) - self.offsettable_cache = None - return table - def gen_zero_gc_pointers(TYPE, v, llops, previous_steps=None): if previous_steps is None: Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/gctypelayout.py Fri Dec 14 15:03:23 2007 @@ -1,12 +1,109 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena +from pypy.rlib.debug import ll_assert +class GCData(object): + """The GC information tables, and the query functions that the GC + calls to decode their content. The encoding of this information + is done by TypeLayoutBuilder.get_type_id(). These two places + should be in sync, obviously, but in principle no other code + should depend on the details of the encoding in TYPE_INFO. + """ + _alloc_flavor_ = 'raw' + + OFFSETS_TO_GC_PTR = lltype.Array(lltype.Signed) + ADDRESS_VOID_FUNC = lltype.FuncType([llmemory.Address], lltype.Void) + FINALIZERTYPE = lltype.Ptr(ADDRESS_VOID_FUNC) + + # structure describing the layout of a typeid + TYPE_INFO = lltype.Struct("type_info", + ("isvarsize", lltype.Bool), + ("gcptrinvarsize", lltype.Bool), + ("gcarrayofgcptr", lltype.Bool), + ("finalizer", FINALIZERTYPE), + ("fixedsize", lltype.Signed), + ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), + ("varitemsize", lltype.Signed), + ("ofstovar", lltype.Signed), + ("ofstolength", lltype.Signed), + ("varofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), + ("weakptrofs", lltype.Signed), + ) + TYPE_INFO_TABLE = lltype.Array(TYPE_INFO) + + def __init__(self, type_info_table): + self.type_info_table = type_info_table + # 'type_info_table' is a list of TYPE_INFO structures when + # running with gcwrapper, or a real TYPE_INFO_TABLE after + # the gctransformer. + + def q_is_varsize(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return self.type_info_table[typeid].isvarsize + + def q_has_gcptr_in_varsize(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return self.type_info_table[typeid].gcptrinvarsize + + def q_is_gcarrayofgcptr(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return self.type_info_table[typeid].gcarrayofgcptr + + def q_finalizer(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return self.type_info_table[typeid].finalizer + + def q_offsets_to_gc_pointers(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return self.type_info_table[typeid].ofstoptrs + + def q_fixed_size(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return self.type_info_table[typeid].fixedsize + + def q_varsize_item_sizes(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return self.type_info_table[typeid].varitemsize + + def q_varsize_offset_to_variable_part(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return self.type_info_table[typeid].ofstovar + + def q_varsize_offset_to_length(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return self.type_info_table[typeid].ofstolength + + def q_varsize_offsets_to_gcpointers_in_var_part(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return self.type_info_table[typeid].varofstoptrs + + def q_weakpointer_offset(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return self.type_info_table[typeid].weakptrofs + + def set_query_functions(self, gc): + gc.set_query_functions( + self.q_is_varsize, + self.q_has_gcptr_in_varsize, + self.q_is_gcarrayofgcptr, + self.q_finalizer, + self.q_offsets_to_gc_pointers, + self.q_fixed_size, + self.q_varsize_item_sizes, + self.q_varsize_offset_to_variable_part, + self.q_varsize_offset_to_length, + self.q_varsize_offsets_to_gcpointers_in_var_part, + self.q_weakpointer_offset) + +# ____________________________________________________________ + class TypeLayoutBuilder(object): can_add_new_types = True def __init__(self): - dummy = {"weakptrofs": -1, - "ofstolength": -1} + dummy = lltype.malloc(GCData.TYPE_INFO, immortal=True, zero=True) + dummy.weakptrofs = -1 + dummy.ofstolength = -1 self.type_info_list = [dummy] # don't use typeid 0, helps debugging self.id_of_type = {} # {LLTYPE: type_id} self.seen_roots = {} @@ -18,6 +115,7 @@ # this lists contains pointers in raw Structs and Arrays self.addresses_of_static_ptrs_in_nongc = [] self.finalizer_funcptrs = {} + self.offsettable_cache = {} def get_type_id(self, TYPE): try: @@ -30,60 +128,81 @@ # Struct("type_info") in flatten_table(). type_id = len(self.type_info_list) assert type_id & 0xffff == type_id # make sure it fits into 2 bytes - info = {} + info = lltype.malloc(GCData.TYPE_INFO, immortal=True, zero=True) self.type_info_list.append(info) self.id_of_type[TYPE] = type_id offsets = offsets_to_gc_pointers(TYPE) - info["ofstoptrs"] = self.offsets2table(offsets, TYPE) - info["finalizer"] = self.make_finalizer_funcptr_for_type(TYPE) - info["weakptrofs"] = weakpointer_offset(TYPE) + info.ofstoptrs = self.offsets2table(offsets, TYPE) + info.finalizer = self.make_finalizer_funcptr_for_type(TYPE) + info.weakptrofs = weakpointer_offset(TYPE) if not TYPE._is_varsize(): - info["isvarsize"] = False - info["gcptrinvarsize"] = False - info["fixedsize"] = llarena.round_up_for_allocation( + info.isvarsize = False + info.gcptrinvarsize = False + info.fixedsize = llarena.round_up_for_allocation( llmemory.sizeof(TYPE)) - info["ofstolength"] = -1 + info.ofstolength = -1 # note about round_up_for_allocation(): in the 'info' table # we put a rounded-up size only for fixed-size objects. For # varsize ones, the GC must anyway compute the size at run-time # and round up that result. else: - info["isvarsize"] = True - info["fixedsize"] = llmemory.sizeof(TYPE, 0) + info.isvarsize = True + info.fixedsize = llmemory.sizeof(TYPE, 0) if isinstance(TYPE, lltype.Struct): ARRAY = TYPE._flds[TYPE._arrayfld] ofs1 = llmemory.offsetof(TYPE, TYPE._arrayfld) - info["ofstolength"] = ofs1 + llmemory.ArrayLengthOffset(ARRAY) + info.ofstolength = ofs1 + llmemory.ArrayLengthOffset(ARRAY) if ARRAY.OF != lltype.Void: - info["ofstovar"] = ofs1 + llmemory.itemoffsetof(ARRAY, 0) + info.ofstovar = ofs1 + llmemory.itemoffsetof(ARRAY, 0) else: - info["fixedsize"] = ofs1 + llmemory.sizeof(lltype.Signed) + info.fixedsize = ofs1 + llmemory.sizeof(lltype.Signed) + # XXX we probably don't need isrpystring any more if ARRAY._hints.get('isrpystring'): - info["fixedsize"] = llmemory.sizeof(TYPE, 1) + info.fixedsize = llmemory.sizeof(TYPE, 1) else: ARRAY = TYPE - info["ofstolength"] = llmemory.ArrayLengthOffset(ARRAY) + info.ofstolength = llmemory.ArrayLengthOffset(ARRAY) if ARRAY.OF != lltype.Void: - info["ofstovar"] = llmemory.itemoffsetof(TYPE, 0) + info.ofstovar = llmemory.itemoffsetof(TYPE, 0) else: - info["fixedsize"] = llmemory.ArrayLengthOffset(ARRAY) + llmemory.sizeof(lltype.Signed) + info.fixedsize = (llmemory.ArrayLengthOffset(ARRAY) + + llmemory.sizeof(lltype.Signed)) assert isinstance(ARRAY, lltype.Array) if ARRAY.OF != lltype.Void: offsets = offsets_to_gc_pointers(ARRAY.OF) else: offsets = () - info["varofstoptrs"] = self.offsets2table(offsets, ARRAY.OF) - info["varitemsize"] = llmemory.sizeof(ARRAY.OF) - info["gcptrinvarsize"] = len(offsets) > 0 + info.varofstoptrs = self.offsets2table(offsets, ARRAY.OF) + info.varitemsize = llmemory.sizeof(ARRAY.OF) + info.gcptrinvarsize = len(offsets) > 0 # if the type is of the shape GcArray(gcptr) then we record, # for now, a flag in the 'info'. XXX could use a bit in typeid - info["gcarrayofgcptr"] = (isinstance(TYPE, lltype.GcArray) - and isinstance(TYPE.OF, lltype.Ptr) - and TYPE.OF.TO._gckind == 'gc') + info.gcarrayofgcptr = (isinstance(TYPE, lltype.GcArray) + and isinstance(TYPE.OF, lltype.Ptr) + and TYPE.OF.TO._gckind == 'gc') return type_id def offsets2table(self, offsets, TYPE): - return offsets + try: + return self.offsettable_cache[TYPE] + except KeyError: + cachedarray = lltype.malloc(GCData.OFFSETS_TO_GC_PTR, + len(offsets), immortal=True) + for i, value in enumerate(offsets): + cachedarray[i] = value + self.offsettable_cache[TYPE] = cachedarray + return cachedarray + + def flatten_table(self): + self.can_add_new_types = False + self.offsettable_cache = None + table = lltype.malloc(GCData.TYPE_INFO_TABLE, len(self.type_info_list), + immortal=True) + fieldnames = GCData.TYPE_INFO._names + for tableentry, newcontent in zip(table, self.type_info_list): + for name in fieldnames: + setattr(tableentry, name, getattr(newcontent, name)) + return table def finalizer_funcptr_for_type(self, TYPE): if TYPE in self.finalizer_funcptrs: @@ -95,62 +214,8 @@ def make_finalizer_funcptr_for_type(self, TYPE): return None # must be overridden for proper finalizer support - def q_is_varsize(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["isvarsize"] - - def q_has_gcptr_in_varsize(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["gcptrinvarsize"] - - def q_is_gcarrayofgcptr(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["gcarrayofgcptr"] - - def q_finalizer(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["finalizer"] - - def q_offsets_to_gc_pointers(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["ofstoptrs"] - - def q_fixed_size(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["fixedsize"] - - def q_varsize_item_sizes(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["varitemsize"] - - def q_varsize_offset_to_variable_part(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["ofstovar"] - - def q_varsize_offset_to_length(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["ofstolength"] - - def q_varsize_offsets_to_gcpointers_in_var_part(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["varofstoptrs"] - - def q_weakpointer_offset(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["weakptrofs"] - - def get_query_functions(self): - return (self.q_is_varsize, - self.q_has_gcptr_in_varsize, - self.q_is_gcarrayofgcptr, - self.q_finalizer, - self.q_offsets_to_gc_pointers, - self.q_fixed_size, - self.q_varsize_item_sizes, - self.q_varsize_offset_to_variable_part, - self.q_varsize_offset_to_length, - self.q_varsize_offsets_to_gcpointers_in_var_part, - self.q_weakpointer_offset) + def initialize_gc_query_function(self, gc): + return GCData(self.type_info_list).set_query_functions(gc) def consider_constant(self, TYPE, value, gc): if value is not lltype.top_container(value): Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Fri Dec 14 15:03:23 2007 @@ -1,5 +1,6 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llheap from pypy.rpython import llinterp +from pypy.rpython.annlowlevel import llhelper from pypy.rpython.memory.support import get_address_linked_list from pypy.rpython.memory import gctypelayout from pypy.objspace.flow.model import Constant @@ -18,7 +19,7 @@ def prepare_graphs(self, flowgraphs): layoutbuilder = DirectRunLayoutBuilder(self.llinterp) self.get_type_id = layoutbuilder.get_type_id - self.gc.set_query_functions(*layoutbuilder.get_query_functions()) + layoutbuilder.initialize_gc_query_function(self.gc) constants = collect_constants(flowgraphs) for obj in constants: @@ -141,7 +142,7 @@ DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0] destrgraph = destrptr._obj.graph else: - return None + return lltype.nullptr(gctypelayout.GCData.FINALIZERTYPE.TO) assert not type_contains_pyobjs(TYPE), "not implemented" def ll_finalizer(addr): @@ -151,7 +152,7 @@ except llinterp.LLException: raise RuntimeError( "a finalizer raised an exception, shouldn't happen") - return ll_finalizer + return llhelper(gctypelayout.GCData.FINALIZERTYPE, ll_finalizer) def collect_constants(graphs): From arigo at codespeak.net Fri Dec 14 15:50:02 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Dec 2007 15:50:02 +0100 (CET) Subject: [pypy-svn] r49784 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20071214145002.5EC9D16850A@codespeak.net> Author: arigo Date: Fri Dec 14 15:50:00 2007 New Revision: 49784 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py Log: Can do this at compile-time instead of run-time, although it doesn't really change anything. Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Fri Dec 14 15:50:00 2007 @@ -131,6 +131,7 @@ StackRootIterator = self.build_stack_root_iterator() gcdata.gc = GCClass(AddressLinkedList, get_roots=StackRootIterator, **GC_PARAMS) + gcdata.set_query_functions(gcdata.gc) self.num_pushs = 0 self.write_barrier_calls = 0 @@ -138,7 +139,6 @@ # run-time initialization code StackRootIterator.setup_root_stack() gcdata.gc.setup() - gcdata.set_query_functions(gcdata.gc) bk = self.translator.annotator.bookkeeper From arigo at codespeak.net Fri Dec 14 15:50:25 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Dec 2007 15:50:25 +0100 (CET) Subject: [pypy-svn] r49785 - pypy/dist/pypy/rpython/memory Message-ID: <20071214145025.69B0416850A@codespeak.net> Author: arigo Date: Fri Dec 14 15:50:25 2007 New Revision: 49785 Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py Log: Experimental: encode the Bool flags as combination of bits on the typeid. Small speed-up only. Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/gctypelayout.py Fri Dec 14 15:50:25 2007 @@ -5,9 +5,9 @@ class GCData(object): """The GC information tables, and the query functions that the GC calls to decode their content. The encoding of this information - is done by TypeLayoutBuilder.get_type_id(). These two places - should be in sync, obviously, but in principle no other code - should depend on the details of the encoding in TYPE_INFO. + is done by encode_type_shape(). These two places should be in sync, + obviously, but in principle no other code should depend on the + details of the encoding in TYPE_INFO. """ _alloc_flavor_ = 'raw' @@ -17,9 +17,6 @@ # structure describing the layout of a typeid TYPE_INFO = lltype.Struct("type_info", - ("isvarsize", lltype.Bool), - ("gcptrinvarsize", lltype.Bool), - ("gcarrayofgcptr", lltype.Bool), ("finalizer", FINALIZERTYPE), ("fixedsize", lltype.Signed), ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), @@ -39,15 +36,16 @@ def q_is_varsize(self, typeid): ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].isvarsize + return (typeid & T_IS_FIXSIZE) == 0 def q_has_gcptr_in_varsize(self, typeid): ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].gcptrinvarsize + return (typeid & (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE)) == 0 def q_is_gcarrayofgcptr(self, typeid): ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].gcarrayofgcptr + return (typeid & + (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE|T_NOT_SIMPLE_GCARRAY)) == 0 def q_finalizer(self, typeid): ll_assert(typeid > 0, "invalid type_id") @@ -95,16 +93,107 @@ self.q_varsize_offsets_to_gcpointers_in_var_part, self.q_weakpointer_offset) +# For the q_xxx functions that return flags, we use bit patterns +# in the typeid instead of entries in the type_info_table. The +# following flag combinations are used (the idea being that it's +# very fast on CPUs to check if all flags in a set are all zero): + +# * if T_IS_FIXSIZE is set, the gc object is not var-sized +# * if T_IS_FIXSIZE and T_NO_GCPTR_IN_VARSIZE are both cleared, +# there are gc ptrs in the var-sized part +# * if T_IS_FIXSIZE, T_NO_GCPTR_IN_VARSIZE and T_NOT_SIMPLE_GCARRAY +# are all cleared, the shape is just like GcArray(gcptr) + +T_IS_FIXSIZE = 0x4 +T_NO_GCPTR_IN_VARSIZE = 0x2 +T_NOT_SIMPLE_GCARRAY = 0x1 + +def get_typeid_bitmask(TYPE): + """Return the bits that we would like to be set or cleared in the type_id + corresponding to TYPE. This returns (mask, expected_value), where + the condition is that 'type_id & mask == expected_value'. + """ + if not TYPE._is_varsize(): + return (T_IS_FIXSIZE, T_IS_FIXSIZE) # not var-sized + + if (isinstance(TYPE, lltype.GcArray) + and isinstance(TYPE.OF, lltype.Ptr) + and TYPE.OF.TO._gckind == 'gc'): + # a simple GcArray(gcptr) + return (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE|T_NOT_SIMPLE_GCARRAY, 0) + + if isinstance(TYPE, lltype.Struct): + ARRAY = TYPE._flds[TYPE._arrayfld] + else: + ARRAY = TYPE + assert isinstance(ARRAY, lltype.Array) + if ARRAY.OF != lltype.Void and len(offsets_to_gc_pointers(ARRAY.OF)) > 0: + # var-sized, with gc pointers in the variable part + return (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE|T_NOT_SIMPLE_GCARRAY, + T_NOT_SIMPLE_GCARRAY) + else: + # var-sized, but no gc pointer in the variable part + return (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE, T_NO_GCPTR_IN_VARSIZE) + + +def encode_type_shape(builder, info, TYPE): + """Encode the shape of the TYPE into the TYPE_INFO structure 'info'.""" + offsets = offsets_to_gc_pointers(TYPE) + info.ofstoptrs = builder.offsets2table(offsets, TYPE) + info.finalizer = builder.make_finalizer_funcptr_for_type(TYPE) + info.weakptrofs = weakpointer_offset(TYPE) + if not TYPE._is_varsize(): + #info.isvarsize = False + #info.gcptrinvarsize = False + info.fixedsize = llarena.round_up_for_allocation( + llmemory.sizeof(TYPE)) + info.ofstolength = -1 + # note about round_up_for_allocation(): in the 'info' table + # we put a rounded-up size only for fixed-size objects. For + # varsize ones, the GC must anyway compute the size at run-time + # and round up that result. + else: + #info.isvarsize = True + info.fixedsize = llmemory.sizeof(TYPE, 0) + if isinstance(TYPE, lltype.Struct): + ARRAY = TYPE._flds[TYPE._arrayfld] + ofs1 = llmemory.offsetof(TYPE, TYPE._arrayfld) + info.ofstolength = ofs1 + llmemory.ArrayLengthOffset(ARRAY) + if ARRAY.OF != lltype.Void: + info.ofstovar = ofs1 + llmemory.itemoffsetof(ARRAY, 0) + else: + info.fixedsize = ofs1 + llmemory.sizeof(lltype.Signed) + # XXX we probably don't need isrpystring any more + if ARRAY._hints.get('isrpystring'): + info.fixedsize = llmemory.sizeof(TYPE, 1) + else: + ARRAY = TYPE + info.ofstolength = llmemory.ArrayLengthOffset(ARRAY) + if ARRAY.OF != lltype.Void: + info.ofstovar = llmemory.itemoffsetof(TYPE, 0) + else: + info.fixedsize = (llmemory.ArrayLengthOffset(ARRAY) + + llmemory.sizeof(lltype.Signed)) + assert isinstance(ARRAY, lltype.Array) + if ARRAY.OF != lltype.Void: + offsets = offsets_to_gc_pointers(ARRAY.OF) + else: + offsets = () + info.varofstoptrs = builder.offsets2table(offsets, ARRAY.OF) + info.varitemsize = llmemory.sizeof(ARRAY.OF) + #info.gcptrinvarsize = len(offsets) > 0 + #info.gcarrayofgcptr = (isinstance(TYPE, lltype.GcArray) + # and isinstance(TYPE.OF, lltype.Ptr) + # and TYPE.OF.TO._gckind == 'gc') + # ____________________________________________________________ + class TypeLayoutBuilder(object): can_add_new_types = True def __init__(self): - dummy = lltype.malloc(GCData.TYPE_INFO, immortal=True, zero=True) - dummy.weakptrofs = -1 - dummy.ofstolength = -1 - self.type_info_list = [dummy] # don't use typeid 0, helps debugging + self.type_info_list = [None] # don't use typeid 0, helps debugging self.id_of_type = {} # {LLTYPE: type_id} self.seen_roots = {} # the following are lists of addresses of gc pointers living inside the @@ -116,6 +205,7 @@ self.addresses_of_static_ptrs_in_nongc = [] self.finalizer_funcptrs = {} self.offsettable_cache = {} + self.next_typeid_cache = {} def get_type_id(self, TYPE): try: @@ -123,63 +213,29 @@ except KeyError: assert self.can_add_new_types assert isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)) - # Record the new type_id description as a small dict for now. - # The framework gc transformer will turn it into a - # Struct("type_info") in flatten_table(). - type_id = len(self.type_info_list) + # Record the new type_id description as a TYPE_INFO structure. + # It goes into a list for now, which will be turned into a + # TYPE_INFO_TABLE in flatten_table() by the gc transformer. + + # pick the next type_id with the correct bits set or cleared + mask, expected = get_typeid_bitmask(TYPE) + type_id = self.next_typeid_cache.get((mask, expected), 1) + while True: + if type_id == len(self.type_info_list): + self.type_info_list.append(None) + if (self.type_info_list[type_id] is None and + (type_id & mask) == expected): + break # can use this type_id + else: + type_id += 1 # continue searching + self.next_typeid_cache[mask, expected] = type_id + 1 assert type_id & 0xffff == type_id # make sure it fits into 2 bytes + + # build the TYPE_INFO structure info = lltype.malloc(GCData.TYPE_INFO, immortal=True, zero=True) - self.type_info_list.append(info) + encode_type_shape(self, info, TYPE) + self.type_info_list[type_id] = info self.id_of_type[TYPE] = type_id - offsets = offsets_to_gc_pointers(TYPE) - info.ofstoptrs = self.offsets2table(offsets, TYPE) - info.finalizer = self.make_finalizer_funcptr_for_type(TYPE) - info.weakptrofs = weakpointer_offset(TYPE) - if not TYPE._is_varsize(): - info.isvarsize = False - info.gcptrinvarsize = False - info.fixedsize = llarena.round_up_for_allocation( - llmemory.sizeof(TYPE)) - info.ofstolength = -1 - # note about round_up_for_allocation(): in the 'info' table - # we put a rounded-up size only for fixed-size objects. For - # varsize ones, the GC must anyway compute the size at run-time - # and round up that result. - else: - info.isvarsize = True - info.fixedsize = llmemory.sizeof(TYPE, 0) - if isinstance(TYPE, lltype.Struct): - ARRAY = TYPE._flds[TYPE._arrayfld] - ofs1 = llmemory.offsetof(TYPE, TYPE._arrayfld) - info.ofstolength = ofs1 + llmemory.ArrayLengthOffset(ARRAY) - if ARRAY.OF != lltype.Void: - info.ofstovar = ofs1 + llmemory.itemoffsetof(ARRAY, 0) - else: - info.fixedsize = ofs1 + llmemory.sizeof(lltype.Signed) - # XXX we probably don't need isrpystring any more - if ARRAY._hints.get('isrpystring'): - info.fixedsize = llmemory.sizeof(TYPE, 1) - else: - ARRAY = TYPE - info.ofstolength = llmemory.ArrayLengthOffset(ARRAY) - if ARRAY.OF != lltype.Void: - info.ofstovar = llmemory.itemoffsetof(TYPE, 0) - else: - info.fixedsize = (llmemory.ArrayLengthOffset(ARRAY) + - llmemory.sizeof(lltype.Signed)) - assert isinstance(ARRAY, lltype.Array) - if ARRAY.OF != lltype.Void: - offsets = offsets_to_gc_pointers(ARRAY.OF) - else: - offsets = () - info.varofstoptrs = self.offsets2table(offsets, ARRAY.OF) - info.varitemsize = llmemory.sizeof(ARRAY.OF) - info.gcptrinvarsize = len(offsets) > 0 - # if the type is of the shape GcArray(gcptr) then we record, - # for now, a flag in the 'info'. XXX could use a bit in typeid - info.gcarrayofgcptr = (isinstance(TYPE, lltype.GcArray) - and isinstance(TYPE.OF, lltype.Ptr) - and TYPE.OF.TO._gckind == 'gc') return type_id def offsets2table(self, offsets, TYPE): @@ -200,8 +256,12 @@ immortal=True) fieldnames = GCData.TYPE_INFO._names for tableentry, newcontent in zip(table, self.type_info_list): - for name in fieldnames: - setattr(tableentry, name, getattr(newcontent, name)) + if newcontent is None: # empty entry + tableentry.weakptrofs = -1 + tableentry.ofstolength = -1 + else: + for name in fieldnames: + setattr(tableentry, name, getattr(newcontent, name)) return table def finalizer_funcptr_for_type(self, TYPE): From cfbolz at codespeak.net Fri Dec 14 16:11:07 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 14 Dec 2007 16:11:07 +0100 (CET) Subject: [pypy-svn] r49786 - pypy/branch/applevel-ctypes Message-ID: <20071214151107.5C5F216850F@codespeak.net> Author: cfbolz Date: Fri Dec 14 16:11:06 2007 New Revision: 49786 Added: pypy/branch/applevel-ctypes/ - copied from r49785, pypy/dist/ Log: a branch for playing around with implementing applevel-ctypes (no promises) From fijal at codespeak.net Fri Dec 14 16:35:18 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 14 Dec 2007 16:35:18 +0100 (CET) Subject: [pypy-svn] r49787 - in pypy/branch/lazy-write-barrier/pypy/rpython/memory: . gc gctransform Message-ID: <20071214153518.6A3721684FA@codespeak.net> Author: fijal Date: Fri Dec 14 16:35:17 2007 New Revision: 49787 Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py Log: Another evil hack in the gctypelayout (C-c C-v programming) Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py Fri Dec 14 16:35:17 2007 @@ -1,4 +1,4 @@ -import sys +import sys, os from pypy.rpython.memory.gc.semispace import SemiSpaceGC, GCFLAGSHIFT, \ GCFLAG_IMMORTAL from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage @@ -38,6 +38,7 @@ max_space_size = max_space_size, get_roots = get_roots) self.nursery_size = nursery_size + self.calls = 0 assert nursery_size <= space_size // 2 def setup(self): @@ -295,10 +296,12 @@ self.move_to_static_roots(addr_struct) def append_to_static_roots(self, pointer, arg): + os.write(2, "3\n") self.get_roots.append_static_root(pointer) def move_to_static_roots(self, addr_struct): - self.header(addr_struct).tid &= ~GCFLAG_NEVER_SET + objhdr = self.header(addr_struct) + objhdr.tid &= ~GCFLAG_NEVER_SET self.trace(addr_struct, self.append_to_static_roots, None) def remember_young_pointer(self, addr_struct, addr): Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py Fri Dec 14 16:35:17 2007 @@ -525,7 +525,7 @@ ll_static_roots_inside[i] = addresses_of_static_ptrs[i] ll_instance.inst_static_root_start = llmemory.cast_ptr_to_adr(ll_static_roots_inside) + llmemory.ArrayItemsOffset(lltype.Array(llmemory.Address)) ll_instance.inst_static_root_nongcstart = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(self.layoutbuilder.addresses_of_static_ptrs) - ll_instance.inst_static_root_end = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(ll_static_roots_inside) + ll_instance.inst_static_root_end = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * (len(ll_static_roots_inside) - additional_ptrs) newgcdependencies = [] newgcdependencies.append(table) Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py Fri Dec 14 16:35:17 2007 @@ -175,7 +175,9 @@ adr = llmemory.cast_ptr_to_adr(value._as_ptr()) if TYPE._gckind == "gc": if gen_gc: - for a in mutable_gc_pointers_inside(value, adr): + import py + py.test.pdb() + for a in gc_pointers_inside(value, adr): self.additional_roots_sources += 1 return else: @@ -221,6 +223,27 @@ return llmemory.offsetof(WEAKREF, "weakptr") return -1 +def gc_pointers_inside(v, adr): + t = lltype.typeOf(v) + if isinstance(t, lltype.Struct): + for n, t2 in t._flds.iteritems(): + if isinstance(t2, lltype.Ptr) and t2.TO._gckind == 'gc': + yield adr + llmemory.offsetof(t, n) + elif isinstance(t2, (lltype.Array, lltype.Struct)): + for a in gc_pointers_inside(getattr(v, n), + adr + llmemory.offsetof(t, n)): + yield a + elif isinstance(t, lltype.Array): + if isinstance(t.OF, lltype.Ptr) and t.OF.TO._gckind == 'gc': + for i in range(len(v.items)): + yield adr + llmemory.itemoffsetof(t, i) + elif isinstance(t.OF, lltype.Struct): + for i in range(len(v.items)): + for a in gc_pointers_inside(v.items[i], + adr + llmemory.itemoffsetof(t, i)): + yield a + + def mutable_gc_pointers_inside(v, adr): t = lltype.typeOf(v) if isinstance(t, lltype.Struct): From fijal at codespeak.net Fri Dec 14 16:36:30 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 14 Dec 2007 16:36:30 +0100 (CET) Subject: [pypy-svn] r49788 - in pypy/branch/lazy-write-barrier/pypy/rpython/memory: . gc Message-ID: <20071214153630.840DC1684FA@codespeak.net> Author: fijal Date: Fri Dec 14 16:36:30 2007 New Revision: 49788 Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py Log: Don't check in pdb Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py Fri Dec 14 16:36:30 2007 @@ -294,9 +294,10 @@ self.remember_young_pointer(addr_struct, newvalue) if self.header(addr_struct).tid & GCFLAG_NEVER_SET: self.move_to_static_roots(addr_struct) + self.calls += 1 def append_to_static_roots(self, pointer, arg): - os.write(2, "3\n") + os.write(2, str(self.calls) + "\n") self.get_roots.append_static_root(pointer) def move_to_static_roots(self, addr_struct): Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py Fri Dec 14 16:36:30 2007 @@ -175,8 +175,6 @@ adr = llmemory.cast_ptr_to_adr(value._as_ptr()) if TYPE._gckind == "gc": if gen_gc: - import py - py.test.pdb() for a in gc_pointers_inside(value, adr): self.additional_roots_sources += 1 return From cfbolz at codespeak.net Fri Dec 14 16:47:53 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 14 Dec 2007 16:47:53 +0100 (CET) Subject: [pypy-svn] r49789 - pypy/branch/applevel-ctypes/pypy/lib/ctypes Message-ID: <20071214154753.9546616850F@codespeak.net> Author: cfbolz Date: Fri Dec 14 16:47:53 2007 New Revision: 49789 Removed: pypy/branch/applevel-ctypes/pypy/lib/ctypes/ Log: get rid of this for now From arigo at codespeak.net Fri Dec 14 16:51:08 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Dec 2007 16:51:08 +0100 (CET) Subject: [pypy-svn] r49790 - in pypy/branch/lazy-write-barrier/pypy/rpython/memory: . gc Message-ID: <20071214155108.1EA3216850F@codespeak.net> Author: arigo Date: Fri Dec 14 16:51:03 2007 New Revision: 49790 Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/gcwrapper.py Log: * Adapt gcwrapper to the new world. * Surely GCFLAG_NEVER_SET should be set on immortal objects? Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py Fri Dec 14 16:51:03 2007 @@ -140,7 +140,8 @@ def init_gc_object(self, addr, typeid, flags=GCFLAG_NO_YOUNG_PTRS): SemiSpaceGC.init_gc_object(self, addr, typeid, flags) - def init_gc_object_immortal(self, addr, typeid, flags=GCFLAG_NO_YOUNG_PTRS): + def init_gc_object_immortal(self, addr, typeid, + flags=GCFLAG_NO_YOUNG_PTRS|GCFLAG_NEVER_SET): SemiSpaceGC.init_gc_object_immortal(self, addr, typeid, flags) def semispace_collect(self, size_changing=False): @@ -292,9 +293,9 @@ def write_barrier(self, oldvalue, newvalue, addr_struct): if self.header(addr_struct).tid & GCFLAG_NO_YOUNG_PTRS: self.remember_young_pointer(addr_struct, newvalue) - if self.header(addr_struct).tid & GCFLAG_NEVER_SET: - self.move_to_static_roots(addr_struct) - self.calls += 1 + if self.header(addr_struct).tid & GCFLAG_NEVER_SET: + self.move_to_static_roots(addr_struct) + self.calls += 1 def append_to_static_roots(self, pointer, arg): os.write(2, str(self.calls) + "\n") Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gcwrapper.py Fri Dec 14 16:51:03 2007 @@ -10,10 +10,13 @@ def __init__(self, llinterp, flowgraphs, gc_class, GC_PARAMS={}): self.AddressLinkedList = get_address_linked_list(10) self.gc = gc_class(self.AddressLinkedList, **GC_PARAMS) - self.gc.get_roots = self.get_roots_from_llinterp + def my_get_roots(with_static=True): + return self.get_roots_from_llinterp(with_static) + self.gc.get_roots = my_get_roots self.llinterp = llinterp self.prepare_graphs(flowgraphs) self.gc.setup() + my_get_roots.append_static_root = self.constantroots.append def prepare_graphs(self, flowgraphs): layoutbuilder = DirectRunLayoutBuilder(self.llinterp) @@ -25,7 +28,7 @@ TYPE = lltype.typeOf(obj) layoutbuilder.consider_constant(TYPE, obj, self.gc) - self.constantroots = layoutbuilder.addresses_of_static_ptrs + self.constantroots = list(layoutbuilder.addresses_of_static_ptrs) self.constantrootsnongc = layoutbuilder.addresses_of_static_ptrs_in_nongc def get_roots_from_llinterp(self, with_static=True): From cfbolz at codespeak.net Fri Dec 14 16:53:50 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 14 Dec 2007 16:53:50 +0100 (CET) Subject: [pypy-svn] r49791 - in pypy/branch/applevel-ctypes/pypy/lib/ctypes: . macholib test Message-ID: <20071214155350.AC829168519@codespeak.net> Author: cfbolz Date: Fri Dec 14 16:53:49 2007 New Revision: 49791 Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/ pypy/branch/applevel-ctypes/pypy/lib/ctypes/__init__.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/_endian.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/ pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/README.ctypes pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/__init__.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/dyld.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/dylib.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/fetch_macholib (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/fetch_macholib.bat pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/framework.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/__init__.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/runtests.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_anon.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_array_in_pointer.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_arrays.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_as_parameter.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_bitfields.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_buffers.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_byteswap.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_callbacks.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_cast.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_cfuncs.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_checkretval.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_errcheck.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_find.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_funcptr.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_functions.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_incomplete.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_init.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_integers.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_internals.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_keeprefs.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_libc.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_loading.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_macholib.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_memfunctions.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_numbers.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_objects.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_parameters.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_pointers.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_prototypes.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_python_api.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_random_things.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_refcounts.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_repr.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_returnfuncptrs.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_simplesubclasses.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_sizes.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_slicing.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_stringptr.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_strings.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_struct_fields.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_structures.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_unaligned_structures.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_unicode.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_values.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_varsize_struct.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_win32.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/util.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/wintypes.py (contents, props changed) Log: clean import of the ctypes package of Python 2.5.1 Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/__init__.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,529 @@ +###################################################################### +# This file should be kept compatible with Python 2.3, see PEP 291. # +###################################################################### +"""create and manipulate C data types in Python""" + +import os as _os, sys as _sys + +__version__ = "1.0.2" + +from _ctypes import Union, Structure, Array +from _ctypes import _Pointer +from _ctypes import CFuncPtr as _CFuncPtr +from _ctypes import __version__ as _ctypes_version +from _ctypes import RTLD_LOCAL, RTLD_GLOBAL +from _ctypes import ArgumentError + +from struct import calcsize as _calcsize + +if __version__ != _ctypes_version: + raise Exception, ("Version number mismatch", __version__, _ctypes_version) + +if _os.name in ("nt", "ce"): + from _ctypes import FormatError + +DEFAULT_MODE = RTLD_LOCAL +if _os.name == "posix" and _sys.platform == "darwin": + import gestalt + + # gestalt.gestalt("sysv") returns the version number of the + # currently active system file as BCD. + # On OS X 10.4.6 -> 0x1046 + # On OS X 10.2.8 -> 0x1028 + # See also http://www.rgaros.nl/gestalt/ + # + # On OS X 10.3, we use RTLD_GLOBAL as default mode + # because RTLD_LOCAL does not work at least on some + # libraries. + + if gestalt.gestalt("sysv") < 0x1040: + DEFAULT_MODE = RTLD_GLOBAL + +from _ctypes import FUNCFLAG_CDECL as _FUNCFLAG_CDECL, \ + FUNCFLAG_PYTHONAPI as _FUNCFLAG_PYTHONAPI + +""" +WINOLEAPI -> HRESULT +WINOLEAPI_(type) + +STDMETHODCALLTYPE + +STDMETHOD(name) +STDMETHOD_(type, name) + +STDAPICALLTYPE +""" + +def create_string_buffer(init, size=None): + """create_string_buffer(aString) -> character array + create_string_buffer(anInteger) -> character array + create_string_buffer(aString, anInteger) -> character array + """ + if isinstance(init, (str, unicode)): + if size is None: + size = len(init)+1 + buftype = c_char * size + buf = buftype() + buf.value = init + return buf + elif isinstance(init, (int, long)): + buftype = c_char * init + buf = buftype() + return buf + raise TypeError, init + +def c_buffer(init, size=None): +## "deprecated, use create_string_buffer instead" +## import warnings +## warnings.warn("c_buffer is deprecated, use create_string_buffer instead", +## DeprecationWarning, stacklevel=2) + return create_string_buffer(init, size) + +_c_functype_cache = {} +def CFUNCTYPE(restype, *argtypes): + """CFUNCTYPE(restype, *argtypes) -> function prototype. + + restype: the result type + argtypes: a sequence specifying the argument types + + The function prototype can be called in different ways to create a + callable object: + + prototype(integer address) -> foreign function + prototype(callable) -> create and return a C callable function from callable + prototype(integer index, method name[, paramflags]) -> foreign function calling a COM method + prototype((ordinal number, dll object)[, paramflags]) -> foreign function exported by ordinal + prototype((function name, dll object)[, paramflags]) -> foreign function exported by name + """ + try: + return _c_functype_cache[(restype, argtypes)] + except KeyError: + class CFunctionType(_CFuncPtr): + _argtypes_ = argtypes + _restype_ = restype + _flags_ = _FUNCFLAG_CDECL + _c_functype_cache[(restype, argtypes)] = CFunctionType + return CFunctionType + +if _os.name in ("nt", "ce"): + from _ctypes import LoadLibrary as _dlopen + from _ctypes import FUNCFLAG_STDCALL as _FUNCFLAG_STDCALL + if _os.name == "ce": + # 'ce' doesn't have the stdcall calling convention + _FUNCFLAG_STDCALL = _FUNCFLAG_CDECL + + _win_functype_cache = {} + def WINFUNCTYPE(restype, *argtypes): + # docstring set later (very similar to CFUNCTYPE.__doc__) + try: + return _win_functype_cache[(restype, argtypes)] + except KeyError: + class WinFunctionType(_CFuncPtr): + _argtypes_ = argtypes + _restype_ = restype + _flags_ = _FUNCFLAG_STDCALL + _win_functype_cache[(restype, argtypes)] = WinFunctionType + return WinFunctionType + if WINFUNCTYPE.__doc__: + WINFUNCTYPE.__doc__ = CFUNCTYPE.__doc__.replace("CFUNCTYPE", "WINFUNCTYPE") + +elif _os.name == "posix": + from _ctypes import dlopen as _dlopen + +from _ctypes import sizeof, byref, addressof, alignment, resize +from _ctypes import _SimpleCData + +def _check_size(typ, typecode=None): + # Check if sizeof(ctypes_type) against struct.calcsize. This + # should protect somewhat against a misconfigured libffi. + from struct import calcsize + if typecode is None: + # Most _type_ codes are the same as used in struct + typecode = typ._type_ + actual, required = sizeof(typ), calcsize(typecode) + if actual != required: + raise SystemError("sizeof(%s) wrong: %d instead of %d" % \ + (typ, actual, required)) + +class py_object(_SimpleCData): + _type_ = "O" + def __repr__(self): + try: + return super(py_object, self).__repr__() + except ValueError: + return "%s()" % type(self).__name__ +_check_size(py_object, "P") + +class c_short(_SimpleCData): + _type_ = "h" +_check_size(c_short) + +class c_ushort(_SimpleCData): + _type_ = "H" +_check_size(c_ushort) + +class c_long(_SimpleCData): + _type_ = "l" +_check_size(c_long) + +class c_ulong(_SimpleCData): + _type_ = "L" +_check_size(c_ulong) + +if _calcsize("i") == _calcsize("l"): + # if int and long have the same size, make c_int an alias for c_long + c_int = c_long + c_uint = c_ulong +else: + class c_int(_SimpleCData): + _type_ = "i" + _check_size(c_int) + + class c_uint(_SimpleCData): + _type_ = "I" + _check_size(c_uint) + +class c_float(_SimpleCData): + _type_ = "f" +_check_size(c_float) + +class c_double(_SimpleCData): + _type_ = "d" +_check_size(c_double) + +if _calcsize("l") == _calcsize("q"): + # if long and long long have the same size, make c_longlong an alias for c_long + c_longlong = c_long + c_ulonglong = c_ulong +else: + class c_longlong(_SimpleCData): + _type_ = "q" + _check_size(c_longlong) + + class c_ulonglong(_SimpleCData): + _type_ = "Q" + ## def from_param(cls, val): + ## return ('d', float(val), val) + ## from_param = classmethod(from_param) + _check_size(c_ulonglong) + +class c_ubyte(_SimpleCData): + _type_ = "B" +c_ubyte.__ctype_le__ = c_ubyte.__ctype_be__ = c_ubyte +# backward compatibility: +##c_uchar = c_ubyte +_check_size(c_ubyte) + +class c_byte(_SimpleCData): + _type_ = "b" +c_byte.__ctype_le__ = c_byte.__ctype_be__ = c_byte +_check_size(c_byte) + +class c_char(_SimpleCData): + _type_ = "c" +c_char.__ctype_le__ = c_char.__ctype_be__ = c_char +_check_size(c_char) + +class c_char_p(_SimpleCData): + _type_ = "z" +_check_size(c_char_p, "P") + +class c_void_p(_SimpleCData): + _type_ = "P" +c_voidp = c_void_p # backwards compatibility (to a bug) +_check_size(c_void_p) + +# This cache maps types to pointers to them. +_pointer_type_cache = {} + +def POINTER(cls): + try: + return _pointer_type_cache[cls] + except KeyError: + pass + if type(cls) is str: + klass = type(_Pointer)("LP_%s" % cls, + (_Pointer,), + {}) + _pointer_type_cache[id(klass)] = klass + return klass + else: + name = "LP_%s" % cls.__name__ + klass = type(_Pointer)(name, + (_Pointer,), + {'_type_': cls}) + _pointer_type_cache[cls] = klass + return klass + +try: + from _ctypes import set_conversion_mode +except ImportError: + pass +else: + if _os.name in ("nt", "ce"): + set_conversion_mode("mbcs", "ignore") + else: + set_conversion_mode("ascii", "strict") + + class c_wchar_p(_SimpleCData): + _type_ = "Z" + + class c_wchar(_SimpleCData): + _type_ = "u" + + POINTER(c_wchar).from_param = c_wchar_p.from_param #_SimpleCData.c_wchar_p_from_param + + def create_unicode_buffer(init, size=None): + """create_unicode_buffer(aString) -> character array + create_unicode_buffer(anInteger) -> character array + create_unicode_buffer(aString, anInteger) -> character array + """ + if isinstance(init, (str, unicode)): + if size is None: + size = len(init)+1 + buftype = c_wchar * size + buf = buftype() + buf.value = init + return buf + elif isinstance(init, (int, long)): + buftype = c_wchar * init + buf = buftype() + return buf + raise TypeError, init + +POINTER(c_char).from_param = c_char_p.from_param #_SimpleCData.c_char_p_from_param + +# XXX Deprecated +def SetPointerType(pointer, cls): + if _pointer_type_cache.get(cls, None) is not None: + raise RuntimeError, \ + "This type already exists in the cache" + if not _pointer_type_cache.has_key(id(pointer)): + raise RuntimeError, \ + "What's this???" + pointer.set_type(cls) + _pointer_type_cache[cls] = pointer + del _pointer_type_cache[id(pointer)] + + +def pointer(inst): + return POINTER(type(inst))(inst) + +# XXX Deprecated +def ARRAY(typ, len): + return typ * len + +################################################################ + + +class CDLL(object): + """An instance of this class represents a loaded dll/shared + library, exporting functions using the standard C calling + convention (named 'cdecl' on Windows). + + The exported functions can be accessed as attributes, or by + indexing with the function name. Examples: + + .qsort -> callable object + ['qsort'] -> callable object + + Calling the functions releases the Python GIL during the call and + reaquires it afterwards. + """ + class _FuncPtr(_CFuncPtr): + _flags_ = _FUNCFLAG_CDECL + _restype_ = c_int # default, can be overridden in instances + + def __init__(self, name, mode=DEFAULT_MODE, handle=None): + self._name = name + if handle is None: + self._handle = _dlopen(self._name, mode) + else: + self._handle = handle + + def __repr__(self): + return "<%s '%s', handle %x at %x>" % \ + (self.__class__.__name__, self._name, + (self._handle & (_sys.maxint*2 + 1)), + id(self) & (_sys.maxint*2 + 1)) + + def __getattr__(self, name): + if name.startswith('__') and name.endswith('__'): + raise AttributeError, name + func = self.__getitem__(name) + setattr(self, name, func) + return func + + def __getitem__(self, name_or_ordinal): + func = self._FuncPtr((name_or_ordinal, self)) + if not isinstance(name_or_ordinal, (int, long)): + func.__name__ = name_or_ordinal + return func + +class PyDLL(CDLL): + """This class represents the Python library itself. It allows to + access Python API functions. The GIL is not released, and + Python exceptions are handled correctly. + """ + class _FuncPtr(_CFuncPtr): + _flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI + _restype_ = c_int # default, can be overridden in instances + +if _os.name in ("nt", "ce"): + + class WinDLL(CDLL): + """This class represents a dll exporting functions using the + Windows stdcall calling convention. + """ + class _FuncPtr(_CFuncPtr): + _flags_ = _FUNCFLAG_STDCALL + _restype_ = c_int # default, can be overridden in instances + + # XXX Hm, what about HRESULT as normal parameter? + # Mustn't it derive from c_long then? + from _ctypes import _check_HRESULT, _SimpleCData + class HRESULT(_SimpleCData): + _type_ = "l" + # _check_retval_ is called with the function's result when it + # is used as restype. It checks for the FAILED bit, and + # raises a WindowsError if it is set. + # + # The _check_retval_ method is implemented in C, so that the + # method definition itself is not included in the traceback + # when it raises an error - that is what we want (and Python + # doesn't have a way to raise an exception in the caller's + # frame). + _check_retval_ = _check_HRESULT + + class OleDLL(CDLL): + """This class represents a dll exporting functions using the + Windows stdcall calling convention, and returning HRESULT. + HRESULT error values are automatically raised as WindowsError + exceptions. + """ + class _FuncPtr(_CFuncPtr): + _flags_ = _FUNCFLAG_STDCALL + _restype_ = HRESULT + +class LibraryLoader(object): + def __init__(self, dlltype): + self._dlltype = dlltype + + def __getattr__(self, name): + if name[0] == '_': + raise AttributeError(name) + dll = self._dlltype(name) + setattr(self, name, dll) + return dll + + def __getitem__(self, name): + return getattr(self, name) + + def LoadLibrary(self, name): + return self._dlltype(name) + +cdll = LibraryLoader(CDLL) +pydll = LibraryLoader(PyDLL) + +if _os.name in ("nt", "ce"): + pythonapi = PyDLL("python dll", None, _sys.dllhandle) +elif _sys.platform == "cygwin": + pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2]) +else: + pythonapi = PyDLL(None) + + +if _os.name in ("nt", "ce"): + windll = LibraryLoader(WinDLL) + oledll = LibraryLoader(OleDLL) + + if _os.name == "nt": + GetLastError = windll.kernel32.GetLastError + else: + GetLastError = windll.coredll.GetLastError + + def WinError(code=None, descr=None): + if code is None: + code = GetLastError() + if descr is None: + descr = FormatError(code).strip() + return WindowsError(code, descr) + +_pointer_type_cache[None] = c_void_p + +if sizeof(c_uint) == sizeof(c_void_p): + c_size_t = c_uint +elif sizeof(c_ulong) == sizeof(c_void_p): + c_size_t = c_ulong + +# functions + +from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr + +## void *memmove(void *, const void *, size_t); +memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr) + +## void *memset(void *, int, size_t) +memset = CFUNCTYPE(c_void_p, c_void_p, c_int, c_size_t)(_memset_addr) + +def PYFUNCTYPE(restype, *argtypes): + class CFunctionType(_CFuncPtr): + _argtypes_ = argtypes + _restype_ = restype + _flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI + return CFunctionType + +_cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr) +def cast(obj, typ): + return _cast(obj, obj, typ) + +_string_at = CFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) +def string_at(ptr, size=-1): + """string_at(addr[, size]) -> string + + Return the string at addr.""" + return _string_at(ptr, size) + +try: + from _ctypes import _wstring_at_addr +except ImportError: + pass +else: + _wstring_at = CFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr) + def wstring_at(ptr, size=-1): + """wstring_at(addr[, size]) -> string + + Return the string at addr.""" + return _wstring_at(ptr, size) + + +if _os.name in ("nt", "ce"): # COM stuff + def DllGetClassObject(rclsid, riid, ppv): + try: + ccom = __import__("comtypes.server.inprocserver", globals(), locals(), ['*']) + except ImportError: + return -2147221231 # CLASS_E_CLASSNOTAVAILABLE + else: + return ccom.DllGetClassObject(rclsid, riid, ppv) + + def DllCanUnloadNow(): + try: + ccom = __import__("comtypes.server.inprocserver", globals(), locals(), ['*']) + except ImportError: + return 0 # S_OK + return ccom.DllCanUnloadNow() + +from ctypes._endian import BigEndianStructure, LittleEndianStructure + +# Fill in specifically-sized types +c_int8 = c_byte +c_uint8 = c_ubyte +for kind in [c_short, c_int, c_long, c_longlong]: + if sizeof(kind) == 2: c_int16 = kind + elif sizeof(kind) == 4: c_int32 = kind + elif sizeof(kind) == 8: c_int64 = kind +for kind in [c_ushort, c_uint, c_ulong, c_ulonglong]: + if sizeof(kind) == 2: c_uint16 = kind + elif sizeof(kind) == 4: c_uint32 = kind + elif sizeof(kind) == 8: c_uint64 = kind +del(kind) Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/_endian.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/_endian.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,60 @@ +###################################################################### +# This file should be kept compatible with Python 2.3, see PEP 291. # +###################################################################### +import sys +from ctypes import * + +_array_type = type(c_int * 3) + +def _other_endian(typ): + """Return the type with the 'other' byte order. Simple types like + c_int and so on already have __ctype_be__ and __ctype_le__ + attributes which contain the types, for more complicated types + only arrays are supported. + """ + try: + return getattr(typ, _OTHER_ENDIAN) + except AttributeError: + if type(typ) == _array_type: + return _other_endian(typ._type_) * typ._length_ + raise TypeError("This type does not support other endian: %s" % typ) + +class _swapped_meta(type(Structure)): + def __setattr__(self, attrname, value): + if attrname == "_fields_": + fields = [] + for desc in value: + name = desc[0] + typ = desc[1] + rest = desc[2:] + fields.append((name, _other_endian(typ)) + rest) + value = fields + super(_swapped_meta, self).__setattr__(attrname, value) + +################################################################ + +# Note: The Structure metaclass checks for the *presence* (not the +# value!) of a _swapped_bytes_ attribute to determine the bit order in +# structures containing bit fields. + +if sys.byteorder == "little": + _OTHER_ENDIAN = "__ctype_be__" + + LittleEndianStructure = Structure + + class BigEndianStructure(Structure): + """Structure with big endian byte order""" + __metaclass__ = _swapped_meta + _swappedbytes_ = None + +elif sys.byteorder == "big": + _OTHER_ENDIAN = "__ctype_le__" + + BigEndianStructure = Structure + class LittleEndianStructure(Structure): + """Structure with little endian byte order""" + __metaclass__ = _swapped_meta + _swappedbytes_ = None + +else: + raise RuntimeError("Invalid byteorder") Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/README.ctypes ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/README.ctypes Fri Dec 14 16:53:49 2007 @@ -0,0 +1,7 @@ +Files in this directory from from Bob Ippolito's py2app. + +License: Any components of the py2app suite may be distributed under +the MIT or PSF open source licenses. + +This is version 1.0, SVN revision 789, from 2006/01/25. +The main repository is http://svn.red-bean.com/bob/macholib/trunk/macholib/ \ No newline at end of file Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/__init__.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,12 @@ +###################################################################### +# This file should be kept compatible with Python 2.3, see PEP 291. # +###################################################################### +""" +Enough Mach-O to make your head spin. + +See the relevant header files in /usr/include/mach-o + +And also Apple's documentation. +""" + +__version__ = '1.0' Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/dyld.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/dyld.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,169 @@ +###################################################################### +# This file should be kept compatible with Python 2.3, see PEP 291. # +###################################################################### +""" +dyld emulation +""" + +import os +from framework import framework_info +from dylib import dylib_info +from itertools import * + +__all__ = [ + 'dyld_find', 'framework_find', + 'framework_info', 'dylib_info', +] + +# These are the defaults as per man dyld(1) +# +DEFAULT_FRAMEWORK_FALLBACK = [ + os.path.expanduser("~/Library/Frameworks"), + "/Library/Frameworks", + "/Network/Library/Frameworks", + "/System/Library/Frameworks", +] + +DEFAULT_LIBRARY_FALLBACK = [ + os.path.expanduser("~/lib"), + "/usr/local/lib", + "/lib", + "/usr/lib", +] + +def ensure_utf8(s): + """Not all of PyObjC and Python understand unicode paths very well yet""" + if isinstance(s, unicode): + return s.encode('utf8') + return s + +def dyld_env(env, var): + if env is None: + env = os.environ + rval = env.get(var) + if rval is None: + return [] + return rval.split(':') + +def dyld_image_suffix(env=None): + if env is None: + env = os.environ + return env.get('DYLD_IMAGE_SUFFIX') + +def dyld_framework_path(env=None): + return dyld_env(env, 'DYLD_FRAMEWORK_PATH') + +def dyld_library_path(env=None): + return dyld_env(env, 'DYLD_LIBRARY_PATH') + +def dyld_fallback_framework_path(env=None): + return dyld_env(env, 'DYLD_FALLBACK_FRAMEWORK_PATH') + +def dyld_fallback_library_path(env=None): + return dyld_env(env, 'DYLD_FALLBACK_LIBRARY_PATH') + +def dyld_image_suffix_search(iterator, env=None): + """For a potential path iterator, add DYLD_IMAGE_SUFFIX semantics""" + suffix = dyld_image_suffix(env) + if suffix is None: + return iterator + def _inject(iterator=iterator, suffix=suffix): + for path in iterator: + if path.endswith('.dylib'): + yield path[:-len('.dylib')] + suffix + '.dylib' + else: + yield path + suffix + yield path + return _inject() + +def dyld_override_search(name, env=None): + # If DYLD_FRAMEWORK_PATH is set and this dylib_name is a + # framework name, use the first file that exists in the framework + # path if any. If there is none go on to search the DYLD_LIBRARY_PATH + # if any. + + framework = framework_info(name) + + if framework is not None: + for path in dyld_framework_path(env): + yield os.path.join(path, framework['name']) + + # If DYLD_LIBRARY_PATH is set then use the first file that exists + # in the path. If none use the original name. + for path in dyld_library_path(env): + yield os.path.join(path, os.path.basename(name)) + +def dyld_executable_path_search(name, executable_path=None): + # If we haven't done any searching and found a library and the + # dylib_name starts with "@executable_path/" then construct the + # library name. + if name.startswith('@executable_path/') and executable_path is not None: + yield os.path.join(executable_path, name[len('@executable_path/'):]) + +def dyld_default_search(name, env=None): + yield name + + framework = framework_info(name) + + if framework is not None: + fallback_framework_path = dyld_fallback_framework_path(env) + for path in fallback_framework_path: + yield os.path.join(path, framework['name']) + + fallback_library_path = dyld_fallback_library_path(env) + for path in fallback_library_path: + yield os.path.join(path, os.path.basename(name)) + + if framework is not None and not fallback_framework_path: + for path in DEFAULT_FRAMEWORK_FALLBACK: + yield os.path.join(path, framework['name']) + + if not fallback_library_path: + for path in DEFAULT_LIBRARY_FALLBACK: + yield os.path.join(path, os.path.basename(name)) + +def dyld_find(name, executable_path=None, env=None): + """ + Find a library or framework using dyld semantics + """ + name = ensure_utf8(name) + executable_path = ensure_utf8(executable_path) + for path in dyld_image_suffix_search(chain( + dyld_override_search(name, env), + dyld_executable_path_search(name, executable_path), + dyld_default_search(name, env), + ), env): + if os.path.isfile(path): + return path + raise ValueError, "dylib %s could not be found" % (name,) + +def framework_find(fn, executable_path=None, env=None): + """ + Find a framework using dyld semantics in a very loose manner. + + Will take input such as: + Python + Python.framework + Python.framework/Versions/Current + """ + try: + return dyld_find(fn, executable_path=executable_path, env=env) + except ValueError, e: + pass + fmwk_index = fn.rfind('.framework') + if fmwk_index == -1: + fmwk_index = len(fn) + fn += '.framework' + fn = os.path.join(fn, os.path.basename(fn[:fmwk_index])) + try: + return dyld_find(fn, executable_path=executable_path, env=env) + except ValueError: + raise e + +def test_dyld_find(): + env = {} + assert dyld_find('libSystem.dylib') == '/usr/lib/libSystem.dylib' + assert dyld_find('System.framework/System') == '/System/Library/Frameworks/System.framework/System' + +if __name__ == '__main__': + test_dyld_find() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/dylib.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/dylib.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,66 @@ +###################################################################### +# This file should be kept compatible with Python 2.3, see PEP 291. # +###################################################################### +""" +Generic dylib path manipulation +""" + +import re + +__all__ = ['dylib_info'] + +DYLIB_RE = re.compile(r"""(?x) +(?P^.*)(?:^|/) +(?P + (?P\w+?) + (?:\.(?P[^._]+))? + (?:_(?P[^._]+))? + \.dylib$ +) +""") + +def dylib_info(filename): + """ + A dylib name can take one of the following four forms: + Location/Name.SomeVersion_Suffix.dylib + Location/Name.SomeVersion.dylib + Location/Name_Suffix.dylib + Location/Name.dylib + + returns None if not found or a mapping equivalent to: + dict( + location='Location', + name='Name.SomeVersion_Suffix.dylib', + shortname='Name', + version='SomeVersion', + suffix='Suffix', + ) + + Note that SomeVersion and Suffix are optional and may be None + if not present. + """ + is_dylib = DYLIB_RE.match(filename) + if not is_dylib: + return None + return is_dylib.groupdict() + + +def test_dylib_info(): + def d(location=None, name=None, shortname=None, version=None, suffix=None): + return dict( + location=location, + name=name, + shortname=shortname, + version=version, + suffix=suffix + ) + assert dylib_info('completely/invalid') is None + assert dylib_info('completely/invalide_debug') is None + assert dylib_info('P/Foo.dylib') == d('P', 'Foo.dylib', 'Foo') + assert dylib_info('P/Foo_debug.dylib') == d('P', 'Foo_debug.dylib', 'Foo', suffix='debug') + assert dylib_info('P/Foo.A.dylib') == d('P', 'Foo.A.dylib', 'Foo', 'A') + assert dylib_info('P/Foo_debug.A.dylib') == d('P', 'Foo_debug.A.dylib', 'Foo_debug', 'A') + assert dylib_info('P/Foo.A_debug.dylib') == d('P', 'Foo.A_debug.dylib', 'Foo', 'A', 'debug') + +if __name__ == '__main__': + test_dylib_info() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/fetch_macholib ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/fetch_macholib Fri Dec 14 16:53:49 2007 @@ -0,0 +1,2 @@ +#!/bin/sh +svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ . Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/fetch_macholib.bat ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/fetch_macholib.bat Fri Dec 14 16:53:49 2007 @@ -0,0 +1 @@ +svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ . Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/framework.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/framework.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,68 @@ +###################################################################### +# This file should be kept compatible with Python 2.3, see PEP 291. # +###################################################################### +""" +Generic framework path manipulation +""" + +import re + +__all__ = ['framework_info'] + +STRICT_FRAMEWORK_RE = re.compile(r"""(?x) +(?P^.*)(?:^|/) +(?P + (?P\w+).framework/ + (?:Versions/(?P[^/]+)/)? + (?P=shortname) + (?:_(?P[^_]+))? +)$ +""") + +def framework_info(filename): + """ + A framework name can take one of the following four forms: + Location/Name.framework/Versions/SomeVersion/Name_Suffix + Location/Name.framework/Versions/SomeVersion/Name + Location/Name.framework/Name_Suffix + Location/Name.framework/Name + + returns None if not found, or a mapping equivalent to: + dict( + location='Location', + name='Name.framework/Versions/SomeVersion/Name_Suffix', + shortname='Name', + version='SomeVersion', + suffix='Suffix', + ) + + Note that SomeVersion and Suffix are optional and may be None + if not present + """ + is_framework = STRICT_FRAMEWORK_RE.match(filename) + if not is_framework: + return None + return is_framework.groupdict() + +def test_framework_info(): + def d(location=None, name=None, shortname=None, version=None, suffix=None): + return dict( + location=location, + name=name, + shortname=shortname, + version=version, + suffix=suffix + ) + assert framework_info('completely/invalid') is None + assert framework_info('completely/invalid/_debug') is None + assert framework_info('P/F.framework') is None + assert framework_info('P/F.framework/_debug') is None + assert framework_info('P/F.framework/F') == d('P', 'F.framework/F', 'F') + assert framework_info('P/F.framework/F_debug') == d('P', 'F.framework/F_debug', 'F', suffix='debug') + assert framework_info('P/F.framework/Versions') is None + assert framework_info('P/F.framework/Versions/A') is None + assert framework_info('P/F.framework/Versions/A/F') == d('P', 'F.framework/Versions/A/F', 'F', 'A') + assert framework_info('P/F.framework/Versions/A/F_debug') == d('P', 'F.framework/Versions/A/F_debug', 'F', 'A', 'debug') + +if __name__ == '__main__': + test_framework_info() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/__init__.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,202 @@ +import glob, os, sys, unittest, getopt, time + +use_resources = [] + +class ResourceDenied(Exception): + """Test skipped because it requested a disallowed resource. + + This is raised when a test calls requires() for a resource that + has not be enabled. Resources are defined by test modules. + """ + +def is_resource_enabled(resource): + """Test whether a resource is enabled. + + If the caller's module is __main__ then automatically return True.""" + if sys._getframe().f_back.f_globals.get("__name__") == "__main__": + return True + result = use_resources is not None and \ + (resource in use_resources or "*" in use_resources) + if not result: + _unavail[resource] = None + return result + +_unavail = {} +def requires(resource, msg=None): + """Raise ResourceDenied if the specified resource is not available. + + If the caller's module is __main__ then automatically return True.""" + # see if the caller's module is __main__ - if so, treat as if + # the resource was set + if sys._getframe().f_back.f_globals.get("__name__") == "__main__": + return + if not is_resource_enabled(resource): + if msg is None: + msg = "Use of the `%s' resource not enabled" % resource + raise ResourceDenied(msg) + +def find_package_modules(package, mask): + import fnmatch + if hasattr(package, "__loader__"): + path = package.__name__.replace(".", os.path.sep) + mask = os.path.join(path, mask) + for fnm in package.__loader__._files.iterkeys(): + if fnmatch.fnmatchcase(fnm, mask): + yield os.path.splitext(fnm)[0].replace(os.path.sep, ".") + else: + path = package.__path__[0] + for fnm in os.listdir(path): + if fnmatch.fnmatchcase(fnm, mask): + yield "%s.%s" % (package.__name__, os.path.splitext(fnm)[0]) + +def get_tests(package, mask, verbosity): + """Return a list of skipped test modules, and a list of test cases.""" + tests = [] + skipped = [] + for modname in find_package_modules(package, mask): + try: + mod = __import__(modname, globals(), locals(), ['*']) + except ResourceDenied, detail: + skipped.append(modname) + if verbosity > 1: + print >> sys.stderr, "Skipped %s: %s" % (modname, detail) + continue + except Exception, detail: + print >> sys.stderr, "Warning: could not import %s: %s" % (modname, detail) + continue + for name in dir(mod): + if name.startswith("_"): + continue + o = getattr(mod, name) + if type(o) is type(unittest.TestCase) and issubclass(o, unittest.TestCase): + tests.append(o) + return skipped, tests + +def usage(): + print __doc__ + return 1 + +def test_with_refcounts(runner, verbosity, testcase): + """Run testcase several times, tracking reference counts.""" + import gc + import ctypes + ptc = ctypes._pointer_type_cache.copy() + cfc = ctypes._c_functype_cache.copy() + wfc = ctypes._win_functype_cache.copy() + + # when searching for refcount leaks, we have to manually reset any + # caches that ctypes has. + def cleanup(): + ctypes._pointer_type_cache = ptc.copy() + ctypes._c_functype_cache = cfc.copy() + ctypes._win_functype_cache = wfc.copy() + gc.collect() + + test = unittest.makeSuite(testcase) + for i in range(5): + rc = sys.gettotalrefcount() + runner.run(test) + cleanup() + COUNT = 5 + refcounts = [None] * COUNT + for i in range(COUNT): + rc = sys.gettotalrefcount() + runner.run(test) + cleanup() + refcounts[i] = sys.gettotalrefcount() - rc + if filter(None, refcounts): + print "%s leaks:\n\t" % testcase, refcounts + elif verbosity: + print "%s: ok." % testcase + +class TestRunner(unittest.TextTestRunner): + def run(self, test, skipped): + "Run the given test case or test suite." + # Same as unittest.TextTestRunner.run, except that it reports + # skipped tests. + result = self._makeResult() + startTime = time.time() + test(result) + stopTime = time.time() + timeTaken = stopTime - startTime + result.printErrors() + self.stream.writeln(result.separator2) + run = result.testsRun + if _unavail: #skipped: + requested = _unavail.keys() + requested.sort() + self.stream.writeln("Ran %d test%s in %.3fs (%s module%s skipped)" % + (run, run != 1 and "s" or "", timeTaken, + len(skipped), + len(skipped) != 1 and "s" or "")) + self.stream.writeln("Unavailable resources: %s" % ", ".join(requested)) + else: + self.stream.writeln("Ran %d test%s in %.3fs" % + (run, run != 1 and "s" or "", timeTaken)) + self.stream.writeln() + if not result.wasSuccessful(): + self.stream.write("FAILED (") + failed, errored = map(len, (result.failures, result.errors)) + if failed: + self.stream.write("failures=%d" % failed) + if errored: + if failed: self.stream.write(", ") + self.stream.write("errors=%d" % errored) + self.stream.writeln(")") + else: + self.stream.writeln("OK") + return result + + +def main(*packages): + try: + opts, args = getopt.getopt(sys.argv[1:], "rqvu:") + except getopt.error: + return usage() + + verbosity = 1 + search_leaks = False + for flag, value in opts: + if flag == "-q": + verbosity -= 1 + elif flag == "-v": + verbosity += 1 + elif flag == "-r": + try: + sys.gettotalrefcount + except AttributeError: + print >> sys.stderr, "-r flag requires Python debug build" + return -1 + search_leaks = True + elif flag == "-u": + use_resources.extend(value.split(",")) + + mask = "test_*.py" + if args: + mask = args[0] + + for package in packages: + run_tests(package, mask, verbosity, search_leaks) + + +def run_tests(package, mask, verbosity, search_leaks): + skipped, testcases = get_tests(package, mask, verbosity) + runner = TestRunner(verbosity=verbosity) + + suites = [unittest.makeSuite(o) for o in testcases] + suite = unittest.TestSuite(suites) + result = runner.run(suite, skipped) + + if search_leaks: + # hunt for refcount leaks + runner = BasicTestRunner() + for t in testcases: + test_with_refcounts(runner, verbosity, t) + + return bool(result.errors) + +class BasicTestRunner: + def run(self, test): + result = unittest.TestResult() + test(result) + return result Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/runtests.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/runtests.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,17 @@ +"""Usage: runtests.py [-q] [-r] [-v] [-u resources] [mask] + +Run all tests found in this directory, and print a summary of the results. +Command line flags: + -q quiet mode: don't prnt anything while the tests are running + -r run tests repeatedly, look for refcount leaks + -u + Add resources to the lits of allowed resources. '*' allows all + resources. + -v verbose mode: print the test currently executed + mask mask to select filenames containing testcases, wildcards allowed +""" +import sys +import ctypes.test + +if __name__ == "__main__": + sys.exit(ctypes.test.main(ctypes.test)) Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_anon.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_anon.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,60 @@ +import unittest +from ctypes import * + +class AnonTest(unittest.TestCase): + + def test_anon(self): + class ANON(Union): + _fields_ = [("a", c_int), + ("b", c_int)] + + class Y(Structure): + _fields_ = [("x", c_int), + ("_", ANON), + ("y", c_int)] + _anonymous_ = ["_"] + + self.failUnlessEqual(Y.a.offset, sizeof(c_int)) + self.failUnlessEqual(Y.b.offset, sizeof(c_int)) + + self.failUnlessEqual(ANON.a.offset, 0) + self.failUnlessEqual(ANON.b.offset, 0) + + def test_anon_nonseq(self): + # TypeError: _anonymous_ must be a sequence + self.failUnlessRaises(TypeError, + lambda: type(Structure)("Name", + (Structure,), + {"_fields_": [], "_anonymous_": 42})) + + def test_anon_nonmember(self): + # AttributeError: type object 'Name' has no attribute 'x' + self.failUnlessRaises(AttributeError, + lambda: type(Structure)("Name", + (Structure,), + {"_fields_": [], + "_anonymous_": ["x"]})) + + def test_nested(self): + class ANON_S(Structure): + _fields_ = [("a", c_int)] + + class ANON_U(Union): + _fields_ = [("_", ANON_S), + ("b", c_int)] + _anonymous_ = ["_"] + + class Y(Structure): + _fields_ = [("x", c_int), + ("_", ANON_U), + ("y", c_int)] + _anonymous_ = ["_"] + + self.failUnlessEqual(Y.x.offset, 0) + self.failUnlessEqual(Y.a.offset, sizeof(c_int)) + self.failUnlessEqual(Y.b.offset, sizeof(c_int)) + self.failUnlessEqual(Y._.offset, sizeof(c_int)) + self.failUnlessEqual(Y.y.offset, sizeof(c_int) * 2) + +if __name__ == "__main__": + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_array_in_pointer.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_array_in_pointer.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,64 @@ +import unittest +from ctypes import * +from binascii import hexlify +import re + +def dump(obj): + # helper function to dump memory contents in hex, with a hyphen + # between the bytes. + h = hexlify(buffer(obj)) + return re.sub(r"(..)", r"\1-", h)[:-1] + + +class Value(Structure): + _fields_ = [("val", c_byte)] + +class Container(Structure): + _fields_ = [("pvalues", POINTER(Value))] + +class Test(unittest.TestCase): + def test(self): + # create an array of 4 values + val_array = (Value * 4)() + + # create a container, which holds a pointer to the pvalues array. + c = Container() + c.pvalues = val_array + + # memory contains 4 NUL bytes now, that's correct + self.failUnlessEqual("00-00-00-00", dump(val_array)) + + # set the values of the array through the pointer: + for i in range(4): + c.pvalues[i].val = i + 1 + + values = [c.pvalues[i].val for i in range(4)] + + # These are the expected results: here s the bug! + self.failUnlessEqual( + (values, dump(val_array)), + ([1, 2, 3, 4], "01-02-03-04") + ) + + def test_2(self): + + val_array = (Value * 4)() + + # memory contains 4 NUL bytes now, that's correct + self.failUnlessEqual("00-00-00-00", dump(val_array)) + + ptr = cast(val_array, POINTER(Value)) + # set the values of the array through the pointer: + for i in range(4): + ptr[i].val = i + 1 + + values = [ptr[i].val for i in range(4)] + + # These are the expected results: here s the bug! + self.failUnlessEqual( + (values, dump(val_array)), + ([1, 2, 3, 4], "01-02-03-04") + ) + +if __name__ == "__main__": + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_arrays.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_arrays.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,112 @@ +import unittest +from ctypes import * + +formats = "bBhHiIlLqQfd" + +formats = c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, \ + c_long, c_ulonglong, c_float, c_double + +class ArrayTestCase(unittest.TestCase): + def test_simple(self): + # create classes holding simple numeric types, and check + # various properties. + + init = range(15, 25) + + for fmt in formats: + alen = len(init) + int_array = ARRAY(fmt, alen) + + ia = int_array(*init) + # length of instance ok? + self.failUnlessEqual(len(ia), alen) + + # slot values ok? + values = [ia[i] for i in range(len(init))] + self.failUnlessEqual(values, init) + + # change the items + from operator import setitem + new_values = range(42, 42+alen) + [setitem(ia, n, new_values[n]) for n in range(alen)] + values = [ia[i] for i in range(len(init))] + self.failUnlessEqual(values, new_values) + + # are the items initialized to 0? + ia = int_array() + values = [ia[i] for i in range(len(init))] + self.failUnlessEqual(values, [0] * len(init)) + + # Too many in itializers should be caught + self.assertRaises(IndexError, int_array, *range(alen*2)) + + CharArray = ARRAY(c_char, 3) + + ca = CharArray("a", "b", "c") + + # Should this work? It doesn't: + # CharArray("abc") + self.assertRaises(TypeError, CharArray, "abc") + + self.failUnlessEqual(ca[0], "a") + self.failUnlessEqual(ca[1], "b") + self.failUnlessEqual(ca[2], "c") + self.failUnlessEqual(ca[-3], "a") + self.failUnlessEqual(ca[-2], "b") + self.failUnlessEqual(ca[-1], "c") + + self.failUnlessEqual(len(ca), 3) + + # slicing is now supported, but not extended slicing (3-argument)! + from operator import getslice, delitem + self.assertRaises(TypeError, getslice, ca, 0, 1, -1) + + # cannot delete items + self.assertRaises(TypeError, delitem, ca, 0) + + def test_numeric_arrays(self): + + alen = 5 + + numarray = ARRAY(c_int, alen) + + na = numarray() + values = [na[i] for i in range(alen)] + self.failUnlessEqual(values, [0] * alen) + + na = numarray(*[c_int()] * alen) + values = [na[i] for i in range(alen)] + self.failUnlessEqual(values, [0]*alen) + + na = numarray(1, 2, 3, 4, 5) + values = [i for i in na] + self.failUnlessEqual(values, [1, 2, 3, 4, 5]) + + na = numarray(*map(c_int, (1, 2, 3, 4, 5))) + values = [i for i in na] + self.failUnlessEqual(values, [1, 2, 3, 4, 5]) + + def test_classcache(self): + self.failUnless(not ARRAY(c_int, 3) is ARRAY(c_int, 4)) + self.failUnless(ARRAY(c_int, 3) is ARRAY(c_int, 3)) + + def test_from_address(self): + # Failed with 0.9.8, reported by JUrner + p = create_string_buffer("foo") + sz = (c_char * 3).from_address(addressof(p)) + self.failUnlessEqual(sz[:], "foo") + self.failUnlessEqual(sz.value, "foo") + + try: + create_unicode_buffer + except NameError: + pass + else: + def test_from_addressW(self): + p = create_unicode_buffer("foo") + sz = (c_wchar * 3).from_address(addressof(p)) + self.failUnlessEqual(sz[:], "foo") + self.failUnlessEqual(sz.value, "foo") + +if __name__ == '__main__': + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_as_parameter.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_as_parameter.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,215 @@ +import unittest +from ctypes import * +import _ctypes_test + +dll = CDLL(_ctypes_test.__file__) + +try: + CALLBACK_FUNCTYPE = WINFUNCTYPE +except NameError: + # fake to enable this test on Linux + CALLBACK_FUNCTYPE = CFUNCTYPE + +class POINT(Structure): + _fields_ = [("x", c_int), ("y", c_int)] + +class BasicWrapTestCase(unittest.TestCase): + def wrap(self, param): + return param + + def test_wchar_parm(self): + try: + c_wchar + except NameError: + return + f = dll._testfunc_i_bhilfd + f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double] + result = f(self.wrap(1), self.wrap(u"x"), self.wrap(3), self.wrap(4), self.wrap(5.0), self.wrap(6.0)) + self.failUnlessEqual(result, 139) + self.failUnless(type(result), int) + + def test_pointers(self): + f = dll._testfunc_p_p + f.restype = POINTER(c_int) + f.argtypes = [POINTER(c_int)] + + # This only works if the value c_int(42) passed to the + # function is still alive while the pointer (the result) is + # used. + + v = c_int(42) + + self.failUnlessEqual(pointer(v).contents.value, 42) + result = f(self.wrap(pointer(v))) + self.failUnlessEqual(type(result), POINTER(c_int)) + self.failUnlessEqual(result.contents.value, 42) + + # This on works... + result = f(self.wrap(pointer(v))) + self.failUnlessEqual(result.contents.value, v.value) + + p = pointer(c_int(99)) + result = f(self.wrap(p)) + self.failUnlessEqual(result.contents.value, 99) + + def test_shorts(self): + f = dll._testfunc_callback_i_if + + args = [] + expected = [262144, 131072, 65536, 32768, 16384, 8192, 4096, 2048, + 1024, 512, 256, 128, 64, 32, 16, 8, 4, 2, 1] + + def callback(v): + args.append(v) + return v + + CallBack = CFUNCTYPE(c_int, c_int) + + cb = CallBack(callback) + f(self.wrap(2**18), self.wrap(cb)) + self.failUnlessEqual(args, expected) + + ################################################################ + + def test_callbacks(self): + f = dll._testfunc_callback_i_if + f.restype = c_int + + MyCallback = CFUNCTYPE(c_int, c_int) + + def callback(value): + #print "called back with", value + return value + + cb = MyCallback(callback) + + result = f(self.wrap(-10), self.wrap(cb)) + self.failUnlessEqual(result, -18) + + # test with prototype + f.argtypes = [c_int, MyCallback] + cb = MyCallback(callback) + + result = f(self.wrap(-10), self.wrap(cb)) + self.failUnlessEqual(result, -18) + + result = f(self.wrap(-10), self.wrap(cb)) + self.failUnlessEqual(result, -18) + + AnotherCallback = CALLBACK_FUNCTYPE(c_int, c_int, c_int, c_int, c_int) + + # check that the prototype works: we call f with wrong + # argument types + cb = AnotherCallback(callback) + self.assertRaises(ArgumentError, f, self.wrap(-10), self.wrap(cb)) + + def test_callbacks_2(self): + # Can also use simple datatypes as argument type specifiers + # for the callback function. + # In this case the call receives an instance of that type + f = dll._testfunc_callback_i_if + f.restype = c_int + + MyCallback = CFUNCTYPE(c_int, c_int) + + f.argtypes = [c_int, MyCallback] + + def callback(value): + #print "called back with", value + self.failUnlessEqual(type(value), int) + return value + + cb = MyCallback(callback) + result = f(self.wrap(-10), self.wrap(cb)) + self.failUnlessEqual(result, -18) + + def test_longlong_callbacks(self): + + f = dll._testfunc_callback_q_qf + f.restype = c_longlong + + MyCallback = CFUNCTYPE(c_longlong, c_longlong) + + f.argtypes = [c_longlong, MyCallback] + + def callback(value): + self.failUnless(isinstance(value, (int, long))) + return value & 0x7FFFFFFF + + cb = MyCallback(callback) + + self.failUnlessEqual(13577625587, int(f(self.wrap(1000000000000), self.wrap(cb)))) + + def test_byval(self): + # without prototype + ptin = POINT(1, 2) + ptout = POINT() + # EXPORT int _testfunc_byval(point in, point *pout) + result = dll._testfunc_byval(ptin, byref(ptout)) + got = result, ptout.x, ptout.y + expected = 3, 1, 2 + self.failUnlessEqual(got, expected) + + # with prototype + ptin = POINT(101, 102) + ptout = POINT() + dll._testfunc_byval.argtypes = (POINT, POINTER(POINT)) + dll._testfunc_byval.restype = c_int + result = dll._testfunc_byval(self.wrap(ptin), byref(ptout)) + got = result, ptout.x, ptout.y + expected = 203, 101, 102 + self.failUnlessEqual(got, expected) + + def test_struct_return_2H(self): + class S2H(Structure): + _fields_ = [("x", c_short), + ("y", c_short)] + dll.ret_2h_func.restype = S2H + dll.ret_2h_func.argtypes = [S2H] + inp = S2H(99, 88) + s2h = dll.ret_2h_func(self.wrap(inp)) + self.failUnlessEqual((s2h.x, s2h.y), (99*2, 88*3)) + + def test_struct_return_8H(self): + class S8I(Structure): + _fields_ = [("a", c_int), + ("b", c_int), + ("c", c_int), + ("d", c_int), + ("e", c_int), + ("f", c_int), + ("g", c_int), + ("h", c_int)] + dll.ret_8i_func.restype = S8I + dll.ret_8i_func.argtypes = [S8I] + inp = S8I(9, 8, 7, 6, 5, 4, 3, 2) + s8i = dll.ret_8i_func(self.wrap(inp)) + self.failUnlessEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h), + (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) + +#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +class AsParamWrapper(object): + def __init__(self, param): + self._as_parameter_ = param + +class AsParamWrapperTestCase(BasicWrapTestCase): + wrap = AsParamWrapper + +#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +class AsParamPropertyWrapper(object): + def __init__(self, param): + self._param = param + + def getParameter(self): + return self._param + _as_parameter_ = property(getParameter) + +class AsParamPropertyWrapperTestCase(BasicWrapTestCase): + wrap = AsParamPropertyWrapper + +#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +if __name__ == '__main__': + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_bitfields.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_bitfields.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,228 @@ +from ctypes import * +import unittest +import os + +import ctypes +import _ctypes_test + +class BITS(Structure): + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9), + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +func = CDLL(_ctypes_test.__file__).unpack_bitfields +func.argtypes = POINTER(BITS), c_char + +##for n in "ABCDEFGHIMNOPQRS": +## print n, hex(getattr(BITS, n).size), getattr(BITS, n).offset + +class C_Test(unittest.TestCase): + + def test_ints(self): + for i in range(512): + for name in "ABCDEFGHI": + b = BITS() + setattr(b, name, i) + self.failUnlessEqual((name, i, getattr(b, name)), (name, i, func(byref(b), name))) + + def test_shorts(self): + for i in range(256): + for name in "MNOPQRS": + b = BITS() + setattr(b, name, i) + self.failUnlessEqual((name, i, getattr(b, name)), (name, i, func(byref(b), name))) + +signed_int_types = (c_byte, c_short, c_int, c_long, c_longlong) +unsigned_int_types = (c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong) +int_types = unsigned_int_types + signed_int_types + +class BitFieldTest(unittest.TestCase): + + def test_longlong(self): + class X(Structure): + _fields_ = [("a", c_longlong, 1), + ("b", c_longlong, 62), + ("c", c_longlong, 1)] + + self.failUnlessEqual(sizeof(X), sizeof(c_longlong)) + x = X() + x.a, x.b, x.c = -1, 7, -1 + self.failUnlessEqual((x.a, x.b, x.c), (-1, 7, -1)) + + def test_ulonglong(self): + class X(Structure): + _fields_ = [("a", c_ulonglong, 1), + ("b", c_ulonglong, 62), + ("c", c_ulonglong, 1)] + + self.failUnlessEqual(sizeof(X), sizeof(c_longlong)) + x = X() + self.failUnlessEqual((x.a, x.b, x.c), (0, 0, 0)) + x.a, x.b, x.c = 7, 7, 7 + self.failUnlessEqual((x.a, x.b, x.c), (1, 7, 1)) + + def test_signed(self): + for c_typ in signed_int_types: + class X(Structure): + _fields_ = [("dummy", c_typ), + ("a", c_typ, 3), + ("b", c_typ, 3), + ("c", c_typ, 1)] + self.failUnlessEqual(sizeof(X), sizeof(c_typ)*2) + + x = X() + self.failUnlessEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) + x.a = -1 + self.failUnlessEqual((c_typ, x.a, x.b, x.c), (c_typ, -1, 0, 0)) + x.a, x.b = 0, -1 + self.failUnlessEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, -1, 0)) + + + def test_unsigned(self): + for c_typ in unsigned_int_types: + class X(Structure): + _fields_ = [("a", c_typ, 3), + ("b", c_typ, 3), + ("c", c_typ, 1)] + self.failUnlessEqual(sizeof(X), sizeof(c_typ)) + + x = X() + self.failUnlessEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) + x.a = -1 + self.failUnlessEqual((c_typ, x.a, x.b, x.c), (c_typ, 7, 0, 0)) + x.a, x.b = 0, -1 + self.failUnlessEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) + + + def fail_fields(self, *fields): + return self.get_except(type(Structure), "X", (), + {"_fields_": fields}) + + def test_nonint_types(self): + # bit fields are not allowed on non-integer types. + result = self.fail_fields(("a", c_char_p, 1)) + self.failUnlessEqual(result, (TypeError, 'bit fields not allowed for type c_char_p')) + + result = self.fail_fields(("a", c_void_p, 1)) + self.failUnlessEqual(result, (TypeError, 'bit fields not allowed for type c_void_p')) + + if c_int != c_long: + result = self.fail_fields(("a", POINTER(c_int), 1)) + self.failUnlessEqual(result, (TypeError, 'bit fields not allowed for type LP_c_int')) + + result = self.fail_fields(("a", c_char, 1)) + self.failUnlessEqual(result, (TypeError, 'bit fields not allowed for type c_char')) + + try: + c_wchar + except NameError: + pass + else: + result = self.fail_fields(("a", c_wchar, 1)) + self.failUnlessEqual(result, (TypeError, 'bit fields not allowed for type c_wchar')) + + class Dummy(Structure): + _fields_ = [] + + result = self.fail_fields(("a", Dummy, 1)) + self.failUnlessEqual(result, (TypeError, 'bit fields not allowed for type Dummy')) + + def test_single_bitfield_size(self): + for c_typ in int_types: + result = self.fail_fields(("a", c_typ, -1)) + self.failUnlessEqual(result, (ValueError, 'number of bits invalid for bit field')) + + result = self.fail_fields(("a", c_typ, 0)) + self.failUnlessEqual(result, (ValueError, 'number of bits invalid for bit field')) + + class X(Structure): + _fields_ = [("a", c_typ, 1)] + self.failUnlessEqual(sizeof(X), sizeof(c_typ)) + + class X(Structure): + _fields_ = [("a", c_typ, sizeof(c_typ)*8)] + self.failUnlessEqual(sizeof(X), sizeof(c_typ)) + + result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) + self.failUnlessEqual(result, (ValueError, 'number of bits invalid for bit field')) + + def test_multi_bitfields_size(self): + class X(Structure): + _fields_ = [("a", c_short, 1), + ("b", c_short, 14), + ("c", c_short, 1)] + self.failUnlessEqual(sizeof(X), sizeof(c_short)) + + class X(Structure): + _fields_ = [("a", c_short, 1), + ("a1", c_short), + ("b", c_short, 14), + ("c", c_short, 1)] + self.failUnlessEqual(sizeof(X), sizeof(c_short)*3) + self.failUnlessEqual(X.a.offset, 0) + self.failUnlessEqual(X.a1.offset, sizeof(c_short)) + self.failUnlessEqual(X.b.offset, sizeof(c_short)*2) + self.failUnlessEqual(X.c.offset, sizeof(c_short)*2) + + class X(Structure): + _fields_ = [("a", c_short, 3), + ("b", c_short, 14), + ("c", c_short, 14)] + self.failUnlessEqual(sizeof(X), sizeof(c_short)*3) + self.failUnlessEqual(X.a.offset, sizeof(c_short)*0) + self.failUnlessEqual(X.b.offset, sizeof(c_short)*1) + self.failUnlessEqual(X.c.offset, sizeof(c_short)*2) + + + def get_except(self, func, *args, **kw): + try: + func(*args, **kw) + except Exception, detail: + return detail.__class__, str(detail) + + def test_mixed_1(self): + class X(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_int, 4)] + if os.name in ("nt", "ce"): + self.failUnlessEqual(sizeof(X), sizeof(c_int)*2) + else: + self.failUnlessEqual(sizeof(X), sizeof(c_int)) + + def test_mixed_2(self): + class X(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_int, 32)] + self.failUnlessEqual(sizeof(X), sizeof(c_int)*2) + + def test_mixed_3(self): + class X(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_ubyte, 4)] + self.failUnlessEqual(sizeof(X), sizeof(c_byte)) + + def test_anon_bitfields(self): + # anonymous bit-fields gave a strange error message + class X(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_ubyte, 4)] + class Y(Structure): + _anonymous_ = ["_"] + _fields_ = [("_", X)] + +if __name__ == "__main__": + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_buffers.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_buffers.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,54 @@ +from ctypes import * +import unittest + +class StringBufferTestCase(unittest.TestCase): + + def test_buffer(self): + b = create_string_buffer(32) + self.failUnlessEqual(len(b), 32) + self.failUnlessEqual(sizeof(b), 32 * sizeof(c_char)) + self.failUnless(type(b[0]) is str) + + b = create_string_buffer("abc") + self.failUnlessEqual(len(b), 4) # trailing nul char + self.failUnlessEqual(sizeof(b), 4 * sizeof(c_char)) + self.failUnless(type(b[0]) is str) + self.failUnlessEqual(b[0], "a") + self.failUnlessEqual(b[:], "abc\0") + + def test_string_conversion(self): + b = create_string_buffer(u"abc") + self.failUnlessEqual(len(b), 4) # trailing nul char + self.failUnlessEqual(sizeof(b), 4 * sizeof(c_char)) + self.failUnless(type(b[0]) is str) + self.failUnlessEqual(b[0], "a") + self.failUnlessEqual(b[:], "abc\0") + + try: + c_wchar + except NameError: + pass + else: + def test_unicode_buffer(self): + b = create_unicode_buffer(32) + self.failUnlessEqual(len(b), 32) + self.failUnlessEqual(sizeof(b), 32 * sizeof(c_wchar)) + self.failUnless(type(b[0]) is unicode) + + b = create_unicode_buffer(u"abc") + self.failUnlessEqual(len(b), 4) # trailing nul char + self.failUnlessEqual(sizeof(b), 4 * sizeof(c_wchar)) + self.failUnless(type(b[0]) is unicode) + self.failUnlessEqual(b[0], u"a") + self.failUnlessEqual(b[:], "abc\0") + + def test_unicode_conversion(self): + b = create_unicode_buffer("abc") + self.failUnlessEqual(len(b), 4) # trailing nul char + self.failUnlessEqual(sizeof(b), 4 * sizeof(c_wchar)) + self.failUnless(type(b[0]) is unicode) + self.failUnlessEqual(b[0], u"a") + self.failUnlessEqual(b[:], "abc\0") + +if __name__ == "__main__": + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_byteswap.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_byteswap.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,280 @@ +import sys, unittest, struct, math +from binascii import hexlify + +from ctypes import * + +def bin(s): + return hexlify(buffer(s)).upper() + +# Each *simple* type that supports different byte orders has an +# __ctype_be__ attribute that specifies the same type in BIG ENDIAN +# byte order, and a __ctype_le__ attribute that is the same type in +# LITTLE ENDIAN byte order. +# +# For Structures and Unions, these types are created on demand. + +class Test(unittest.TestCase): + def X_test(self): + print >> sys.stderr, sys.byteorder + for i in range(32): + bits = BITS() + setattr(bits, "i%s" % i, 1) + dump(bits) + + def test_endian_short(self): + if sys.byteorder == "little": + self.failUnless(c_short.__ctype_le__ is c_short) + self.failUnless(c_short.__ctype_be__.__ctype_le__ is c_short) + else: + self.failUnless(c_short.__ctype_be__ is c_short) + self.failUnless(c_short.__ctype_le__.__ctype_be__ is c_short) + s = c_short.__ctype_be__(0x1234) + self.failUnlessEqual(bin(struct.pack(">h", 0x1234)), "1234") + self.failUnlessEqual(bin(s), "1234") + self.failUnlessEqual(s.value, 0x1234) + + s = c_short.__ctype_le__(0x1234) + self.failUnlessEqual(bin(struct.pack("h", 0x1234)), "1234") + self.failUnlessEqual(bin(s), "1234") + self.failUnlessEqual(s.value, 0x1234) + + s = c_ushort.__ctype_le__(0x1234) + self.failUnlessEqual(bin(struct.pack("i", 0x12345678)), "12345678") + self.failUnlessEqual(bin(s), "12345678") + self.failUnlessEqual(s.value, 0x12345678) + + s = c_int.__ctype_le__(0x12345678) + self.failUnlessEqual(bin(struct.pack("I", 0x12345678)), "12345678") + self.failUnlessEqual(bin(s), "12345678") + self.failUnlessEqual(s.value, 0x12345678) + + s = c_uint.__ctype_le__(0x12345678) + self.failUnlessEqual(bin(struct.pack("q", 0x1234567890ABCDEF)), "1234567890ABCDEF") + self.failUnlessEqual(bin(s), "1234567890ABCDEF") + self.failUnlessEqual(s.value, 0x1234567890ABCDEF) + + s = c_longlong.__ctype_le__(0x1234567890ABCDEF) + self.failUnlessEqual(bin(struct.pack("Q", 0x1234567890ABCDEF)), "1234567890ABCDEF") + self.failUnlessEqual(bin(s), "1234567890ABCDEF") + self.failUnlessEqual(s.value, 0x1234567890ABCDEF) + + s = c_ulonglong.__ctype_le__(0x1234567890ABCDEF) + self.failUnlessEqual(bin(struct.pack("f", math.pi)), bin(s)) + + def test_endian_double(self): + if sys.byteorder == "little": + self.failUnless(c_double.__ctype_le__ is c_double) + self.failUnless(c_double.__ctype_be__.__ctype_le__ is c_double) + else: + self.failUnless(c_double.__ctype_be__ is c_double) + self.failUnless(c_double.__ctype_le__.__ctype_be__ is c_double) + s = c_double(math.pi) + self.failUnlessEqual(s.value, math.pi) + self.failUnlessEqual(bin(struct.pack("d", math.pi)), bin(s)) + s = c_double.__ctype_le__(math.pi) + self.failUnlessEqual(s.value, math.pi) + self.failUnlessEqual(bin(struct.pack("d", math.pi)), bin(s)) + + def test_endian_other(self): + self.failUnless(c_byte.__ctype_le__ is c_byte) + self.failUnless(c_byte.__ctype_be__ is c_byte) + + self.failUnless(c_ubyte.__ctype_le__ is c_ubyte) + self.failUnless(c_ubyte.__ctype_be__ is c_ubyte) + + self.failUnless(c_char.__ctype_le__ is c_char) + self.failUnless(c_char.__ctype_be__ is c_char) + + def test_struct_fields_1(self): + if sys.byteorder == "little": + base = BigEndianStructure + else: + base = LittleEndianStructure + + class T(base): + pass + _fields_ = [("a", c_ubyte), + ("b", c_byte), + ("c", c_short), + ("d", c_ushort), + ("e", c_int), + ("f", c_uint), + ("g", c_long), + ("h", c_ulong), + ("i", c_longlong), + ("k", c_ulonglong), + ("l", c_float), + ("m", c_double), + ("n", c_char), + + ("b1", c_byte, 3), + ("b2", c_byte, 3), + ("b3", c_byte, 2), + ("a", c_int * 3 * 3 * 3)] + T._fields_ = _fields_ + + # these fields do not support different byte order: + for typ in c_wchar, c_void_p, POINTER(c_int): + _fields_.append(("x", typ)) + class T(base): + pass + self.assertRaises(TypeError, setattr, T, "_fields_", [("x", typ)]) + + def test_struct_struct(self): + # Nested structures with different byte order not (yet) supported + if sys.byteorder == "little": + base = BigEndianStructure + else: + base = LittleEndianStructure + + class T(Structure): + _fields_ = [("a", c_int), + ("b", c_int)] + class S(base): + pass + self.assertRaises(TypeError, setattr, S, "_fields_", [("s", T)]) + + def test_struct_fields_2(self): + # standard packing in struct uses no alignment. + # So, we have to align using pad bytes. + # + # Unaligned accesses will crash Python (on those platforms that + # don't allow it, like sparc solaris). + if sys.byteorder == "little": + base = BigEndianStructure + fmt = ">bxhid" + else: + base = LittleEndianStructure + fmt = " float -> double + import math + self.check_type(c_float, math.e) + self.check_type(c_float, -math.e) + + def test_double(self): + self.check_type(c_double, 3.14) + self.check_type(c_double, -3.14) + + def test_char(self): + self.check_type(c_char, "x") + self.check_type(c_char, "a") + + # disabled: would now (correctly) raise a RuntimeWarning about + # a memory leak. A callback function cannot return a non-integral + # C type without causing a memory leak. +## def test_char_p(self): +## self.check_type(c_char_p, "abc") +## self.check_type(c_char_p, "def") + + def test_pyobject(self): + o = () + from sys import getrefcount as grc + for o in (), [], object(): + initial = grc(o) + # This call leaks a reference to 'o'... + self.check_type(py_object, o) + before = grc(o) + # ...but this call doesn't leak any more. Where is the refcount? + self.check_type(py_object, o) + after = grc(o) + self.failUnlessEqual((after, o), (before, o)) + + def test_unsupported_restype_1(self): + # Only "fundamental" result types are supported for callback + # functions, the type must have a non-NULL stgdict->setfunc. + # POINTER(c_double), for example, is not supported. + + prototype = self.functype.im_func(POINTER(c_double)) + # The type is checked when the prototype is called + self.assertRaises(TypeError, prototype, lambda: None) + + def test_unsupported_restype_2(self): + prototype = self.functype.im_func(object) + self.assertRaises(TypeError, prototype, lambda: None) + +try: + WINFUNCTYPE +except NameError: + pass +else: + class StdcallCallbacks(Callbacks): + functype = WINFUNCTYPE + +################################################################ + +class SampleCallbacksTestCase(unittest.TestCase): + + def test_integrate(self): + # Derived from some then non-working code, posted by David Foster + dll = CDLL(_ctypes_test.__file__) + + # The function prototype called by 'integrate': double func(double); + CALLBACK = CFUNCTYPE(c_double, c_double) + + # The integrate function itself, exposed from the _ctypes_test dll + integrate = dll.integrate + integrate.argtypes = (c_double, c_double, CALLBACK, c_long) + integrate.restype = c_double + + def func(x): + return x**2 + + result = integrate(0.0, 1.0, CALLBACK(func), 10) + diff = abs(result - 1./3.) + + self.failUnless(diff < 0.01, "%s not less than 0.01" % diff) + +################################################################ + +if __name__ == '__main__': + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_cast.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_cast.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,77 @@ +from ctypes import * +import unittest +import sys + +class Test(unittest.TestCase): + + def test_array2pointer(self): + array = (c_int * 3)(42, 17, 2) + + # casting an array to a pointer works. + ptr = cast(array, POINTER(c_int)) + self.failUnlessEqual([ptr[i] for i in range(3)], [42, 17, 2]) + + if 2*sizeof(c_short) == sizeof(c_int): + ptr = cast(array, POINTER(c_short)) + if sys.byteorder == "little": + self.failUnlessEqual([ptr[i] for i in range(6)], + [42, 0, 17, 0, 2, 0]) + else: + self.failUnlessEqual([ptr[i] for i in range(6)], + [0, 42, 0, 17, 0, 2]) + + def test_address2pointer(self): + array = (c_int * 3)(42, 17, 2) + + address = addressof(array) + ptr = cast(c_void_p(address), POINTER(c_int)) + self.failUnlessEqual([ptr[i] for i in range(3)], [42, 17, 2]) + + ptr = cast(address, POINTER(c_int)) + self.failUnlessEqual([ptr[i] for i in range(3)], [42, 17, 2]) + + def test_p2a_objects(self): + array = (c_char_p * 5)() + self.failUnlessEqual(array._objects, None) + array[0] = "foo bar" + self.failUnlessEqual(array._objects, {'0': "foo bar"}) + + p = cast(array, POINTER(c_char_p)) + # array and p share a common _objects attribute + self.failUnless(p._objects is array._objects) + self.failUnlessEqual(array._objects, {'0': "foo bar", id(array): array}) + p[0] = "spam spam" + self.failUnlessEqual(p._objects, {'0': "spam spam", id(array): array}) + self.failUnless(array._objects is p._objects) + p[1] = "foo bar" + self.failUnlessEqual(p._objects, {'1': 'foo bar', '0': "spam spam", id(array): array}) + self.failUnless(array._objects is p._objects) + + def test_other(self): + p = cast((c_int * 4)(1, 2, 3, 4), POINTER(c_int)) + self.failUnlessEqual(p[:4], [1,2, 3, 4]) + c_int() + self.failUnlessEqual(p[:4], [1, 2, 3, 4]) + p[2] = 96 + self.failUnlessEqual(p[:4], [1, 2, 96, 4]) + c_int() + self.failUnlessEqual(p[:4], [1, 2, 96, 4]) + + def test_char_p(self): + # This didn't work: bad argument to internal function + s = c_char_p("hiho") + self.failUnlessEqual(cast(cast(s, c_void_p), c_char_p).value, + "hiho") + + try: + c_wchar_p + except NameError: + pass + else: + def test_wchar_p(self): + s = c_wchar_p("hiho") + self.failUnlessEqual(cast(cast(s, c_void_p), c_wchar_p).value, + "hiho") + +if __name__ == "__main__": + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_cfuncs.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_cfuncs.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,199 @@ +# A lot of failures in these tests on Mac OS X. +# Byte order related? + +import unittest +from ctypes import * + +import _ctypes_test + +class CFunctions(unittest.TestCase): + _dll = CDLL(_ctypes_test.__file__) + + def S(self): + return c_longlong.in_dll(self._dll, "last_tf_arg_s").value + def U(self): + return c_ulonglong.in_dll(self._dll, "last_tf_arg_u").value + + def test_byte(self): + self._dll.tf_b.restype = c_byte + self._dll.tf_b.argtypes = (c_byte,) + self.failUnlessEqual(self._dll.tf_b(-126), -42) + self.failUnlessEqual(self.S(), -126) + + def test_byte_plus(self): + self._dll.tf_bb.restype = c_byte + self._dll.tf_bb.argtypes = (c_byte, c_byte) + self.failUnlessEqual(self._dll.tf_bb(0, -126), -42) + self.failUnlessEqual(self.S(), -126) + + def test_ubyte(self): + self._dll.tf_B.restype = c_ubyte + self._dll.tf_B.argtypes = (c_ubyte,) + self.failUnlessEqual(self._dll.tf_B(255), 85) + self.failUnlessEqual(self.U(), 255) + + def test_ubyte_plus(self): + self._dll.tf_bB.restype = c_ubyte + self._dll.tf_bB.argtypes = (c_byte, c_ubyte) + self.failUnlessEqual(self._dll.tf_bB(0, 255), 85) + self.failUnlessEqual(self.U(), 255) + + def test_short(self): + self._dll.tf_h.restype = c_short + self._dll.tf_h.argtypes = (c_short,) + self.failUnlessEqual(self._dll.tf_h(-32766), -10922) + self.failUnlessEqual(self.S(), -32766) + + def test_short_plus(self): + self._dll.tf_bh.restype = c_short + self._dll.tf_bh.argtypes = (c_byte, c_short) + self.failUnlessEqual(self._dll.tf_bh(0, -32766), -10922) + self.failUnlessEqual(self.S(), -32766) + + def test_ushort(self): + self._dll.tf_H.restype = c_ushort + self._dll.tf_H.argtypes = (c_ushort,) + self.failUnlessEqual(self._dll.tf_H(65535), 21845) + self.failUnlessEqual(self.U(), 65535) + + def test_ushort_plus(self): + self._dll.tf_bH.restype = c_ushort + self._dll.tf_bH.argtypes = (c_byte, c_ushort) + self.failUnlessEqual(self._dll.tf_bH(0, 65535), 21845) + self.failUnlessEqual(self.U(), 65535) + + def test_int(self): + self._dll.tf_i.restype = c_int + self._dll.tf_i.argtypes = (c_int,) + self.failUnlessEqual(self._dll.tf_i(-2147483646), -715827882) + self.failUnlessEqual(self.S(), -2147483646) + + def test_int_plus(self): + self._dll.tf_bi.restype = c_int + self._dll.tf_bi.argtypes = (c_byte, c_int) + self.failUnlessEqual(self._dll.tf_bi(0, -2147483646), -715827882) + self.failUnlessEqual(self.S(), -2147483646) + + def test_uint(self): + self._dll.tf_I.restype = c_uint + self._dll.tf_I.argtypes = (c_uint,) + self.failUnlessEqual(self._dll.tf_I(4294967295), 1431655765) + self.failUnlessEqual(self.U(), 4294967295) + + def test_uint_plus(self): + self._dll.tf_bI.restype = c_uint + self._dll.tf_bI.argtypes = (c_byte, c_uint) + self.failUnlessEqual(self._dll.tf_bI(0, 4294967295), 1431655765) + self.failUnlessEqual(self.U(), 4294967295) + + def test_long(self): + self._dll.tf_l.restype = c_long + self._dll.tf_l.argtypes = (c_long,) + self.failUnlessEqual(self._dll.tf_l(-2147483646), -715827882) + self.failUnlessEqual(self.S(), -2147483646) + + def test_long_plus(self): + self._dll.tf_bl.restype = c_long + self._dll.tf_bl.argtypes = (c_byte, c_long) + self.failUnlessEqual(self._dll.tf_bl(0, -2147483646), -715827882) + self.failUnlessEqual(self.S(), -2147483646) + + def test_ulong(self): + self._dll.tf_L.restype = c_ulong + self._dll.tf_L.argtypes = (c_ulong,) + self.failUnlessEqual(self._dll.tf_L(4294967295), 1431655765) + self.failUnlessEqual(self.U(), 4294967295) + + def test_ulong_plus(self): + self._dll.tf_bL.restype = c_ulong + self._dll.tf_bL.argtypes = (c_char, c_ulong) + self.failUnlessEqual(self._dll.tf_bL(' ', 4294967295), 1431655765) + self.failUnlessEqual(self.U(), 4294967295) + + def test_longlong(self): + self._dll.tf_q.restype = c_longlong + self._dll.tf_q.argtypes = (c_longlong, ) + self.failUnlessEqual(self._dll.tf_q(-9223372036854775806), -3074457345618258602) + self.failUnlessEqual(self.S(), -9223372036854775806) + + def test_longlong_plus(self): + self._dll.tf_bq.restype = c_longlong + self._dll.tf_bq.argtypes = (c_byte, c_longlong) + self.failUnlessEqual(self._dll.tf_bq(0, -9223372036854775806), -3074457345618258602) + self.failUnlessEqual(self.S(), -9223372036854775806) + + def test_ulonglong(self): + self._dll.tf_Q.restype = c_ulonglong + self._dll.tf_Q.argtypes = (c_ulonglong, ) + self.failUnlessEqual(self._dll.tf_Q(18446744073709551615), 6148914691236517205) + self.failUnlessEqual(self.U(), 18446744073709551615) + + def test_ulonglong_plus(self): + self._dll.tf_bQ.restype = c_ulonglong + self._dll.tf_bQ.argtypes = (c_byte, c_ulonglong) + self.failUnlessEqual(self._dll.tf_bQ(0, 18446744073709551615), 6148914691236517205) + self.failUnlessEqual(self.U(), 18446744073709551615) + + def test_float(self): + self._dll.tf_f.restype = c_float + self._dll.tf_f.argtypes = (c_float,) + self.failUnlessEqual(self._dll.tf_f(-42.), -14.) + self.failUnlessEqual(self.S(), -42) + + def test_float_plus(self): + self._dll.tf_bf.restype = c_float + self._dll.tf_bf.argtypes = (c_byte, c_float) + self.failUnlessEqual(self._dll.tf_bf(0, -42.), -14.) + self.failUnlessEqual(self.S(), -42) + + def test_double(self): + self._dll.tf_d.restype = c_double + self._dll.tf_d.argtypes = (c_double,) + self.failUnlessEqual(self._dll.tf_d(42.), 14.) + self.failUnlessEqual(self.S(), 42) + + def test_double_plus(self): + self._dll.tf_bd.restype = c_double + self._dll.tf_bd.argtypes = (c_byte, c_double) + self.failUnlessEqual(self._dll.tf_bd(0, 42.), 14.) + self.failUnlessEqual(self.S(), 42) + + def test_callwithresult(self): + def process_result(result): + return result * 2 + self._dll.tf_i.restype = process_result + self._dll.tf_i.argtypes = (c_int,) + self.failUnlessEqual(self._dll.tf_i(42), 28) + self.failUnlessEqual(self.S(), 42) + self.failUnlessEqual(self._dll.tf_i(-42), -28) + self.failUnlessEqual(self.S(), -42) + + def test_void(self): + self._dll.tv_i.restype = None + self._dll.tv_i.argtypes = (c_int,) + self.failUnlessEqual(self._dll.tv_i(42), None) + self.failUnlessEqual(self.S(), 42) + self.failUnlessEqual(self._dll.tv_i(-42), None) + self.failUnlessEqual(self.S(), -42) + +# The following repeates the above tests with stdcall functions (where +# they are available) +try: + WinDLL +except NameError: + pass +else: + class stdcall_dll(WinDLL): + def __getattr__(self, name): + if name[:2] == '__' and name[-2:] == '__': + raise AttributeError, name + func = self._FuncPtr(("s_" + name, self)) + setattr(self, name, func) + return func + + class stdcallCFunctions(CFunctions): + _dll = stdcall_dll(_ctypes_test.__file__) + pass + +if __name__ == '__main__': + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_checkretval.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_checkretval.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,40 @@ +import unittest +import sys + +from ctypes import * + +class CHECKED(c_int): + def _check_retval_(value): + # Receives a CHECKED instance. + return str(value.value) + _check_retval_ = staticmethod(_check_retval_) + +class Test(unittest.TestCase): + + def test_checkretval(self): + + import _ctypes_test + dll = CDLL(_ctypes_test.__file__) + self.failUnlessEqual(42, dll._testfunc_p_p(42)) + + dll._testfunc_p_p.restype = CHECKED + self.failUnlessEqual("42", dll._testfunc_p_p(42)) + + dll._testfunc_p_p.restype = None + self.failUnlessEqual(None, dll._testfunc_p_p(42)) + + del dll._testfunc_p_p.restype + self.failUnlessEqual(42, dll._testfunc_p_p(42)) + + try: + oledll + except NameError: + pass + else: + def test_oledll(self): + self.failUnlessRaises(WindowsError, + oledll.oleaut32.CreateTypeLib2, + 0, 0, 0) + +if __name__ == "__main__": + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_errcheck.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_errcheck.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,19 @@ +import sys +from ctypes import * + +##class HMODULE(Structure): +## _fields_ = [("value", c_void_p)] + +## def __repr__(self): +## return "" % self.value + +##windll.kernel32.GetModuleHandleA.restype = HMODULE + +##print windll.kernel32.GetModuleHandleA("python23.dll") +##print hex(sys.dllhandle) + +##def nonzero(handle): +## return (GetLastError(), handle) + +##windll.kernel32.GetModuleHandleA.errcheck = nonzero +##print windll.kernel32.GetModuleHandleA("spam") Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_find.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_find.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,104 @@ +import unittest +import os, sys +from ctypes import * +from ctypes.util import find_library +from ctypes.test import is_resource_enabled + +if sys.platform == "win32": + lib_gl = find_library("OpenGL32") + lib_glu = find_library("Glu32") + lib_glut = find_library("glut32") + lib_gle = None +elif sys.platform == "darwin": + lib_gl = lib_glu = find_library("OpenGL") + lib_glut = find_library("GLUT") + lib_gle = None +else: + lib_gl = find_library("GL") + lib_glu = find_library("GLU") + lib_glut = find_library("glut") + lib_gle = find_library("gle") + +## print, for debugging +if is_resource_enabled("printing"): + if lib_gl or lib_glu or lib_glut or lib_gle: + print "OpenGL libraries:" + for item in (("GL", lib_gl), + ("GLU", lib_glu), + ("glut", lib_glut), + ("gle", lib_gle)): + print "\t", item + + +# On some systems, loading the OpenGL libraries needs the RTLD_GLOBAL mode. +class Test_OpenGL_libs(unittest.TestCase): + def setUp(self): + self.gl = self.glu = self.gle = self.glut = None + if lib_gl: + self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL) + if lib_glu: + self.glu = CDLL(lib_glu, RTLD_GLOBAL) + if lib_glut: + # On some systems, additional libraries seem to be + # required, loading glut fails with + # "OSError: /usr/lib/libglut.so.3: undefined symbol: XGetExtensionVersion" + # I cannot figure out how to repair the test on these + # systems (red hat), so we ignore it when the glut or gle + # libraries cannot be loaded. See also: + # https://sourceforge.net/tracker/?func=detail&atid=105470&aid=1478253&group_id=5470 + # http://mail.python.org/pipermail/python-dev/2006-May/064789.html + try: + self.glut = CDLL(lib_glut) + except OSError: + pass + if lib_gle: + try: + self.gle = CDLL(lib_gle) + except OSError: + pass + + if lib_gl: + def test_gl(self): + if self.gl: + self.gl.glClearIndex + + if lib_glu: + def test_glu(self): + if self.glu: + self.glu.gluBeginCurve + + if lib_glut: + def test_glut(self): + if self.glut: + self.glut.glutWireTetrahedron + + if lib_gle: + def test_gle(self): + if self.gle: + self.gle.gleGetJoinStyle + +##if os.name == "posix" and sys.platform != "darwin": + +## # On platforms where the default shared library suffix is '.so', +## # at least some libraries can be loaded as attributes of the cdll +## # object, since ctypes now tries loading the lib again +## # with '.so' appended of the first try fails. +## # +## # Won't work for libc, unfortunately. OTOH, it isn't +## # needed for libc since this is already mapped into the current +## # process (?) +## # +## # On MAC OSX, it won't work either, because dlopen() needs a full path, +## # and the default suffix is either none or '.dylib'. + +## class LoadLibs(unittest.TestCase): +## def test_libm(self): +## import math +## libm = cdll.libm +## sqrt = libm.sqrt +## sqrt.argtypes = (c_double,) +## sqrt.restype = c_double +## self.failUnlessEqual(sqrt(2), math.sqrt(2)) + +if __name__ == "__main__": + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_funcptr.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_funcptr.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,127 @@ +import os, unittest +from ctypes import * + +try: + WINFUNCTYPE +except NameError: + # fake to enable this test on Linux + WINFUNCTYPE = CFUNCTYPE + +import _ctypes_test +lib = CDLL(_ctypes_test.__file__) + +class CFuncPtrTestCase(unittest.TestCase): + def test_basic(self): + X = WINFUNCTYPE(c_int, c_int, c_int) + + def func(*args): + return len(args) + + x = X(func) + self.failUnlessEqual(x.restype, c_int) + self.failUnlessEqual(x.argtypes, (c_int, c_int)) + self.failUnlessEqual(sizeof(x), sizeof(c_voidp)) + self.failUnlessEqual(sizeof(X), sizeof(c_voidp)) + + def test_first(self): + StdCallback = WINFUNCTYPE(c_int, c_int, c_int) + CdeclCallback = CFUNCTYPE(c_int, c_int, c_int) + + def func(a, b): + return a + b + + s = StdCallback(func) + c = CdeclCallback(func) + + self.failUnlessEqual(s(1, 2), 3) + self.failUnlessEqual(c(1, 2), 3) + # The following no longer raises a TypeError - it is now + # possible, as in C, to call cdecl functions with more parameters. + #self.assertRaises(TypeError, c, 1, 2, 3) + self.failUnlessEqual(c(1, 2, 3, 4, 5, 6), 3) + if not WINFUNCTYPE is CFUNCTYPE and os.name != "ce": + self.assertRaises(TypeError, s, 1, 2, 3) + + def test_structures(self): + WNDPROC = WINFUNCTYPE(c_long, c_int, c_int, c_int, c_int) + + def wndproc(hwnd, msg, wParam, lParam): + return hwnd + msg + wParam + lParam + + HINSTANCE = c_int + HICON = c_int + HCURSOR = c_int + LPCTSTR = c_char_p + + class WNDCLASS(Structure): + _fields_ = [("style", c_uint), + ("lpfnWndProc", WNDPROC), + ("cbClsExtra", c_int), + ("cbWndExtra", c_int), + ("hInstance", HINSTANCE), + ("hIcon", HICON), + ("hCursor", HCURSOR), + ("lpszMenuName", LPCTSTR), + ("lpszClassName", LPCTSTR)] + + wndclass = WNDCLASS() + wndclass.lpfnWndProc = WNDPROC(wndproc) + + WNDPROC_2 = WINFUNCTYPE(c_long, c_int, c_int, c_int, c_int) + + # This is no longer true, now that WINFUNCTYPE caches created types internally. + ## # CFuncPtr subclasses are compared by identity, so this raises a TypeError: + ## self.assertRaises(TypeError, setattr, wndclass, + ## "lpfnWndProc", WNDPROC_2(wndproc)) + # instead: + + self.failUnless(WNDPROC is WNDPROC_2) + # 'wndclass.lpfnWndProc' leaks 94 references. Why? + self.failUnlessEqual(wndclass.lpfnWndProc(1, 2, 3, 4), 10) + + + f = wndclass.lpfnWndProc + + del wndclass + del wndproc + + self.failUnlessEqual(f(10, 11, 12, 13), 46) + + def test_dllfunctions(self): + + def NoNullHandle(value): + if not value: + raise WinError() + return value + + strchr = lib.my_strchr + strchr.restype = c_char_p + strchr.argtypes = (c_char_p, c_char) + self.failUnlessEqual(strchr("abcdefghi", "b"), "bcdefghi") + self.failUnlessEqual(strchr("abcdefghi", "x"), None) + + + strtok = lib.my_strtok + strtok.restype = c_char_p + # Neither of this does work: strtok changes the buffer it is passed +## strtok.argtypes = (c_char_p, c_char_p) +## strtok.argtypes = (c_string, c_char_p) + + def c_string(init): + size = len(init) + 1 + return (c_char*size)(*init) + + s = "a\nb\nc" + b = c_string(s) + +## b = (c_char * (len(s)+1))() +## b.value = s + +## b = c_string(s) + self.failUnlessEqual(strtok(b, "\n"), "a") + self.failUnlessEqual(strtok(None, "\n"), "b") + self.failUnlessEqual(strtok(None, "\n"), "c") + self.failUnlessEqual(strtok(None, "\n"), None) + +if __name__ == '__main__': + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_functions.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_functions.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,395 @@ +""" +Here is probably the place to write the docs, since the test-cases +show how the type behave. + +Later... +""" + +from ctypes import * +import sys, unittest + +try: + WINFUNCTYPE +except NameError: + # fake to enable this test on Linux + WINFUNCTYPE = CFUNCTYPE + +import _ctypes_test +dll = CDLL(_ctypes_test.__file__) +if sys.platform == "win32": + windll = WinDLL(_ctypes_test.__file__) + +class POINT(Structure): + _fields_ = [("x", c_int), ("y", c_int)] +class RECT(Structure): + _fields_ = [("left", c_int), ("top", c_int), + ("right", c_int), ("bottom", c_int)] +class FunctionTestCase(unittest.TestCase): + + def test_mro(self): + # in Python 2.3, this raises TypeError: MRO conflict among bases classes, + # in Python 2.2 it works. + # + # But in early versions of _ctypes.c, the result of tp_new + # wasn't checked, and it even crashed Python. + # Found by Greg Chapman. + + try: + class X(object, Array): + _length_ = 5 + _type_ = "i" + except TypeError: + pass + + + from _ctypes import _Pointer + try: + class X(object, _Pointer): + pass + except TypeError: + pass + + from _ctypes import _SimpleCData + try: + class X(object, _SimpleCData): + _type_ = "i" + except TypeError: + pass + + try: + class X(object, Structure): + _fields_ = [] + except TypeError: + pass + + + def test_wchar_parm(self): + try: + c_wchar + except NameError: + return + f = dll._testfunc_i_bhilfd + f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double] + result = f(1, u"x", 3, 4, 5.0, 6.0) + self.failUnlessEqual(result, 139) + self.failUnlessEqual(type(result), int) + + def test_wchar_result(self): + try: + c_wchar + except NameError: + return + f = dll._testfunc_i_bhilfd + f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] + f.restype = c_wchar + result = f(0, 0, 0, 0, 0, 0) + self.failUnlessEqual(result, u'\x00') + + def test_voidresult(self): + f = dll._testfunc_v + f.restype = None + f.argtypes = [c_int, c_int, POINTER(c_int)] + result = c_int() + self.failUnlessEqual(None, f(1, 2, byref(result))) + self.failUnlessEqual(result.value, 3) + + def test_intresult(self): + f = dll._testfunc_i_bhilfd + f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] + f.restype = c_int + result = f(1, 2, 3, 4, 5.0, 6.0) + self.failUnlessEqual(result, 21) + self.failUnlessEqual(type(result), int) + + result = f(-1, -2, -3, -4, -5.0, -6.0) + self.failUnlessEqual(result, -21) + self.failUnlessEqual(type(result), int) + + # If we declare the function to return a short, + # is the high part split off? + f.restype = c_short + result = f(1, 2, 3, 4, 5.0, 6.0) + self.failUnlessEqual(result, 21) + self.failUnlessEqual(type(result), int) + + result = f(1, 2, 3, 0x10004, 5.0, 6.0) + self.failUnlessEqual(result, 21) + self.failUnlessEqual(type(result), int) + + # You cannot assing character format codes as restype any longer + self.assertRaises(TypeError, setattr, f, "restype", "i") + + def test_floatresult(self): + f = dll._testfunc_f_bhilfd + f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] + f.restype = c_float + result = f(1, 2, 3, 4, 5.0, 6.0) + self.failUnlessEqual(result, 21) + self.failUnlessEqual(type(result), float) + + result = f(-1, -2, -3, -4, -5.0, -6.0) + self.failUnlessEqual(result, -21) + self.failUnlessEqual(type(result), float) + + def test_doubleresult(self): + f = dll._testfunc_d_bhilfd + f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] + f.restype = c_double + result = f(1, 2, 3, 4, 5.0, 6.0) + self.failUnlessEqual(result, 21) + self.failUnlessEqual(type(result), float) + + result = f(-1, -2, -3, -4, -5.0, -6.0) + self.failUnlessEqual(result, -21) + self.failUnlessEqual(type(result), float) + + def test_longlongresult(self): + try: + c_longlong + except NameError: + return + f = dll._testfunc_q_bhilfd + f.restype = c_longlong + f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] + result = f(1, 2, 3, 4, 5.0, 6.0) + self.failUnlessEqual(result, 21) + + f = dll._testfunc_q_bhilfdq + f.restype = c_longlong + f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double, c_longlong] + result = f(1, 2, 3, 4, 5.0, 6.0, 21) + self.failUnlessEqual(result, 42) + + def test_stringresult(self): + f = dll._testfunc_p_p + f.argtypes = None + f.restype = c_char_p + result = f("123") + self.failUnlessEqual(result, "123") + + result = f(None) + self.failUnlessEqual(result, None) + + def test_pointers(self): + f = dll._testfunc_p_p + f.restype = POINTER(c_int) + f.argtypes = [POINTER(c_int)] + + # This only works if the value c_int(42) passed to the + # function is still alive while the pointer (the result) is + # used. + + v = c_int(42) + + self.failUnlessEqual(pointer(v).contents.value, 42) + result = f(pointer(v)) + self.failUnlessEqual(type(result), POINTER(c_int)) + self.failUnlessEqual(result.contents.value, 42) + + # This on works... + result = f(pointer(v)) + self.failUnlessEqual(result.contents.value, v.value) + + p = pointer(c_int(99)) + result = f(p) + self.failUnlessEqual(result.contents.value, 99) + + arg = byref(v) + result = f(arg) + self.failIfEqual(result.contents, v.value) + + self.assertRaises(ArgumentError, f, byref(c_short(22))) + + # It is dangerous, however, because you don't control the lifetime + # of the pointer: + result = f(byref(c_int(99))) + self.failIfEqual(result.contents, 99) + + def test_errors(self): + f = dll._testfunc_p_p + f.restype = c_int + + class X(Structure): + _fields_ = [("y", c_int)] + + self.assertRaises(TypeError, f, X()) #cannot convert parameter + + ################################################################ + def test_shorts(self): + f = dll._testfunc_callback_i_if + + args = [] + expected = [262144, 131072, 65536, 32768, 16384, 8192, 4096, 2048, + 1024, 512, 256, 128, 64, 32, 16, 8, 4, 2, 1] + + def callback(v): + args.append(v) + return v + + CallBack = CFUNCTYPE(c_int, c_int) + + cb = CallBack(callback) + f(2**18, cb) + self.failUnlessEqual(args, expected) + + ################################################################ + + + def test_callbacks(self): + f = dll._testfunc_callback_i_if + f.restype = c_int + + MyCallback = CFUNCTYPE(c_int, c_int) + + def callback(value): + #print "called back with", value + return value + + cb = MyCallback(callback) + result = f(-10, cb) + self.failUnlessEqual(result, -18) + + # test with prototype + f.argtypes = [c_int, MyCallback] + cb = MyCallback(callback) + result = f(-10, cb) + self.failUnlessEqual(result, -18) + + AnotherCallback = WINFUNCTYPE(c_int, c_int, c_int, c_int, c_int) + + # check that the prototype works: we call f with wrong + # argument types + cb = AnotherCallback(callback) + self.assertRaises(ArgumentError, f, -10, cb) + + + def test_callbacks_2(self): + # Can also use simple datatypes as argument type specifiers + # for the callback function. + # In this case the call receives an instance of that type + f = dll._testfunc_callback_i_if + f.restype = c_int + + MyCallback = CFUNCTYPE(c_int, c_int) + + f.argtypes = [c_int, MyCallback] + + def callback(value): + #print "called back with", value + self.failUnlessEqual(type(value), int) + return value + + cb = MyCallback(callback) + result = f(-10, cb) + self.failUnlessEqual(result, -18) + + def test_longlong_callbacks(self): + + f = dll._testfunc_callback_q_qf + f.restype = c_longlong + + MyCallback = CFUNCTYPE(c_longlong, c_longlong) + + f.argtypes = [c_longlong, MyCallback] + + def callback(value): + self.failUnless(isinstance(value, (int, long))) + return value & 0x7FFFFFFF + + cb = MyCallback(callback) + + self.failUnlessEqual(13577625587, f(1000000000000, cb)) + + def test_errors(self): + self.assertRaises(AttributeError, getattr, dll, "_xxx_yyy") + self.assertRaises(ValueError, c_int.in_dll, dll, "_xxx_yyy") + + def test_byval(self): + + # without prototype + ptin = POINT(1, 2) + ptout = POINT() + # EXPORT int _testfunc_byval(point in, point *pout) + result = dll._testfunc_byval(ptin, byref(ptout)) + got = result, ptout.x, ptout.y + expected = 3, 1, 2 + self.failUnlessEqual(got, expected) + + # with prototype + ptin = POINT(101, 102) + ptout = POINT() + dll._testfunc_byval.argtypes = (POINT, POINTER(POINT)) + dll._testfunc_byval.restype = c_int + result = dll._testfunc_byval(ptin, byref(ptout)) + got = result, ptout.x, ptout.y + expected = 203, 101, 102 + self.failUnlessEqual(got, expected) + + def test_struct_return_2H(self): + class S2H(Structure): + _fields_ = [("x", c_short), + ("y", c_short)] + dll.ret_2h_func.restype = S2H + dll.ret_2h_func.argtypes = [S2H] + inp = S2H(99, 88) + s2h = dll.ret_2h_func(inp) + self.failUnlessEqual((s2h.x, s2h.y), (99*2, 88*3)) + + if sys.platform == "win32": + def test_struct_return_2H_stdcall(self): + class S2H(Structure): + _fields_ = [("x", c_short), + ("y", c_short)] + + windll.s_ret_2h_func.restype = S2H + windll.s_ret_2h_func.argtypes = [S2H] + s2h = windll.s_ret_2h_func(S2H(99, 88)) + self.failUnlessEqual((s2h.x, s2h.y), (99*2, 88*3)) + + def test_struct_return_8H(self): + class S8I(Structure): + _fields_ = [("a", c_int), + ("b", c_int), + ("c", c_int), + ("d", c_int), + ("e", c_int), + ("f", c_int), + ("g", c_int), + ("h", c_int)] + dll.ret_8i_func.restype = S8I + dll.ret_8i_func.argtypes = [S8I] + inp = S8I(9, 8, 7, 6, 5, 4, 3, 2) + s8i = dll.ret_8i_func(inp) + self.failUnlessEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h), + (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) + + if sys.platform == "win32": + def test_struct_return_8H_stdcall(self): + class S8I(Structure): + _fields_ = [("a", c_int), + ("b", c_int), + ("c", c_int), + ("d", c_int), + ("e", c_int), + ("f", c_int), + ("g", c_int), + ("h", c_int)] + windll.s_ret_8i_func.restype = S8I + windll.s_ret_8i_func.argtypes = [S8I] + inp = S8I(9, 8, 7, 6, 5, 4, 3, 2) + s8i = windll.s_ret_8i_func(inp) + self.failUnlessEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h), + (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) + + def test_sf1651235(self): + # see http://www.python.org/sf/1651235 + + proto = CFUNCTYPE(c_int, RECT, POINT) + def callback(*args): + return 0 + + callback = proto(callback) + self.failUnlessRaises(ArgumentError, lambda: callback((1, 2, 3, 4), POINT())) + +if __name__ == '__main__': + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_incomplete.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_incomplete.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,42 @@ +import unittest +from ctypes import * + +################################################################ +# +# The incomplete pointer example from the tutorial +# + +class MyTestCase(unittest.TestCase): + + def test_incomplete_example(self): + lpcell = POINTER("cell") + class cell(Structure): + _fields_ = [("name", c_char_p), + ("next", lpcell)] + + SetPointerType(lpcell, cell) + + c1 = cell() + c1.name = "foo" + c2 = cell() + c2.name = "bar" + + c1.next = pointer(c2) + c2.next = pointer(c1) + + p = c1 + + result = [] + for i in range(8): + result.append(p.name) + p = p.next[0] + self.failUnlessEqual(result, ["foo", "bar"] * 4) + + # to not leak references, we must clean _pointer_type_cache + from ctypes import _pointer_type_cache + del _pointer_type_cache[cell] + +################################################################ + +if __name__ == '__main__': + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_init.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_init.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,40 @@ +from ctypes import * +import unittest + +class X(Structure): + _fields_ = [("a", c_int), + ("b", c_int)] + new_was_called = False + + def __new__(cls): + result = super(X, cls).__new__(cls) + result.new_was_called = True + return result + + def __init__(self): + self.a = 9 + self.b = 12 + +class Y(Structure): + _fields_ = [("x", X)] + + +class InitTest(unittest.TestCase): + def test_get(self): + # make sure the only accessing a nested structure + # doesn't call the structure's __new__ and __init__ + y = Y() + self.failUnlessEqual((y.x.a, y.x.b), (0, 0)) + self.failUnlessEqual(y.x.new_was_called, False) + + # But explicitely creating an X structure calls __new__ and __init__, of course. + x = X() + self.failUnlessEqual((x.a, x.b), (9, 12)) + self.failUnlessEqual(x.new_was_called, True) + + y.x = x + self.failUnlessEqual((y.x.a, y.x.b), (9, 12)) + self.failUnlessEqual(y.x.new_was_called, False) + +if __name__ == "__main__": + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_integers.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_integers.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,5 @@ +# superseeded by test_numbers.py +import unittest + +if __name__ == '__main__': + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_internals.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_internals.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,103 @@ +# This tests the internal _objects attribute +import unittest +from ctypes import * +from sys import getrefcount as grc + +# XXX This test must be reviewed for correctness!!! + +""" +ctypes' types are container types. + +They have an internal memory block, which only consists of some bytes, +but it has to keep references to other objects as well. This is not +really needed for trivial C types like int or char, but it is important +for aggregate types like strings or pointers in particular. + +What about pointers? + +""" + +class ObjectsTestCase(unittest.TestCase): + def failUnlessSame(self, a, b): + self.failUnlessEqual(id(a), id(b)) + + def test_ints(self): + i = 42000123 + self.failUnlessEqual(3, grc(i)) + ci = c_int(i) + self.failUnlessEqual(3, grc(i)) + self.failUnlessEqual(ci._objects, None) + + def test_c_char_p(self): + s = "Hello, World" + self.failUnlessEqual(3, grc(s)) + cs = c_char_p(s) + self.failUnlessEqual(4, grc(s)) + self.failUnlessSame(cs._objects, s) + + def test_simple_struct(self): + class X(Structure): + _fields_ = [("a", c_int), ("b", c_int)] + + a = 421234 + b = 421235 + x = X() + self.failUnlessEqual(x._objects, None) + x.a = a + x.b = b + self.failUnlessEqual(x._objects, None) + + def test_embedded_structs(self): + class X(Structure): + _fields_ = [("a", c_int), ("b", c_int)] + + class Y(Structure): + _fields_ = [("x", X), ("y", X)] + + y = Y() + self.failUnlessEqual(y._objects, None) + + x1, x2 = X(), X() + y.x, y.y = x1, x2 + self.failUnlessEqual(y._objects, {"0": {}, "1": {}}) + x1.a, x2.b = 42, 93 + self.failUnlessEqual(y._objects, {"0": {}, "1": {}}) + + def test_xxx(self): + class X(Structure): + _fields_ = [("a", c_char_p), ("b", c_char_p)] + + class Y(Structure): + _fields_ = [("x", X), ("y", X)] + + s1 = "Hello, World" + s2 = "Hallo, Welt" + + x = X() + x.a = s1 + x.b = s2 + self.failUnlessEqual(x._objects, {"0": s1, "1": s2}) + + y = Y() + y.x = x + self.failUnlessEqual(y._objects, {"0": {"0": s1, "1": s2}}) +## x = y.x +## del y +## print x._b_base_._objects + + def test_ptr_struct(self): + class X(Structure): + _fields_ = [("data", POINTER(c_int))] + + A = c_int*4 + a = A(11, 22, 33, 44) + self.failUnlessEqual(a._objects, None) + + x = X() + x.data = a +##XXX print x._objects +##XXX print x.data[0] +##XXX print x.data._objects + +if __name__ == '__main__': + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_keeprefs.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_keeprefs.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,152 @@ +from ctypes import * +import unittest + +class SimpleTestCase(unittest.TestCase): + def test_cint(self): + x = c_int() + self.assertEquals(x._objects, None) + x.value = 42 + self.assertEquals(x._objects, None) + x = c_int(99) + self.assertEquals(x._objects, None) + + def test_ccharp(self): + x = c_char_p() + self.assertEquals(x._objects, None) + x.value = "abc" + self.assertEquals(x._objects, "abc") + x = c_char_p("spam") + self.assertEquals(x._objects, "spam") + +class StructureTestCase(unittest.TestCase): + def test_cint_struct(self): + class X(Structure): + _fields_ = [("a", c_int), + ("b", c_int)] + + x = X() + self.assertEquals(x._objects, None) + x.a = 42 + x.b = 99 + self.assertEquals(x._objects, None) + + def test_ccharp_struct(self): + class X(Structure): + _fields_ = [("a", c_char_p), + ("b", c_char_p)] + x = X() + self.assertEquals(x._objects, None) + + x.a = "spam" + x.b = "foo" + self.assertEquals(x._objects, {"0": "spam", "1": "foo"}) + + def test_struct_struct(self): + class POINT(Structure): + _fields_ = [("x", c_int), ("y", c_int)] + class RECT(Structure): + _fields_ = [("ul", POINT), ("lr", POINT)] + + r = RECT() + r.ul.x = 0 + r.ul.y = 1 + r.lr.x = 2 + r.lr.y = 3 + self.assertEquals(r._objects, None) + + r = RECT() + pt = POINT(1, 2) + r.ul = pt + self.assertEquals(r._objects, {'0': {}}) + r.ul.x = 22 + r.ul.y = 44 + self.assertEquals(r._objects, {'0': {}}) + r.lr = POINT() + self.assertEquals(r._objects, {'0': {}, '1': {}}) + +class ArrayTestCase(unittest.TestCase): + def test_cint_array(self): + INTARR = c_int * 3 + + ia = INTARR() + self.assertEquals(ia._objects, None) + ia[0] = 1 + ia[1] = 2 + ia[2] = 3 + self.assertEquals(ia._objects, None) + + class X(Structure): + _fields_ = [("x", c_int), + ("a", INTARR)] + + x = X() + x.x = 1000 + x.a[0] = 42 + x.a[1] = 96 + self.assertEquals(x._objects, None) + x.a = ia + self.assertEquals(x._objects, {'1': {}}) + +class PointerTestCase(unittest.TestCase): + def test_p_cint(self): + i = c_int(42) + x = pointer(i) + self.failUnlessEqual(x._objects, {'1': i}) + +class DeletePointerTestCase(unittest.TestCase): + def X_test(self): + class X(Structure): + _fields_ = [("p", POINTER(c_char_p))] + x = X() + i = c_char_p("abc def") + from sys import getrefcount as grc + print "2?", grc(i) + x.p = pointer(i) + print "3?", grc(i) + for i in range(320): + c_int(99) + x.p[0] + print x.p[0] +## del x +## print "2?", grc(i) +## del i + import gc + gc.collect() + for i in range(320): + c_int(99) + x.p[0] + print x.p[0] + print x.p.contents +## print x._objects + + x.p[0] = "spam spam" +## print x.p[0] + print "+" * 42 + print x._objects + +class PointerToStructure(unittest.TestCase): + def test(self): + class POINT(Structure): + _fields_ = [("x", c_int), ("y", c_int)] + class RECT(Structure): + _fields_ = [("a", POINTER(POINT)), + ("b", POINTER(POINT))] + r = RECT() + p1 = POINT(1, 2) + + r.a = pointer(p1) + r.b = pointer(p1) +## from pprint import pprint as pp +## pp(p1._objects) +## pp(r._objects) + + r.a[0].x = 42 + r.a[0].y = 99 + + # to avoid leaking when tests are run several times + # clean up the types left in the cache. + from ctypes import _pointer_type_cache + del _pointer_type_cache[POINT] + +if __name__ == "__main__": + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_libc.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_libc.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,30 @@ +import sys, os +import unittest + +from ctypes import * +import _ctypes_test + +lib = CDLL(_ctypes_test.__file__) + +class LibTest(unittest.TestCase): + def test_sqrt(self): + lib.my_sqrt.argtypes = c_double, + lib.my_sqrt.restype = c_double + self.failUnlessEqual(lib.my_sqrt(4.0), 2.0) + import math + self.failUnlessEqual(lib.my_sqrt(2.0), math.sqrt(2.0)) + + def test_qsort(self): + comparefunc = CFUNCTYPE(c_int, POINTER(c_char), POINTER(c_char)) + lib.my_qsort.argtypes = c_void_p, c_size_t, c_size_t, comparefunc + lib.my_qsort.restype = None + + def sort(a, b): + return cmp(a[0], b[0]) + + chars = create_string_buffer("spam, spam, and spam") + lib.my_qsort(chars, len(chars)-1, sizeof(c_char), comparefunc(sort)) + self.failUnlessEqual(chars.raw, " ,,aaaadmmmnpppsss\x00") + +if __name__ == "__main__": + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_loading.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_loading.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,78 @@ +from ctypes import * +import sys, unittest +import os, StringIO +from ctypes.util import find_library +from ctypes.test import is_resource_enabled + +libc_name = None +if os.name == "nt": + libc_name = "msvcrt" +elif os.name == "ce": + libc_name = "coredll" +elif sys.platform == "cygwin": + libc_name = "cygwin1.dll" +else: + libc_name = find_library("c") + +if True or is_resource_enabled("printing"): + print >> sys.stderr, "\tfind_library('c') -> ", find_library('c') + print >> sys.stderr, "\tfind_library('m') -> ", find_library('m') + +class LoaderTest(unittest.TestCase): + + unknowndll = "xxrandomnamexx" + + if libc_name is not None: + def test_load(self): + CDLL(libc_name) + CDLL(os.path.basename(libc_name)) + self.assertRaises(OSError, CDLL, self.unknowndll) + + if libc_name is not None and os.path.basename(libc_name) == "libc.so.6": + def test_load_version(self): + cdll.LoadLibrary("libc.so.6") + # linux uses version, libc 9 should not exist + self.assertRaises(OSError, cdll.LoadLibrary, "libc.so.9") + self.assertRaises(OSError, cdll.LoadLibrary, self.unknowndll) + + def test_find(self): + for name in ("c", "m"): + lib = find_library(name) + if lib: + cdll.LoadLibrary(lib) + CDLL(lib) + + if os.name in ("nt", "ce"): + def test_load_library(self): + if is_resource_enabled("printing"): + print find_library("kernel32") + print find_library("user32") + + if os.name == "nt": + windll.kernel32.GetModuleHandleW + windll["kernel32"].GetModuleHandleW + windll.LoadLibrary("kernel32").GetModuleHandleW + WinDLL("kernel32").GetModuleHandleW + elif os.name == "ce": + windll.coredll.GetModuleHandleW + windll["coredll"].GetModuleHandleW + windll.LoadLibrary("coredll").GetModuleHandleW + WinDLL("coredll").GetModuleHandleW + + def test_load_ordinal_functions(self): + import _ctypes_test + dll = WinDLL(_ctypes_test.__file__) + # We load the same function both via ordinal and name + func_ord = dll[2] + func_name = dll.GetString + # addressof gets the address where the function pointer is stored + a_ord = addressof(func_ord) + a_name = addressof(func_name) + f_ord_addr = c_void_p.from_address(a_ord).value + f_name_addr = c_void_p.from_address(a_name).value + self.failUnlessEqual(hex(f_ord_addr), hex(f_name_addr)) + + self.failUnlessRaises(AttributeError, dll.__getitem__, 1234) + +if __name__ == "__main__": + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_macholib.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_macholib.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,62 @@ +import os +import sys +import unittest + +# Bob Ippolito: +""" +Ok.. the code to find the filename for __getattr__ should look +something like: + +import os +from macholib.dyld import dyld_find + +def find_lib(name): + possible = ['lib'+name+'.dylib', name+'.dylib', + name+'.framework/'+name] + for dylib in possible: + try: + return os.path.realpath(dyld_find(dylib)) + except ValueError: + pass + raise ValueError, "%s not found" % (name,) + +It'll have output like this: + + >>> find_lib('pthread') +'/usr/lib/libSystem.B.dylib' + >>> find_lib('z') +'/usr/lib/libz.1.dylib' + >>> find_lib('IOKit') +'/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit' + +-bob + +""" + +from ctypes.macholib.dyld import dyld_find + +def find_lib(name): + possible = ['lib'+name+'.dylib', name+'.dylib', name+'.framework/'+name] + for dylib in possible: + try: + return os.path.realpath(dyld_find(dylib)) + except ValueError: + pass + raise ValueError, "%s not found" % (name,) + +class MachOTest(unittest.TestCase): + if sys.platform == "darwin": + def test_find(self): + + self.failUnlessEqual(find_lib('pthread'), + '/usr/lib/libSystem.B.dylib') + + result = find_lib('z') + self.failUnless(result.startswith('/usr/lib/libz.1')) + self.failUnless(result.endswith('.dylib')) + + self.failUnlessEqual(find_lib('IOKit'), + '/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit') + +if __name__ == "__main__": + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_memfunctions.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_memfunctions.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,61 @@ +import sys +import unittest +from ctypes import * + +class MemFunctionsTest(unittest.TestCase): + def test_memmove(self): + # large buffers apparently increase the chance that the memory + # is allocated in high address space. + a = create_string_buffer(1000000) + p = "Hello, World" + result = memmove(a, p, len(p)) + self.failUnlessEqual(a.value, "Hello, World") + + self.failUnlessEqual(string_at(result), "Hello, World") + self.failUnlessEqual(string_at(result, 5), "Hello") + self.failUnlessEqual(string_at(result, 16), "Hello, World\0\0\0\0") + self.failUnlessEqual(string_at(result, 0), "") + + def test_memset(self): + a = create_string_buffer(1000000) + result = memset(a, ord('x'), 16) + self.failUnlessEqual(a.value, "xxxxxxxxxxxxxxxx") + + self.failUnlessEqual(string_at(result), "xxxxxxxxxxxxxxxx") + self.failUnlessEqual(string_at(a), "xxxxxxxxxxxxxxxx") + self.failUnlessEqual(string_at(a, 20), "xxxxxxxxxxxxxxxx\0\0\0\0") + + def test_cast(self): + a = (c_ubyte * 32)(*map(ord, "abcdef")) + self.failUnlessEqual(cast(a, c_char_p).value, "abcdef") + self.failUnlessEqual(cast(a, POINTER(c_byte))[:7], + [97, 98, 99, 100, 101, 102, 0]) + + def test_string_at(self): + s = string_at("foo bar") + # XXX The following may be wrong, depending on how Python + # manages string instances + self.failUnlessEqual(2, sys.getrefcount(s)) + self.failUnless(s, "foo bar") + + self.failUnlessEqual(string_at("foo bar", 8), "foo bar\0") + self.failUnlessEqual(string_at("foo bar", 3), "foo") + + try: + create_unicode_buffer + except NameError: + pass + else: + def test_wstring_at(self): + p = create_unicode_buffer("Hello, World") + a = create_unicode_buffer(1000000) + result = memmove(a, p, len(p) * sizeof(c_wchar)) + self.failUnlessEqual(a.value, "Hello, World") + + self.failUnlessEqual(wstring_at(a), "Hello, World") + self.failUnlessEqual(wstring_at(a, 5), "Hello") + self.failUnlessEqual(wstring_at(a, 16), "Hello, World\0\0\0\0") + self.failUnlessEqual(wstring_at(a, 0), "") + +if __name__ == "__main__": + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_numbers.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_numbers.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,230 @@ +from ctypes import * +import unittest +import sys, struct + +def valid_ranges(*types): + # given a sequence of numeric types, collect their _type_ + # attribute, which is a single format character compatible with + # the struct module, use the struct module to calculate the + # minimum and maximum value allowed for this format. + # Returns a list of (min, max) values. + result = [] + for t in types: + fmt = t._type_ + size = struct.calcsize(fmt) + a = struct.unpack(fmt, ("\x00"*32)[:size])[0] + b = struct.unpack(fmt, ("\xFF"*32)[:size])[0] + c = struct.unpack(fmt, ("\x7F"+"\x00"*32)[:size])[0] + d = struct.unpack(fmt, ("\x80"+"\xFF"*32)[:size])[0] + result.append((min(a, b, c, d), max(a, b, c, d))) + return result + +ArgType = type(byref(c_int(0))) + +unsigned_types = [c_ubyte, c_ushort, c_uint, c_ulong] +signed_types = [c_byte, c_short, c_int, c_long, c_longlong] + +float_types = [c_double, c_float] + +try: + c_ulonglong + c_longlong +except NameError: + pass +else: + unsigned_types.append(c_ulonglong) + signed_types.append(c_longlong) + +unsigned_ranges = valid_ranges(*unsigned_types) +signed_ranges = valid_ranges(*signed_types) + +################################################################ + +class NumberTestCase(unittest.TestCase): + + def test_default_init(self): + # default values are set to zero + for t in signed_types + unsigned_types + float_types: + self.failUnlessEqual(t().value, 0) + + def test_unsigned_values(self): + # the value given to the constructor is available + # as the 'value' attribute + for t, (l, h) in zip(unsigned_types, unsigned_ranges): + self.failUnlessEqual(t(l).value, l) + self.failUnlessEqual(t(h).value, h) + + def test_signed_values(self): + # see above + for t, (l, h) in zip(signed_types, signed_ranges): + self.failUnlessEqual(t(l).value, l) + self.failUnlessEqual(t(h).value, h) + + def test_typeerror(self): + # Only numbers are allowed in the contructor, + # otherwise TypeError is raised + for t in signed_types + unsigned_types + float_types: + self.assertRaises(TypeError, t, "") + self.assertRaises(TypeError, t, None) + +## def test_valid_ranges(self): +## # invalid values of the correct type +## # raise ValueError (not OverflowError) +## for t, (l, h) in zip(unsigned_types, unsigned_ranges): +## self.assertRaises(ValueError, t, l-1) +## self.assertRaises(ValueError, t, h+1) + + def test_from_param(self): + # the from_param class method attribute always + # returns PyCArgObject instances + for t in signed_types + unsigned_types + float_types: + self.failUnlessEqual(ArgType, type(t.from_param(0))) + + def test_byref(self): + # calling byref returns also a PyCArgObject instance + for t in signed_types + unsigned_types + float_types: + parm = byref(t()) + self.failUnlessEqual(ArgType, type(parm)) + + + def test_floats(self): + # c_float and c_double can be created from + # Python int, long and float + for t in float_types: + self.failUnlessEqual(t(2.0).value, 2.0) + self.failUnlessEqual(t(2).value, 2.0) + self.failUnlessEqual(t(2L).value, 2.0) + + def test_integers(self): + # integers cannot be constructed from floats + for t in signed_types + unsigned_types: + self.assertRaises(TypeError, t, 3.14) + + def test_sizes(self): + for t in signed_types + unsigned_types + float_types: + size = struct.calcsize(t._type_) + # sizeof of the type... + self.failUnlessEqual(sizeof(t), size) + # and sizeof of an instance + self.failUnlessEqual(sizeof(t()), size) + + def test_alignments(self): + for t in signed_types + unsigned_types + float_types: + code = t._type_ # the typecode + align = struct.calcsize("c%c" % code) - struct.calcsize(code) + + # alignment of the type... + self.failUnlessEqual((code, alignment(t)), + (code, align)) + # and alignment of an instance + self.failUnlessEqual((code, alignment(t())), + (code, align)) + + def test_int_from_address(self): + from array import array + for t in signed_types + unsigned_types: + # the array module doesn't suppport all format codes + # (no 'q' or 'Q') + try: + array(t._type_) + except ValueError: + continue + a = array(t._type_, [100]) + + # v now is an integer at an 'external' memory location + v = t.from_address(a.buffer_info()[0]) + self.failUnlessEqual(v.value, a[0]) + self.failUnlessEqual(type(v), t) + + # changing the value at the memory location changes v's value also + a[0] = 42 + self.failUnlessEqual(v.value, a[0]) + + + def test_float_from_address(self): + from array import array + for t in float_types: + a = array(t._type_, [3.14]) + v = t.from_address(a.buffer_info()[0]) + self.failUnlessEqual(v.value, a[0]) + self.failUnless(type(v) is t) + a[0] = 2.3456e17 + self.failUnlessEqual(v.value, a[0]) + self.failUnless(type(v) is t) + + def test_char_from_address(self): + from ctypes import c_char + from array import array + + a = array('c', 'x') + v = c_char.from_address(a.buffer_info()[0]) + self.failUnlessEqual(v.value, a[0]) + self.failUnless(type(v) is c_char) + + a[0] = '?' + self.failUnlessEqual(v.value, a[0]) + + def test_init(self): + # c_int() can be initialized from Python's int, and c_int. + # Not from c_long or so, which seems strange, abd should + # probably be changed: + self.assertRaises(TypeError, c_int, c_long(42)) + +## def test_perf(self): +## check_perf() + +from ctypes import _SimpleCData +class c_int_S(_SimpleCData): + _type_ = "i" + __slots__ = [] + +def run_test(rep, msg, func, arg=None): +## items = [None] * rep + items = range(rep) + from time import clock + if arg is not None: + start = clock() + for i in items: + func(arg); func(arg); func(arg); func(arg); func(arg) + stop = clock() + else: + start = clock() + for i in items: + func(); func(); func(); func(); func() + stop = clock() + print "%15s: %.2f us" % (msg, ((stop-start)*1e6/5/rep)) + +def check_perf(): + # Construct 5 objects + from ctypes import c_int + + REP = 200000 + + run_test(REP, "int()", int) + run_test(REP, "int(999)", int) + run_test(REP, "c_int()", c_int) + run_test(REP, "c_int(999)", c_int) + run_test(REP, "c_int_S()", c_int_S) + run_test(REP, "c_int_S(999)", c_int_S) + +# Python 2.3 -OO, win2k, P4 700 MHz: +# +# int(): 0.87 us +# int(999): 0.87 us +# c_int(): 3.35 us +# c_int(999): 3.34 us +# c_int_S(): 3.23 us +# c_int_S(999): 3.24 us + +# Python 2.2 -OO, win2k, P4 700 MHz: +# +# int(): 0.89 us +# int(999): 0.89 us +# c_int(): 9.99 us +# c_int(999): 10.02 us +# c_int_S(): 9.87 us +# c_int_S(999): 9.85 us + +if __name__ == '__main__': +## check_perf() + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_objects.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_objects.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,70 @@ +r''' +This tests the '_objects' attribute of ctypes instances. '_objects' +holds references to objects that must be kept alive as long as the +ctypes instance, to make sure that the memory buffer is valid. + +WARNING: The '_objects' attribute is exposed ONLY for debugging ctypes itself, +it MUST NEVER BE MODIFIED! + +'_objects' is initialized to a dictionary on first use, before that it +is None. + +Here is an array of string pointers: + +>>> from ctypes import * +>>> array = (c_char_p * 5)() +>>> print array._objects +None +>>> + +The memory block stores pointers to strings, and the strings itself +assigned from Python must be kept. + +>>> array[4] = 'foo bar' +>>> array._objects +{'4': 'foo bar'} +>>> array[4] +'foo bar' +>>> + +It gets more complicated when the ctypes instance itself is contained +in a 'base' object. + +>>> class X(Structure): +... _fields_ = [("x", c_int), ("y", c_int), ("array", c_char_p * 5)] +... +>>> x = X() +>>> print x._objects +None +>>> + +The'array' attribute of the 'x' object shares part of the memory buffer +of 'x' ('_b_base_' is either None, or the root object owning the memory block): + +>>> print x.array._b_base_ # doctest: +ELLIPSIS + +>>> + +>>> x.array[0] = 'spam spam spam' +>>> x._objects +{'0:2': 'spam spam spam'} +>>> x.array._b_base_._objects +{'0:2': 'spam spam spam'} +>>> + +''' + +import unittest, doctest, sys + +import ctypes.test.test_objects + +class TestCase(unittest.TestCase): + if sys.hexversion > 0x02040000: + # Python 2.3 has no ELLIPSIS flag, so we don't test with this + # version: + def test(self): + doctest.testmod(ctypes.test.test_objects) + +if __name__ == '__main__': + if sys.hexversion > 0x02040000: + doctest.testmod(ctypes.test.test_objects) Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_parameters.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_parameters.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,188 @@ +import unittest, sys + +class SimpleTypesTestCase(unittest.TestCase): + + def setUp(self): + import ctypes + try: + from _ctypes import set_conversion_mode + except ImportError: + pass + else: + self.prev_conv_mode = set_conversion_mode("ascii", "strict") + + def tearDown(self): + try: + from _ctypes import set_conversion_mode + except ImportError: + pass + else: + set_conversion_mode(*self.prev_conv_mode) + + + def test_subclasses(self): + from ctypes import c_void_p, c_char_p + # ctypes 0.9.5 and before did overwrite from_param in SimpleType_new + class CVOIDP(c_void_p): + def from_param(cls, value): + return value * 2 + from_param = classmethod(from_param) + + class CCHARP(c_char_p): + def from_param(cls, value): + return value * 4 + from_param = classmethod(from_param) + + self.failUnlessEqual(CVOIDP.from_param("abc"), "abcabc") + self.failUnlessEqual(CCHARP.from_param("abc"), "abcabcabcabc") + + try: + from ctypes import c_wchar_p + except ImportError: + return + + class CWCHARP(c_wchar_p): + def from_param(cls, value): + return value * 3 + from_param = classmethod(from_param) + + self.failUnlessEqual(CWCHARP.from_param("abc"), "abcabcabc") + + # XXX Replace by c_char_p tests + def test_cstrings(self): + from ctypes import c_char_p, byref + + # c_char_p.from_param on a Python String packs the string + # into a cparam object + s = "123" + self.failUnless(c_char_p.from_param(s)._obj is s) + + # new in 0.9.1: convert (encode) unicode to ascii + self.failUnlessEqual(c_char_p.from_param(u"123")._obj, "123") + self.assertRaises(UnicodeEncodeError, c_char_p.from_param, u"123\377") + + self.assertRaises(TypeError, c_char_p.from_param, 42) + + # calling c_char_p.from_param with a c_char_p instance + # returns the argument itself: + a = c_char_p("123") + self.failUnless(c_char_p.from_param(a) is a) + + def test_cw_strings(self): + from ctypes import byref + try: + from ctypes import c_wchar_p + except ImportError: +## print "(No c_wchar_p)" + return + s = u"123" + if sys.platform == "win32": + self.failUnless(c_wchar_p.from_param(s)._obj is s) + self.assertRaises(TypeError, c_wchar_p.from_param, 42) + + # new in 0.9.1: convert (decode) ascii to unicode + self.failUnlessEqual(c_wchar_p.from_param("123")._obj, u"123") + self.assertRaises(UnicodeDecodeError, c_wchar_p.from_param, "123\377") + + pa = c_wchar_p.from_param(c_wchar_p(u"123")) + self.failUnlessEqual(type(pa), c_wchar_p) + + def test_int_pointers(self): + from ctypes import c_short, c_uint, c_int, c_long, POINTER, pointer + LPINT = POINTER(c_int) + +## p = pointer(c_int(42)) +## x = LPINT.from_param(p) + x = LPINT.from_param(pointer(c_int(42))) + self.failUnlessEqual(x.contents.value, 42) + self.failUnlessEqual(LPINT(c_int(42)).contents.value, 42) + + self.failUnlessEqual(LPINT.from_param(None), 0) + + if c_int != c_long: + self.assertRaises(TypeError, LPINT.from_param, pointer(c_long(42))) + self.assertRaises(TypeError, LPINT.from_param, pointer(c_uint(42))) + self.assertRaises(TypeError, LPINT.from_param, pointer(c_short(42))) + + def test_byref_pointer(self): + # The from_param class method of POINTER(typ) classes accepts what is + # returned by byref(obj), it type(obj) == typ + from ctypes import c_short, c_uint, c_int, c_long, pointer, POINTER, byref + LPINT = POINTER(c_int) + + LPINT.from_param(byref(c_int(42))) + + self.assertRaises(TypeError, LPINT.from_param, byref(c_short(22))) + if c_int != c_long: + self.assertRaises(TypeError, LPINT.from_param, byref(c_long(22))) + self.assertRaises(TypeError, LPINT.from_param, byref(c_uint(22))) + + def test_byref_pointerpointer(self): + # See above + from ctypes import c_short, c_uint, c_int, c_long, pointer, POINTER, byref + + LPLPINT = POINTER(POINTER(c_int)) + LPLPINT.from_param(byref(pointer(c_int(42)))) + + self.assertRaises(TypeError, LPLPINT.from_param, byref(pointer(c_short(22)))) + if c_int != c_long: + self.assertRaises(TypeError, LPLPINT.from_param, byref(pointer(c_long(22)))) + self.assertRaises(TypeError, LPLPINT.from_param, byref(pointer(c_uint(22)))) + + def test_array_pointers(self): + from ctypes import c_short, c_uint, c_int, c_long, POINTER + INTARRAY = c_int * 3 + ia = INTARRAY() + self.failUnlessEqual(len(ia), 3) + self.failUnlessEqual([ia[i] for i in range(3)], [0, 0, 0]) + + # Pointers are only compatible with arrays containing items of + # the same type! + LPINT = POINTER(c_int) + LPINT.from_param((c_int*3)()) + self.assertRaises(TypeError, LPINT.from_param, c_short*3) + self.assertRaises(TypeError, LPINT.from_param, c_long*3) + self.assertRaises(TypeError, LPINT.from_param, c_uint*3) + +## def test_performance(self): +## check_perf() + + def test_noctypes_argtype(self): + import _ctypes_test + from ctypes import CDLL, c_void_p, ArgumentError + + func = CDLL(_ctypes_test.__file__)._testfunc_p_p + func.restype = c_void_p + # TypeError: has no from_param method + self.assertRaises(TypeError, setattr, func, "argtypes", (object,)) + + class Adapter(object): + def from_param(cls, obj): + return None + + func.argtypes = (Adapter(),) + self.failUnlessEqual(func(None), None) + self.failUnlessEqual(func(object()), None) + + class Adapter(object): + def from_param(cls, obj): + return obj + + func.argtypes = (Adapter(),) + # don't know how to convert parameter 1 + self.assertRaises(ArgumentError, func, object()) + self.failUnlessEqual(func(c_void_p(42)), 42) + + class Adapter(object): + def from_param(cls, obj): + raise ValueError(obj) + + func.argtypes = (Adapter(),) + # ArgumentError: argument 1: ValueError: 99 + self.assertRaises(ArgumentError, func, 99) + + +################################################################ + +if __name__ == '__main__': + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_pointers.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_pointers.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,179 @@ +import unittest + +from ctypes import * +import _ctypes_test + +ctype_types = [c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, + c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float] +python_types = [int, int, int, int, int, long, + int, long, long, long, float, float] + +class PointersTestCase(unittest.TestCase): + + def test_pointer_crash(self): + + class A(POINTER(c_ulong)): + pass + + POINTER(c_ulong)(c_ulong(22)) + # Pointer can't set contents: has no _type_ + self.failUnlessRaises(TypeError, A, c_ulong(33)) + + def test_pass_pointers(self): + dll = CDLL(_ctypes_test.__file__) + func = dll._testfunc_p_p + func.restype = c_long + + i = c_int(12345678) +## func.argtypes = (POINTER(c_int),) + address = func(byref(i)) + self.failUnlessEqual(c_int.from_address(address).value, 12345678) + + func.restype = POINTER(c_int) + res = func(pointer(i)) + self.failUnlessEqual(res.contents.value, 12345678) + self.failUnlessEqual(res[0], 12345678) + + def test_change_pointers(self): + dll = CDLL(_ctypes_test.__file__) + func = dll._testfunc_p_p + + i = c_int(87654) + func.restype = POINTER(c_int) + func.argtypes = (POINTER(c_int),) + + res = func(pointer(i)) + self.failUnlessEqual(res[0], 87654) + self.failUnlessEqual(res.contents.value, 87654) + + # C code: *res = 54345 + res[0] = 54345 + self.failUnlessEqual(i.value, 54345) + + # C code: + # int x = 12321; + # res = &x + res.contents = c_int(12321) + self.failUnlessEqual(i.value, 54345) + + def test_callbacks_with_pointers(self): + # a function type receiving a pointer + PROTOTYPE = CFUNCTYPE(c_int, POINTER(c_int)) + + self.result = [] + + def func(arg): + for i in range(10): +## print arg[i], + self.result.append(arg[i]) +## print + return 0 + callback = PROTOTYPE(func) + + dll = CDLL(_ctypes_test.__file__) + # This function expects a function pointer, + # and calls this with an integer pointer as parameter. + # The int pointer points to a table containing the numbers 1..10 + doit = dll._testfunc_callback_with_pointer + +## i = c_int(42) +## callback(byref(i)) +## self.failUnless(i.value == 84) + + doit(callback) +## print self.result + doit(callback) +## print self.result + + def test_basics(self): + from operator import delitem + for ct, pt in zip(ctype_types, python_types): + i = ct(42) + p = pointer(i) +## print type(p.contents), ct + self.failUnless(type(p.contents) is ct) + # p.contents is the same as p[0] +## print p.contents +## self.failUnless(p.contents == 42) +## self.failUnless(p[0] == 42) + + self.assertRaises(TypeError, delitem, p, 0) + + def test_from_address(self): + from array import array + a = array('i', [100, 200, 300, 400, 500]) + addr = a.buffer_info()[0] + + p = POINTER(POINTER(c_int)) +## print dir(p) +## print p.from_address +## print p.from_address(addr)[0][0] + + def test_other(self): + class Table(Structure): + _fields_ = [("a", c_int), + ("b", c_int), + ("c", c_int)] + + pt = pointer(Table(1, 2, 3)) + + self.failUnlessEqual(pt.contents.a, 1) + self.failUnlessEqual(pt.contents.b, 2) + self.failUnlessEqual(pt.contents.c, 3) + + pt.contents.c = 33 + + from ctypes import _pointer_type_cache + del _pointer_type_cache[Table] + + def test_basic(self): + p = pointer(c_int(42)) + # Although a pointer can be indexed, it ha no length + self.assertRaises(TypeError, len, p) + self.failUnlessEqual(p[0], 42) + self.failUnlessEqual(p.contents.value, 42) + + def test_charpp(self): + """Test that a character pointer-to-pointer is correctly passed""" + dll = CDLL(_ctypes_test.__file__) + func = dll._testfunc_c_p_p + func.restype = c_char_p + argv = (c_char_p * 2)() + argc = c_int( 2 ) + argv[0] = 'hello' + argv[1] = 'world' + result = func( byref(argc), argv ) + assert result == 'world', result + + def test_bug_1467852(self): + # http://sourceforge.net/tracker/?func=detail&atid=532154&aid=1467852&group_id=71702 + x = c_int(5) + dummy = [] + for i in range(32000): + dummy.append(c_int(i)) + y = c_int(6) + p = pointer(x) + pp = pointer(p) + q = pointer(y) + pp[0] = q # <== + self.failUnlessEqual(p[0], 6) + def test_c_void_p(self): + # http://sourceforge.net/tracker/?func=detail&aid=1518190&group_id=5470&atid=105470 + if sizeof(c_void_p) == 4: + self.failUnlessEqual(c_void_p(0xFFFFFFFFL).value, + c_void_p(-1).value) + self.failUnlessEqual(c_void_p(0xFFFFFFFFFFFFFFFFL).value, + c_void_p(-1).value) + elif sizeof(c_void_p) == 8: + self.failUnlessEqual(c_void_p(0xFFFFFFFFL).value, + 0xFFFFFFFFL) + self.failUnlessEqual(c_void_p(0xFFFFFFFFFFFFFFFFL).value, + c_void_p(-1).value) + self.failUnlessEqual(c_void_p(0xFFFFFFFFFFFFFFFFFFFFFFFFL).value, + c_void_p(-1).value) + + self.assertRaises(TypeError, c_void_p, 3.14) # make sure floats are NOT accepted + self.assertRaises(TypeError, c_void_p, object()) # nor other objects + +if __name__ == '__main__': + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_prototypes.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_prototypes.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,203 @@ +from ctypes import * +import unittest + +# IMPORTANT INFO: +# +# Consider this call: +# func.restype = c_char_p +# func(c_char_p("123")) +# It returns +# "123" +# +# WHY IS THIS SO? +# +# argument tuple (c_char_p("123"), ) is destroyed after the function +# func is called, but NOT before the result is actually built. +# +# If the arglist would be destroyed BEFORE the result has been built, +# the c_char_p("123") object would already have a zero refcount, +# and the pointer passed to (and returned by) the function would +# probably point to deallocated space. +# +# In this case, there would have to be an additional reference to the argument... + +import _ctypes_test +testdll = CDLL(_ctypes_test.__file__) + +# Return machine address `a` as a (possibly long) non-negative integer. +# Starting with Python 2.5, id(anything) is always non-negative, and +# the ctypes addressof() inherits that via PyLong_FromVoidPtr(). +def positive_address(a): + if a >= 0: + return a + # View the bits in `a` as unsigned instead. + import struct + num_bits = struct.calcsize("P") * 8 # num bits in native machine address + a += 1L << num_bits + assert a >= 0 + return a + +def c_wbuffer(init): + n = len(init) + 1 + return (c_wchar * n)(*init) + +class CharPointersTestCase(unittest.TestCase): + + def setUp(self): + func = testdll._testfunc_p_p + func.restype = c_long + func.argtypes = None + + def test_int_pointer_arg(self): + func = testdll._testfunc_p_p + func.restype = c_long + self.failUnlessEqual(0, func(0)) + + ci = c_int(0) + + func.argtypes = POINTER(c_int), + self.failUnlessEqual(positive_address(addressof(ci)), + positive_address(func(byref(ci)))) + + func.argtypes = c_char_p, + self.assertRaises(ArgumentError, func, byref(ci)) + + func.argtypes = POINTER(c_short), + self.assertRaises(ArgumentError, func, byref(ci)) + + func.argtypes = POINTER(c_double), + self.assertRaises(ArgumentError, func, byref(ci)) + + def test_POINTER_c_char_arg(self): + func = testdll._testfunc_p_p + func.restype = c_char_p + func.argtypes = POINTER(c_char), + + self.failUnlessEqual(None, func(None)) + self.failUnlessEqual("123", func("123")) + self.failUnlessEqual(None, func(c_char_p(None))) + self.failUnlessEqual("123", func(c_char_p("123"))) + + self.failUnlessEqual("123", func(c_buffer("123"))) + ca = c_char("a") + self.failUnlessEqual("a", func(pointer(ca))[0]) + self.failUnlessEqual("a", func(byref(ca))[0]) + + def test_c_char_p_arg(self): + func = testdll._testfunc_p_p + func.restype = c_char_p + func.argtypes = c_char_p, + + self.failUnlessEqual(None, func(None)) + self.failUnlessEqual("123", func("123")) + self.failUnlessEqual(None, func(c_char_p(None))) + self.failUnlessEqual("123", func(c_char_p("123"))) + + self.failUnlessEqual("123", func(c_buffer("123"))) + ca = c_char("a") + self.failUnlessEqual("a", func(pointer(ca))[0]) + self.failUnlessEqual("a", func(byref(ca))[0]) + + def test_c_void_p_arg(self): + func = testdll._testfunc_p_p + func.restype = c_char_p + func.argtypes = c_void_p, + + self.failUnlessEqual(None, func(None)) + self.failUnlessEqual("123", func("123")) + self.failUnlessEqual("123", func(c_char_p("123"))) + self.failUnlessEqual(None, func(c_char_p(None))) + + self.failUnlessEqual("123", func(c_buffer("123"))) + ca = c_char("a") + self.failUnlessEqual("a", func(pointer(ca))[0]) + self.failUnlessEqual("a", func(byref(ca))[0]) + + func(byref(c_int())) + func(pointer(c_int())) + func((c_int * 3)()) + + try: + func.restype = c_wchar_p + except NameError: + pass + else: + self.failUnlessEqual(None, func(c_wchar_p(None))) + self.failUnlessEqual(u"123", func(c_wchar_p(u"123"))) + + def test_instance(self): + func = testdll._testfunc_p_p + func.restype = c_void_p + + class X: + _as_parameter_ = None + + func.argtypes = c_void_p, + self.failUnlessEqual(None, func(X())) + + func.argtypes = None + self.failUnlessEqual(None, func(X())) + +try: + c_wchar +except NameError: + pass +else: + class WCharPointersTestCase(unittest.TestCase): + + def setUp(self): + func = testdll._testfunc_p_p + func.restype = c_int + func.argtypes = None + + + def test_POINTER_c_wchar_arg(self): + func = testdll._testfunc_p_p + func.restype = c_wchar_p + func.argtypes = POINTER(c_wchar), + + self.failUnlessEqual(None, func(None)) + self.failUnlessEqual(u"123", func(u"123")) + self.failUnlessEqual(None, func(c_wchar_p(None))) + self.failUnlessEqual(u"123", func(c_wchar_p(u"123"))) + + self.failUnlessEqual(u"123", func(c_wbuffer(u"123"))) + ca = c_wchar("a") + self.failUnlessEqual(u"a", func(pointer(ca))[0]) + self.failUnlessEqual(u"a", func(byref(ca))[0]) + + def test_c_wchar_p_arg(self): + func = testdll._testfunc_p_p + func.restype = c_wchar_p + func.argtypes = c_wchar_p, + + c_wchar_p.from_param(u"123") + + self.failUnlessEqual(None, func(None)) + self.failUnlessEqual("123", func(u"123")) + self.failUnlessEqual(None, func(c_wchar_p(None))) + self.failUnlessEqual("123", func(c_wchar_p("123"))) + + # XXX Currently, these raise TypeErrors, although they shouldn't: + self.failUnlessEqual("123", func(c_wbuffer("123"))) + ca = c_wchar("a") + self.failUnlessEqual("a", func(pointer(ca))[0]) + self.failUnlessEqual("a", func(byref(ca))[0]) + +class ArrayTest(unittest.TestCase): + def test(self): + func = testdll._testfunc_ai8 + func.restype = POINTER(c_int) + func.argtypes = c_int * 8, + + func((c_int * 8)(1, 2, 3, 4, 5, 6, 7, 8)) + + # This did crash before: + + def func(): pass + CFUNCTYPE(None, c_int * 3)(func) + +################################################################ + +if __name__ == '__main__': + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_python_api.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_python_api.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,87 @@ +from ctypes import * +import unittest, sys +from ctypes.test import is_resource_enabled + +################################################################ +# This section should be moved into ctypes\__init__.py, when it's ready. + +from _ctypes import PyObj_FromPtr + +################################################################ + +from sys import getrefcount as grc + + +class PythonAPITestCase(unittest.TestCase): + + def test_PyString_FromStringAndSize(self): + PyString_FromStringAndSize = pythonapi.PyString_FromStringAndSize + + PyString_FromStringAndSize.restype = py_object + PyString_FromStringAndSize.argtypes = c_char_p, c_int + + self.failUnlessEqual(PyString_FromStringAndSize("abcdefghi", 3), "abc") + + def test_PyString_FromString(self): + pythonapi.PyString_FromString.restype = py_object + pythonapi.PyString_FromString.argtypes = (c_char_p,) + + s = "abc" + refcnt = grc(s) + pyob = pythonapi.PyString_FromString(s) + self.failUnlessEqual(grc(s), refcnt) + self.failUnlessEqual(s, pyob) + del pyob + self.failUnlessEqual(grc(s), refcnt) + + if is_resource_enabled("refcount"): + # This test is unreliable, because it is possible that code in + # unittest changes the refcount of the '42' integer. So, it + # is disabled by default. + def test_PyInt_Long(self): + ref42 = grc(42) + pythonapi.PyInt_FromLong.restype = py_object + self.failUnlessEqual(pythonapi.PyInt_FromLong(42), 42) + + self.failUnlessEqual(grc(42), ref42) + + pythonapi.PyInt_AsLong.argtypes = (py_object,) + pythonapi.PyInt_AsLong.restype = c_long + + res = pythonapi.PyInt_AsLong(42) + self.failUnlessEqual(grc(res), ref42 + 1) + del res + self.failUnlessEqual(grc(42), ref42) + + def test_PyObj_FromPtr(self): + s = "abc def ghi jkl" + ref = grc(s) + # id(python-object) is the address + pyobj = PyObj_FromPtr(id(s)) + self.failUnless(s is pyobj) + + self.failUnlessEqual(grc(s), ref + 1) + del pyobj + self.failUnlessEqual(grc(s), ref) + + def test_PyOS_snprintf(self): + PyOS_snprintf = pythonapi.PyOS_snprintf + PyOS_snprintf.argtypes = POINTER(c_char), c_int, c_char_p + + buf = c_buffer(256) + PyOS_snprintf(buf, sizeof(buf), "Hello from %s", "ctypes") + self.failUnlessEqual(buf.value, "Hello from ctypes") + + PyOS_snprintf(buf, sizeof(buf), "Hello from %s", "ctypes", 1, 2, 3) + self.failUnlessEqual(buf.value, "Hello from ctypes") + + # not enough arguments + self.failUnlessRaises(TypeError, PyOS_snprintf, buf) + + def test_pyobject_repr(self): + self.failUnlessEqual(repr(py_object()), "py_object()") + self.failUnlessEqual(repr(py_object(42)), "py_object(42)") + self.failUnlessEqual(repr(py_object(object)), "py_object(%r)" % object) + +if __name__ == "__main__": + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_random_things.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_random_things.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,71 @@ +from ctypes import * +import unittest, sys + +def callback_func(arg): + 42 / arg + raise ValueError, arg + +if sys.platform == "win32": + + class call_function_TestCase(unittest.TestCase): + # _ctypes.call_function is deprecated and private, but used by + # Gary Bishp's readline module. If we have it, we must test it as well. + + def test(self): + from _ctypes import call_function + hdll = windll.kernel32.LoadLibraryA("kernel32") + funcaddr = windll.kernel32.GetProcAddress(hdll, "GetModuleHandleA") + + self.failUnlessEqual(call_function(funcaddr, (None,)), + windll.kernel32.GetModuleHandleA(None)) + +class CallbackTracbackTestCase(unittest.TestCase): + # When an exception is raised in a ctypes callback function, the C + # code prints a traceback. + # + # This test makes sure the exception types *and* the exception + # value is printed correctly. + # + # Changed in 0.9.3: No longer is '(in callback)' prepended to the + # error message - instead a additional frame for the C code is + # created, then a full traceback printed. When SystemExit is + # raised in a callback function, the interpreter exits. + + def capture_stderr(self, func, *args, **kw): + # helper - call function 'func', and return the captured stderr + import StringIO + old_stderr = sys.stderr + logger = sys.stderr = StringIO.StringIO() + try: + func(*args, **kw) + finally: + sys.stderr = old_stderr + return logger.getvalue() + + def test_ValueError(self): + cb = CFUNCTYPE(c_int, c_int)(callback_func) + out = self.capture_stderr(cb, 42) + self.failUnlessEqual(out.splitlines()[-1], + "ValueError: 42") + + def test_IntegerDivisionError(self): + cb = CFUNCTYPE(c_int, c_int)(callback_func) + out = self.capture_stderr(cb, 0) + self.failUnlessEqual(out.splitlines()[-1][:19], + "ZeroDivisionError: ") + + def test_FloatDivisionError(self): + cb = CFUNCTYPE(c_int, c_double)(callback_func) + out = self.capture_stderr(cb, 0.0) + self.failUnlessEqual(out.splitlines()[-1][:19], + "ZeroDivisionError: ") + + def test_TypeErrorDivisionError(self): + cb = CFUNCTYPE(c_int, c_char_p)(callback_func) + out = self.capture_stderr(cb, "spam") + self.failUnlessEqual(out.splitlines()[-1], + "TypeError: " + "unsupported operand type(s) for /: 'int' and 'str'") + +if __name__ == '__main__': + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_refcounts.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_refcounts.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,98 @@ +import unittest +import ctypes +import gc + +MyCallback = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int) +OtherCallback = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_ulonglong) + +import _ctypes_test +dll = ctypes.CDLL(_ctypes_test.__file__) + +class RefcountTestCase(unittest.TestCase): + + def test_1(self): + from sys import getrefcount as grc + + f = dll._testfunc_callback_i_if + f.restype = ctypes.c_int + f.argtypes = [ctypes.c_int, MyCallback] + + def callback(value): + #print "called back with", value + return value + + self.failUnlessEqual(grc(callback), 2) + cb = MyCallback(callback) + + self.failUnless(grc(callback) > 2) + result = f(-10, cb) + self.failUnlessEqual(result, -18) + cb = None + + gc.collect() + + self.failUnlessEqual(grc(callback), 2) + + + def test_refcount(self): + from sys import getrefcount as grc + def func(*args): + pass + # this is the standard refcount for func + self.failUnlessEqual(grc(func), 2) + + # the CFuncPtr instance holds atr least one refcount on func: + f = OtherCallback(func) + self.failUnless(grc(func) > 2) + + # and may release it again + del f + self.failUnless(grc(func) >= 2) + + # but now it must be gone + gc.collect() + self.failUnless(grc(func) == 2) + + class X(ctypes.Structure): + _fields_ = [("a", OtherCallback)] + x = X() + x.a = OtherCallback(func) + + # the CFuncPtr instance holds atr least one refcount on func: + self.failUnless(grc(func) > 2) + + # and may release it again + del x + self.failUnless(grc(func) >= 2) + + # and now it must be gone again + gc.collect() + self.failUnlessEqual(grc(func), 2) + + f = OtherCallback(func) + + # the CFuncPtr instance holds atr least one refcount on func: + self.failUnless(grc(func) > 2) + + # create a cycle + f.cycle = f + + del f + gc.collect() + self.failUnlessEqual(grc(func), 2) + +class AnotherLeak(unittest.TestCase): + def test_callback(self): + import sys + + proto = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_int) + def func(a, b): + return a * b * 2 + f = proto(func) + + a = sys.getrefcount(ctypes.c_int) + f(1, 2) + self.failUnlessEqual(sys.getrefcount(ctypes.c_int), a) + +if __name__ == '__main__': + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_repr.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_repr.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,29 @@ +from ctypes import * +import unittest + +subclasses = [] +for base in [c_byte, c_short, c_int, c_long, c_longlong, + c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong, + c_float, c_double]: + class X(base): + pass + subclasses.append(X) + +class X(c_char): + pass + +# This test checks if the __repr__ is correct for subclasses of simple types + +class ReprTest(unittest.TestCase): + def test_numbers(self): + for typ in subclasses: + base = typ.__bases__[0] + self.failUnless(repr(base(42)).startswith(base.__name__)) + self.failUnlessEqual(": " + "expected string or Unicode object, int found") + else: + self.failUnlessEqual(msg, + "(Phone) exceptions.TypeError: " + "expected string or Unicode object, int found") + + cls, msg = self.get_except(Person, "Someone", ("a", "b", "c")) + self.failUnlessEqual(cls, RuntimeError) + if issubclass(Exception, object): + self.failUnlessEqual(msg, + "(Phone) : too many initializers") + else: + self.failUnlessEqual(msg, "(Phone) exceptions.ValueError: too many initializers") + + + def get_except(self, func, *args): + try: + func(*args) + except Exception, detail: + return detail.__class__, str(detail) + + +## def test_subclass_creation(self): +## meta = type(Structure) +## # same as 'class X(Structure): pass' +## # fails, since we need either a _fields_ or a _abstract_ attribute +## cls, msg = self.get_except(meta, "X", (Structure,), {}) +## self.failUnlessEqual((cls, msg), +## (AttributeError, "class must define a '_fields_' attribute")) + + def test_abstract_class(self): + class X(Structure): + _abstract_ = "something" + # try 'X()' + cls, msg = self.get_except(eval, "X()", locals()) + self.failUnlessEqual((cls, msg), (TypeError, "abstract class")) + + def test_methods(self): +## class X(Structure): +## _fields_ = [] + + self.failUnless("in_dll" in dir(type(Structure))) + self.failUnless("from_address" in dir(type(Structure))) + self.failUnless("in_dll" in dir(type(Structure))) + +class PointerMemberTestCase(unittest.TestCase): + + def test(self): + # a Structure with a POINTER field + class S(Structure): + _fields_ = [("array", POINTER(c_int))] + + s = S() + # We can assign arrays of the correct type + s.array = (c_int * 3)(1, 2, 3) + items = [s.array[i] for i in range(3)] + self.failUnlessEqual(items, [1, 2, 3]) + + # The following are bugs, but are included here because the unittests + # also describe the current behaviour. + # + # This fails with SystemError: bad arg to internal function + # or with IndexError (with a patch I have) + + s.array[0] = 42 + + items = [s.array[i] for i in range(3)] + self.failUnlessEqual(items, [42, 2, 3]) + + s.array[0] = 1 + +## s.array[1] = 42 + + items = [s.array[i] for i in range(3)] + self.failUnlessEqual(items, [1, 2, 3]) + + def test_none_to_pointer_fields(self): + class S(Structure): + _fields_ = [("x", c_int), + ("p", POINTER(c_int))] + + s = S() + s.x = 12345678 + s.p = None + self.failUnlessEqual(s.x, 12345678) + +class TestRecursiveStructure(unittest.TestCase): + def test_contains_itself(self): + class Recursive(Structure): + pass + + try: + Recursive._fields_ = [("next", Recursive)] + except AttributeError, details: + self.failUnless("Structure or union cannot contain itself" in + str(details)) + else: + self.fail("Structure or union cannot contain itself") + + + def test_vice_versa(self): + class First(Structure): + pass + class Second(Structure): + pass + + First._fields_ = [("second", Second)] + + try: + Second._fields_ = [("first", First)] + except AttributeError, details: + self.failUnless("_fields_ is final" in + str(details)) + else: + self.fail("AttributeError not raised") + +if __name__ == '__main__': + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_unaligned_structures.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_unaligned_structures.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,45 @@ +import sys, unittest +from ctypes import * + +structures = [] +byteswapped_structures = [] + + +if sys.byteorder == "little": + SwappedStructure = BigEndianStructure +else: + SwappedStructure = LittleEndianStructure + +for typ in [c_short, c_int, c_long, c_longlong, + c_float, c_double, + c_ushort, c_uint, c_ulong, c_ulonglong]: + class X(Structure): + _pack_ = 1 + _fields_ = [("pad", c_byte), + ("value", typ)] + class Y(SwappedStructure): + _pack_ = 1 + _fields_ = [("pad", c_byte), + ("value", typ)] + structures.append(X) + byteswapped_structures.append(Y) + +class TestStructures(unittest.TestCase): + def test_native(self): + for typ in structures: +## print typ.value + self.failUnlessEqual(typ.value.offset, 1) + o = typ() + o.value = 4 + self.failUnlessEqual(o.value, 4) + + def test_swapped(self): + for typ in byteswapped_structures: +## print >> sys.stderr, typ.value + self.failUnlessEqual(typ.value.offset, 1) + o = typ() + o.value = 4 + self.failUnlessEqual(o.value, 4) + +if __name__ == '__main__': + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_unicode.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_unicode.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,115 @@ +# coding: latin-1 +import unittest +import ctypes + +try: + ctypes.c_wchar +except AttributeError: + pass +else: + import _ctypes_test + dll = ctypes.CDLL(_ctypes_test.__file__) + wcslen = dll.my_wcslen + wcslen.argtypes = [ctypes.c_wchar_p] + + + class UnicodeTestCase(unittest.TestCase): + def setUp(self): + self.prev_conv_mode = ctypes.set_conversion_mode("ascii", "strict") + + def tearDown(self): + ctypes.set_conversion_mode(*self.prev_conv_mode) + + def test_ascii_strict(self): + ctypes.set_conversion_mode("ascii", "strict") + # no conversions take place with unicode arguments + self.failUnlessEqual(wcslen(u"abc"), 3) + self.failUnlessEqual(wcslen(u"ab\u2070"), 3) + # string args are converted + self.failUnlessEqual(wcslen("abc"), 3) + self.failUnlessRaises(ctypes.ArgumentError, wcslen, "ab?") + + def test_ascii_replace(self): + ctypes.set_conversion_mode("ascii", "replace") + self.failUnlessEqual(wcslen(u"abc"), 3) + self.failUnlessEqual(wcslen(u"ab\u2070"), 3) + self.failUnlessEqual(wcslen("abc"), 3) + self.failUnlessEqual(wcslen("ab?"), 3) + + def test_ascii_ignore(self): + ctypes.set_conversion_mode("ascii", "ignore") + self.failUnlessEqual(wcslen(u"abc"), 3) + self.failUnlessEqual(wcslen(u"ab\u2070"), 3) + # ignore error mode skips non-ascii characters + self.failUnlessEqual(wcslen("abc"), 3) + self.failUnlessEqual(wcslen("????"), 0) + + def test_latin1_strict(self): + ctypes.set_conversion_mode("latin-1", "strict") + self.failUnlessEqual(wcslen(u"abc"), 3) + self.failUnlessEqual(wcslen(u"ab\u2070"), 3) + self.failUnlessEqual(wcslen("abc"), 3) + self.failUnlessEqual(wcslen("????"), 4) + + def test_buffers(self): + ctypes.set_conversion_mode("ascii", "strict") + buf = ctypes.create_unicode_buffer("abc") + self.failUnlessEqual(len(buf), 3+1) + + ctypes.set_conversion_mode("ascii", "replace") + buf = ctypes.create_unicode_buffer("ab???") + self.failUnlessEqual(buf[:], u"ab\uFFFD\uFFFD\uFFFD\0") + + ctypes.set_conversion_mode("ascii", "ignore") + buf = ctypes.create_unicode_buffer("ab???") + # is that correct? not sure. But with 'ignore', you get what you pay for.. + self.failUnlessEqual(buf[:], u"ab\0\0\0\0") + + import _ctypes_test + func = ctypes.CDLL(_ctypes_test.__file__)._testfunc_p_p + + class StringTestCase(UnicodeTestCase): + def setUp(self): + self.prev_conv_mode = ctypes.set_conversion_mode("ascii", "strict") + func.argtypes = [ctypes.c_char_p] + func.restype = ctypes.c_char_p + + def tearDown(self): + ctypes.set_conversion_mode(*self.prev_conv_mode) + func.argtypes = None + func.restype = ctypes.c_int + + def test_ascii_replace(self): + ctypes.set_conversion_mode("ascii", "strict") + self.failUnlessEqual(func("abc"), "abc") + self.failUnlessEqual(func(u"abc"), "abc") + self.assertRaises(ctypes.ArgumentError, func, u"ab?") + + def test_ascii_ignore(self): + ctypes.set_conversion_mode("ascii", "ignore") + self.failUnlessEqual(func("abc"), "abc") + self.failUnlessEqual(func(u"abc"), "abc") + self.failUnlessEqual(func(u"????"), "") + + def test_ascii_replace(self): + ctypes.set_conversion_mode("ascii", "replace") + self.failUnlessEqual(func("abc"), "abc") + self.failUnlessEqual(func(u"abc"), "abc") + self.failUnlessEqual(func(u"????"), "????") + + def test_buffers(self): + ctypes.set_conversion_mode("ascii", "strict") + buf = ctypes.create_string_buffer(u"abc") + self.failUnlessEqual(len(buf), 3+1) + + ctypes.set_conversion_mode("ascii", "replace") + buf = ctypes.create_string_buffer(u"ab???") + self.failUnlessEqual(buf[:], "ab???\0") + + ctypes.set_conversion_mode("ascii", "ignore") + buf = ctypes.create_string_buffer(u"ab???") + # is that correct? not sure. But with 'ignore', you get what you pay for.. + self.failUnlessEqual(buf[:], "ab\0\0\0\0") + +if __name__ == '__main__': + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_values.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_values.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,82 @@ +""" +A testcase which accesses *values* in a dll. +""" + +import unittest +from ctypes import * + +import _ctypes_test + +class ValuesTestCase(unittest.TestCase): + + def test_an_integer(self): + ctdll = CDLL(_ctypes_test.__file__) + an_integer = c_int.in_dll(ctdll, "an_integer") + x = an_integer.value + self.failUnlessEqual(x, ctdll.get_an_integer()) + an_integer.value *= 2 + self.failUnlessEqual(x*2, ctdll.get_an_integer()) + + def test_undefined(self): + ctdll = CDLL(_ctypes_test.__file__) + self.assertRaises(ValueError, c_int.in_dll, ctdll, "Undefined_Symbol") + + class Win_ValuesTestCase(unittest.TestCase): + """This test only works when python itself is a dll/shared library""" + + def test_optimizeflag(self): + # This test accesses the Py_OptimizeFlag intger, which is + # exported by the Python dll. + + # It's value is set depending on the -O and -OO flags: + # if not given, it is 0 and __debug__ is 1. + # If -O is given, the flag is 1, for -OO it is 2. + # docstrings are also removed in the latter case. + opt = c_int.in_dll(pydll, "Py_OptimizeFlag").value + if __debug__: + self.failUnlessEqual(opt, 0) + elif ValuesTestCase.__doc__ is not None: + self.failUnlessEqual(opt, 1) + else: + self.failUnlessEqual(opt, 2) + + def test_frozentable(self): + # Python exports a PyImport_FrozenModules symbol. This is a + # pointer to an array of struct _frozen entries. The end of the + # array is marked by an entry containing a NULL name and zero + # size. + + # In standard Python, this table contains a __hello__ + # module, and a __phello__ package containing a spam + # module. + class struct_frozen(Structure): + _fields_ = [("name", c_char_p), + ("code", POINTER(c_ubyte)), + ("size", c_int)] + FrozenTable = POINTER(struct_frozen) + + ft = FrozenTable.in_dll(pydll, "PyImport_FrozenModules") + # ft is a pointer to the struct_frozen entries: + items = [] + for entry in ft: + # This is dangerous. We *can* iterate over a pointer, but + # the loop will not terminate (maybe with an access + # violation;-) because the pointer instance has no size. + if entry.name is None: + break + items.append((entry.name, entry.size)) + import sys + if sys.version_info[:2] >= (2, 3): + expected = [("__hello__", 104), ("__phello__", -104), ("__phello__.spam", 104)] + else: + expected = [("__hello__", 100), ("__phello__", -100), ("__phello__.spam", 100)] + self.failUnlessEqual(items, expected) + + from ctypes import _pointer_type_cache + del _pointer_type_cache[struct_frozen] + + def test_undefined(self): + self.assertRaises(ValueError, c_int.in_dll, pydll, "Undefined_Symbol") + +if __name__ == '__main__': + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_varsize_struct.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_varsize_struct.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,50 @@ +from ctypes import * +import unittest + +class VarSizeTest(unittest.TestCase): + def test_resize(self): + class X(Structure): + _fields_ = [("item", c_int), + ("array", c_int * 1)] + + self.failUnlessEqual(sizeof(X), sizeof(c_int) * 2) + x = X() + x.item = 42 + x.array[0] = 100 + self.failUnlessEqual(sizeof(x), sizeof(c_int) * 2) + + # make room for one additional item + new_size = sizeof(X) + sizeof(c_int) * 1 + resize(x, new_size) + self.failUnlessEqual(sizeof(x), new_size) + self.failUnlessEqual((x.item, x.array[0]), (42, 100)) + + # make room for 10 additional items + new_size = sizeof(X) + sizeof(c_int) * 9 + resize(x, new_size) + self.failUnlessEqual(sizeof(x), new_size) + self.failUnlessEqual((x.item, x.array[0]), (42, 100)) + + # make room for one additional item + new_size = sizeof(X) + sizeof(c_int) * 1 + resize(x, new_size) + self.failUnlessEqual(sizeof(x), new_size) + self.failUnlessEqual((x.item, x.array[0]), (42, 100)) + + def test_array_invalid_length(self): + # cannot create arrays with non-positive size + self.failUnlessRaises(ValueError, lambda: c_int * -1) + self.failUnlessRaises(ValueError, lambda: c_int * -3) + + def test_zerosized_array(self): + array = (c_int * 0)() + # accessing elements of zero-sized arrays raise IndexError + self.failUnlessRaises(IndexError, array.__setitem__, 0, None) + self.failUnlessRaises(IndexError, array.__getitem__, 0) + self.failUnlessRaises(IndexError, array.__setitem__, 1, None) + self.failUnlessRaises(IndexError, array.__getitem__, 1) + self.failUnlessRaises(IndexError, array.__setitem__, -1, None) + self.failUnlessRaises(IndexError, array.__getitem__, -1) + +if __name__ == "__main__": + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_win32.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_win32.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,60 @@ +# Windows specific tests + +from ctypes import * +from ctypes.test import is_resource_enabled +import unittest, sys + +import _ctypes_test + +if sys.platform == "win32": + + class WindowsTestCase(unittest.TestCase): + def test_callconv_1(self): + # Testing stdcall function + + IsWindow = windll.user32.IsWindow + # ValueError: Procedure probably called with not enough arguments (4 bytes missing) + self.assertRaises(ValueError, IsWindow) + + # This one should succeeed... + self.failUnlessEqual(0, IsWindow(0)) + + # ValueError: Procedure probably called with too many arguments (8 bytes in excess) + self.assertRaises(ValueError, IsWindow, 0, 0, 0) + + def test_callconv_2(self): + # Calling stdcall function as cdecl + + IsWindow = cdll.user32.IsWindow + + # ValueError: Procedure called with not enough arguments (4 bytes missing) + # or wrong calling convention + self.assertRaises(ValueError, IsWindow, None) + + if is_resource_enabled("SEH"): + def test_SEH(self): + # Call functions with invalid arguments, and make sure that access violations + # are trapped and raise an exception. + self.assertRaises(WindowsError, windll.kernel32.GetModuleHandleA, 32) + +class Structures(unittest.TestCase): + + def test_struct_by_value(self): + class POINT(Structure): + _fields_ = [("x", c_long), + ("y", c_long)] + + class RECT(Structure): + _fields_ = [("left", c_long), + ("top", c_long), + ("right", c_long), + ("bottom", c_long)] + + dll = CDLL(_ctypes_test.__file__) + + pt = POINT(10, 10) + rect = RECT(0, 0, 20, 20) + self.failUnlessEqual(1, dll.PointInRect(byref(rect), pt)) + +if __name__ == '__main__': + unittest.main() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/util.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/util.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,154 @@ +###################################################################### +# This file should be kept compatible with Python 2.3, see PEP 291. # +###################################################################### +import sys, os + +# find_library(name) returns the pathname of a library, or None. +if os.name == "nt": + def find_library(name): + # See MSDN for the REAL search order. + for directory in os.environ['PATH'].split(os.pathsep): + fname = os.path.join(directory, name) + if os.path.exists(fname): + return fname + if fname.lower().endswith(".dll"): + continue + fname = fname + ".dll" + if os.path.exists(fname): + return fname + return None + +if os.name == "ce": + # search path according to MSDN: + # - absolute path specified by filename + # - The .exe launch directory + # - the Windows directory + # - ROM dll files (where are they?) + # - OEM specified search path: HKLM\Loader\SystemPath + def find_library(name): + return name + +if os.name == "posix" and sys.platform == "darwin": + from ctypes.macholib.dyld import dyld_find as _dyld_find + def find_library(name): + possible = ['lib%s.dylib' % name, + '%s.dylib' % name, + '%s.framework/%s' % (name, name)] + for name in possible: + try: + return _dyld_find(name) + except ValueError: + continue + return None + +elif os.name == "posix": + # Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump + import re, tempfile, errno + + def _findLib_gcc(name): + expr = r'[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name) + fdout, ccout = tempfile.mkstemp() + os.close(fdout) + cmd = 'if type gcc >/dev/null 2>&1; then CC=gcc; else CC=cc; fi;' \ + '$CC -Wl,-t -o ' + ccout + ' 2>&1 -l' + name + try: + f = os.popen(cmd) + trace = f.read() + f.close() + finally: + try: + os.unlink(ccout) + except OSError, e: + if e.errno != errno.ENOENT: + raise + res = re.search(expr, trace) + if not res: + return None + return res.group(0) + + def _get_soname(f): + # assuming GNU binutils / ELF + if not f: + return None + cmd = "objdump -p -j .dynamic 2>/dev/null " + f + res = re.search(r'\sSONAME\s+([^\s]+)', os.popen(cmd).read()) + if not res: + return None + return res.group(1) + + if (sys.platform.startswith("freebsd") + or sys.platform.startswith("openbsd") + or sys.platform.startswith("dragonfly")): + + def _num_version(libname): + # "libxyz.so.MAJOR.MINOR" => [ MAJOR, MINOR ] + parts = libname.split(".") + nums = [] + try: + while parts: + nums.insert(0, int(parts.pop())) + except ValueError: + pass + return nums or [ sys.maxint ] + + def find_library(name): + ename = re.escape(name) + expr = r':-l%s\.\S+ => \S*/(lib%s\.\S+)' % (ename, ename) + res = re.findall(expr, + os.popen('/sbin/ldconfig -r 2>/dev/null').read()) + if not res: + return _get_soname(_findLib_gcc(name)) + res.sort(cmp= lambda x,y: cmp(_num_version(x), _num_version(y))) + return res[-1] + + else: + + def _findLib_ldconfig(name): + # XXX assuming GLIBC's ldconfig (with option -p) + expr = r'/[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name) + res = re.search(expr, + os.popen('/sbin/ldconfig -p 2>/dev/null').read()) + if not res: + # Hm, this works only for libs needed by the python executable. + cmd = 'ldd %s 2>/dev/null' % sys.executable + res = re.search(expr, os.popen(cmd).read()) + if not res: + return None + return res.group(0) + + def find_library(name): + return _get_soname(_findLib_ldconfig(name) or _findLib_gcc(name)) + +################################################################ +# test code + +def test(): + from ctypes import cdll + if os.name == "nt": + print cdll.msvcrt + print cdll.load("msvcrt") + print find_library("msvcrt") + + if os.name == "posix": + # find and load_version + print find_library("m") + print find_library("c") + print find_library("bz2") + + # getattr +## print cdll.m +## print cdll.bz2 + + # load + if sys.platform == "darwin": + print cdll.LoadLibrary("libm.dylib") + print cdll.LoadLibrary("libcrypto.dylib") + print cdll.LoadLibrary("libSystem.dylib") + print cdll.LoadLibrary("System.framework/System") + else: + print cdll.LoadLibrary("libm.so") + print cdll.LoadLibrary("libcrypt.so") + print find_library("crypt") + +if __name__ == "__main__": + test() Added: pypy/branch/applevel-ctypes/pypy/lib/ctypes/wintypes.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/wintypes.py Fri Dec 14 16:53:49 2007 @@ -0,0 +1,172 @@ +###################################################################### +# This file should be kept compatible with Python 2.3, see PEP 291. # +###################################################################### + +# The most useful windows datatypes +from ctypes import * + +BYTE = c_byte +WORD = c_ushort +DWORD = c_ulong + +WCHAR = c_wchar +UINT = c_uint + +DOUBLE = c_double + +BOOLEAN = BYTE +BOOL = c_long + +from ctypes import _SimpleCData +class VARIANT_BOOL(_SimpleCData): + _type_ = "v" + def __repr__(self): + return "%s(%r)" % (self.__class__.__name__, self.value) + +ULONG = c_ulong +LONG = c_long + +# in the windows header files, these are structures. +_LARGE_INTEGER = LARGE_INTEGER = c_longlong +_ULARGE_INTEGER = ULARGE_INTEGER = c_ulonglong + +LPCOLESTR = LPOLESTR = OLESTR = c_wchar_p +LPCWSTR = LPWSTR = c_wchar_p +LPCSTR = LPSTR = c_char_p + +WPARAM = c_uint +LPARAM = c_long + +ATOM = WORD +LANGID = WORD + +COLORREF = DWORD +LGRPID = DWORD +LCTYPE = DWORD + +LCID = DWORD + +################################################################ +# HANDLE types +HANDLE = c_ulong # in the header files: void * + +HACCEL = HANDLE +HBITMAP = HANDLE +HBRUSH = HANDLE +HCOLORSPACE = HANDLE +HDC = HANDLE +HDESK = HANDLE +HDWP = HANDLE +HENHMETAFILE = HANDLE +HFONT = HANDLE +HGDIOBJ = HANDLE +HGLOBAL = HANDLE +HHOOK = HANDLE +HICON = HANDLE +HINSTANCE = HANDLE +HKEY = HANDLE +HKL = HANDLE +HLOCAL = HANDLE +HMENU = HANDLE +HMETAFILE = HANDLE +HMODULE = HANDLE +HMONITOR = HANDLE +HPALETTE = HANDLE +HPEN = HANDLE +HRGN = HANDLE +HRSRC = HANDLE +HSTR = HANDLE +HTASK = HANDLE +HWINSTA = HANDLE +HWND = HANDLE +SC_HANDLE = HANDLE +SERVICE_STATUS_HANDLE = HANDLE + +################################################################ +# Some important structure definitions + +class RECT(Structure): + _fields_ = [("left", c_long), + ("top", c_long), + ("right", c_long), + ("bottom", c_long)] +tagRECT = _RECTL = RECTL = RECT + +class _SMALL_RECT(Structure): + _fields_ = [('Left', c_short), + ('Top', c_short), + ('Right', c_short), + ('Bottom', c_short)] +SMALL_RECT = _SMALL_RECT + +class _COORD(Structure): + _fields_ = [('X', c_short), + ('Y', c_short)] + +class POINT(Structure): + _fields_ = [("x", c_long), + ("y", c_long)] +tagPOINT = _POINTL = POINTL = POINT + +class SIZE(Structure): + _fields_ = [("cx", c_long), + ("cy", c_long)] +tagSIZE = SIZEL = SIZE + +def RGB(red, green, blue): + return red + (green << 8) + (blue << 16) + +class FILETIME(Structure): + _fields_ = [("dwLowDateTime", DWORD), + ("dwHighDateTime", DWORD)] +_FILETIME = FILETIME + +class MSG(Structure): + _fields_ = [("hWnd", HWND), + ("message", c_uint), + ("wParam", WPARAM), + ("lParam", LPARAM), + ("time", DWORD), + ("pt", POINT)] +tagMSG = MSG +MAX_PATH = 260 + +class WIN32_FIND_DATAA(Structure): + _fields_ = [("dwFileAttributes", DWORD), + ("ftCreationTime", FILETIME), + ("ftLastAccessTime", FILETIME), + ("ftLastWriteTime", FILETIME), + ("nFileSizeHigh", DWORD), + ("nFileSizeLow", DWORD), + ("dwReserved0", DWORD), + ("dwReserved1", DWORD), + ("cFileName", c_char * MAX_PATH), + ("cAlternameFileName", c_char * 14)] + +class WIN32_FIND_DATAW(Structure): + _fields_ = [("dwFileAttributes", DWORD), + ("ftCreationTime", FILETIME), + ("ftLastAccessTime", FILETIME), + ("ftLastWriteTime", FILETIME), + ("nFileSizeHigh", DWORD), + ("nFileSizeLow", DWORD), + ("dwReserved0", DWORD), + ("dwReserved1", DWORD), + ("cFileName", c_wchar * MAX_PATH), + ("cAlternameFileName", c_wchar * 14)] + +__all__ = ['ATOM', 'BOOL', 'BOOLEAN', 'BYTE', 'COLORREF', 'DOUBLE', + 'DWORD', 'FILETIME', 'HACCEL', 'HANDLE', 'HBITMAP', 'HBRUSH', + 'HCOLORSPACE', 'HDC', 'HDESK', 'HDWP', 'HENHMETAFILE', 'HFONT', + 'HGDIOBJ', 'HGLOBAL', 'HHOOK', 'HICON', 'HINSTANCE', 'HKEY', + 'HKL', 'HLOCAL', 'HMENU', 'HMETAFILE', 'HMODULE', 'HMONITOR', + 'HPALETTE', 'HPEN', 'HRGN', 'HRSRC', 'HSTR', 'HTASK', 'HWINSTA', + 'HWND', 'LANGID', 'LARGE_INTEGER', 'LCID', 'LCTYPE', 'LGRPID', + 'LONG', 'LPARAM', 'LPCOLESTR', 'LPCSTR', 'LPCWSTR', 'LPOLESTR', + 'LPSTR', 'LPWSTR', 'MAX_PATH', 'MSG', 'OLESTR', 'POINT', + 'POINTL', 'RECT', 'RECTL', 'RGB', 'SC_HANDLE', + 'SERVICE_STATUS_HANDLE', 'SIZE', 'SIZEL', 'SMALL_RECT', 'UINT', + 'ULARGE_INTEGER', 'ULONG', 'VARIANT_BOOL', 'WCHAR', + 'WIN32_FIND_DATAA', 'WIN32_FIND_DATAW', 'WORD', 'WPARAM', '_COORD', + '_FILETIME', '_LARGE_INTEGER', '_POINTL', '_RECTL', '_SMALL_RECT', + '_ULARGE_INTEGER', 'tagMSG', 'tagPOINT', 'tagRECT', 'tagSIZE'] From arigo at codespeak.net Fri Dec 14 16:54:44 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Dec 2007 16:54:44 +0100 (CET) Subject: [pypy-svn] r49792 - pypy/branch/lazy-write-barrier/pypy/rpython/memory Message-ID: <20071214155444.65997168519@codespeak.net> Author: arigo Date: Fri Dec 14 16:54:44 2007 New Revision: 49792 Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py Log: Don't overwrite the flags provided by GenerationGC.init_gc_object_immortal(). Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py Fri Dec 14 16:54:44 2007 @@ -153,7 +153,6 @@ # XXX hack, a lot of gengc details here from pypy.rpython.memory.gc.generation import GenerationGC - from pypy.rpython.memory.gc.generation import GCFLAG_NEVER_SET if isinstance(gc, GenerationGC): gen_gc = True else: @@ -163,10 +162,7 @@ typeid = self.get_type_id(TYPE) hdr = gc.gcheaderbuilder.new_header(value) adr = llmemory.cast_ptr_to_adr(hdr) - flags = 0 - if gen_gc: - flags = GCFLAG_NEVER_SET - gc.init_gc_object_immortal(adr, typeid, flags=flags) + gc.init_gc_object_immortal(adr, typeid) # The following collects the addresses of all the fields that have # a GC Pointer type, inside the current prebuilt object. All such From cfbolz at codespeak.net Fri Dec 14 16:58:10 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 14 Dec 2007 16:58:10 +0100 (CET) Subject: [pypy-svn] r49793 - in pypy/branch/applevel-ctypes/pypy/lib/_ctypes: . test Message-ID: <20071214155810.C9893168520@codespeak.net> Author: cfbolz Date: Fri Dec 14 16:58:10 2007 New Revision: 49793 Added: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/ pypy/branch/applevel-ctypes/pypy/lib/_ctypes/__init__.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/_ctypes/test/ pypy/branch/applevel-ctypes/pypy/lib/_ctypes/test/__init__.py (contents, props changed) Log: empty _ctypes module Added: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/__init__.py ============================================================================== Added: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/test/__init__.py ============================================================================== From arigo at codespeak.net Fri Dec 14 17:04:50 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Dec 2007 17:04:50 +0100 (CET) Subject: [pypy-svn] r49794 - pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc Message-ID: <20071214160450.978D9168520@codespeak.net> Author: arigo Date: Fri Dec 14 17:04:47 2007 New Revision: 49794 Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py Log: * This expression doesn't give a single bit :-) fix misleading previous line too. * Move logic out of write_barrier() itself, because it gets inlined abolutely everywhere in the program. Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py Fri Dec 14 17:04:47 2007 @@ -11,11 +11,11 @@ # in the nursery. It is initially set on all prebuilt and old objects, # and gets cleared by the write_barrier() when we write in them a # pointer to a young object. -GCFLAG_NO_YOUNG_PTRS = 2 << GCFLAGSHIFT +GCFLAG_NO_YOUNG_PTRS = 1 << (GCFLAGSHIFT+1) # The following flag is set for static roots which are not on the list # of static roots yet, but will appear with write barrier -GCFLAG_NEVER_SET = 3 << GCFLAGSHIFT +GCFLAG_NEVER_SET = 1 << (GCFLAGSHIFT+2) DEBUG_PRINT = False @@ -293,9 +293,6 @@ def write_barrier(self, oldvalue, newvalue, addr_struct): if self.header(addr_struct).tid & GCFLAG_NO_YOUNG_PTRS: self.remember_young_pointer(addr_struct, newvalue) - if self.header(addr_struct).tid & GCFLAG_NEVER_SET: - self.move_to_static_roots(addr_struct) - self.calls += 1 def append_to_static_roots(self, pointer, arg): os.write(2, str(self.calls) + "\n") @@ -309,9 +306,12 @@ def remember_young_pointer(self, addr_struct, addr): ll_assert(not self.is_in_nursery(addr_struct), "nursery object with GCFLAG_NO_YOUNG_PTRS") + oldhdr = self.header(addr_struct) if self.is_in_nursery(addr): - oldhdr = self.header(addr_struct) oldhdr.forw = self.old_objects_pointing_to_young self.old_objects_pointing_to_young = addr_struct oldhdr.tid &= ~GCFLAG_NO_YOUNG_PTRS + if oldhdr.tid & GCFLAG_NEVER_SET: + self.move_to_static_roots(addr_struct) + self.calls += 1 remember_young_pointer.dont_inline = True From arigo at codespeak.net Fri Dec 14 17:40:57 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Dec 2007 17:40:57 +0100 (CET) Subject: [pypy-svn] r49795 - pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc Message-ID: <20071214164057.F2BA1168508@codespeak.net> Author: arigo Date: Fri Dec 14 17:40:56 2007 New Revision: 49795 Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py Log: Oups - cannot use os.write() in the GC, because that allocates RPython strings! Need to use llop.debug_print(). (Removing the prints now makes sense too.) Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py Fri Dec 14 17:40:56 2007 @@ -38,7 +38,6 @@ max_space_size = max_space_size, get_roots = get_roots) self.nursery_size = nursery_size - self.calls = 0 assert nursery_size <= space_size // 2 def setup(self): @@ -295,7 +294,6 @@ self.remember_young_pointer(addr_struct, newvalue) def append_to_static_roots(self, pointer, arg): - os.write(2, str(self.calls) + "\n") self.get_roots.append_static_root(pointer) def move_to_static_roots(self, addr_struct): @@ -313,5 +311,4 @@ oldhdr.tid &= ~GCFLAG_NO_YOUNG_PTRS if oldhdr.tid & GCFLAG_NEVER_SET: self.move_to_static_roots(addr_struct) - self.calls += 1 remember_young_pointer.dont_inline = True From jacob at codespeak.net Fri Dec 14 18:16:35 2007 From: jacob at codespeak.net (jacob at codespeak.net) Date: Fri, 14 Dec 2007 18:16:35 +0100 (CET) Subject: [pypy-svn] r49796 - pypy/extradoc/planning/roadmap Message-ID: <20071214171635.A43C61684FE@codespeak.net> Author: jacob Date: Fri Dec 14 18:16:34 2007 New Revision: 49796 Added: pypy/extradoc/planning/roadmap/ pypy/extradoc/planning/roadmap/cpython_replacement.txt Log: New roadmap file for becoming a viable replacement to CPython. Added: pypy/extradoc/planning/roadmap/cpython_replacement.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/cpython_replacement.txt Fri Dec 14 18:16:34 2007 @@ -0,0 +1,119 @@ + + +Refactor the JIT +================ + +The JIT is the result of programming under severe time pressure. This +means that the existing JIT framework needs a fair bit of cleanup and +restructuring. Armin also has some ideas for a refactoring based on an +interpreter, rather than annotated graphs. +XXX maybe phrase this a bit more clearly. + +Add optimizations to the JIT +============================ + +A base set of optimizations include common operations on floats and +strings as well as general call optimization. Fixing bugs in the +current JIT implementation is also needed. + +Separate compilation +==================== + +To make porting third party modules reasonable, it should not be necessary +to recompile all of PyPy every time you want to integrate a new extension +module. This requires supporting separate compilation. + +Support a tool like Swig or Boost +================================= + +To make it simpler to integrate C extensions with PyPy, we should support +at least one tool that is already in use in the Python community. + +Google uses Swig heavily, so in a scenario where Google pays for some +development, it would be natural to do Swig. Once PyPy gets traction +we expect proponents of other tools to make their own ports. + +In addition to Swig, a PyPy version of ctypes should be implemented. +An attempt to build a layer on top of the existing ctypes +implementation for CPython has failed. Ctypes was not written with +this kind of reuse in mind, and it would be harder to change it +than to make an RPython implementation from scratch. + +XXX Some example use case needed here. + +Port more modules to rffi style +=============================== + +Some standard modules written in C have already been ported to the +rffi style, but more remain. +XXX Flesh out with some more details. Examples for example. + +Move modules from app to interpreter level +========================================== + +Some modules are implemented in Python at application level. For +preformance reasons, they should be implemented at interpreter level. + +XXX Which ones? + +Write new modules +================= + +There are still some modules in the standard library that need writing +before PyPy is a reasonable replacement for CPython. Exactly which +ones needs some investigation. +XXX Flesh out with some more details. Examples for example. + +Hand optimizations +================== + +There are still some hand optimizations of the Python interpreter that +are well worth doing, since they can be expected to yield significant +performance improvements. For instance, some of our mircobenchmarks +show particularly poor performance in some areas. The use of better +algorithms should improve our performance numbers significantly for +some of the benchmarks. + +Parser cleanup +============== + +The parser is in a rather sorry state. Making it support various +versions of Python is currently a rather nasty job. It needs replacing +with a more robust design. The estimates for this are probably too low, +since they were based on the idea of improving the existing parser. +Subsequent attempts to improve the parser have prompted the idea of +a rewrite. In the parser work, there is also time needed to upgrade +the interpreter to handle all of Python 2.5. + +Multi-platform support +====================== + +In order to be a viable alternative to CPython, PyPy needs to be +tested and run cleanly on more platforms than it currently does. Amd64 +with Linux is probably the most urgent one, but there are probably +others that have some problems. + +It probably makes sense to put this item off until we are closer to +having a production ready system for the i386. We also have a possible +volunteer effort under way. It remains to be seen if it takes off. + +Support a GUI +============= + +One of the C extension modules that we need to have in order to get +traction is support for one of the multi-platform GUI toolkits. The +choice is probably between Qt and wxWindows, with Tkinter as a +possible third. Someone with more understanding of the Python +community than us should make the pick. Which one would bring more +users to PyPy? Would any one of them prompt proponents of other +GUI toolkits to make their own ports? + +Third party module support +========================== + +There are some third party modules that would bring early adopters to +PyPy. They would also serve as code coverage cases. Our time estimate +is a guess on how much time we would need to spend to get enough +traction in the community. Exactly what modules to support needs to +be determined after some of the infrastructure in other tasks is +in place. From arigo at codespeak.net Fri Dec 14 18:31:01 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Dec 2007 18:31:01 +0100 (CET) Subject: [pypy-svn] r49797 - pypy/dist/pypy/translator/cli Message-ID: <20071214173101.B05EC168508@codespeak.net> Author: arigo Date: Fri Dec 14 18:31:00 2007 New Revision: 49797 Modified: pypy/dist/pypy/translator/cli/dotnet.py Log: Fix unbox(). Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Fri Dec 14 18:31:00 2007 @@ -402,11 +402,13 @@ return OverloadingResolver.lltype_to_annotation(TYPE) def specialize_call(self, hop): - v_obj, v_type = hop.inputargs(*hop.args_r) - if v_type.value is ootype.String or isinstance(v_type.value, (type, types.ClassType)): + TYPE = hop.args_v[1].value + v_obj = hop.inputarg(hop.args_r[0], arg=0) + if TYPE is ootype.String or isinstance(TYPE, (type, types.ClassType)): return hop.genop('oodowncast', [v_obj], hop.r_result.lowleveltype) else: - return hop.genop('cliunbox', [v_obj, v_type], hop.r_result.lowleveltype) + c_type = hop.inputconst(ootype.Void, TYPE) + return hop.genop('cliunbox', [v_obj, c_type], hop.r_result.lowleveltype) native_exc_cache = {} From arigo at codespeak.net Fri Dec 14 18:38:15 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Dec 2007 18:38:15 +0100 (CET) Subject: [pypy-svn] r49798 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20071214173815.BC36016852B@codespeak.net> Author: arigo Date: Fri Dec 14 18:38:15 2007 New Revision: 49798 Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py Log: Provide the FUNCTYPE early, as we know it. Solves an issue with finalizers in gctypelayout.py. Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/transform.py Fri Dec 14 18:38:15 2007 @@ -279,7 +279,8 @@ self.need_minimal_transform(graph) if inline: self.graphs_to_inline[graph] = True - return self.mixlevelannotator.graph2delayed(graph) + FUNCTYPE = lltype.FuncType(ll_args, ll_result) + return self.mixlevelannotator.graph2delayed(graph, FUNCTYPE=FUNCTYPE) def inittime_helper(self, ll_helper, ll_args, ll_result, inline=True): ptr = self.annotate_helper(ll_helper, ll_args, ll_result, inline=inline) From xoraxax at codespeak.net Fri Dec 14 18:38:31 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Fri, 14 Dec 2007 18:38:31 +0100 (CET) Subject: [pypy-svn] r49799 - pypy/extradoc/planning/roadmap Message-ID: <20071214173831.25EC416852B@codespeak.net> Author: xoraxax Date: Fri Dec 14 18:38:30 2007 New Revision: 49799 Modified: pypy/extradoc/planning/roadmap/cpython_replacement.txt Log: Add gtk and a small paragraph to separate module compilation. Modified: pypy/extradoc/planning/roadmap/cpython_replacement.txt ============================================================================== --- pypy/extradoc/planning/roadmap/cpython_replacement.txt (original) +++ pypy/extradoc/planning/roadmap/cpython_replacement.txt Fri Dec 14 18:38:30 2007 @@ -23,6 +23,11 @@ to recompile all of PyPy every time you want to integrate a new extension module. This requires supporting separate compilation. +Also this would be interesting ground work for a try to compile cpython modules +with a pypy-supplied Python.h file and link them to pypy. This would require +writing a lot of new functions (for the CPython API), though. Also it is not +clear how large the expected speed impact would be (because of e. g. pinning). + Support a tool like Swig or Boost ================================= @@ -102,7 +107,8 @@ One of the C extension modules that we need to have in order to get traction is support for one of the multi-platform GUI toolkits. The -choice is probably between Qt and wxWindows, with Tkinter as a +choice is probably between gtk (in use by olpc and others), Qt (written in c++, +using SIP to get a cpython package) and wxWindows, with Tkinter as a possible third. Someone with more understanding of the Python community than us should make the pick. Which one would bring more users to PyPy? Would any one of them prompt proponents of other From xoraxax at codespeak.net Fri Dec 14 18:42:37 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Fri, 14 Dec 2007 18:42:37 +0100 (CET) Subject: [pypy-svn] r49800 - pypy/extradoc/planning/roadmap Message-ID: <20071214174237.16F2F16852E@codespeak.net> Author: xoraxax Date: Fri Dec 14 18:42:36 2007 New Revision: 49800 Modified: pypy/extradoc/planning/roadmap/cpython_replacement.txt Log: Make the cpython api task a separate one. Modified: pypy/extradoc/planning/roadmap/cpython_replacement.txt ============================================================================== --- pypy/extradoc/planning/roadmap/cpython_replacement.txt (original) +++ pypy/extradoc/planning/roadmap/cpython_replacement.txt Fri Dec 14 18:42:36 2007 @@ -23,10 +23,13 @@ to recompile all of PyPy every time you want to integrate a new extension module. This requires supporting separate compilation. -Also this would be interesting ground work for a try to compile cpython modules -with a pypy-supplied Python.h file and link them to pypy. This would require -writing a lot of new functions (for the CPython API), though. Also it is not -clear how large the expected speed impact would be (because of e. g. pinning). +Providing the CPython C-API +=========================== +Separate compilation would be interesting ground work for a try to compile +cpython modules with a pypy-supplied Python.h file and link them to pypy. This +would require writing a lot of new functions (for the CPython API), though. +Also it is not clear how large the expected speed impact would be (because of +e.g. pinning). Support a tool like Swig or Boost ================================= From jacob at codespeak.net Fri Dec 14 19:01:29 2007 From: jacob at codespeak.net (jacob at codespeak.net) Date: Fri, 14 Dec 2007 19:01:29 +0100 (CET) Subject: [pypy-svn] r49801 - pypy/extradoc/planning/roadmap Message-ID: <20071214180129.0888C168434@codespeak.net> Author: jacob Date: Fri Dec 14 19:01:29 2007 New Revision: 49801 Modified: pypy/extradoc/planning/roadmap/cpython_replacement.txt Log: Introductory blurb. Modified: pypy/extradoc/planning/roadmap/cpython_replacement.txt ============================================================================== --- pypy/extradoc/planning/roadmap/cpython_replacement.txt (original) +++ pypy/extradoc/planning/roadmap/cpython_replacement.txt Fri Dec 14 19:01:29 2007 @@ -1,4 +1,12 @@ +This is a roadmap for the specific target of +"Making PyPy a viable replacement for CPython" + +This does not include things like porting every C extension that +works for CPython, but it does include porting enough modules that +enough people can use PyPy in their daily production that they will +do the porting of modules and other tasks that are beyond the +resources of the core project. Refactor the JIT ================ From arigo at codespeak.net Fri Dec 14 19:04:06 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Dec 2007 19:04:06 +0100 (CET) Subject: [pypy-svn] r49802 - in pypy/branch/pypy-more-delayedptr: . rpython/memory rpython/memory/gctransform translator/c translator/cli Message-ID: <20071214180406.754C5168508@codespeak.net> Author: arigo Date: Fri Dec 14 19:04:05 2007 New Revision: 49802 Added: pypy/branch/pypy-more-delayedptr/ - copied from r49782, pypy/dist/pypy/ pypy/branch/pypy-more-delayedptr/rpython/memory/gctransform/framework.py - copied, changed from r49784, pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/branch/pypy-more-delayedptr/rpython/memory/gctransform/transform.py - copied unchanged from r49798, pypy/dist/pypy/rpython/memory/gctransform/transform.py pypy/branch/pypy-more-delayedptr/rpython/memory/gctypelayout.py - copied unchanged from r49785, pypy/dist/pypy/rpython/memory/gctypelayout.py pypy/branch/pypy-more-delayedptr/rpython/memory/gcwrapper.py - copied unchanged from r49783, pypy/dist/pypy/rpython/memory/gcwrapper.py pypy/branch/pypy-more-delayedptr/translator/cli/dotnet.py - copied unchanged from r49797, pypy/dist/pypy/translator/cli/dotnet.py Modified: pypy/branch/pypy-more-delayedptr/translator/c/database.py Log: Trying to use a delayed ptr instead of the hacking done inside the gcdata for the type_info_table. This needs some fragile changes in translator/c/database.py so I'm making a branch to run many tests on wyvern. Modified: pypy/branch/pypy-more-delayedptr/translator/c/database.py ============================================================================== --- pypy/dist/pypy/translator/c/database.py (original) +++ pypy/branch/pypy-more-delayedptr/translator/c/database.py Fri Dec 14 19:04:05 2007 @@ -42,8 +42,7 @@ self.pendingsetupnodes = [] self.containernodes = {} self.containerlist = [] - self.delayedfunctionnames = {} - self.delayedfunctionptrs = [] + self.delayedobjptrs = [] self.completedcontainers = 0 self.containerstats = {} self.externalfuncs = {} @@ -188,31 +187,16 @@ if obj: # test if the ptr is non-NULL try: container = obj._obj - except lltype.DelayedPointer: + except lltype.DelayedPointer, e: # hack hack hack name = obj._obj0 assert name.startswith('delayed!') n = len('delayed!') if len(name) == n: raise - if id(obj) in self.delayedfunctionnames: - return self.delayedfunctionnames[id(obj)][0] - funcname = name[n:] - funcname = self.namespace.uniquename('g_' + funcname) - self.delayedfunctionnames[id(obj)] = funcname, obj - self.delayedfunctionptrs.append(obj) - return funcname - # /hack hack hack - else: - # hack hack hack - if id(obj) in self.delayedfunctionnames: - # this used to be a delayed function, - # make sure we use the same name - forcename = self.delayedfunctionnames[id(obj)][0] - node = self.getcontainernode(container, - forcename=forcename) - assert node.ptrname == forcename - return forcename + self.delayedobjptrs.append(obj) + return e # not a string - the caller can't really use + # this. We don't have a name to provide yet # /hack hack hack if isinstance(container, int): @@ -315,15 +299,15 @@ dump() show_i += 1000 - if self.delayedfunctionptrs: - lst = self.delayedfunctionptrs - self.delayedfunctionptrs = [] + if self.delayedobjptrs: + lst = self.delayedobjptrs + self.delayedobjptrs = [] progress = False for fnptr in lst: try: fnptr._obj except lltype.DelayedPointer: # still not resolved - self.delayedfunctionptrs.append(fnptr) + self.delayedobjptrs.append(fnptr) else: self.get(fnptr) progress = True @@ -339,7 +323,7 @@ break # database is now complete - assert not self.delayedfunctionptrs + assert not self.delayedobjptrs self.completed = True if show_progress: dump() From arigo at codespeak.net Fri Dec 14 19:21:16 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Dec 2007 19:21:16 +0100 (CET) Subject: [pypy-svn] r49803 - pypy/branch/pypy-more-delayedptr/translator/c Message-ID: <20071214182116.4F5C2168503@codespeak.net> Author: arigo Date: Fri Dec 14 19:21:16 2007 New Revision: 49803 Modified: pypy/branch/pypy-more-delayedptr/translator/c/database.py Log: Restore the original logic about delayed func ptr, just skip some of it for delayed non-func ptrs. Modified: pypy/branch/pypy-more-delayedptr/translator/c/database.py ============================================================================== --- pypy/branch/pypy-more-delayedptr/translator/c/database.py (original) +++ pypy/branch/pypy-more-delayedptr/translator/c/database.py Fri Dec 14 19:21:16 2007 @@ -42,7 +42,8 @@ self.pendingsetupnodes = [] self.containernodes = {} self.containerlist = [] - self.delayedobjptrs = [] + self.delayedfunctionnames = {} + self.delayedfunctionptrs = [] self.completedcontainers = 0 self.containerstats = {} self.externalfuncs = {} @@ -187,16 +188,35 @@ if obj: # test if the ptr is non-NULL try: container = obj._obj - except lltype.DelayedPointer, e: + except lltype.DelayedPointer: # hack hack hack name = obj._obj0 assert name.startswith('delayed!') n = len('delayed!') if len(name) == n: raise - self.delayedobjptrs.append(obj) - return e # not a string - the caller can't really use - # this. We don't have a name to provide yet + if isinstance(lltype.typeOf(obj).TO, lltype.FuncType): + if id(obj) in self.delayedfunctionnames: + return self.delayedfunctionnames[id(obj)][0] + funcname = name[n:] + funcname = self.namespace.uniquename('g_'+funcname) + self.delayedfunctionnames[id(obj)] = funcname, obj + else: + funcname = None # can't use the name of a + # delayed non-function ptr + self.delayedfunctionptrs.append(obj) + return funcname + # /hack hack hack + else: + # hack hack hack + if id(obj) in self.delayedfunctionnames: + # this used to be a delayed function, + # make sure we use the same name + forcename = self.delayedfunctionnames[id(obj)][0] + node = self.getcontainernode(container, + forcename=forcename) + assert node.ptrname == forcename + return forcename # /hack hack hack if isinstance(container, int): @@ -299,15 +319,15 @@ dump() show_i += 1000 - if self.delayedobjptrs: - lst = self.delayedobjptrs - self.delayedobjptrs = [] + if self.delayedfunctionptrs: + lst = self.delayedfunctionptrs + self.delayedfunctionptrs = [] progress = False for fnptr in lst: try: fnptr._obj except lltype.DelayedPointer: # still not resolved - self.delayedobjptrs.append(fnptr) + self.delayedfunctionptrs.append(fnptr) else: self.get(fnptr) progress = True @@ -323,7 +343,7 @@ break # database is now complete - assert not self.delayedobjptrs + assert not self.delayedfunctionptrs self.completed = True if show_progress: dump() From cfbolz at codespeak.net Fri Dec 14 19:38:56 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 14 Dec 2007 19:38:56 +0100 (CET) Subject: [pypy-svn] r49804 - in pypy/dist/pypy/module/__builtin__: . test Message-ID: <20071214183856.6275916852B@codespeak.net> Author: cfbolz Date: Fri Dec 14 19:38:55 2007 New Revision: 49804 Modified: pypy/dist/pypy/module/__builtin__/interp_classobj.py pypy/dist/pypy/module/__builtin__/test/test_classobj.py Log: fix the issue with the missing next method Modified: pypy/dist/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/dist/pypy/module/__builtin__/interp_classobj.py Fri Dec 14 19:38:55 2007 @@ -591,6 +591,12 @@ return space.call_function(w_func, w_other, w_modulo) return space.w_NotImplemented + def descr_next(self, space): + w_func = self.getattr(space, space.wrap('next'), False) + if w_func is None: + raise OperationError(space.w_TypeError, + space.wrap("instance has no next() method")) + return space.call_function(w_func) rawdict = {} @@ -678,6 +684,8 @@ unwrap_spec=['self', ObjSpace, W_Root, W_Root]), __rpow__ = interp2app(W_InstanceObject.descr_rpow, unwrap_spec=['self', ObjSpace, W_Root, W_Root]), + next = interp2app(W_InstanceObject.descr_next, + unwrap_spec=['self', ObjSpace]), __weakref__ = make_weakref_descr(W_InstanceObject), **rawdict ) Modified: pypy/dist/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/dist/pypy/module/__builtin__/test/test_classobj.py Fri Dec 14 19:38:55 2007 @@ -638,3 +638,16 @@ gc.collect() gc.collect() assert ref() is None + + def test_next(self): + class X: + def __iter__(self): + return Y() + + class Y: + def next(self): + return 3 + + for i in X(): + print i, + break From arigo at codespeak.net Fri Dec 14 19:56:16 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Dec 2007 19:56:16 +0100 (CET) Subject: [pypy-svn] r49805 - in pypy/dist/pypy: rpython/memory/gctransform translator/c Message-ID: <20071214185616.F22BB16850C@codespeak.net> Author: arigo Date: Fri Dec 14 19:56:16 2007 New Revision: 49805 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/translator/c/database.py Log: Use a delayed pointer for the type_info_table. Requires a bit of further hacking in c/database.py, but otherwise it's a much cleaner approach and it gives better performance because the type_info_table attribute can now be considered as a constant. Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Fri Dec 14 19:56:16 2007 @@ -418,9 +418,10 @@ # replace the type_info_table pointer in gcdata -- at this point, # the database is in principle complete, so it has already seen - # the old (empty) array. We need to force it to consider the new - # array now. It's a bit hackish as the old empty array will also - # be generated in the C source, but that's a rather minor problem. + # the delayed pointer, but it still remembers it and will look + # again after we "resolve" it to a real pointer. + + self.gcdata.type_info_table._become(table) # XXX because we call inputconst already in replace_malloc, we can't # modify the instance, we have to modify the 'rtyped instance' @@ -430,8 +431,6 @@ self.gcdata) r_gcdata = self.translator.rtyper.getrepr(s_gcdata) ll_instance = rmodel.inputconst(r_gcdata, self.gcdata).value - ll_instance.inst_type_info_table = table - #self.gcdata.type_info_table = table addresses_of_static_ptrs = ( self.layoutbuilder.addresses_of_static_ptrs + @@ -447,7 +446,6 @@ ll_instance.inst_static_root_end = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(ll_static_roots_inside) newgcdependencies = [] - newgcdependencies.append(table) newgcdependencies.append(ll_static_roots_inside) self.write_typeid_list() return newgcdependencies Modified: pypy/dist/pypy/translator/c/database.py ============================================================================== --- pypy/dist/pypy/translator/c/database.py (original) +++ pypy/dist/pypy/translator/c/database.py Fri Dec 14 19:56:16 2007 @@ -195,11 +195,15 @@ n = len('delayed!') if len(name) == n: raise - if id(obj) in self.delayedfunctionnames: - return self.delayedfunctionnames[id(obj)][0] - funcname = name[n:] - funcname = self.namespace.uniquename('g_' + funcname) - self.delayedfunctionnames[id(obj)] = funcname, obj + if isinstance(lltype.typeOf(obj).TO, lltype.FuncType): + if id(obj) in self.delayedfunctionnames: + return self.delayedfunctionnames[id(obj)][0] + funcname = name[n:] + funcname = self.namespace.uniquename('g_'+funcname) + self.delayedfunctionnames[id(obj)] = funcname, obj + else: + funcname = None # can't use the name of a + # delayed non-function ptr self.delayedfunctionptrs.append(obj) return funcname # /hack hack hack From arigo at codespeak.net Fri Dec 14 20:07:59 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Dec 2007 20:07:59 +0100 (CET) Subject: [pypy-svn] r49806 - pypy/dist/pypy/translator/c Message-ID: <20071214190759.F2ACB16851A@codespeak.net> Author: arigo Date: Fri Dec 14 20:07:59 2007 New Revision: 49806 Modified: pypy/dist/pypy/translator/c/database.py Log: Can't really use this assert any more. Modified: pypy/dist/pypy/translator/c/database.py ============================================================================== --- pypy/dist/pypy/translator/c/database.py (original) +++ pypy/dist/pypy/translator/c/database.py Fri Dec 14 20:07:59 2007 @@ -164,9 +164,9 @@ self.containerstats[kind] = self.containerstats.get(kind, 0) + 1 self.containerlist.append(node) if self.completed: - assert not node.globalcontainer - # non-global containers are found very late, e.g. _subarrays - # via addresses introduced by the GC transformer + pass # we would like to fail here, but a few containers + # are found very late, e.g. _subarrays via addresses + # introduced by the GC transformer, or the type_info_table return node def get(self, obj): From arigo at codespeak.net Fri Dec 14 20:14:01 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Dec 2007 20:14:01 +0100 (CET) Subject: [pypy-svn] r49807 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20071214191401.4FD88168519@codespeak.net> Author: arigo Date: Fri Dec 14 20:14:00 2007 New Revision: 49807 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py Log: I'm quite confused. For some reason this helps, but the generated C code doesn't look efficient. Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Fri Dec 14 20:14:00 2007 @@ -446,6 +446,7 @@ ll_instance.inst_static_root_end = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(ll_static_roots_inside) newgcdependencies = [] + newgcdependencies.append(self.gcdata.type_info_table) newgcdependencies.append(ll_static_roots_inside) self.write_typeid_list() return newgcdependencies From arigo at codespeak.net Fri Dec 14 20:18:20 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 14 Dec 2007 20:18:20 +0100 (CET) Subject: [pypy-svn] r49808 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20071214191820.78290168500@codespeak.net> Author: arigo Date: Fri Dec 14 20:18:20 2007 New Revision: 49808 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py Log: Uh? I messed up the branch merge in some way. Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Fri Dec 14 20:18:20 2007 @@ -113,14 +113,17 @@ self.get_type_id = self.layoutbuilder.get_type_id # set up dummy a table, to be overwritten with the real one in finish() - type_info_table = lltype.malloc(gctypelayout.GCData.TYPE_INFO_TABLE, 0, - immortal=True) + type_info_table = lltype._ptr( + lltype.Ptr(gctypelayout.GCData.TYPE_INFO_TABLE), + "delayed!type_info_table", solid=True) gcdata = gctypelayout.GCData(type_info_table) # initialize the following two fields with a random non-NULL address, # to make the annotator happy. The fields are patched in finish() - # to point to a real array (not 'type_info_table', another one). - a_random_address = llmemory.cast_ptr_to_adr(type_info_table) + # to point to a real array. + foo = lltype.malloc(lltype.FixedSizeArray(llmemory.Address, 1), + immortal=True, zero=True) + a_random_address = llmemory.cast_ptr_to_adr(foo) gcdata.static_root_start = a_random_address # patched in finish() gcdata.static_root_nongcstart = a_random_address # patched in finish() gcdata.static_root_end = a_random_address # patched in finish() @@ -143,12 +146,9 @@ bk = self.translator.annotator.bookkeeper # the point of this little dance is to not annotate - # self.gcdata.type_info_table as a constant. + # self.gcdata.static_root_xyz as constants. XXX is it still needed?? data_classdef = bk.getuniqueclassdef(gctypelayout.GCData) data_classdef.generalize_attr( - 'type_info_table', - annmodel.SomePtr(lltype.Ptr(gctypelayout.GCData.TYPE_INFO_TABLE))) - data_classdef.generalize_attr( 'static_root_start', annmodel.SomeAddress()) data_classdef.generalize_attr( @@ -418,8 +418,8 @@ # replace the type_info_table pointer in gcdata -- at this point, # the database is in principle complete, so it has already seen - # the delayed pointer, but it still remembers it and will look - # again after we "resolve" it to a real pointer. + # the delayed pointer. We need to force it to consider the new + # array now. self.gcdata.type_info_table._become(table) From cfbolz at codespeak.net Fri Dec 14 20:30:21 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 14 Dec 2007 20:30:21 +0100 (CET) Subject: [pypy-svn] r49809 - pypy/extradoc/planning/roadmap Message-ID: <20071214193021.18C8D16850C@codespeak.net> Author: cfbolz Date: Fri Dec 14 20:30:19 2007 New Revision: 49809 Added: pypy/extradoc/planning/roadmap/goal_cpython_replacement.txt - copied, changed from r49806, pypy/extradoc/planning/roadmap/cpython_replacement.txt pypy/extradoc/planning/roadmap/task_cpython_api.txt (contents, props changed) pypy/extradoc/planning/roadmap/task_gui_support.txt (contents, props changed) pypy/extradoc/planning/roadmap/task_manual_optimizations.txt (contents, props changed) pypy/extradoc/planning/roadmap/task_modules_3rdparty.txt (contents, props changed) pypy/extradoc/planning/roadmap/task_modules_rffi.txt (contents, props changed) pypy/extradoc/planning/roadmap/task_multi_platform.txt (contents, props changed) pypy/extradoc/planning/roadmap/task_optimize_jit.txt (contents, props changed) pypy/extradoc/planning/roadmap/task_parser_cleanup.txt (contents, props changed) pypy/extradoc/planning/roadmap/task_refactor_jit.txt (contents, props changed) pypy/extradoc/planning/roadmap/task_separate_compilation.txt (contents, props changed) pypy/extradoc/planning/roadmap/task_wrapper_generator.txt (contents, props changed) Removed: pypy/extradoc/planning/roadmap/cpython_replacement.txt Log: make one file per task, to allow easier parallel working on them. Also we can add other goals which include certain tasks. Added: pypy/extradoc/planning/roadmap/task_cpython_api.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_cpython_api.txt Fri Dec 14 20:30:19 2007 @@ -0,0 +1,7 @@ +Providing the CPython C-API +=========================== +Separate compilation would be interesting ground work for a try to compile +cpython modules with a pypy-supplied Python.h file and link them to pypy. This +would require writing a lot of new functions (for the CPython API), though. +Also it is not clear how large the expected speed impact would be (because of +e.g. pinning). Added: pypy/extradoc/planning/roadmap/task_gui_support.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_gui_support.txt Fri Dec 14 20:30:19 2007 @@ -0,0 +1,11 @@ +Support a GUI +============= + +One of the C extension modules that we need to have in order to get +traction is support for one of the multi-platform GUI toolkits. The +choice is probably between gtk (in use by olpc and others), Qt (written in c++, +using SIP to get a cpython package) and wxWindows, with Tkinter as a +possible third. Someone with more understanding of the Python +community than us should make the pick. Which one would bring more +users to PyPy? Would any one of them prompt proponents of other +GUI toolkits to make their own ports? Added: pypy/extradoc/planning/roadmap/task_manual_optimizations.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_manual_optimizations.txt Fri Dec 14 20:30:19 2007 @@ -0,0 +1,9 @@ +Hand optimizations +================== + +There are still some hand optimizations of the Python interpreter that +are well worth doing, since they can be expected to yield significant +performance improvements. For instance, some of our mircobenchmarks +show particularly poor performance in some areas. The use of better +algorithms should improve our performance numbers significantly for +some of the benchmarks. Added: pypy/extradoc/planning/roadmap/task_modules_3rdparty.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_modules_3rdparty.txt Fri Dec 14 20:30:19 2007 @@ -0,0 +1,9 @@ +Third party module support +========================== + +There are some third party modules that would bring early adopters to +PyPy. They would also serve as code coverage cases. Our time estimate +is a guess on how much time we would need to spend to get enough +traction in the community. Exactly what modules to support needs to +be determined after some of the infrastructure in other tasks is +in place. Added: pypy/extradoc/planning/roadmap/task_modules_rffi.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_modules_rffi.txt Fri Dec 14 20:30:19 2007 @@ -0,0 +1,25 @@ +Port more modules to rffi style +=============================== + +Some standard modules written in C have already been ported to the +rffi style, but more remain. +XXX Flesh out with some more details. Examples for example. + + +Move modules from app to interpreter level +========================================== + +Some modules are implemented in Python at application level. For +preformance reasons, they should be implemented at interpreter level. + +XXX Which ones? + +Write new modules +================= + +There are still some modules in the standard library that need writing +before PyPy is a reasonable replacement for CPython. Exactly which +ones needs some investigation. +XXX Flesh out with some more details. Examples for example. + +XXX these tasks should be merged, I think Added: pypy/extradoc/planning/roadmap/task_multi_platform.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_multi_platform.txt Fri Dec 14 20:30:19 2007 @@ -0,0 +1,11 @@ +Multi-platform support +====================== + +In order to be a viable alternative to CPython, PyPy needs to be +tested and run cleanly on more platforms than it currently does. Amd64 +with Linux is probably the most urgent one, but there are probably +others that have some problems. + +It probably makes sense to put this item off until we are closer to +having a production ready system for the i386. We also have a possible +volunteer effort under way. It remains to be seen if it takes off. Added: pypy/extradoc/planning/roadmap/task_optimize_jit.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_optimize_jit.txt Fri Dec 14 20:30:19 2007 @@ -0,0 +1,6 @@ +Add optimizations to the JIT +============================ + +A base set of optimizations include common operations on floats and +strings as well as general call optimization. Fixing bugs in the +current JIT implementation is also needed. Added: pypy/extradoc/planning/roadmap/task_parser_cleanup.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_parser_cleanup.txt Fri Dec 14 20:30:19 2007 @@ -0,0 +1,10 @@ +Parser cleanup +============== + +The parser is in a rather sorry state. Making it support various +versions of Python is currently a rather nasty job. It needs replacing +with a more robust design. The estimates for this are probably too low, +since they were based on the idea of improving the existing parser. +Subsequent attempts to improve the parser have prompted the idea of +a rewrite. In the parser work, there is also time needed to upgrade +the interpreter to handle all of Python 2.5. Added: pypy/extradoc/planning/roadmap/task_refactor_jit.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_refactor_jit.txt Fri Dec 14 20:30:19 2007 @@ -0,0 +1,8 @@ +Refactor the JIT +================ + +The JIT is the result of programming under severe time pressure. This +means that the existing JIT framework needs a fair bit of cleanup and +restructuring. Armin also has some ideas for a refactoring based on an +interpreter, rather than annotated graphs. +XXX maybe phrase this a bit more clearly. Added: pypy/extradoc/planning/roadmap/task_separate_compilation.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_separate_compilation.txt Fri Dec 14 20:30:19 2007 @@ -0,0 +1,7 @@ + +Separate compilation +==================== + +To make porting third party modules reasonable, it should not be necessary +to recompile all of PyPy every time you want to integrate a new extension +module. This requires supporting separate compilation. Added: pypy/extradoc/planning/roadmap/task_wrapper_generator.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_wrapper_generator.txt Fri Dec 14 20:30:19 2007 @@ -0,0 +1,17 @@ +Support a tool like Swig or Boost +================================= + +To make it simpler to integrate C extensions with PyPy, we should support +at least one tool that is already in use in the Python community. + +Google uses Swig heavily, so in a scenario where Google pays for some +development, it would be natural to do Swig. Once PyPy gets traction +we expect proponents of other tools to make their own ports. + +In addition to Swig, a PyPy version of ctypes should be implemented. +An attempt to build a layer on top of the existing ctypes +implementation for CPython has failed. Ctypes was not written with +this kind of reuse in mind, and it would be harder to change it +than to make an RPython implementation from scratch. + +XXX Some example use case needed here. From fijal at codespeak.net Fri Dec 14 21:41:36 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 14 Dec 2007 21:41:36 +0100 (CET) Subject: [pypy-svn] r49810 - pypy/extradoc/planning/roadmap Message-ID: <20071214204136.5E447168502@codespeak.net> Author: fijal Date: Fri Dec 14 21:41:34 2007 New Revision: 49810 Added: pypy/extradoc/planning/roadmap/task_ctypes.txt (contents, props changed) Modified: pypy/extradoc/planning/roadmap/task_wrapper_generator.txt Log: Add a separate ctypes task Added: pypy/extradoc/planning/roadmap/task_ctypes.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_ctypes.txt Fri Dec 14 21:41:34 2007 @@ -0,0 +1,7 @@ +Support app-level ctypes +======================== + +As a first approach for providing C-level bindings, we should implement +the ctypes module on top of already existing _ffi wrapper. This would +allow to access quite a bit of libraries which use ctypes already +(like SDL wrapper). Modified: pypy/extradoc/planning/roadmap/task_wrapper_generator.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_wrapper_generator.txt (original) +++ pypy/extradoc/planning/roadmap/task_wrapper_generator.txt Fri Dec 14 21:41:34 2007 @@ -8,10 +8,4 @@ development, it would be natural to do Swig. Once PyPy gets traction we expect proponents of other tools to make their own ports. -In addition to Swig, a PyPy version of ctypes should be implemented. -An attempt to build a layer on top of the existing ctypes -implementation for CPython has failed. Ctypes was not written with -this kind of reuse in mind, and it would be harder to change it -than to make an RPython implementation from scratch. -XXX Some example use case needed here. From regmee at codespeak.net Fri Dec 14 23:11:41 2007 From: regmee at codespeak.net (regmee at codespeak.net) Date: Fri, 14 Dec 2007 23:11:41 +0100 (CET) Subject: [pypy-svn] r49811 - in pypy/branch/clr-module-improvements/pypy/module/clr: . test Message-ID: <20071214221141.6C62316851D@codespeak.net> Author: regmee Date: Fri Dec 14 23:11:39 2007 New Revision: 49811 Modified: pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py pypy/branch/clr-module-improvements/pypy/module/clr/app_importer.py pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py Log: Support for Generic classes with a single import for System.Collection.Generic, Few test cases, Import needs slight improvement Modified: pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/__init__.py Fri Dec 14 23:11:39 2007 @@ -14,7 +14,8 @@ '_CliObject_internal': 'interp_clr.W_CliObject', 'call_staticmethod': 'interp_clr.call_staticmethod', 'load_cli_class': 'interp_clr.load_cli_class', - 'load_valid_namespaces': 'interp_clr.load_valid_namespaces', + 'list_of_valid_namespaces': 'interp_clr.list_of_valid_namespaces', + 'list_of_generic_classes': 'interp_clr.list_of_generic_classes', 'isDotNetType': 'interp_clr.isDotNetType', 'load_assembly': 'interp_clr.load_assembly', 'list_of_loadedAssemblies': 'interp_clr.list_of_loadedAssemblies', Modified: pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/app_clr.py Fri Dec 14 23:11:39 2007 @@ -78,6 +78,46 @@ def __get__(self, obj, type_): return self.fget() +class MetaGenericCliClassWrapper(type): + def __setattr__(cls, name, value): + obj = cls.__dict__.get(name, None) + if isinstance(obj, StaticProperty): + obj.fset(value) + else: + type.__setattr__(cls, name, value) + + def __getitem__(cls,*args): + #cls.__cliclass__ ='System.Collections.Generic.Dictionary`2' + rightDot = cls.__cliclass__.rfind('.') + rightTilde = cls.__cliclass__.rfind('`') + load_cli_class_leftArg = cls.__cliclass__[:rightDot] + genClassName = cls.__cliclass__[rightDot+1: rightTilde] + genClassNumArgs = int(cls.__cliclass__[rightTilde +1 :]) + import clr + try: + ln = len(args[0]) + # put a check for the number of arguments passed for the Generic class + if ln != genClassNumArgs: + raise "InvalidArgumentList" + else: + lindex = str(args[0][0]).find('\'') + rindex = str(args[0][0]).rfind('\'') + load_cli_class_rightArg = genClassName + load_cli_class_rightArg += "`%s[%s"%(ln,str(args[0][0])[lindex+1:rindex]) + for i in range(1,ln): + lindex = str(args[0][i]).find('\'') + rindex = str(args[0][i]).rfind('\'') + load_cli_class_rightArg += ",%s"%str(args[0][i])[lindex+1:rindex] + load_cli_class_rightArg += "]" + return clr.load_cli_class(load_cli_class_leftArg,load_cli_class_rightArg) + except: + # it's a single arg passed + lindex = str(args[0]).find('\'') + rindex = str(args[0]).rfind('\'') + load_cli_class_rightArg = genClassName + load_cli_class_rightArg += "`1[%s]"%(str(args[0])[lindex+1:rindex]) + return clr.load_cli_class(load_cli_class_leftArg,load_cli_class_rightArg) + class MetaCliClassWrapper(type): def __setattr__(cls, name, value): obj = cls.__dict__.get(name, None) @@ -115,7 +155,7 @@ obj.__cliobj__ = cliobj return obj -def build_wrapper(namespace, classname, staticmethods, methods, properties, indexers, hasIEnumerable): +def build_wrapper(namespace, classname, staticmethods, methods, properties, indexers, hasIEnumerable, isClassGeneric): fullname = '%s.%s' % (namespace, classname) d = {'__cliclass__': fullname, '__module__': namespace} @@ -136,7 +176,10 @@ d['__getitem__'] = d[getter] if setter: d['__setitem__'] = d[setter] - cls = MetaCliClassWrapper(classname, (CliClassWrapper,), d) + if isClassGeneric: + cls = MetaGenericCliClassWrapper(classname, (CliClassWrapper,), d) + else: + cls = MetaCliClassWrapper(classname, (CliClassWrapper,), d) # we must add properties *after* the class has been created # because we need to store UnboundMethods as getters and setters Modified: pypy/branch/clr-module-improvements/pypy/module/clr/app_importer.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/app_importer.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/app_importer.py Fri Dec 14 23:11:39 2007 @@ -62,6 +62,13 @@ # add it to the modules list sys.modules[fullname] = mod + # treating System.Collections.Generic specially here. + # this treatment is done after the empty module insertion + if fullname == "System.Collections.Generic": + genericClassList = clr.list_of_generic_classes() + for genericClass in genericClassList: + sys.modules[genericClass[: genericClass.find('`')]] = clr.load_cli_class("System.Collections.Generic", genericClass) + return sys.modules[fullname] class importer(object): @@ -78,7 +85,7 @@ def __init__(self): import clr # this might not be the correct place to load the valid NameSpaces - self.ValidNameSpaces = clr.load_valid_namespaces() + self.ValidNameSpaces = clr.list_of_valid_namespaces() self.loader = loader() def find_module(self, fullname, path = None): Modified: pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/interp_clr.py Fri Dec 14 23:11:39 2007 @@ -185,7 +185,7 @@ return self.cache.get(fullname, None) CliClassCache = _CliClassCache() -def load_valid_namespaces(space): +def list_of_valid_namespaces(space): """ We use this function to play with reflection and then return useful data to the app_importer module @@ -206,6 +206,36 @@ w_listOfNamespaces = wrap_list_of_strings(space, listOfNamespaces) return w_listOfNamespaces +def list_of_generic_classes(space): + """ + use reflection to get a list of generic classes + + Return: List of generic classes + e.g. [Dictionary`2 , List`1 , IEnumerator`1 , IEnumerable`1] + """ + listOfGenericTypes = [] + currentDomain = System.AppDomain.get_CurrentDomain() + assems = currentDomain.GetAssemblies() + for loadedAssembly in assems: + typesInAssembly = loadedAssembly.GetTypes() + for type in typesInAssembly: + namespace = type.get_Namespace() + type_str = type.ToString() + if namespace == "System.Collections.Generic": + rightDot = type_str.rfind('.') + rightTilde = type_str.rfind('`') + firstSqBracket = type_str.find('[') + firstPlus = type_str.find('+') + if rightDot != -1 and rightTilde != -1: + if firstPlus == -1: + nameToPush = type_str[rightDot+1 : firstSqBracket] + else: + nameToPush = type_str[rightDot+1 : firstPlus] + if nameToPush not in listOfGenericTypes: + listOfGenericTypes.append(nameToPush) + w_listOfGenericTypes = wrap_list_of_strings(space, listOfGenericTypes) + return w_listOfGenericTypes + def isDotNetType(space, nameFromImporter): """ determines if the string input is a DotNetType @@ -302,6 +332,12 @@ if interface.ToString() == "System.Collections.IEnumerable": hasIEnumerable = True + # this is where we test if the class is Generic + # set the flag isClassGeneric + isClassGeneric = False + if b_type.IsGenericType: + isClassGeneric = True + w_staticmethods, w_methods = get_methods(space, b_type) w_properties, w_indexers = get_properties(space, b_type) return build_wrapper(space, @@ -311,7 +347,8 @@ w_methods, w_properties, w_indexers, - space.wrap(hasIEnumerable)) + space.wrap(hasIEnumerable), + space.wrap(isClassGeneric)) class W_CliObject(Wrappable): Modified: pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py ============================================================================== --- pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py (original) +++ pypy/branch/clr-module-improvements/pypy/module/clr/test/test_clr.py Fri Dec 14 23:11:39 2007 @@ -160,7 +160,7 @@ enum = x.GetEnumerator() assert enum.MoveNext() is False - def test_iteration(self): + def test_iteration_arrayList(self): import clr ArrayList = clr.load_cli_class('System.Collections', 'ArrayList') x = ArrayList() @@ -215,3 +215,18 @@ raises(TypeError, x.__setitem__, 4, 4.453) raises(TypeError, x.__setitem__, "test", 3) + def test_generic_class_with_single_import(self): + import clr + import System.Collections.Generic + import List + import System.Int32 + l2 = List[System.Int32]() + l2.Add(3) + raises(TypeError, l2.Add, "test") + + import Dictionary + import System.String + d1 = Dictionary[System.Int32,System.String]() + d1[1]="test" + raises(TypeError, d1.__setitem__, 3, 3) + From antocuni at codespeak.net Sat Dec 15 12:35:34 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 15 Dec 2007 12:35:34 +0100 (CET) Subject: [pypy-svn] r49813 - in pypy/dist/pypy: annotation rpython/ootypesystem rpython/ootypesystem/test Message-ID: <20071215113534.7894D168508@codespeak.net> Author: antocuni Date: Sat Dec 15 12:35:33 2007 New Revision: 49813 Modified: pypy/dist/pypy/annotation/builtin.py pypy/dist/pypy/rpython/ootypesystem/ootype.py pypy/dist/pypy/rpython/ootypesystem/rbuiltin.py pypy/dist/pypy/rpython/ootypesystem/test/test_oortype.py Log: add annotator/rtyper support for oodowncast and ooupcast Modified: pypy/dist/pypy/annotation/builtin.py ============================================================================== --- pypy/dist/pypy/annotation/builtin.py (original) +++ pypy/dist/pypy/annotation/builtin.py Sat Dec 15 12:35:33 2007 @@ -16,6 +16,7 @@ from pypy.annotation.bookkeeper import getbookkeeper from pypy.annotation import description from pypy.objspace.flow.model import Constant +from pypy.tool.error import AnnotatorError import pypy.rlib.rarithmetic import pypy.rlib.objectmodel @@ -547,6 +548,20 @@ assert isinstance(i, SomeOOInstance) return SomeInteger() +def ooupcast(I, i): + assert isinstance(I.const, ootype.Instance) + if ootype.isSubclass(i.ootype, I.const): + return SomeOOInstance(I.const) + else: + raise AnnotatorError, 'Cannot cast %s to %s' % (i.ootype, I.const) + +def oodowncast(I, i): + assert isinstance(I.const, ootype.Instance) + if ootype.isSubclass(I.const, i.ootype): + return SomeOOInstance(I.const) + else: + raise AnnotatorError, 'Cannot cast %s to %s' % (i.ootype, I.const) + BUILTIN_ANALYZERS[ootype.instanceof] = instanceof BUILTIN_ANALYZERS[ootype.new] = new BUILTIN_ANALYZERS[ootype.null] = null @@ -554,6 +569,8 @@ BUILTIN_ANALYZERS[ootype.classof] = classof BUILTIN_ANALYZERS[ootype.subclassof] = subclassof BUILTIN_ANALYZERS[ootype.ooidentityhash] = ooidentityhash +BUILTIN_ANALYZERS[ootype.ooupcast] = ooupcast +BUILTIN_ANALYZERS[ootype.oodowncast] = oodowncast #________________________________ # weakrefs Modified: pypy/dist/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/ootype.py Sat Dec 15 12:35:33 2007 @@ -263,11 +263,12 @@ # We try to keep Record as similar to Instance as possible, so backends # can treat them polymorphically, if they choose to do so. - def __init__(self, fields): + def __init__(self, fields, _hints={}): self._fields = frozendict() for name, ITEMTYPE in fields.items(): self._fields[name] = ITEMTYPE, ITEMTYPE._defl() self._null = _null_record(self) + self._hints = frozendict(_hints) def _defl(self): return self._null Modified: pypy/dist/pypy/rpython/ootypesystem/rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rbuiltin.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rbuiltin.py Sat Dec 15 12:35:33 2007 @@ -40,6 +40,18 @@ return hop.genop('ooidentityhash', vlist, resulttype = ootype.Signed) +def rtype_ooupcast(hop): + assert isinstance(hop.args_s[0].const, ootype.Instance) + assert isinstance(hop.args_s[1], annmodel.SomeOOInstance) + v_inst = hop.inputarg(hop.args_r[1], arg=1) + return hop.genop('ooupcast', [v_inst], resulttype = hop.r_result.lowleveltype) + +def rtype_oodowncast(hop): + assert isinstance(hop.args_s[0].const, ootype.Instance) + assert isinstance(hop.args_s[1], annmodel.SomeOOInstance) + v_inst = hop.inputarg(hop.args_r[1], arg=1) + return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype) + def rtype_builtin_isinstance(hop): if hop.s_result.is_constant(): return hop.inputconst(ootype.Bool, hop.s_result.const) @@ -98,6 +110,8 @@ BUILTIN_TYPER[ootype.subclassof] = rtype_subclassof BUILTIN_TYPER[ootype.runtimenew] = rtype_runtimenew BUILTIN_TYPER[ootype.ooidentityhash] = rtype_ooidentityhash +BUILTIN_TYPER[ootype.ooupcast] = rtype_ooupcast +BUILTIN_TYPER[ootype.oodowncast] = rtype_oodowncast BUILTIN_TYPER[isinstance] = rtype_builtin_isinstance BUILTIN_TYPER[objectmodel.r_dict] = rtype_r_dict BUILTIN_TYPER[objectmodel.instantiate] = rtype_instantiate Modified: pypy/dist/pypy/rpython/ootypesystem/test/test_oortype.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/test/test_oortype.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/test/test_oortype.py Sat Dec 15 12:35:33 2007 @@ -9,6 +9,7 @@ from pypy.translator.translator import TranslationContext, graphof from pypy.rpython.test.test_llinterp import interpret from pypy.rlib.objectmodel import r_dict +from pypy.tool.error import AnnotatorError from pypy.rpython.ootypesystem import ooregistry # side effects def gengraph(f, args=[], viewBefore=False, viewAfter=False, mangle=True): @@ -300,3 +301,40 @@ res = interpret(oof, [], type_system='ootype') assert res == 42 + +def test_ooupcast(): + A = Instance('A', ootype.ROOT, {}) + B = Instance('B', A, {}) + C = Instance('C', ootype.ROOT) + + def fn(): + b = new(B) + return ooupcast(A, b) + + res = interpret(fn, [], type_system='ootype') + assert typeOf(res) is A + + def fn(): + c = new(C) + return ooupcast(A, c) + + py.test.raises(AnnotatorError, interpret, fn, [], type_system='ootype') + +def test_oodowncast(): + A = Instance('A', ootype.ROOT, {}) + B = Instance('B', A, {}) + C = Instance('C', ootype.ROOT) + + def fn(): + b = new(B) + a = ooupcast(A, b) + return oodowncast(B, a) + + res = interpret(fn, [], type_system='ootype') + assert typeOf(res) is B + + def fn(): + c = new(C) + return oodowncast(A, c) + + py.test.raises(AnnotatorError, interpret, fn, [], type_system='ootype') From antocuni at codespeak.net Sat Dec 15 13:53:33 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 15 Dec 2007 13:53:33 +0100 (CET) Subject: [pypy-svn] r49814 - in pypy/dist/pypy/jit/hintannotator: . test Message-ID: <20071215125333.DD15316850E@codespeak.net> Author: antocuni Date: Sat Dec 15 13:53:32 2007 New Revision: 49814 Modified: pypy/dist/pypy/jit/hintannotator/annotator.py pypy/dist/pypy/jit/hintannotator/model.py pypy/dist/pypy/jit/hintannotator/test/test_annotator.py pypy/dist/pypy/jit/hintannotator/test/test_toy.py Log: port hintannotator to ootype. Relevant tests in test_annotator pass, but test_toy still fails Modified: pypy/dist/pypy/jit/hintannotator/annotator.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/annotator.py (original) +++ pypy/dist/pypy/jit/hintannotator/annotator.py Sat Dec 15 13:53:32 2007 @@ -6,7 +6,8 @@ from pypy.jit.hintannotator.bookkeeper import HintBookkeeper from pypy.jit.hintannotator.policy import HintAnnotatorPolicy from pypy.rpython.lltypesystem import lltype - +from pypy.rpython.ootypesystem import ootype +from pypy.translator.simplify import get_funcobj class HintAnnotator(RPythonAnnotator): @@ -28,6 +29,15 @@ def getuserclassdefinitions(self): return [] + def consider_op_new(self, hs_TYPE): + TYPE = hs_TYPE.const + if self.policy.novirtualcontainer: + return hintmodel.SomeLLAbstractVariable(TYPE) + else: + # XXX: ootype + vstructdef = self.bookkeeper.getvirtualcontainerdef(TYPE) + return hintmodel.SomeLLAbstractContainer(vstructdef) + def consider_op_malloc(self, hs_TYPE, hs_flags): TYPE = hs_TYPE.const flags = hs_flags.const @@ -65,10 +75,15 @@ def consider_op_ts_metacall(self, hs_f1, hs_metadesccls, *args_hs): bookkeeper = self.bookkeeper - fnobj = hs_f1.const._obj + fnobj = get_funcobj(hs_f1.const) return hintmodel.cannot_follow_call(bookkeeper, fnobj.graph, args_hs, lltype.typeOf(fnobj).RESULT) + def consider_op_oosend(self, hs_name, *args_hs): + assert hs_name.concretetype is ootype.Void + hs_obj, args_hs = args_hs[0], args_hs[1:] + return hs_obj.oosend(hs_name, *args_hs) + def simplify(self): RPythonAnnotator.simplify(self, extra_passes=[]) Modified: pypy/dist/pypy/jit/hintannotator/model.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/model.py (original) +++ pypy/dist/pypy/jit/hintannotator/model.py Sat Dec 15 13:53:32 2007 @@ -2,6 +2,8 @@ from pypy.tool.pairtype import pair, pairtype from pypy.jit.hintannotator.bookkeeper import getbookkeeper from pypy.rpython.lltypesystem import lltype, lloperation +from pypy.rpython.ootypesystem import ootype +from pypy.translator.simplify import get_funcobj UNARY_OPERATIONS = """same_as hint getfield setfield getsubstruct getarraysize getinteriorfield getinteriorarraysize setinteriorfield @@ -20,6 +22,14 @@ ptr_nonzero ptr_iszero is_early_constant + oogetfield + oosetfield + oononnull + ooupcast + oodowncast + oois + subclassof + instanceof """.split() BINARY_OPERATIONS = """int_add int_sub int_mul int_mod int_and int_rshift @@ -244,7 +254,11 @@ def __init__(self, contentdef): self.contentdef = contentdef - self.concretetype = lltype.Ptr(contentdef.T) + T = contentdef.T + if isinstance(T, ootype.OOType): + self.concretetype = T + else: + self.concretetype = lltype.Ptr(T) def annotationcolor(self): """Compute the color of the variables with this annotation @@ -339,9 +353,16 @@ FIELD_TYPE = getattr(S, hs_fieldname.const) return variableoftype(FIELD_TYPE, hs_v1.deepfrozen) + def oogetfield(hs_v1, hs_fieldname): + _, FIELD_TYPE = hs_v1.concretetype._lookup_field(hs_fieldname.const) + return variableoftype(FIELD_TYPE, hs_v1.deepfrozen) + def setfield(hs_v1, hs_fieldname, hs_value): pass + def oosetfield(hs_v1, hs_fieldname, hs_value): + pass + def getsubstruct(hs_v1, hs_fieldname): S = hs_v1.concretetype.TO FIELD_TYPE = getattr(S, hs_fieldname.const) @@ -406,6 +427,9 @@ # function return annmodel.unionof(hs_res, bookkeeper.current_op_binding()) + def oosend(hs_v1, hs_name, *args_hs): + RESTYPE = getbookkeeper().current_op_concretetype() + return SomeLLAbstractVariable(RESTYPE) class __extend__(SomeLLAbstractConstant): @@ -428,7 +452,7 @@ def direct_call(hs_f1, *args_hs): bookkeeper = getbookkeeper() - fnobj = hs_f1.const._obj + fnobj = get_funcobj(hs_f1.const) if (bookkeeper.annotator.policy.oopspec and hasattr(fnobj._callable, 'oopspec')): # try to handle the call as a high-level operation @@ -473,9 +497,34 @@ # function return annmodel.unionof(hs_res, bookkeeper.current_op_binding()) + def oosend(hs_c1, hs_name, *args_hs): + TYPE = hs_c1.concretetype + name = hs_name.const + graph_list = TYPE._lookup_graphs(name) + if not graph_list: + # it's a method of a BuiltinType + bk = getbookkeeper() + origin = bk.myorigin() + d = setadd(hs_c1.origins, origin) + RESTYPE = bk.current_op_concretetype() + hs_res = SomeLLAbstractConstant(RESTYPE, d, + eager_concrete = hs_c1.eager_concrete, + myorigin = origin) + # if hs_c1.is_constant(): ... + return hs_res + #import pdb;pdb.set_trace() + def getfield(hs_c1, hs_fieldname): S = hs_c1.concretetype.TO FIELD_TYPE = getattr(S, hs_fieldname.const) + return hs_c1.getfield_impl(S, FIELD_TYPE) + + def oogetfield(hs_c1, hs_fieldname): + S = hs_c1.concretetype + _, FIELD_TYPE = S._lookup_field(hs_fieldname.const) + return hs_c1.getfield_impl(S, FIELD_TYPE) + + def getfield_impl(hs_c1, S, FIELD_TYPE): if S._hints.get('immutable', False) or hs_c1.deepfrozen: origin = getbookkeeper().myorigin() d = setadd(hs_c1.origins, origin) Modified: pypy/dist/pypy/jit/hintannotator/test/test_annotator.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/test/test_annotator.py (original) +++ pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Sat Dec 15 13:53:32 2007 @@ -20,956 +20,1002 @@ P_NOVIRTUAL = HintAnnotatorPolicy(novirtualcontainer=True, entrypoint_returns_red=False) -def hannotate(func, argtypes, policy=P_DEFAULT, annotator=False, inline=None, - backendoptimize=False): - # build the normal ll graphs for ll_function - t = TranslationContext() - a = t.buildannotator() - a.build_types(func, argtypes) - rtyper = t.buildrtyper() - rtyper.specialize() - if inline: - auto_inlining(t, threshold=inline) - if backendoptimize: - from pypy.translator.backendopt.all import backend_optimizations - backend_optimizations(t) - graph1 = graphof(t, func) - - # build hint annotator types - hannotator = HintAnnotator(base_translator=t, policy=policy) - hs = hannotator.build_types(graph1, [SomeLLAbstractConstant(v.concretetype, - {OriginFlags(): True}) - for v in graph1.getargs()]) - hannotator.simplify() - t = hannotator.translator - if conftest.option.view: - t.view() - if annotator: - return hs, hannotator - else: - return hs - -def test_simple(): - def ll_function(x, y): - return x + y - hs = hannotate(ll_function, [int, int]) - assert isinstance(hs, SomeLLAbstractConstant) - assert len(hs.origins) == 3 - assert hs.concretetype == lltype.Signed - -def test_join(): - def ll_function(cond, x,y): - if cond: - z = x+y - else: - z = x-y - return z - hs = hannotate(ll_function, [bool, int, int]) - assert isinstance(hs, SomeLLAbstractConstant) - assert len(hs.origins) == 4 - assert hs.concretetype == lltype.Signed +class AbstractAnnotatorTest: + type_system = None + + def hannotate(self, func, argtypes, policy=P_DEFAULT, annotator=False, inline=None, + backendoptimize=False): + # build the normal ll graphs for ll_function + t = TranslationContext() + a = t.buildannotator() + a.build_types(func, argtypes) + rtyper = t.buildrtyper(type_system = self.type_system) + rtyper.specialize() + if inline: + auto_inlining(t, threshold=inline) + if backendoptimize: + from pypy.translator.backendopt.all import backend_optimizations + backend_optimizations(t) + graph1 = graphof(t, func) + + # build hint annotator types + hannotator = HintAnnotator(base_translator=t, policy=policy) + hs = hannotator.build_types(graph1, [SomeLLAbstractConstant(v.concretetype, + {OriginFlags(): True}) + for v in graph1.getargs()]) + hannotator.simplify() + t = hannotator.translator + if conftest.option.view: + t.view() + if annotator: + return hs, hannotator + else: + return hs + + +class BaseAnnotatorTest(AbstractAnnotatorTest): + + def test_simple(self): + def ll_function(x, y): + return x + y + hs = self.hannotate(ll_function, [int, int]) + assert isinstance(hs, SomeLLAbstractConstant) + assert len(hs.origins) == 3 + assert hs.concretetype == lltype.Signed + + def test_join(self): + def ll_function(cond, x,y): + if cond: + z = x+y + else: + z = x-y + return z + hs = self.hannotate(ll_function, [bool, int, int]) + assert isinstance(hs, SomeLLAbstractConstant) + assert len(hs.origins) == 4 + assert hs.concretetype == lltype.Signed -def test_simple_hint_result(): - def ll_function(cond, x,y): - if cond: - z = x+y - else: - z = x-y - z = hint(z, concrete=True) - return z - hs = hannotate(ll_function, [bool, int, int]) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.eager_concrete - assert hs.concretetype == lltype.Signed + def test_simple_hint_result(self): + def ll_function(cond, x,y): + if cond: + z = x+y + else: + z = x-y + z = hint(z, concrete=True) + return z + hs = self.hannotate(ll_function, [bool, int, int]) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.eager_concrete + assert hs.concretetype == lltype.Signed + + def test_deepfreeze(self): + + A = lltype.GcArray(lltype.Signed) + + def ll_function(a, i): + a = hint(a, deepfreeze=True) + res = a[i] + res = hint(res, concrete=True) + + res = hint(res, variable=True) + return res + + hs = self.hannotate(ll_function, [annmodel.SomePtr(lltype.Ptr(A)), int]) + assert type(hs) is SomeLLAbstractVariable + assert hs.concretetype == lltype.Signed + + def test_lists_deepfreeze(self): + + l1 = [1,2,3,4,5] + l2 = [6,7,8,9,10] + + def getlist(n): + if n: + return l1 + else: + return l2 -def test_deepfreeze(): + def ll_function(n, i): + l = getlist(n) + l = hint(l, deepfreeze=True) - A = lltype.GcArray(lltype.Signed) - - def ll_function(a, i): - a = hint(a, deepfreeze=True) - res = a[i] - res = hint(res, concrete=True) - - res = hint(res, variable=True) - return res - - hs = hannotate(ll_function, [annmodel.SomePtr(lltype.Ptr(A)), int]) - assert type(hs) is SomeLLAbstractVariable - assert hs.concretetype == lltype.Signed + res = l[i] + res = hint(res, concrete=True) -def test_lists_deepfreeze(): + res = hint(res, variable=True) + return res - l1 = [1,2,3,4,5] - l2 = [6,7,8,9,10] - - def getlist(n): - if n: - return l1 - else: - return l2 - - def ll_function(n, i): - l = getlist(n) - l = hint(l, deepfreeze=True) - - res = l[i] - res = hint(res, concrete=True) - - res = hint(res, variable=True) - return res + hs = self.hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) + assert hs.concretetype == lltype.Signed - hs = hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) - assert hs.concretetype == lltype.Signed + def test_dicts_deepfreeze(self): -def test_dicts_deepfreeze(): + d1 = {1:2, 2:3} + d2 = {2:3, 3:4} - d1 = {1:2, 2:3} - d2 = {2:3, 3:4} - - def getdict(n): - if n: - return d1 - else: - return d2 - - def ll_function(n, i): - d = getdict(n) - d = hint(d, deepfreeze=True) - - res = d[i] - res = hint(res, concrete=True) - - res = hint(res, variable=True) - return res - - # must backendoptimize to remove the mallocs related to the interior ptrs - hs = hannotate(ll_function, [int, int], policy=P_NOVIRTUAL, - backendoptimize=True) - assert hs.concretetype == lltype.Signed - - -def test_simple_hint_origins(): - def ll_function(cond, x,y): - if cond: - z = x+y - else: - z = x-y - z1 = hint(z, concrete=True) - return z # origin of z1 - hs, ha = hannotate(ll_function, [bool, int, int], annotator=True) - assert isinstance(hs, SomeLLAbstractConstant) - assert len(hs.origins) == 4 - assert hs.is_fixed() - assert hs.concretetype == lltype.Signed - ll_function_graph = graphof(ha.base_translator, ll_function) - gdesc = ha.bookkeeper.getdesc(ll_function_graph) - _, x_v, y_v = gdesc._cache[None].getargs() - assert ha.binding(x_v).is_fixed() - assert ha.binding(y_v).is_fixed() - -def test_simple_variable(): - def ll_function(x,y): - x = hint(x, variable=True) # special hint only for testing purposes!!! - return x + y - hs = hannotate(ll_function, [int, int]) - assert type(hs) is SomeLLAbstractVariable - assert hs.concretetype == lltype.Signed - -def test_simple_concrete_propagation(): - def ll_function(x,y): - x = hint(x, concrete=True) - return x + y - hs = hannotate(ll_function, [int, int]) - assert type(hs) is SomeLLAbstractConstant - assert hs.eager_concrete - assert hs.concretetype == lltype.Signed - -def test_union(): - unionof = annmodel.unionof - av1, av2 = SomeLLAbstractVariable(lltype.Signed), SomeLLAbstractVariable(lltype.Signed) - cv1, cv2 = SomeLLAbstractConstant(lltype.Signed, {}, eager_concrete=True), SomeLLAbstractConstant(lltype.Signed, {}, eager_concrete=True) - ac1, ac2 = SomeLLAbstractConstant(lltype.Signed, {}), SomeLLAbstractConstant(lltype.Signed, {}) - ac3 = SomeLLAbstractConstant(lltype.Signed, {}) - ac3.const = 3 - ac4 = SomeLLAbstractConstant(lltype.Signed, {}) - ac4.const = 4 - assert unionof(av1, av2) == av1 - assert unionof(cv1, cv2) == cv2 - assert unionof(ac1, ac2) == ac1 - assert unionof(ac3, ac3) == ac3 - assert unionof(ac3, ac2) == ac1 - assert unionof(ac4, ac3) == ac1 - # degenerating cases - py.test.raises(annmodel.UnionError, "unionof(cv1, av1)") - py.test.raises(annmodel.UnionError, "unionof(av1, cv1)") - - # MAYBE... - #py.test.raises(annmodel.UnionError, "unionof(ac1, cv1)") - #py.test.raises(annmodel.UnionError, "unionof(cv1, ac1)") - assert unionof(cv1, ac1) == ac1 - assert unionof(ac1, cv1) == ac1 - - # constant with values - assert unionof(av1, ac1) == av1 - assert unionof(ac1, av1) == av1 - assert unionof(ac3, av1) == av1 - assert unionof(av2, ac4) == av1 - -def test_op_meet(): - def meet(hs1, hs2): - bk = HintBookkeeper(None) - block = flowmodel.Block([]) - block.operations.append(flowmodel.SpaceOperation('x', [], - flowmodel.Variable())) - bk.enter(("graph", block, 0)) - bk.current_op_concretetype = lambda: lltype.Signed # hack - return pair(hs1, hs2).int_add() - av1, av2 = SomeLLAbstractVariable(lltype.Signed), SomeLLAbstractVariable(lltype.Signed) - cv1, cv2 = SomeLLAbstractConstant(lltype.Signed, {}, True), SomeLLAbstractConstant(lltype.Signed, {}, True) - ac1, ac2 = SomeLLAbstractConstant(lltype.Signed, {}), SomeLLAbstractConstant(lltype.Signed, {}) - assert meet(av1, av2) == av1 - res = meet(cv1, cv2) - assert res.eager_concrete - assert isinstance(meet(ac1, ac2), SomeLLAbstractConstant) - assert meet(ac1, cv1).eager_concrete - assert meet(cv1, ac1).eager_concrete - assert meet(av1, cv1) == av1 - assert meet(cv1, av1) == av1 - assert meet(ac1, av1) == av1 - assert meet(av1, ac1) == av1 - -def test_loop(): - def ll_function(x, y): - while x > 0: - y += x - x -= 1 - return y - hs = hannotate(ll_function, [int, int]) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.concretetype == lltype.Signed - assert len(hs.origins) == 4 - -def test_loop1(): - def ll_function(x, y): - while x > 0: - x1 = hint(x, concrete=True) - if x1 == 7: + def getdict(n): + if n: + return d1 + else: + return d2 + + def ll_function(n, i): + d = getdict(n) + d = hint(d, deepfreeze=True) + + res = d[i] + res = hint(res, concrete=True) + + res = hint(res, variable=True) + return res + + # must backendoptimize to remove the mallocs related to the interior ptrs + hs = self.hannotate(ll_function, [int, int], policy=P_NOVIRTUAL, + backendoptimize=True) + assert hs.concretetype == lltype.Signed + + + def test_simple_hint_origins(self): + def ll_function(cond, x,y): + if cond: + z = x+y + else: + z = x-y + z1 = hint(z, concrete=True) + return z # origin of z1 + hs, ha = self.hannotate(ll_function, [bool, int, int], annotator=True) + assert isinstance(hs, SomeLLAbstractConstant) + assert len(hs.origins) == 4 + assert hs.is_fixed() + assert hs.concretetype == lltype.Signed + ll_function_graph = graphof(ha.base_translator, ll_function) + gdesc = ha.bookkeeper.getdesc(ll_function_graph) + _, x_v, y_v = gdesc._cache[None].getargs() + assert ha.binding(x_v).is_fixed() + assert ha.binding(y_v).is_fixed() + + def test_simple_variable(self): + def ll_function(x,y): + x = hint(x, variable=True) # special hint only for testing purposes!!! + return x + y + hs = self.hannotate(ll_function, [int, int]) + assert type(hs) is SomeLLAbstractVariable + assert hs.concretetype == lltype.Signed + + def test_simple_concrete_propagation(self): + def ll_function(x,y): + x = hint(x, concrete=True) + return x + y + hs = self.hannotate(ll_function, [int, int]) + assert type(hs) is SomeLLAbstractConstant + assert hs.eager_concrete + assert hs.concretetype == lltype.Signed + + def test_union(self): + unionof = annmodel.unionof + av1, av2 = SomeLLAbstractVariable(lltype.Signed), SomeLLAbstractVariable(lltype.Signed) + cv1, cv2 = SomeLLAbstractConstant(lltype.Signed, {}, eager_concrete=True), SomeLLAbstractConstant(lltype.Signed, {}, eager_concrete=True) + ac1, ac2 = SomeLLAbstractConstant(lltype.Signed, {}), SomeLLAbstractConstant(lltype.Signed, {}) + ac3 = SomeLLAbstractConstant(lltype.Signed, {}) + ac3.const = 3 + ac4 = SomeLLAbstractConstant(lltype.Signed, {}) + ac4.const = 4 + assert unionof(av1, av2) == av1 + assert unionof(cv1, cv2) == cv2 + assert unionof(ac1, ac2) == ac1 + assert unionof(ac3, ac3) == ac3 + assert unionof(ac3, ac2) == ac1 + assert unionof(ac4, ac3) == ac1 + # degenerating cases + py.test.raises(annmodel.UnionError, "unionof(cv1, av1)") + py.test.raises(annmodel.UnionError, "unionof(av1, cv1)") + + # MAYBE... + #py.test.raises(annmodel.UnionError, "unionof(ac1, cv1)") + #py.test.raises(annmodel.UnionError, "unionof(cv1, ac1)") + assert unionof(cv1, ac1) == ac1 + assert unionof(ac1, cv1) == ac1 + + # constant with values + assert unionof(av1, ac1) == av1 + assert unionof(ac1, av1) == av1 + assert unionof(ac3, av1) == av1 + assert unionof(av2, ac4) == av1 + + def test_op_meet(self): + def meet(hs1, hs2): + bk = HintBookkeeper(None) + block = flowmodel.Block([]) + block.operations.append(flowmodel.SpaceOperation('x', [], + flowmodel.Variable())) + bk.enter(("graph", block, 0)) + bk.current_op_concretetype = lambda: lltype.Signed # hack + return pair(hs1, hs2).int_add() + av1, av2 = SomeLLAbstractVariable(lltype.Signed), SomeLLAbstractVariable(lltype.Signed) + cv1, cv2 = SomeLLAbstractConstant(lltype.Signed, {}, True), SomeLLAbstractConstant(lltype.Signed, {}, True) + ac1, ac2 = SomeLLAbstractConstant(lltype.Signed, {}), SomeLLAbstractConstant(lltype.Signed, {}) + assert meet(av1, av2) == av1 + res = meet(cv1, cv2) + assert res.eager_concrete + assert isinstance(meet(ac1, ac2), SomeLLAbstractConstant) + assert meet(ac1, cv1).eager_concrete + assert meet(cv1, ac1).eager_concrete + assert meet(av1, cv1) == av1 + assert meet(cv1, av1) == av1 + assert meet(ac1, av1) == av1 + assert meet(av1, ac1) == av1 + + def test_loop(self): + def ll_function(x, y): + while x > 0: y += x - x -= 1 - return y - hs = hannotate(ll_function, [int, int]) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.concretetype == lltype.Signed - assert len(hs.origins) == 4 - -def test_simple_struct(): - S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), - ('world', lltype.Signed), - hints={'immutable': True}) - def ll_function(s): - return s.hello * s.world - hs = hannotate(ll_function, [annmodel.SomePtr(lltype.Ptr(S))]) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.concretetype == lltype.Signed - assert len(hs.origins) == 4 - -def test_simple_struct_malloc(): - S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), - ('world', lltype.Signed)) - def ll_function(x): - s = lltype.malloc(S) - s.hello = x - return s.hello + s.world - - hs = hannotate(ll_function, [int]) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.concretetype == lltype.Signed - assert len(hs.origins) == 2 - -def test_container_union(): - S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), - ('world', lltype.Signed)) - def ll_function(cond, x, y): - if cond: - s = lltype.malloc(S) + x -= 1 + return y + hs = self.hannotate(ll_function, [int, int]) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + assert len(hs.origins) == 4 + + def test_loop1(self): + def ll_function(x, y): + while x > 0: + x1 = hint(x, concrete=True) + if x1 == 7: + y += x + x -= 1 + return y + hs = self.hannotate(ll_function, [int, int]) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + assert len(hs.origins) == 4 + + def test_simple_struct(self): + S = self.make_struct('helloworld', ('hello', lltype.Signed), + ('world', lltype.Signed), + hints={'immutable': True}) + def ll_function(s): + return s.hello * s.world + hs = self.hannotate(ll_function, [self.annotate_struct(S)]) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + assert len(hs.origins) == 4 + + def test_simple_struct_malloc(self): + S = self.make_struct('helloworld', ('hello', lltype.Signed), + ('world', lltype.Signed)) + malloc = self.malloc + def ll_function(x): + s = malloc(S) s.hello = x - else: - s = lltype.malloc(S) - s.world = y - return s.hello + s.world + return s.hello + s.world - hs = hannotate(ll_function, [bool, int, int]) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.concretetype == lltype.Signed - assert len(hs.origins) == 3 - -def test_simple_call(): - def ll2(x, y, z): - return x + (y + 42) - def ll1(x, y, z): - return ll2(x, y - z, x + y + z) - hs = hannotate(ll1, [int, int, int]) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.concretetype == lltype.Signed - assert len(hs.origins) == 5 - -def test_simple_list_operations(): - def ll_function(x, y, index): - l = [x] - l.append(y) - return l[index] - hs = hannotate(ll_function, [int, int, int], policy=P_OOPSPEC) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.concretetype == lltype.Signed - assert len(hs.origins) == 4 - -def test_some_more_list_operations(): - def ll_function(x, y, index): - l = [] - l.append(x) - l[0] = y - return (l+list(l))[index] - hs = hannotate(ll_function, [int, int, int], policy=P_OOPSPEC) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.concretetype == lltype.Signed - assert len(hs.origins) == 4 - -def test_make_a_list(): - def ll_function(x, y): - return [x, y] - hs = hannotate(ll_function, [int, int], policy=P_OOPSPEC) - assert isinstance(hs, SomeLLAbstractContainer) - -def test_frozen_list(): - lst = [5, 7, 9] - def ll_function(x): - mylist = hint(lst, deepfreeze=True) - z = mylist[x] - hint(z, concrete=True) - return z - hs = hannotate(ll_function, [int], policy=P_OOPSPEC_NOVIRTUAL) - assert hs.is_green() - -def test_simple_cast_pointer(): - GCS1 = lltype.GcStruct('s1', ('x', lltype.Signed)) - GCS2 = lltype.GcStruct('s2', ('sub', GCS1), ('y', lltype.Signed)) - PGCS1 = lltype.Ptr(GCS1) - PGCS2 = lltype.Ptr(GCS2) - def ll1(): - s2 = lltype.malloc(GCS2) - return lltype.cast_pointer(PGCS1, s2) - hs = hannotate(ll1, []) - assert isinstance(hs, SomeLLAbstractContainer) - assert hs.concretetype == PGCS1 - def ll1(): - s2 = lltype.malloc(GCS2) - s1 = s2.sub - return lltype.cast_pointer(PGCS2, s1) - hs = hannotate(ll1, []) - assert isinstance(hs, SomeLLAbstractContainer) - assert hs.concretetype == PGCS2 - -def test_getarrayitem(): - A = lltype.GcArray(lltype.Signed, hints={'immutable': True}) - a = lltype.malloc(A, 10) - def ll1(n): - v = a[n] - v = hint(v, concrete=True) - return v - hs, ha = hannotate(ll1, [int], annotator=True) - assert hs.eager_concrete - g1 = graphof(ha.translator, ll1) - hs_n = ha.binding(g1.getargs()[0]) - assert hs_n.origins.keys()[0].fixed - -def test_getvarrayitem(): - A = lltype.GcArray(lltype.Signed, hints={'immutable': True}) - def ll1(n): - a = lltype.malloc(A, 10) - v = a[n] - v = hint(v, concrete=True) - return v - hs, ha = hannotate(ll1, [int], annotator=True) - assert hs.eager_concrete - g1 = graphof(ha.translator, ll1) - hs_n = ha.binding(g1.getargs()[0]) - assert hs_n.origins.keys()[0].fixed - -def test_prebuilt_structure(): - S = lltype.GcStruct('S', ('n', lltype.Signed)) - s = lltype.malloc(S) - def ll1(n): - s.n = n - return s.n - hs = hannotate(ll1, [int]) - assert isinstance(hs, SomeLLAbstractVariable) - -def test_degenerated_merge_substructure(): - S = lltype.GcStruct('S', ('n', lltype.Signed)) - T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) - - def ll_function(flag): - t = lltype.malloc(T) - t.s.n = 3 - s = lltype.malloc(S) - s.n = 4 - if flag: - s = t.s - return s, t - hs = hannotate(ll_function, [bool]) - assert isinstance(hs, SomeLLAbstractContainer) - assert not hs.contentdef.degenerated - assert len(hs.contentdef.fields) == 2 - hs0 = hs.contentdef.fields['item0'].s_value # 's' - assert isinstance(hs0, SomeLLAbstractContainer) - assert hs0.contentdef.degenerated - hs1 = hs.contentdef.fields['item1'].s_value # 't' - assert isinstance(hs1, SomeLLAbstractContainer) - assert hs1.contentdef.degenerated - -def test_degenerated_merge_cross_substructure(): - py.test.skip("no longer a valid test") - from pypy.rlib import objectmodel - S = lltype.Struct('S', ('n', lltype.Signed)) - T = lltype.GcStruct('T', ('s', S), ('s1', S), ('n', lltype.Float)) - - def ll_function(flag): - t = lltype.malloc(T) - t.s.n = 3 - t.s1.n = 3 - if flag: - s = t.s - else: - s = t.s1 - objectmodel.keepalive_until_here(t) - return s, t - hs = hannotate(ll_function, [bool]) - assert isinstance(hs, SomeLLAbstractContainer) - assert not hs.contentdef.degenerated - assert len(hs.contentdef.fields) == 2 - hs0 = hs.contentdef.fields['item0'].s_value # 's' - assert isinstance(hs0, SomeLLAbstractContainer) - assert hs0.contentdef.degenerated - hs1 = hs.contentdef.fields['item1'].s_value # 't' - assert isinstance(hs1, SomeLLAbstractContainer) - assert hs1.contentdef.degenerated - - -def test_simple_fixed_call(): - def ll_help(cond, x, y): - if cond: - z = x+y - else: - z = x-y - return z - def ll_function(cond, x,y, x1, y1): - z1 = ll_help(cond, x1, y1) - z = ll_help(cond, x, y) - z = hint(z, concrete=True) - return z - hs, ha = hannotate(ll_function, [bool, int, int, int, int], annotator=True) - assert hs.eager_concrete - assert hs.concretetype == lltype.Signed - ll_help_graph = graphof(ha.base_translator, ll_help) - gdesc = ha.bookkeeper.getdesc(ll_help_graph) - assert not ha.binding(gdesc._cache[None].getreturnvar()).is_fixed() - assert len(gdesc._cache) == 2 - assert ha.binding(gdesc._cache['fixed'].getreturnvar()).is_fixed() - -def test_specialize_calls(): - def ll_add(x, y): - return x+y - def ll_function(x,y): - z0 = ll_add(y, 2) - z1 = ll_add(x, y) - x1 = hint(x, concrete=True) - z2 = ll_add(x1, y) - return z2 - hs, ha = hannotate(ll_function, [int, int], annotator=True) - assert hs.eager_concrete - assert hs.concretetype == lltype.Signed - ll_add_graph = graphof(ha.base_translator, ll_add) - gdesc = ha.bookkeeper.getdesc(ll_add_graph) - assert len(gdesc._cache) == 2 - assert 'Exxx' in gdesc._cache - v1, v2 = gdesc._cache['Exxx'].getargs() - - assert isinstance(ha.binding(v1), SomeLLAbstractConstant) - assert isinstance(ha.binding(v2), SomeLLAbstractConstant) - assert ha.binding(v1).eager_concrete - assert not ha.binding(v2).is_fixed() - -def test_specialize_deepfreeze_calls(): + hs = self.hannotate(ll_function, [int]) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + assert len(hs.origins) == 2 + + def test_container_union(self): + S = self.make_struct('helloworld', ('hello', lltype.Signed), + ('world', lltype.Signed)) + malloc = self.malloc + def ll_function(cond, x, y): + if cond: + s = malloc(S) + s.hello = x + else: + s = malloc(S) + s.world = y + return s.hello + s.world + + hs = self.hannotate(ll_function, [bool, int, int]) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + assert len(hs.origins) == 3 + + def test_simple_call(self): + def ll2(x, y, z): + return x + (y + 42) + def ll1(x, y, z): + return ll2(x, y - z, x + y + z) + hs = self.hannotate(ll1, [int, int, int]) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + assert len(hs.origins) == 5 + + def test_simple_list_operations(self): + def ll_function(x, y, index): + l = [x] + l.append(y) + return l[index] + hs = self.hannotate(ll_function, [int, int, int], policy=P_OOPSPEC) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + assert len(hs.origins) == 4 + + def test_some_more_list_operations(self): + def ll_function(x, y, index): + l = [] + l.append(x) + l[0] = y + return (l+list(l))[index] + hs = self.hannotate(ll_function, [int, int, int], policy=P_OOPSPEC) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + assert len(hs.origins) == 4 + + def test_make_a_list(self): + def ll_function(x, y): + return [x, y] + hs = self.hannotate(ll_function, [int, int], policy=P_OOPSPEC) + assert isinstance(hs, SomeLLAbstractContainer) + + def test_frozen_list(self): + lst = [5, 7, 9] + def ll_function(x): + mylist = hint(lst, deepfreeze=True) + z = mylist[x] + hint(z, concrete=True) + return z + hs = self.hannotate(ll_function, [int], policy=P_OOPSPEC_NOVIRTUAL) + assert hs.is_green() + + def test_prebuilt_structure(self): + S = self.make_struct('S', ('n', lltype.Signed)) + s = self.malloc(S) + def ll1(n): + s.n = n + return s.n + hs = self.hannotate(ll1, [int]) + assert isinstance(hs, SomeLLAbstractVariable) + + def test_simple_fixed_call(self): + def ll_help(cond, x, y): + if cond: + z = x+y + else: + z = x-y + return z + def ll_function(cond, x,y, x1, y1): + z1 = ll_help(cond, x1, y1) + z = ll_help(cond, x, y) + z = hint(z, concrete=True) + return z + hs, ha = self.hannotate(ll_function, [bool, int, int, int, int], annotator=True) + assert hs.eager_concrete + assert hs.concretetype == lltype.Signed + ll_help_graph = graphof(ha.base_translator, ll_help) + gdesc = ha.bookkeeper.getdesc(ll_help_graph) + assert not ha.binding(gdesc._cache[None].getreturnvar()).is_fixed() + assert len(gdesc._cache) == 2 + assert ha.binding(gdesc._cache['fixed'].getreturnvar()).is_fixed() + + def test_specialize_calls(self): + def ll_add(x, y): + return x+y + def ll_function(x,y): + z0 = ll_add(y, 2) + z1 = ll_add(x, y) + x1 = hint(x, concrete=True) + z2 = ll_add(x1, y) + return z2 + hs, ha = self.hannotate(ll_function, [int, int], annotator=True) + assert hs.eager_concrete + assert hs.concretetype == lltype.Signed + ll_add_graph = graphof(ha.base_translator, ll_add) + gdesc = ha.bookkeeper.getdesc(ll_add_graph) + assert len(gdesc._cache) == 2 + assert 'Exxx' in gdesc._cache + v1, v2 = gdesc._cache['Exxx'].getargs() + + assert isinstance(ha.binding(v1), SomeLLAbstractConstant) + assert isinstance(ha.binding(v2), SomeLLAbstractConstant) + assert ha.binding(v1).eager_concrete + assert not ha.binding(v2).is_fixed() + + def test_specialize_deepfreeze_calls(self): + + l1 = [1,2,3,4,5] + l2 = [6,7,8,9,10] + + def getlist(n): + if n: + return l1 + else: + return l2 - l1 = [1,2,3,4,5] - l2 = [6,7,8,9,10] - - def getlist(n): - if n: - return l1 - else: - return l2 + def ll_get(l, i): + return l[i] - def ll_get(l, i): - return l[i] + def ll_function(n, i): + l = getlist(n) - def ll_function(n, i): - l = getlist(n) + l2 = ll_get(l, 0) + + l = hint(l, deepfreeze=True) + res = ll_get(l, i) + return res + + hs, ha = self.hannotate(ll_function, [int, int], annotator=True, policy=P_NOVIRTUAL) + assert hs.deepfrozen + assert hs.concretetype == lltype.Signed + ll_get_graph = graphof(ha.base_translator, ll_get) + gdesc = ha.bookkeeper.getdesc(ll_get_graph) + assert len(gdesc._cache) == 2 + assert 'xDxx' in gdesc._cache + v1, v2 = gdesc._cache['xDxx'].getargs() + + assert isinstance(ha.binding(v1), SomeLLAbstractConstant) + assert isinstance(ha.binding(v2), SomeLLAbstractConstant) + assert ha.binding(v1).deepfrozen + + def test_deepfreeze_variables(self): + l1 = [[1], [2, 3], [4], []] + def ll_function(i): + i = hint(i, variable=True) + l = hint(l1, deepfreeze=True) + return l[i] + + hs, ha = self.hannotate(ll_function, [int], annotator=True, policy=P_NOVIRTUAL) + assert isinstance(hs, SomeLLAbstractVariable) + assert hs.deepfrozen + + def test_propagate_fixing_across_func_arguments(self): + def ll_func2(z): + z = hint(z, concrete=True) + return z + 1 + def ll_function(cond, x,y): + if cond: + z = x+y + else: + z = x-y + z = ll_func2(z) + return z + hs, ha = self.hannotate(ll_function, [bool, int, int], annotator=True) + assert hs.eager_concrete + assert hs.concretetype == lltype.Signed + ll_function_graph = graphof(ha.base_translator, ll_function) + gdesc = ha.bookkeeper.getdesc(ll_function_graph) + _, x_v, y_v = gdesc._cache[None].getargs() + assert ha.binding(x_v).is_fixed() + assert ha.binding(y_v).is_fixed() + + def test_hannotate_plus_minus(self): + def ll_plus_minus(s, x, y): + acc = x + n = len(s) + pc = 0 + while pc < n: + op = s[pc] + op = hint(op, concrete=True) + if op == '+': + acc += y + elif op == '-': + acc -= y + pc += 1 + return acc + assert ll_plus_minus("+-+", 0, 2) == 2 + self.hannotate(ll_plus_minus, [str, int, int]) + self.hannotate(ll_plus_minus, [str, int, int], inline=100000) + + def test_invalid_hint_1(self): + S = self.make_struct('S', ('x', lltype.Signed)) + def ll_getitem_switch(s): + n = s.x # -> variable + return hint(n, concrete=True) + py.test.raises(HintError, self.hannotate, + ll_getitem_switch, [self.annotate_struct(S)]) + + def undecided_relevance_test_invalid_hint_2(self): + S = lltype.GcStruct('S', ('x', lltype.Signed)) + def ll_getitem_switch(s): + if s.x > 0: # variable exitswitch + sign = 1 + else: + sign = -1 + return hint(sign, concrete=True) + py.test.skip("in-progress: I think we expect a HintError here, do we?") + py.test.raises(HintError, hannotate, + ll_getitem_switch, [annmodel.SomePtr(lltype.Ptr(S))]) - l2 = ll_get(l, 0) - - l = hint(l, deepfreeze=True) - res = ll_get(l, i) - return res - - hs, ha = hannotate(ll_function, [int, int], annotator=True, policy=P_NOVIRTUAL) - assert hs.deepfrozen - assert hs.concretetype == lltype.Signed - ll_get_graph = graphof(ha.base_translator, ll_get) - gdesc = ha.bookkeeper.getdesc(ll_get_graph) - assert len(gdesc._cache) == 2 - assert 'xDxx' in gdesc._cache - v1, v2 = gdesc._cache['xDxx'].getargs() - - assert isinstance(ha.binding(v1), SomeLLAbstractConstant) - assert isinstance(ha.binding(v2), SomeLLAbstractConstant) - assert ha.binding(v1).deepfrozen - -def test_deepfreeze_variables(): - l1 = [[1], [2, 3], [4], []] - def ll_function(i): - i = hint(i, variable=True) - l = hint(l1, deepfreeze=True) - return l[i] - - hs, ha = hannotate(ll_function, [int], annotator=True, policy=P_NOVIRTUAL) - assert isinstance(hs, SomeLLAbstractVariable) - assert hs.deepfrozen - -def test_propagate_fixing_across_func_arguments(): - def ll_func2(z): - z = hint(z, concrete=True) - return z + 1 - def ll_function(cond, x,y): - if cond: - z = x+y - else: - z = x-y - z = ll_func2(z) - return z - hs, ha = hannotate(ll_function, [bool, int, int], annotator=True) - assert hs.eager_concrete - assert hs.concretetype == lltype.Signed - ll_function_graph = graphof(ha.base_translator, ll_function) - gdesc = ha.bookkeeper.getdesc(ll_function_graph) - _, x_v, y_v = gdesc._cache[None].getargs() - assert ha.binding(x_v).is_fixed() - assert ha.binding(y_v).is_fixed() - -def test_hannotate_plus_minus(): - def ll_plus_minus(s, x, y): - acc = x - n = len(s) - pc = 0 - while pc < n: - op = s[pc] - op = hint(op, concrete=True) - if op == '+': - acc += y - elif op == '-': - acc -= y - pc += 1 - return acc - assert ll_plus_minus("+-+", 0, 2) == 2 - hannotate(ll_plus_minus, [str, int, int]) - hannotate(ll_plus_minus, [str, int, int], inline=100000) - -def test_invalid_hint_1(): - S = lltype.GcStruct('S', ('x', lltype.Signed)) - def ll_getitem_switch(s): - n = s.x # -> variable - return hint(n, concrete=True) - py.test.raises(HintError, hannotate, - ll_getitem_switch, [annmodel.SomePtr(lltype.Ptr(S))]) - -def undecided_relevance_test_invalid_hint_2(): - S = lltype.GcStruct('S', ('x', lltype.Signed)) - def ll_getitem_switch(s): - if s.x > 0: # variable exitswitch - sign = 1 - else: - sign = -1 - return hint(sign, concrete=True) - py.test.skip("in-progress: I think we expect a HintError here, do we?") - py.test.raises(HintError, hannotate, - ll_getitem_switch, [annmodel.SomePtr(lltype.Ptr(S))]) - - -def test_raise_exc(): - class E(Exception): - pass - def f1(): - raise E - hannotate(f1, [], policy=P_OOPSPEC_NOVIRTUAL) - - def f2(): - e = E() - e.a = 3 - raise e - hannotate(f2, [], policy=P_OOPSPEC_NOVIRTUAL) - -def test_raise_and_catch_exc(): - class E(Exception): - pass - def f(flag): - if flag: + def test_raise_exc(self): + class E(Exception): + pass + def f1(): raise E + self.hannotate(f1, [], policy=P_OOPSPEC_NOVIRTUAL) - def g(flag): - try: - f(flag) - except E: - return -1 - return 0 - - hs = hannotate(g, [bool], policy=P_OOPSPEC_NOVIRTUAL) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.concretetype == lltype.Signed - - def f(flag): - if flag: + def f2(): e = E() e.a = 3 raise e - - hs = hannotate(g, [bool], policy=P_OOPSPEC_NOVIRTUAL) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.concretetype == lltype.Signed - - -def test_more_green(): - def f(x): - z = x + 1 - x2 = hint(x, concrete=True) - return z - - hs = hannotate(f, [int]) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.is_green() - assert not hs.is_fixed() - -def test_blue_simple_meth(): - py.test.skip("with abstract containers this test explode in the cast_pointer annotation logic") - class Base(object): - - def m(self): - raise NotImplementedError - - class Concrete(Base): - - def m(self): - return 42 - - def f(flag): - if flag: - o = Base() - else: - o = Concrete() - return o.m() + self.hannotate(f2, [], policy=P_OOPSPEC_NOVIRTUAL) - hs = hannotate(f, [bool], policy=P_OOPSPEC) + def test_raise_and_catch_exc(self): + class E(Exception): + pass + def f(flag): + if flag: + raise E + + def g(flag): + try: + f(flag) + except E: + return -1 + return 0 + + hs = self.hannotate(g, [bool], policy=P_OOPSPEC_NOVIRTUAL) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + + def f(flag): + if flag: + e = E() + e.a = 3 + raise e + + hs = self.hannotate(g, [bool], policy=P_OOPSPEC_NOVIRTUAL) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + + + def test_more_green(self): + def f(x): + z = x + 1 + x2 = hint(x, concrete=True) + return z + + hs = self.hannotate(f, [int]) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.is_green() + assert not hs.is_fixed() + + def test_blue_simple_meth(self): + py.test.skip("with abstract containers this test explode in the cast_pointer annotation logic") + class Base(object): + + def m(self): + raise NotImplementedError + + class Concrete(Base): + + def m(self): + return 42 + + def f(flag): + if flag: + o = Base() + else: + o = Concrete() + return o.m() -def test_simple_meth(): - class Base(object): - def m(self): - raise NotImplementedError - pass # for inspect.getsource() bugs - - class Concrete(Base): - def m(self): - return 42 - pass # for inspect.getsource() bugs - - def f(flag): - if flag: - o = Base() - else: - o = Concrete() - return o.m() + hs = self.hannotate(f, [bool], policy=P_OOPSPEC) - hs = hannotate(f, [bool], policy=P_OOPSPEC_NOVIRTUAL) + def test_simple_meth(self): + class Base(object): + def m(self): + raise NotImplementedError + pass # for inspect.getsource() bugs + + class Concrete(Base): + def m(self): + return 42 + pass # for inspect.getsource() bugs + + def f(flag): + if flag: + o = Base() + else: + o = Concrete() + return o.m() -def test_green_isinstance(): - class Base(object): - pass - class Concrete(Base): - pass + hs = self.hannotate(f, [bool], policy=P_OOPSPEC_NOVIRTUAL) - def f(o): - hint(o, concrete=True) - return isinstance(o, Concrete) - hs = hannotate(f, [Base], policy=P_OOPSPEC_NOVIRTUAL) - assert hs.is_green() + def test_green_isinstance(self): + class Base(object): + pass + class Concrete(Base): + pass + def f(o): + hint(o, concrete=True) + return isinstance(o, Concrete) -def test_cast_pointer_keeps_deepfreeze(): + hs = self.hannotate(f, [Base], policy=P_OOPSPEC_NOVIRTUAL) + assert hs.is_green() - class A(object): - pass - class B(A): - pass - - def getinstance(n): - if n: - return A() - else: - return B() - - def ll_function(n): - a = getinstance(n) - a = hint(a, promote=True) - a = hint(a, deepfreeze=True) - if isinstance(a, B): - return a - return None + def test_cast_pointer_keeps_deepfreeze(self): - hs = hannotate(ll_function, [int], policy=P_NOVIRTUAL) - assert hs.deepfrozen + class A(object): + pass + class B(A): + pass + def getinstance(n): + if n: + return A() + else: + return B() -def test_concrete_fnptr_for_green_call(): + def ll_function(n): + a = getinstance(n) + a = hint(a, promote=True) + a = hint(a, deepfreeze=True) - def h1(n): - return n * 10 - - def h2(n): - return n + 20 - - lst = [h1, h2] - - def ll_function(n, m): - h = hint(lst, deepfreeze=True)[m] - res = h(n) - hint(res, concrete=True) # so 'h' gets green, so 'm' gets green - return m - - hs = hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) - assert hs.is_green() - - -def test_indirect_yellow_call(): - - def h1(n): - return 123 - - def h2(n): - return 456 - - lst = [h1, h2] - - def ll_function(n, m): - h = hint(lst, deepfreeze=True)[m] - return h(n) - - hs = hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) - assert not hs.is_green() - - -def test_indirect_sometimes_residual_pure_red_call(): - def h1(x): - return x-2 - def h2(x): - return x*4 - l = [h1, h2] - def f(n, x): - frozenl = hint(l, deepfreeze=True) - h = frozenl[n&1] - return h(x) - - P = StopAtXPolicy(h1) - P.oopspec = True - P.entrypoint_returns_red = False - hs, hannotator = hannotate(f, [int, int], policy=P, annotator=True) - assert not hs.is_green() - assert isinstance(hs, SomeLLAbstractConstant) - - tsgraph = graphof(hannotator.translator, h2) - hs = hannotator.binding(tsgraph.getargs()[0]) - assert not hs.is_green() - - -def test_indirect_sometimes_residual_red_call(): - class Stuff: - pass - stuff = Stuff() - def h1(x): - stuff.hello = 123 - return x-2 - def h2(x): - return x*4 - l = [h1, h2] - def f(n, x): - frozenl = hint(l, deepfreeze=True) - h = frozenl[n&1] - return h(x) - - P = StopAtXPolicy(h1) - P.oopspec = True - P.entrypoint_returns_red = False - hs, hannotator = hannotate(f, [int, int], policy=P, annotator=True) - assert not hs.is_green() - - tsgraph = graphof(hannotator.translator, h2) - hs = hannotator.binding(tsgraph.getargs()[0]) - assert not hs.is_green() - - -def test_indirect_sometimes_residual_pure_but_fixed_red_call(): - def h1(x): - return x-2 - def h2(x): - return x*4 - l = [h1, h2] - def f(n, x): - frozenl = hint(l, deepfreeze=True) - h = frozenl[n&1] - z = h(x) - hint(z, concrete=True) - return z - - P = StopAtXPolicy(h1) - P.oopspec = True - P.entrypoint_returns_red = False - hs, hannotator = hannotate(f, [int, int], policy=P, annotator=True) - assert hs.is_green() - - #tsgraph = graphof(hannotator.translator, h2) - #hs = hannotator.binding(tsgraph.getargs()[0]) - #assert hs.is_green() - - tsgraph = graphof(hannotator.translator, f) - hs = hannotator.binding(tsgraph.getargs()[0]) - assert hs.is_green() - hs = hannotator.binding(tsgraph.getargs()[1]) - assert hs.is_green() - -def test_ignore_nonjitted_path(): - def f(n): - if we_are_jitted(): - return 5 - else: - return n - hs = hannotate(f, [int]) - assert hs.is_green() - - def g(n): - if not we_are_jitted(): - return n - else: - return 5 - hs = hannotate(g, [int]) - assert hs.is_green() - - def g(n): - if not we_are_jitted(): - return n - else: - return 5 - hs = hannotate(g, [int], backendoptimize=True) - assert hs.is_green() - - -def test_substitute_graph(): - class MetaG: - pass # the details are only used by the timeshifter - - def g(m): - return m * 17 - - def f(n, m): - x = g(n) - y = g(m) - hint(y, concrete=True) - return g(m) - - class MyPolicy(HintAnnotatorPolicy): - entrypoint_returns_red = False - def look_inside_graph(self, graph): - if graph.func is g: - return MetaG # replaces g with a meta-call to metafunc() - else: - return True - - hs, hannotator = hannotate(f, [int, int], policy=MyPolicy(), - annotator=True) - assert hs.is_green() - for graph in hannotator.translator.graphs: - assert 'int_mul' not in flowmodel.summary(graph) - - -def test_cast_ptr_to_int(): - GCS1 = lltype.GcStruct('s1', ('x', lltype.Signed)) - def f(): - p = lltype.malloc(GCS1) - return lltype.cast_ptr_to_int(p) - - hs = hannotate(f, [], policy=P_NOVIRTUAL) - assert not hs.is_green() - -def test_strange_green_result_after_red_switch(): - py.test.skip("is this right?") - class LinkedRules(object): - _immutable_ = True - def __init__(self, data, next=None): - self.data = data - self.next = next - - def find_applicable_rule(self, query): - # self is green, query isn't - # should the result really be green? - while self: - data = self.data - hint(data, concrete=True) - j = 0 - if self.data == query: - return self - self = self.next + if isinstance(a, B): + return a return None - chain = LinkedRules(1, LinkedRules(2, LinkedRules(0))) - def f(x): - rulechain = chain.find_applicable_rule(x) - return rulechain - hs = hannotate(f, [int], policy=P_OOPSPEC_NOVIRTUAL) - assert isinstance(hs, SomeLLAbstractVariable) - -def test_manual_marking_of_pure_functions(): - d = {} - def h1(s): - try: - return d[s] - except KeyError: - d[s] = r = hash(s) - return r - h1._pure_function_ = True - def f(n): - hint(n, concrete=True) - if n == 0: - s = "abc" - else: - s = "123" - a = h1(s) - return a - - P = StopAtXPolicy(h1) - P.oopspec = True - P.entrypoint_returns_red = False - hs = hannotate(f, [int], policy=P) - assert hs.is_green() + hs = self.hannotate(ll_function, [int], policy=P_NOVIRTUAL) + assert hs.deepfrozen + + + def test_concrete_fnptr_for_green_call(self): + + def h1(n): + return n * 10 + + def h2(n): + return n + 20 + + lst = [h1, h2] + + def ll_function(n, m): + h = hint(lst, deepfreeze=True)[m] + res = h(n) + hint(res, concrete=True) # so 'h' gets green, so 'm' gets green + return m + + hs = self.hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) + assert hs.is_green() + + + def test_indirect_yellow_call(self): + + def h1(n): + return 123 + + def h2(n): + return 456 + + lst = [h1, h2] + + def ll_function(n, m): + h = hint(lst, deepfreeze=True)[m] + return h(n) + + hs = self.hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) + assert not hs.is_green() + + + def test_indirect_sometimes_residual_pure_red_call(self): + def h1(x): + return x-2 + def h2(x): + return x*4 + l = [h1, h2] + def f(n, x): + frozenl = hint(l, deepfreeze=True) + h = frozenl[n&1] + return h(x) + + P = StopAtXPolicy(h1) + P.oopspec = True + P.entrypoint_returns_red = False + hs, hannotator = self.hannotate(f, [int, int], policy=P, annotator=True) + assert not hs.is_green() + assert isinstance(hs, SomeLLAbstractConstant) + + tsgraph = graphof(hannotator.translator, h2) + hs = hannotator.binding(tsgraph.getargs()[0]) + assert not hs.is_green() + + + def test_indirect_sometimes_residual_red_call(self): + class Stuff: + pass + stuff = Stuff() + def h1(x): + stuff.hello = 123 + return x-2 + def h2(x): + return x*4 + l = [h1, h2] + def f(n, x): + frozenl = hint(l, deepfreeze=True) + h = frozenl[n&1] + return h(x) + + P = StopAtXPolicy(h1) + P.oopspec = True + P.entrypoint_returns_red = False + hs, hannotator = self.hannotate(f, [int, int], policy=P, annotator=True) + assert not hs.is_green() + + tsgraph = graphof(hannotator.translator, h2) + hs = hannotator.binding(tsgraph.getargs()[0]) + assert not hs.is_green() + + + def test_indirect_sometimes_residual_pure_but_fixed_red_call(self): + def h1(x): + return x-2 + def h2(x): + return x*4 + l = [h1, h2] + def f(n, x): + frozenl = hint(l, deepfreeze=True) + h = frozenl[n&1] + z = h(x) + hint(z, concrete=True) + return z + + P = StopAtXPolicy(h1) + P.oopspec = True + P.entrypoint_returns_red = False + hs, hannotator = self.hannotate(f, [int, int], policy=P, annotator=True) + assert hs.is_green() + + #tsgraph = graphof(hannotator.translator, h2) + #hs = hannotator.binding(tsgraph.getargs()[0]) + #assert hs.is_green() + + tsgraph = graphof(hannotator.translator, f) + hs = hannotator.binding(tsgraph.getargs()[0]) + assert hs.is_green() + hs = hannotator.binding(tsgraph.getargs()[1]) + assert hs.is_green() + + def test_ignore_nonjitted_path(self): + def f(n): + if we_are_jitted(): + return 5 + else: + return n + hs = self.hannotate(f, [int]) + assert hs.is_green() + + def g(n): + if not we_are_jitted(): + return n + else: + return 5 + hs = self.hannotate(g, [int]) + assert hs.is_green() + + def g(n): + if not we_are_jitted(): + return n + else: + return 5 + hs = self.hannotate(g, [int], backendoptimize=True) + assert hs.is_green() + + + def test_substitute_graph(self): + class MetaG: + pass # the details are only used by the timeshifter + + def g(m): + return m * 17 + + def f(n, m): + x = g(n) + y = g(m) + hint(y, concrete=True) + return g(m) + + class MyPolicy(HintAnnotatorPolicy): + entrypoint_returns_red = False + def look_inside_graph(self, graph): + if graph.func is g: + return MetaG # replaces g with a meta-call to metafunc() + else: + return True + + hs, hannotator = self.hannotate(f, [int, int], policy=MyPolicy(), + annotator=True) + assert hs.is_green() + for graph in hannotator.translator.graphs: + assert 'int_mul' not in flowmodel.summary(graph) + + def test_strange_green_result_after_red_switch(self): + py.test.skip("is this right?") + class LinkedRules(object): + _immutable_ = True + def __init__(self, data, next=None): + self.data = data + self.next = next + + def find_applicable_rule(self, query): + # self is green, query isn't + # should the result really be green? + while self: + data = self.data + hint(data, concrete=True) + j = 0 + if self.data == query: + return self + self = self.next + return None + + chain = LinkedRules(1, LinkedRules(2, LinkedRules(0))) + def f(x): + rulechain = chain.find_applicable_rule(x) + return rulechain + hs = self.hannotate(f, [int], policy=P_OOPSPEC_NOVIRTUAL) + assert isinstance(hs, SomeLLAbstractVariable) + + def test_manual_marking_of_pure_functions(self): + d = {} + def h1(s): + try: + return d[s] + except KeyError: + d[s] = r = hash(s) + return r + h1._pure_function_ = True + def f(n): + hint(n, concrete=True) + if n == 0: + s = "abc" + else: + s = "123" + a = h1(s) + return a + + P = StopAtXPolicy(h1) + P.oopspec = True + P.entrypoint_returns_red = False + hs = self.hannotate(f, [int], policy=P) + assert hs.is_green() + +class TestLLType(BaseAnnotatorTest): + type_system = 'lltype' + + malloc = property(lambda self: lltype.malloc) + + def make_struct(self, name, *fields, **kwds): + return lltype.GcStruct(name, *fields, **kwds) + + def annotate_struct(self, S): + return annmodel.SomePtr(lltype.Ptr(S)) + + def test_simple_cast_pointer(self): + GCS1 = lltype.GcStruct('s1', ('x', lltype.Signed)) + GCS2 = lltype.GcStruct('s2', ('sub', GCS1), ('y', lltype.Signed)) + PGCS1 = lltype.Ptr(GCS1) + PGCS2 = lltype.Ptr(GCS2) + def ll1(): + s2 = lltype.malloc(GCS2) + return lltype.cast_pointer(PGCS1, s2) + hs = self.hannotate(ll1, []) + assert isinstance(hs, SomeLLAbstractContainer) + assert hs.concretetype == PGCS1 + def ll1(): + s2 = lltype.malloc(GCS2) + s1 = s2.sub + return lltype.cast_pointer(PGCS2, s1) + hs = self.hannotate(ll1, []) + assert isinstance(hs, SomeLLAbstractContainer) + assert hs.concretetype == PGCS2 + + def test_getarrayitem(self): + A = lltype.GcArray(lltype.Signed, hints={'immutable': True}) + a = lltype.malloc(A, 10) + def ll1(n): + v = a[n] + v = hint(v, concrete=True) + return v + hs, ha = self.hannotate(ll1, [int], annotator=True) + assert hs.eager_concrete + g1 = graphof(ha.translator, ll1) + hs_n = ha.binding(g1.getargs()[0]) + assert hs_n.origins.keys()[0].fixed + + def test_getvarrayitem(self): + A = lltype.GcArray(lltype.Signed, hints={'immutable': True}) + def ll1(n): + a = lltype.malloc(A, 10) + v = a[n] + v = hint(v, concrete=True) + return v + hs, ha = self.hannotate(ll1, [int], annotator=True) + assert hs.eager_concrete + g1 = graphof(ha.translator, ll1) + hs_n = ha.binding(g1.getargs()[0]) + assert hs_n.origins.keys()[0].fixed + + def test_degenerated_merge_substructure(self): + S = lltype.GcStruct('S', ('n', lltype.Signed)) + T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) + + def ll_function(flag): + t = lltype.malloc(T) + t.s.n = 3 + s = lltype.malloc(S) + s.n = 4 + if flag: + s = t.s + return s, t + hs = self.hannotate(ll_function, [bool]) + assert isinstance(hs, SomeLLAbstractContainer) + assert not hs.contentdef.degenerated + assert len(hs.contentdef.fields) == 2 + hs0 = hs.contentdef.fields['item0'].s_value # 's' + assert isinstance(hs0, SomeLLAbstractContainer) + assert hs0.contentdef.degenerated + hs1 = hs.contentdef.fields['item1'].s_value # 't' + assert isinstance(hs1, SomeLLAbstractContainer) + assert hs1.contentdef.degenerated + + def test_degenerated_merge_cross_substructure(self): + py.test.skip("no longer a valid test") + from pypy.rlib import objectmodel + S = lltype.Struct('S', ('n', lltype.Signed)) + T = lltype.GcStruct('T', ('s', S), ('s1', S), ('n', lltype.Float)) + + def ll_function(flag): + t = lltype.malloc(T) + t.s.n = 3 + t.s1.n = 3 + if flag: + s = t.s + else: + s = t.s1 + objectmodel.keepalive_until_here(t) + return s, t + hs = self.hannotate(ll_function, [bool]) + assert isinstance(hs, SomeLLAbstractContainer) + assert not hs.contentdef.degenerated + assert len(hs.contentdef.fields) == 2 + hs0 = hs.contentdef.fields['item0'].s_value # 's' + assert isinstance(hs0, SomeLLAbstractContainer) + assert hs0.contentdef.degenerated + hs1 = hs.contentdef.fields['item1'].s_value # 't' + assert isinstance(hs1, SomeLLAbstractContainer) + assert hs1.contentdef.degenerated + + def test_cast_ptr_to_int(self): + GCS1 = lltype.GcStruct('s1', ('x', lltype.Signed)) + def f(): + p = lltype.malloc(GCS1) + return lltype.cast_ptr_to_int(p) + + hs = self.hannotate(f, [], policy=P_NOVIRTUAL) + assert not hs.is_green() + + +class TestOOType(BaseAnnotatorTest): + type_system = 'ootype' + + malloc = property(lambda self: ootype.new) + + def make_struct(self, name, *fields, **kwds): + fields = dict(fields) + hints = kwds.pop('hints', None) + if hints: + kwds['_hints'] = hints + return ootype.Instance(name, ootype.ROOT, fields, **kwds) + + def annotate_struct(self, S): + return annmodel.SomeOOInstance(S) + + def skip_policy(self): + py.test.skip('fixme? (This policy is not relevant for now)') + + test_simple_list_operations = skip_policy + test_some_more_list_operations = skip_policy + test_make_a_list = skip_policy + test_simple_struct_malloc = skip_policy + test_container_union = skip_policy + test_specialize_deepfreeze_calls = skip_policy + test_deepfreeze_variables = skip_policy + test_cast_pointer_keeps_deepfreeze = skip_policy + test_concrete_fnptr_for_green_call = skip_policy Modified: pypy/dist/pypy/jit/hintannotator/test/test_toy.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/test/test_toy.py (original) +++ pypy/dist/pypy/jit/hintannotator/test/test_toy.py Sat Dec 15 13:53:32 2007 @@ -1,20 +1,27 @@ -from pypy.jit.hintannotator.test.test_annotator import hannotate, P_OOPSPEC -from pypy.jit.hintannotator.test.test_annotator import P_OOPSPEC_NOVIRTUAL +from pypy.jit.hintannotator.test.test_annotator import AbstractAnnotatorTest +from pypy.jit.hintannotator.test.test_annotator import P_OOPSPEC, P_OOPSPEC_NOVIRTUAL +class BaseToyTest(AbstractAnnotatorTest): + def test_hannotate_tl(self): + from pypy.jit.tl import tl + self.hannotate(tl.interp, [str, int, int], policy=P_OOPSPEC) -def test_hannotate_tl(): - from pypy.jit.tl import tl - hannotate(tl.interp, [str, int, int], policy=P_OOPSPEC) - -def test_hannotate_tl_novirtual(): - from pypy.jit.tl import tl - hannotate(tl.interp, [str, int, int], policy=P_OOPSPEC_NOVIRTUAL) - -def test_hannotate_tlr_novirtual(): - from pypy.jit.tl import tlr - hannotate(tlr.interpret, [str, int], policy=P_OOPSPEC_NOVIRTUAL) - -def test_hannotate_tlc_novirtual(): - from pypy.jit.tl import tlc - hannotate(tlc.interp_without_call, [str, int, int], - policy=P_OOPSPEC_NOVIRTUAL, backendoptimize=True) + def test_hannotate_tl_novirtual(self): + from pypy.jit.tl import tl + self.hannotate(tl.interp, [str, int, int], policy=P_OOPSPEC_NOVIRTUAL) + + def test_hannotate_tlr_novirtual(self): + from pypy.jit.tl import tlr + self.hannotate(tlr.interpret, [str, int], policy=P_OOPSPEC_NOVIRTUAL) + + def test_hannotate_tlc_novirtual(self): + from pypy.jit.tl import tlc + self.hannotate(tlc.interp_without_call, [str, int, int], + policy=P_OOPSPEC_NOVIRTUAL, backendoptimize=True) + +class TestLLType(BaseToyTest): + type_system = 'lltype' + +## XXX: all tests fail :-( +##class TestOOType(BaseToyTest): +## type_system = 'ootype' From fijal at codespeak.net Sat Dec 15 13:57:31 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 15 Dec 2007 13:57:31 +0100 (CET) Subject: [pypy-svn] r49815 - in pypy/branch/lazy-write-barrier/pypy: annotation doc interpreter jit/hintannotator jit/hintannotator/test lib module/__builtin__ module/__builtin__/test module/_random module/crypt objspace/std rpython rpython/lltypesystem rpython/lltypesystem/test rpython/memory rpython/memory/gc rpython/memory/gctransform rpython/ootypesystem rpython/ootypesystem/test translator/c translator/c/test translator/cli translator/goal Message-ID: <20071215125731.4127C16850C@codespeak.net> Author: fijal Date: Sat Dec 15 13:57:29 2007 New Revision: 49815 Added: pypy/branch/lazy-write-barrier/pypy/module/__builtin__/interp_classobj.py - copied unchanged from r49814, pypy/dist/pypy/module/__builtin__/interp_classobj.py pypy/branch/lazy-write-barrier/pypy/module/__builtin__/test/test_classobj.py - copied unchanged from r49814, pypy/dist/pypy/module/__builtin__/test/test_classobj.py Removed: pypy/branch/lazy-write-barrier/pypy/lib/_classobj.py pypy/branch/lazy-write-barrier/pypy/module/__builtin__/app_sets.py Modified: pypy/branch/lazy-write-barrier/pypy/annotation/builtin.py pypy/branch/lazy-write-barrier/pypy/doc/prolog-interpreter.txt pypy/branch/lazy-write-barrier/pypy/interpreter/gateway.py pypy/branch/lazy-write-barrier/pypy/jit/hintannotator/annotator.py pypy/branch/lazy-write-barrier/pypy/jit/hintannotator/model.py pypy/branch/lazy-write-barrier/pypy/jit/hintannotator/test/test_annotator.py pypy/branch/lazy-write-barrier/pypy/jit/hintannotator/test/test_toy.py pypy/branch/lazy-write-barrier/pypy/module/__builtin__/__init__.py pypy/branch/lazy-write-barrier/pypy/module/_random/interp_random.py pypy/branch/lazy-write-barrier/pypy/module/crypt/interp_crypt.py pypy/branch/lazy-write-barrier/pypy/objspace/std/objspace.py pypy/branch/lazy-write-barrier/pypy/rpython/lltypesystem/llmemory.py pypy/branch/lazy-write-barrier/pypy/rpython/lltypesystem/test/test_llmemory.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/base.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/transform.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/gcwrapper.py pypy/branch/lazy-write-barrier/pypy/rpython/ootypesystem/ootype.py pypy/branch/lazy-write-barrier/pypy/rpython/ootypesystem/rbuiltin.py pypy/branch/lazy-write-barrier/pypy/rpython/ootypesystem/test/test_oortype.py pypy/branch/lazy-write-barrier/pypy/rpython/rpbc.py pypy/branch/lazy-write-barrier/pypy/translator/c/database.py pypy/branch/lazy-write-barrier/pypy/translator/c/test/test_boehm.py pypy/branch/lazy-write-barrier/pypy/translator/cli/dotnet.py pypy/branch/lazy-write-barrier/pypy/translator/goal/bench-cronjob.py Log: Merge dist -> branch Modified: pypy/branch/lazy-write-barrier/pypy/annotation/builtin.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/annotation/builtin.py (original) +++ pypy/branch/lazy-write-barrier/pypy/annotation/builtin.py Sat Dec 15 13:57:29 2007 @@ -16,6 +16,7 @@ from pypy.annotation.bookkeeper import getbookkeeper from pypy.annotation import description from pypy.objspace.flow.model import Constant +from pypy.tool.error import AnnotatorError import pypy.rlib.rarithmetic import pypy.rlib.objectmodel @@ -547,6 +548,20 @@ assert isinstance(i, SomeOOInstance) return SomeInteger() +def ooupcast(I, i): + assert isinstance(I.const, ootype.Instance) + if ootype.isSubclass(i.ootype, I.const): + return SomeOOInstance(I.const) + else: + raise AnnotatorError, 'Cannot cast %s to %s' % (i.ootype, I.const) + +def oodowncast(I, i): + assert isinstance(I.const, ootype.Instance) + if ootype.isSubclass(I.const, i.ootype): + return SomeOOInstance(I.const) + else: + raise AnnotatorError, 'Cannot cast %s to %s' % (i.ootype, I.const) + BUILTIN_ANALYZERS[ootype.instanceof] = instanceof BUILTIN_ANALYZERS[ootype.new] = new BUILTIN_ANALYZERS[ootype.null] = null @@ -554,6 +569,8 @@ BUILTIN_ANALYZERS[ootype.classof] = classof BUILTIN_ANALYZERS[ootype.subclassof] = subclassof BUILTIN_ANALYZERS[ootype.ooidentityhash] = ooidentityhash +BUILTIN_ANALYZERS[ootype.ooupcast] = ooupcast +BUILTIN_ANALYZERS[ootype.oodowncast] = oodowncast #________________________________ # weakrefs Modified: pypy/branch/lazy-write-barrier/pypy/doc/prolog-interpreter.txt ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/doc/prolog-interpreter.txt (original) +++ pypy/branch/lazy-write-barrier/pypy/doc/prolog-interpreter.txt Sat Dec 15 13:57:29 2007 @@ -25,7 +25,7 @@ .. _`Learn Prolog Now!`: http://www.coli.uni-saarland.de/~kris/learn-prolog-now/ .. _`A Prolog interpreter in Python`: http://codespeak.net/pypy/extradoc/paper/prolog-in-python.pdf -Example useage +Example usage ============== First some simple examples that show simple unification:: Modified: pypy/branch/lazy-write-barrier/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/interpreter/gateway.py (original) +++ pypy/branch/lazy-write-barrier/pypy/interpreter/gateway.py Sat Dec 15 13:57:29 2007 @@ -328,7 +328,7 @@ def visit__object(self, typ): if typ not in (int, str, float, unicode, r_longlong): - assert False, "unsupported basic type in uwnrap_spec" + assert False, "unsupported basic type in unwrap_spec" self.unwrap.append("space.%s_w(%s)" % (typ.__name__, self.nextarg())) Modified: pypy/branch/lazy-write-barrier/pypy/jit/hintannotator/annotator.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/jit/hintannotator/annotator.py (original) +++ pypy/branch/lazy-write-barrier/pypy/jit/hintannotator/annotator.py Sat Dec 15 13:57:29 2007 @@ -6,7 +6,8 @@ from pypy.jit.hintannotator.bookkeeper import HintBookkeeper from pypy.jit.hintannotator.policy import HintAnnotatorPolicy from pypy.rpython.lltypesystem import lltype - +from pypy.rpython.ootypesystem import ootype +from pypy.translator.simplify import get_funcobj class HintAnnotator(RPythonAnnotator): @@ -28,6 +29,15 @@ def getuserclassdefinitions(self): return [] + def consider_op_new(self, hs_TYPE): + TYPE = hs_TYPE.const + if self.policy.novirtualcontainer: + return hintmodel.SomeLLAbstractVariable(TYPE) + else: + # XXX: ootype + vstructdef = self.bookkeeper.getvirtualcontainerdef(TYPE) + return hintmodel.SomeLLAbstractContainer(vstructdef) + def consider_op_malloc(self, hs_TYPE, hs_flags): TYPE = hs_TYPE.const flags = hs_flags.const @@ -65,10 +75,15 @@ def consider_op_ts_metacall(self, hs_f1, hs_metadesccls, *args_hs): bookkeeper = self.bookkeeper - fnobj = hs_f1.const._obj + fnobj = get_funcobj(hs_f1.const) return hintmodel.cannot_follow_call(bookkeeper, fnobj.graph, args_hs, lltype.typeOf(fnobj).RESULT) + def consider_op_oosend(self, hs_name, *args_hs): + assert hs_name.concretetype is ootype.Void + hs_obj, args_hs = args_hs[0], args_hs[1:] + return hs_obj.oosend(hs_name, *args_hs) + def simplify(self): RPythonAnnotator.simplify(self, extra_passes=[]) Modified: pypy/branch/lazy-write-barrier/pypy/jit/hintannotator/model.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/jit/hintannotator/model.py (original) +++ pypy/branch/lazy-write-barrier/pypy/jit/hintannotator/model.py Sat Dec 15 13:57:29 2007 @@ -2,6 +2,8 @@ from pypy.tool.pairtype import pair, pairtype from pypy.jit.hintannotator.bookkeeper import getbookkeeper from pypy.rpython.lltypesystem import lltype, lloperation +from pypy.rpython.ootypesystem import ootype +from pypy.translator.simplify import get_funcobj UNARY_OPERATIONS = """same_as hint getfield setfield getsubstruct getarraysize getinteriorfield getinteriorarraysize setinteriorfield @@ -20,6 +22,14 @@ ptr_nonzero ptr_iszero is_early_constant + oogetfield + oosetfield + oononnull + ooupcast + oodowncast + oois + subclassof + instanceof """.split() BINARY_OPERATIONS = """int_add int_sub int_mul int_mod int_and int_rshift @@ -244,7 +254,11 @@ def __init__(self, contentdef): self.contentdef = contentdef - self.concretetype = lltype.Ptr(contentdef.T) + T = contentdef.T + if isinstance(T, ootype.OOType): + self.concretetype = T + else: + self.concretetype = lltype.Ptr(T) def annotationcolor(self): """Compute the color of the variables with this annotation @@ -339,9 +353,16 @@ FIELD_TYPE = getattr(S, hs_fieldname.const) return variableoftype(FIELD_TYPE, hs_v1.deepfrozen) + def oogetfield(hs_v1, hs_fieldname): + _, FIELD_TYPE = hs_v1.concretetype._lookup_field(hs_fieldname.const) + return variableoftype(FIELD_TYPE, hs_v1.deepfrozen) + def setfield(hs_v1, hs_fieldname, hs_value): pass + def oosetfield(hs_v1, hs_fieldname, hs_value): + pass + def getsubstruct(hs_v1, hs_fieldname): S = hs_v1.concretetype.TO FIELD_TYPE = getattr(S, hs_fieldname.const) @@ -406,6 +427,9 @@ # function return annmodel.unionof(hs_res, bookkeeper.current_op_binding()) + def oosend(hs_v1, hs_name, *args_hs): + RESTYPE = getbookkeeper().current_op_concretetype() + return SomeLLAbstractVariable(RESTYPE) class __extend__(SomeLLAbstractConstant): @@ -428,7 +452,7 @@ def direct_call(hs_f1, *args_hs): bookkeeper = getbookkeeper() - fnobj = hs_f1.const._obj + fnobj = get_funcobj(hs_f1.const) if (bookkeeper.annotator.policy.oopspec and hasattr(fnobj._callable, 'oopspec')): # try to handle the call as a high-level operation @@ -473,9 +497,34 @@ # function return annmodel.unionof(hs_res, bookkeeper.current_op_binding()) + def oosend(hs_c1, hs_name, *args_hs): + TYPE = hs_c1.concretetype + name = hs_name.const + graph_list = TYPE._lookup_graphs(name) + if not graph_list: + # it's a method of a BuiltinType + bk = getbookkeeper() + origin = bk.myorigin() + d = setadd(hs_c1.origins, origin) + RESTYPE = bk.current_op_concretetype() + hs_res = SomeLLAbstractConstant(RESTYPE, d, + eager_concrete = hs_c1.eager_concrete, + myorigin = origin) + # if hs_c1.is_constant(): ... + return hs_res + #import pdb;pdb.set_trace() + def getfield(hs_c1, hs_fieldname): S = hs_c1.concretetype.TO FIELD_TYPE = getattr(S, hs_fieldname.const) + return hs_c1.getfield_impl(S, FIELD_TYPE) + + def oogetfield(hs_c1, hs_fieldname): + S = hs_c1.concretetype + _, FIELD_TYPE = S._lookup_field(hs_fieldname.const) + return hs_c1.getfield_impl(S, FIELD_TYPE) + + def getfield_impl(hs_c1, S, FIELD_TYPE): if S._hints.get('immutable', False) or hs_c1.deepfrozen: origin = getbookkeeper().myorigin() d = setadd(hs_c1.origins, origin) Modified: pypy/branch/lazy-write-barrier/pypy/jit/hintannotator/test/test_annotator.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/jit/hintannotator/test/test_annotator.py (original) +++ pypy/branch/lazy-write-barrier/pypy/jit/hintannotator/test/test_annotator.py Sat Dec 15 13:57:29 2007 @@ -20,956 +20,1002 @@ P_NOVIRTUAL = HintAnnotatorPolicy(novirtualcontainer=True, entrypoint_returns_red=False) -def hannotate(func, argtypes, policy=P_DEFAULT, annotator=False, inline=None, - backendoptimize=False): - # build the normal ll graphs for ll_function - t = TranslationContext() - a = t.buildannotator() - a.build_types(func, argtypes) - rtyper = t.buildrtyper() - rtyper.specialize() - if inline: - auto_inlining(t, threshold=inline) - if backendoptimize: - from pypy.translator.backendopt.all import backend_optimizations - backend_optimizations(t) - graph1 = graphof(t, func) - - # build hint annotator types - hannotator = HintAnnotator(base_translator=t, policy=policy) - hs = hannotator.build_types(graph1, [SomeLLAbstractConstant(v.concretetype, - {OriginFlags(): True}) - for v in graph1.getargs()]) - hannotator.simplify() - t = hannotator.translator - if conftest.option.view: - t.view() - if annotator: - return hs, hannotator - else: - return hs - -def test_simple(): - def ll_function(x, y): - return x + y - hs = hannotate(ll_function, [int, int]) - assert isinstance(hs, SomeLLAbstractConstant) - assert len(hs.origins) == 3 - assert hs.concretetype == lltype.Signed - -def test_join(): - def ll_function(cond, x,y): - if cond: - z = x+y - else: - z = x-y - return z - hs = hannotate(ll_function, [bool, int, int]) - assert isinstance(hs, SomeLLAbstractConstant) - assert len(hs.origins) == 4 - assert hs.concretetype == lltype.Signed +class AbstractAnnotatorTest: + type_system = None + + def hannotate(self, func, argtypes, policy=P_DEFAULT, annotator=False, inline=None, + backendoptimize=False): + # build the normal ll graphs for ll_function + t = TranslationContext() + a = t.buildannotator() + a.build_types(func, argtypes) + rtyper = t.buildrtyper(type_system = self.type_system) + rtyper.specialize() + if inline: + auto_inlining(t, threshold=inline) + if backendoptimize: + from pypy.translator.backendopt.all import backend_optimizations + backend_optimizations(t) + graph1 = graphof(t, func) + + # build hint annotator types + hannotator = HintAnnotator(base_translator=t, policy=policy) + hs = hannotator.build_types(graph1, [SomeLLAbstractConstant(v.concretetype, + {OriginFlags(): True}) + for v in graph1.getargs()]) + hannotator.simplify() + t = hannotator.translator + if conftest.option.view: + t.view() + if annotator: + return hs, hannotator + else: + return hs + + +class BaseAnnotatorTest(AbstractAnnotatorTest): + + def test_simple(self): + def ll_function(x, y): + return x + y + hs = self.hannotate(ll_function, [int, int]) + assert isinstance(hs, SomeLLAbstractConstant) + assert len(hs.origins) == 3 + assert hs.concretetype == lltype.Signed + + def test_join(self): + def ll_function(cond, x,y): + if cond: + z = x+y + else: + z = x-y + return z + hs = self.hannotate(ll_function, [bool, int, int]) + assert isinstance(hs, SomeLLAbstractConstant) + assert len(hs.origins) == 4 + assert hs.concretetype == lltype.Signed -def test_simple_hint_result(): - def ll_function(cond, x,y): - if cond: - z = x+y - else: - z = x-y - z = hint(z, concrete=True) - return z - hs = hannotate(ll_function, [bool, int, int]) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.eager_concrete - assert hs.concretetype == lltype.Signed + def test_simple_hint_result(self): + def ll_function(cond, x,y): + if cond: + z = x+y + else: + z = x-y + z = hint(z, concrete=True) + return z + hs = self.hannotate(ll_function, [bool, int, int]) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.eager_concrete + assert hs.concretetype == lltype.Signed + + def test_deepfreeze(self): + + A = lltype.GcArray(lltype.Signed) + + def ll_function(a, i): + a = hint(a, deepfreeze=True) + res = a[i] + res = hint(res, concrete=True) + + res = hint(res, variable=True) + return res + + hs = self.hannotate(ll_function, [annmodel.SomePtr(lltype.Ptr(A)), int]) + assert type(hs) is SomeLLAbstractVariable + assert hs.concretetype == lltype.Signed + + def test_lists_deepfreeze(self): + + l1 = [1,2,3,4,5] + l2 = [6,7,8,9,10] + + def getlist(n): + if n: + return l1 + else: + return l2 -def test_deepfreeze(): + def ll_function(n, i): + l = getlist(n) + l = hint(l, deepfreeze=True) - A = lltype.GcArray(lltype.Signed) - - def ll_function(a, i): - a = hint(a, deepfreeze=True) - res = a[i] - res = hint(res, concrete=True) - - res = hint(res, variable=True) - return res - - hs = hannotate(ll_function, [annmodel.SomePtr(lltype.Ptr(A)), int]) - assert type(hs) is SomeLLAbstractVariable - assert hs.concretetype == lltype.Signed + res = l[i] + res = hint(res, concrete=True) -def test_lists_deepfreeze(): + res = hint(res, variable=True) + return res - l1 = [1,2,3,4,5] - l2 = [6,7,8,9,10] - - def getlist(n): - if n: - return l1 - else: - return l2 - - def ll_function(n, i): - l = getlist(n) - l = hint(l, deepfreeze=True) - - res = l[i] - res = hint(res, concrete=True) - - res = hint(res, variable=True) - return res + hs = self.hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) + assert hs.concretetype == lltype.Signed - hs = hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) - assert hs.concretetype == lltype.Signed + def test_dicts_deepfreeze(self): -def test_dicts_deepfreeze(): + d1 = {1:2, 2:3} + d2 = {2:3, 3:4} - d1 = {1:2, 2:3} - d2 = {2:3, 3:4} - - def getdict(n): - if n: - return d1 - else: - return d2 - - def ll_function(n, i): - d = getdict(n) - d = hint(d, deepfreeze=True) - - res = d[i] - res = hint(res, concrete=True) - - res = hint(res, variable=True) - return res - - # must backendoptimize to remove the mallocs related to the interior ptrs - hs = hannotate(ll_function, [int, int], policy=P_NOVIRTUAL, - backendoptimize=True) - assert hs.concretetype == lltype.Signed - - -def test_simple_hint_origins(): - def ll_function(cond, x,y): - if cond: - z = x+y - else: - z = x-y - z1 = hint(z, concrete=True) - return z # origin of z1 - hs, ha = hannotate(ll_function, [bool, int, int], annotator=True) - assert isinstance(hs, SomeLLAbstractConstant) - assert len(hs.origins) == 4 - assert hs.is_fixed() - assert hs.concretetype == lltype.Signed - ll_function_graph = graphof(ha.base_translator, ll_function) - gdesc = ha.bookkeeper.getdesc(ll_function_graph) - _, x_v, y_v = gdesc._cache[None].getargs() - assert ha.binding(x_v).is_fixed() - assert ha.binding(y_v).is_fixed() - -def test_simple_variable(): - def ll_function(x,y): - x = hint(x, variable=True) # special hint only for testing purposes!!! - return x + y - hs = hannotate(ll_function, [int, int]) - assert type(hs) is SomeLLAbstractVariable - assert hs.concretetype == lltype.Signed - -def test_simple_concrete_propagation(): - def ll_function(x,y): - x = hint(x, concrete=True) - return x + y - hs = hannotate(ll_function, [int, int]) - assert type(hs) is SomeLLAbstractConstant - assert hs.eager_concrete - assert hs.concretetype == lltype.Signed - -def test_union(): - unionof = annmodel.unionof - av1, av2 = SomeLLAbstractVariable(lltype.Signed), SomeLLAbstractVariable(lltype.Signed) - cv1, cv2 = SomeLLAbstractConstant(lltype.Signed, {}, eager_concrete=True), SomeLLAbstractConstant(lltype.Signed, {}, eager_concrete=True) - ac1, ac2 = SomeLLAbstractConstant(lltype.Signed, {}), SomeLLAbstractConstant(lltype.Signed, {}) - ac3 = SomeLLAbstractConstant(lltype.Signed, {}) - ac3.const = 3 - ac4 = SomeLLAbstractConstant(lltype.Signed, {}) - ac4.const = 4 - assert unionof(av1, av2) == av1 - assert unionof(cv1, cv2) == cv2 - assert unionof(ac1, ac2) == ac1 - assert unionof(ac3, ac3) == ac3 - assert unionof(ac3, ac2) == ac1 - assert unionof(ac4, ac3) == ac1 - # degenerating cases - py.test.raises(annmodel.UnionError, "unionof(cv1, av1)") - py.test.raises(annmodel.UnionError, "unionof(av1, cv1)") - - # MAYBE... - #py.test.raises(annmodel.UnionError, "unionof(ac1, cv1)") - #py.test.raises(annmodel.UnionError, "unionof(cv1, ac1)") - assert unionof(cv1, ac1) == ac1 - assert unionof(ac1, cv1) == ac1 - - # constant with values - assert unionof(av1, ac1) == av1 - assert unionof(ac1, av1) == av1 - assert unionof(ac3, av1) == av1 - assert unionof(av2, ac4) == av1 - -def test_op_meet(): - def meet(hs1, hs2): - bk = HintBookkeeper(None) - block = flowmodel.Block([]) - block.operations.append(flowmodel.SpaceOperation('x', [], - flowmodel.Variable())) - bk.enter(("graph", block, 0)) - bk.current_op_concretetype = lambda: lltype.Signed # hack - return pair(hs1, hs2).int_add() - av1, av2 = SomeLLAbstractVariable(lltype.Signed), SomeLLAbstractVariable(lltype.Signed) - cv1, cv2 = SomeLLAbstractConstant(lltype.Signed, {}, True), SomeLLAbstractConstant(lltype.Signed, {}, True) - ac1, ac2 = SomeLLAbstractConstant(lltype.Signed, {}), SomeLLAbstractConstant(lltype.Signed, {}) - assert meet(av1, av2) == av1 - res = meet(cv1, cv2) - assert res.eager_concrete - assert isinstance(meet(ac1, ac2), SomeLLAbstractConstant) - assert meet(ac1, cv1).eager_concrete - assert meet(cv1, ac1).eager_concrete - assert meet(av1, cv1) == av1 - assert meet(cv1, av1) == av1 - assert meet(ac1, av1) == av1 - assert meet(av1, ac1) == av1 - -def test_loop(): - def ll_function(x, y): - while x > 0: - y += x - x -= 1 - return y - hs = hannotate(ll_function, [int, int]) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.concretetype == lltype.Signed - assert len(hs.origins) == 4 - -def test_loop1(): - def ll_function(x, y): - while x > 0: - x1 = hint(x, concrete=True) - if x1 == 7: + def getdict(n): + if n: + return d1 + else: + return d2 + + def ll_function(n, i): + d = getdict(n) + d = hint(d, deepfreeze=True) + + res = d[i] + res = hint(res, concrete=True) + + res = hint(res, variable=True) + return res + + # must backendoptimize to remove the mallocs related to the interior ptrs + hs = self.hannotate(ll_function, [int, int], policy=P_NOVIRTUAL, + backendoptimize=True) + assert hs.concretetype == lltype.Signed + + + def test_simple_hint_origins(self): + def ll_function(cond, x,y): + if cond: + z = x+y + else: + z = x-y + z1 = hint(z, concrete=True) + return z # origin of z1 + hs, ha = self.hannotate(ll_function, [bool, int, int], annotator=True) + assert isinstance(hs, SomeLLAbstractConstant) + assert len(hs.origins) == 4 + assert hs.is_fixed() + assert hs.concretetype == lltype.Signed + ll_function_graph = graphof(ha.base_translator, ll_function) + gdesc = ha.bookkeeper.getdesc(ll_function_graph) + _, x_v, y_v = gdesc._cache[None].getargs() + assert ha.binding(x_v).is_fixed() + assert ha.binding(y_v).is_fixed() + + def test_simple_variable(self): + def ll_function(x,y): + x = hint(x, variable=True) # special hint only for testing purposes!!! + return x + y + hs = self.hannotate(ll_function, [int, int]) + assert type(hs) is SomeLLAbstractVariable + assert hs.concretetype == lltype.Signed + + def test_simple_concrete_propagation(self): + def ll_function(x,y): + x = hint(x, concrete=True) + return x + y + hs = self.hannotate(ll_function, [int, int]) + assert type(hs) is SomeLLAbstractConstant + assert hs.eager_concrete + assert hs.concretetype == lltype.Signed + + def test_union(self): + unionof = annmodel.unionof + av1, av2 = SomeLLAbstractVariable(lltype.Signed), SomeLLAbstractVariable(lltype.Signed) + cv1, cv2 = SomeLLAbstractConstant(lltype.Signed, {}, eager_concrete=True), SomeLLAbstractConstant(lltype.Signed, {}, eager_concrete=True) + ac1, ac2 = SomeLLAbstractConstant(lltype.Signed, {}), SomeLLAbstractConstant(lltype.Signed, {}) + ac3 = SomeLLAbstractConstant(lltype.Signed, {}) + ac3.const = 3 + ac4 = SomeLLAbstractConstant(lltype.Signed, {}) + ac4.const = 4 + assert unionof(av1, av2) == av1 + assert unionof(cv1, cv2) == cv2 + assert unionof(ac1, ac2) == ac1 + assert unionof(ac3, ac3) == ac3 + assert unionof(ac3, ac2) == ac1 + assert unionof(ac4, ac3) == ac1 + # degenerating cases + py.test.raises(annmodel.UnionError, "unionof(cv1, av1)") + py.test.raises(annmodel.UnionError, "unionof(av1, cv1)") + + # MAYBE... + #py.test.raises(annmodel.UnionError, "unionof(ac1, cv1)") + #py.test.raises(annmodel.UnionError, "unionof(cv1, ac1)") + assert unionof(cv1, ac1) == ac1 + assert unionof(ac1, cv1) == ac1 + + # constant with values + assert unionof(av1, ac1) == av1 + assert unionof(ac1, av1) == av1 + assert unionof(ac3, av1) == av1 + assert unionof(av2, ac4) == av1 + + def test_op_meet(self): + def meet(hs1, hs2): + bk = HintBookkeeper(None) + block = flowmodel.Block([]) + block.operations.append(flowmodel.SpaceOperation('x', [], + flowmodel.Variable())) + bk.enter(("graph", block, 0)) + bk.current_op_concretetype = lambda: lltype.Signed # hack + return pair(hs1, hs2).int_add() + av1, av2 = SomeLLAbstractVariable(lltype.Signed), SomeLLAbstractVariable(lltype.Signed) + cv1, cv2 = SomeLLAbstractConstant(lltype.Signed, {}, True), SomeLLAbstractConstant(lltype.Signed, {}, True) + ac1, ac2 = SomeLLAbstractConstant(lltype.Signed, {}), SomeLLAbstractConstant(lltype.Signed, {}) + assert meet(av1, av2) == av1 + res = meet(cv1, cv2) + assert res.eager_concrete + assert isinstance(meet(ac1, ac2), SomeLLAbstractConstant) + assert meet(ac1, cv1).eager_concrete + assert meet(cv1, ac1).eager_concrete + assert meet(av1, cv1) == av1 + assert meet(cv1, av1) == av1 + assert meet(ac1, av1) == av1 + assert meet(av1, ac1) == av1 + + def test_loop(self): + def ll_function(x, y): + while x > 0: y += x - x -= 1 - return y - hs = hannotate(ll_function, [int, int]) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.concretetype == lltype.Signed - assert len(hs.origins) == 4 - -def test_simple_struct(): - S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), - ('world', lltype.Signed), - hints={'immutable': True}) - def ll_function(s): - return s.hello * s.world - hs = hannotate(ll_function, [annmodel.SomePtr(lltype.Ptr(S))]) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.concretetype == lltype.Signed - assert len(hs.origins) == 4 - -def test_simple_struct_malloc(): - S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), - ('world', lltype.Signed)) - def ll_function(x): - s = lltype.malloc(S) - s.hello = x - return s.hello + s.world - - hs = hannotate(ll_function, [int]) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.concretetype == lltype.Signed - assert len(hs.origins) == 2 - -def test_container_union(): - S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), - ('world', lltype.Signed)) - def ll_function(cond, x, y): - if cond: - s = lltype.malloc(S) + x -= 1 + return y + hs = self.hannotate(ll_function, [int, int]) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + assert len(hs.origins) == 4 + + def test_loop1(self): + def ll_function(x, y): + while x > 0: + x1 = hint(x, concrete=True) + if x1 == 7: + y += x + x -= 1 + return y + hs = self.hannotate(ll_function, [int, int]) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + assert len(hs.origins) == 4 + + def test_simple_struct(self): + S = self.make_struct('helloworld', ('hello', lltype.Signed), + ('world', lltype.Signed), + hints={'immutable': True}) + def ll_function(s): + return s.hello * s.world + hs = self.hannotate(ll_function, [self.annotate_struct(S)]) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + assert len(hs.origins) == 4 + + def test_simple_struct_malloc(self): + S = self.make_struct('helloworld', ('hello', lltype.Signed), + ('world', lltype.Signed)) + malloc = self.malloc + def ll_function(x): + s = malloc(S) s.hello = x - else: - s = lltype.malloc(S) - s.world = y - return s.hello + s.world + return s.hello + s.world - hs = hannotate(ll_function, [bool, int, int]) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.concretetype == lltype.Signed - assert len(hs.origins) == 3 - -def test_simple_call(): - def ll2(x, y, z): - return x + (y + 42) - def ll1(x, y, z): - return ll2(x, y - z, x + y + z) - hs = hannotate(ll1, [int, int, int]) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.concretetype == lltype.Signed - assert len(hs.origins) == 5 - -def test_simple_list_operations(): - def ll_function(x, y, index): - l = [x] - l.append(y) - return l[index] - hs = hannotate(ll_function, [int, int, int], policy=P_OOPSPEC) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.concretetype == lltype.Signed - assert len(hs.origins) == 4 - -def test_some_more_list_operations(): - def ll_function(x, y, index): - l = [] - l.append(x) - l[0] = y - return (l+list(l))[index] - hs = hannotate(ll_function, [int, int, int], policy=P_OOPSPEC) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.concretetype == lltype.Signed - assert len(hs.origins) == 4 - -def test_make_a_list(): - def ll_function(x, y): - return [x, y] - hs = hannotate(ll_function, [int, int], policy=P_OOPSPEC) - assert isinstance(hs, SomeLLAbstractContainer) - -def test_frozen_list(): - lst = [5, 7, 9] - def ll_function(x): - mylist = hint(lst, deepfreeze=True) - z = mylist[x] - hint(z, concrete=True) - return z - hs = hannotate(ll_function, [int], policy=P_OOPSPEC_NOVIRTUAL) - assert hs.is_green() - -def test_simple_cast_pointer(): - GCS1 = lltype.GcStruct('s1', ('x', lltype.Signed)) - GCS2 = lltype.GcStruct('s2', ('sub', GCS1), ('y', lltype.Signed)) - PGCS1 = lltype.Ptr(GCS1) - PGCS2 = lltype.Ptr(GCS2) - def ll1(): - s2 = lltype.malloc(GCS2) - return lltype.cast_pointer(PGCS1, s2) - hs = hannotate(ll1, []) - assert isinstance(hs, SomeLLAbstractContainer) - assert hs.concretetype == PGCS1 - def ll1(): - s2 = lltype.malloc(GCS2) - s1 = s2.sub - return lltype.cast_pointer(PGCS2, s1) - hs = hannotate(ll1, []) - assert isinstance(hs, SomeLLAbstractContainer) - assert hs.concretetype == PGCS2 - -def test_getarrayitem(): - A = lltype.GcArray(lltype.Signed, hints={'immutable': True}) - a = lltype.malloc(A, 10) - def ll1(n): - v = a[n] - v = hint(v, concrete=True) - return v - hs, ha = hannotate(ll1, [int], annotator=True) - assert hs.eager_concrete - g1 = graphof(ha.translator, ll1) - hs_n = ha.binding(g1.getargs()[0]) - assert hs_n.origins.keys()[0].fixed - -def test_getvarrayitem(): - A = lltype.GcArray(lltype.Signed, hints={'immutable': True}) - def ll1(n): - a = lltype.malloc(A, 10) - v = a[n] - v = hint(v, concrete=True) - return v - hs, ha = hannotate(ll1, [int], annotator=True) - assert hs.eager_concrete - g1 = graphof(ha.translator, ll1) - hs_n = ha.binding(g1.getargs()[0]) - assert hs_n.origins.keys()[0].fixed - -def test_prebuilt_structure(): - S = lltype.GcStruct('S', ('n', lltype.Signed)) - s = lltype.malloc(S) - def ll1(n): - s.n = n - return s.n - hs = hannotate(ll1, [int]) - assert isinstance(hs, SomeLLAbstractVariable) - -def test_degenerated_merge_substructure(): - S = lltype.GcStruct('S', ('n', lltype.Signed)) - T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) - - def ll_function(flag): - t = lltype.malloc(T) - t.s.n = 3 - s = lltype.malloc(S) - s.n = 4 - if flag: - s = t.s - return s, t - hs = hannotate(ll_function, [bool]) - assert isinstance(hs, SomeLLAbstractContainer) - assert not hs.contentdef.degenerated - assert len(hs.contentdef.fields) == 2 - hs0 = hs.contentdef.fields['item0'].s_value # 's' - assert isinstance(hs0, SomeLLAbstractContainer) - assert hs0.contentdef.degenerated - hs1 = hs.contentdef.fields['item1'].s_value # 't' - assert isinstance(hs1, SomeLLAbstractContainer) - assert hs1.contentdef.degenerated - -def test_degenerated_merge_cross_substructure(): - py.test.skip("no longer a valid test") - from pypy.rlib import objectmodel - S = lltype.Struct('S', ('n', lltype.Signed)) - T = lltype.GcStruct('T', ('s', S), ('s1', S), ('n', lltype.Float)) - - def ll_function(flag): - t = lltype.malloc(T) - t.s.n = 3 - t.s1.n = 3 - if flag: - s = t.s - else: - s = t.s1 - objectmodel.keepalive_until_here(t) - return s, t - hs = hannotate(ll_function, [bool]) - assert isinstance(hs, SomeLLAbstractContainer) - assert not hs.contentdef.degenerated - assert len(hs.contentdef.fields) == 2 - hs0 = hs.contentdef.fields['item0'].s_value # 's' - assert isinstance(hs0, SomeLLAbstractContainer) - assert hs0.contentdef.degenerated - hs1 = hs.contentdef.fields['item1'].s_value # 't' - assert isinstance(hs1, SomeLLAbstractContainer) - assert hs1.contentdef.degenerated - - -def test_simple_fixed_call(): - def ll_help(cond, x, y): - if cond: - z = x+y - else: - z = x-y - return z - def ll_function(cond, x,y, x1, y1): - z1 = ll_help(cond, x1, y1) - z = ll_help(cond, x, y) - z = hint(z, concrete=True) - return z - hs, ha = hannotate(ll_function, [bool, int, int, int, int], annotator=True) - assert hs.eager_concrete - assert hs.concretetype == lltype.Signed - ll_help_graph = graphof(ha.base_translator, ll_help) - gdesc = ha.bookkeeper.getdesc(ll_help_graph) - assert not ha.binding(gdesc._cache[None].getreturnvar()).is_fixed() - assert len(gdesc._cache) == 2 - assert ha.binding(gdesc._cache['fixed'].getreturnvar()).is_fixed() - -def test_specialize_calls(): - def ll_add(x, y): - return x+y - def ll_function(x,y): - z0 = ll_add(y, 2) - z1 = ll_add(x, y) - x1 = hint(x, concrete=True) - z2 = ll_add(x1, y) - return z2 - hs, ha = hannotate(ll_function, [int, int], annotator=True) - assert hs.eager_concrete - assert hs.concretetype == lltype.Signed - ll_add_graph = graphof(ha.base_translator, ll_add) - gdesc = ha.bookkeeper.getdesc(ll_add_graph) - assert len(gdesc._cache) == 2 - assert 'Exxx' in gdesc._cache - v1, v2 = gdesc._cache['Exxx'].getargs() - - assert isinstance(ha.binding(v1), SomeLLAbstractConstant) - assert isinstance(ha.binding(v2), SomeLLAbstractConstant) - assert ha.binding(v1).eager_concrete - assert not ha.binding(v2).is_fixed() - -def test_specialize_deepfreeze_calls(): + hs = self.hannotate(ll_function, [int]) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + assert len(hs.origins) == 2 + + def test_container_union(self): + S = self.make_struct('helloworld', ('hello', lltype.Signed), + ('world', lltype.Signed)) + malloc = self.malloc + def ll_function(cond, x, y): + if cond: + s = malloc(S) + s.hello = x + else: + s = malloc(S) + s.world = y + return s.hello + s.world + + hs = self.hannotate(ll_function, [bool, int, int]) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + assert len(hs.origins) == 3 + + def test_simple_call(self): + def ll2(x, y, z): + return x + (y + 42) + def ll1(x, y, z): + return ll2(x, y - z, x + y + z) + hs = self.hannotate(ll1, [int, int, int]) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + assert len(hs.origins) == 5 + + def test_simple_list_operations(self): + def ll_function(x, y, index): + l = [x] + l.append(y) + return l[index] + hs = self.hannotate(ll_function, [int, int, int], policy=P_OOPSPEC) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + assert len(hs.origins) == 4 + + def test_some_more_list_operations(self): + def ll_function(x, y, index): + l = [] + l.append(x) + l[0] = y + return (l+list(l))[index] + hs = self.hannotate(ll_function, [int, int, int], policy=P_OOPSPEC) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + assert len(hs.origins) == 4 + + def test_make_a_list(self): + def ll_function(x, y): + return [x, y] + hs = self.hannotate(ll_function, [int, int], policy=P_OOPSPEC) + assert isinstance(hs, SomeLLAbstractContainer) + + def test_frozen_list(self): + lst = [5, 7, 9] + def ll_function(x): + mylist = hint(lst, deepfreeze=True) + z = mylist[x] + hint(z, concrete=True) + return z + hs = self.hannotate(ll_function, [int], policy=P_OOPSPEC_NOVIRTUAL) + assert hs.is_green() + + def test_prebuilt_structure(self): + S = self.make_struct('S', ('n', lltype.Signed)) + s = self.malloc(S) + def ll1(n): + s.n = n + return s.n + hs = self.hannotate(ll1, [int]) + assert isinstance(hs, SomeLLAbstractVariable) + + def test_simple_fixed_call(self): + def ll_help(cond, x, y): + if cond: + z = x+y + else: + z = x-y + return z + def ll_function(cond, x,y, x1, y1): + z1 = ll_help(cond, x1, y1) + z = ll_help(cond, x, y) + z = hint(z, concrete=True) + return z + hs, ha = self.hannotate(ll_function, [bool, int, int, int, int], annotator=True) + assert hs.eager_concrete + assert hs.concretetype == lltype.Signed + ll_help_graph = graphof(ha.base_translator, ll_help) + gdesc = ha.bookkeeper.getdesc(ll_help_graph) + assert not ha.binding(gdesc._cache[None].getreturnvar()).is_fixed() + assert len(gdesc._cache) == 2 + assert ha.binding(gdesc._cache['fixed'].getreturnvar()).is_fixed() + + def test_specialize_calls(self): + def ll_add(x, y): + return x+y + def ll_function(x,y): + z0 = ll_add(y, 2) + z1 = ll_add(x, y) + x1 = hint(x, concrete=True) + z2 = ll_add(x1, y) + return z2 + hs, ha = self.hannotate(ll_function, [int, int], annotator=True) + assert hs.eager_concrete + assert hs.concretetype == lltype.Signed + ll_add_graph = graphof(ha.base_translator, ll_add) + gdesc = ha.bookkeeper.getdesc(ll_add_graph) + assert len(gdesc._cache) == 2 + assert 'Exxx' in gdesc._cache + v1, v2 = gdesc._cache['Exxx'].getargs() + + assert isinstance(ha.binding(v1), SomeLLAbstractConstant) + assert isinstance(ha.binding(v2), SomeLLAbstractConstant) + assert ha.binding(v1).eager_concrete + assert not ha.binding(v2).is_fixed() + + def test_specialize_deepfreeze_calls(self): + + l1 = [1,2,3,4,5] + l2 = [6,7,8,9,10] + + def getlist(n): + if n: + return l1 + else: + return l2 - l1 = [1,2,3,4,5] - l2 = [6,7,8,9,10] - - def getlist(n): - if n: - return l1 - else: - return l2 + def ll_get(l, i): + return l[i] - def ll_get(l, i): - return l[i] + def ll_function(n, i): + l = getlist(n) - def ll_function(n, i): - l = getlist(n) + l2 = ll_get(l, 0) + + l = hint(l, deepfreeze=True) + res = ll_get(l, i) + return res + + hs, ha = self.hannotate(ll_function, [int, int], annotator=True, policy=P_NOVIRTUAL) + assert hs.deepfrozen + assert hs.concretetype == lltype.Signed + ll_get_graph = graphof(ha.base_translator, ll_get) + gdesc = ha.bookkeeper.getdesc(ll_get_graph) + assert len(gdesc._cache) == 2 + assert 'xDxx' in gdesc._cache + v1, v2 = gdesc._cache['xDxx'].getargs() + + assert isinstance(ha.binding(v1), SomeLLAbstractConstant) + assert isinstance(ha.binding(v2), SomeLLAbstractConstant) + assert ha.binding(v1).deepfrozen + + def test_deepfreeze_variables(self): + l1 = [[1], [2, 3], [4], []] + def ll_function(i): + i = hint(i, variable=True) + l = hint(l1, deepfreeze=True) + return l[i] + + hs, ha = self.hannotate(ll_function, [int], annotator=True, policy=P_NOVIRTUAL) + assert isinstance(hs, SomeLLAbstractVariable) + assert hs.deepfrozen + + def test_propagate_fixing_across_func_arguments(self): + def ll_func2(z): + z = hint(z, concrete=True) + return z + 1 + def ll_function(cond, x,y): + if cond: + z = x+y + else: + z = x-y + z = ll_func2(z) + return z + hs, ha = self.hannotate(ll_function, [bool, int, int], annotator=True) + assert hs.eager_concrete + assert hs.concretetype == lltype.Signed + ll_function_graph = graphof(ha.base_translator, ll_function) + gdesc = ha.bookkeeper.getdesc(ll_function_graph) + _, x_v, y_v = gdesc._cache[None].getargs() + assert ha.binding(x_v).is_fixed() + assert ha.binding(y_v).is_fixed() + + def test_hannotate_plus_minus(self): + def ll_plus_minus(s, x, y): + acc = x + n = len(s) + pc = 0 + while pc < n: + op = s[pc] + op = hint(op, concrete=True) + if op == '+': + acc += y + elif op == '-': + acc -= y + pc += 1 + return acc + assert ll_plus_minus("+-+", 0, 2) == 2 + self.hannotate(ll_plus_minus, [str, int, int]) + self.hannotate(ll_plus_minus, [str, int, int], inline=100000) + + def test_invalid_hint_1(self): + S = self.make_struct('S', ('x', lltype.Signed)) + def ll_getitem_switch(s): + n = s.x # -> variable + return hint(n, concrete=True) + py.test.raises(HintError, self.hannotate, + ll_getitem_switch, [self.annotate_struct(S)]) + + def undecided_relevance_test_invalid_hint_2(self): + S = lltype.GcStruct('S', ('x', lltype.Signed)) + def ll_getitem_switch(s): + if s.x > 0: # variable exitswitch + sign = 1 + else: + sign = -1 + return hint(sign, concrete=True) + py.test.skip("in-progress: I think we expect a HintError here, do we?") + py.test.raises(HintError, hannotate, + ll_getitem_switch, [annmodel.SomePtr(lltype.Ptr(S))]) - l2 = ll_get(l, 0) - - l = hint(l, deepfreeze=True) - res = ll_get(l, i) - return res - - hs, ha = hannotate(ll_function, [int, int], annotator=True, policy=P_NOVIRTUAL) - assert hs.deepfrozen - assert hs.concretetype == lltype.Signed - ll_get_graph = graphof(ha.base_translator, ll_get) - gdesc = ha.bookkeeper.getdesc(ll_get_graph) - assert len(gdesc._cache) == 2 - assert 'xDxx' in gdesc._cache - v1, v2 = gdesc._cache['xDxx'].getargs() - - assert isinstance(ha.binding(v1), SomeLLAbstractConstant) - assert isinstance(ha.binding(v2), SomeLLAbstractConstant) - assert ha.binding(v1).deepfrozen - -def test_deepfreeze_variables(): - l1 = [[1], [2, 3], [4], []] - def ll_function(i): - i = hint(i, variable=True) - l = hint(l1, deepfreeze=True) - return l[i] - - hs, ha = hannotate(ll_function, [int], annotator=True, policy=P_NOVIRTUAL) - assert isinstance(hs, SomeLLAbstractVariable) - assert hs.deepfrozen - -def test_propagate_fixing_across_func_arguments(): - def ll_func2(z): - z = hint(z, concrete=True) - return z + 1 - def ll_function(cond, x,y): - if cond: - z = x+y - else: - z = x-y - z = ll_func2(z) - return z - hs, ha = hannotate(ll_function, [bool, int, int], annotator=True) - assert hs.eager_concrete - assert hs.concretetype == lltype.Signed - ll_function_graph = graphof(ha.base_translator, ll_function) - gdesc = ha.bookkeeper.getdesc(ll_function_graph) - _, x_v, y_v = gdesc._cache[None].getargs() - assert ha.binding(x_v).is_fixed() - assert ha.binding(y_v).is_fixed() - -def test_hannotate_plus_minus(): - def ll_plus_minus(s, x, y): - acc = x - n = len(s) - pc = 0 - while pc < n: - op = s[pc] - op = hint(op, concrete=True) - if op == '+': - acc += y - elif op == '-': - acc -= y - pc += 1 - return acc - assert ll_plus_minus("+-+", 0, 2) == 2 - hannotate(ll_plus_minus, [str, int, int]) - hannotate(ll_plus_minus, [str, int, int], inline=100000) - -def test_invalid_hint_1(): - S = lltype.GcStruct('S', ('x', lltype.Signed)) - def ll_getitem_switch(s): - n = s.x # -> variable - return hint(n, concrete=True) - py.test.raises(HintError, hannotate, - ll_getitem_switch, [annmodel.SomePtr(lltype.Ptr(S))]) - -def undecided_relevance_test_invalid_hint_2(): - S = lltype.GcStruct('S', ('x', lltype.Signed)) - def ll_getitem_switch(s): - if s.x > 0: # variable exitswitch - sign = 1 - else: - sign = -1 - return hint(sign, concrete=True) - py.test.skip("in-progress: I think we expect a HintError here, do we?") - py.test.raises(HintError, hannotate, - ll_getitem_switch, [annmodel.SomePtr(lltype.Ptr(S))]) - - -def test_raise_exc(): - class E(Exception): - pass - def f1(): - raise E - hannotate(f1, [], policy=P_OOPSPEC_NOVIRTUAL) - - def f2(): - e = E() - e.a = 3 - raise e - hannotate(f2, [], policy=P_OOPSPEC_NOVIRTUAL) - -def test_raise_and_catch_exc(): - class E(Exception): - pass - def f(flag): - if flag: + def test_raise_exc(self): + class E(Exception): + pass + def f1(): raise E + self.hannotate(f1, [], policy=P_OOPSPEC_NOVIRTUAL) - def g(flag): - try: - f(flag) - except E: - return -1 - return 0 - - hs = hannotate(g, [bool], policy=P_OOPSPEC_NOVIRTUAL) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.concretetype == lltype.Signed - - def f(flag): - if flag: + def f2(): e = E() e.a = 3 raise e - - hs = hannotate(g, [bool], policy=P_OOPSPEC_NOVIRTUAL) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.concretetype == lltype.Signed - - -def test_more_green(): - def f(x): - z = x + 1 - x2 = hint(x, concrete=True) - return z - - hs = hannotate(f, [int]) - assert isinstance(hs, SomeLLAbstractConstant) - assert hs.is_green() - assert not hs.is_fixed() - -def test_blue_simple_meth(): - py.test.skip("with abstract containers this test explode in the cast_pointer annotation logic") - class Base(object): - - def m(self): - raise NotImplementedError - - class Concrete(Base): - - def m(self): - return 42 - - def f(flag): - if flag: - o = Base() - else: - o = Concrete() - return o.m() + self.hannotate(f2, [], policy=P_OOPSPEC_NOVIRTUAL) - hs = hannotate(f, [bool], policy=P_OOPSPEC) + def test_raise_and_catch_exc(self): + class E(Exception): + pass + def f(flag): + if flag: + raise E + + def g(flag): + try: + f(flag) + except E: + return -1 + return 0 + + hs = self.hannotate(g, [bool], policy=P_OOPSPEC_NOVIRTUAL) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + + def f(flag): + if flag: + e = E() + e.a = 3 + raise e + + hs = self.hannotate(g, [bool], policy=P_OOPSPEC_NOVIRTUAL) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + + + def test_more_green(self): + def f(x): + z = x + 1 + x2 = hint(x, concrete=True) + return z + + hs = self.hannotate(f, [int]) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.is_green() + assert not hs.is_fixed() + + def test_blue_simple_meth(self): + py.test.skip("with abstract containers this test explode in the cast_pointer annotation logic") + class Base(object): + + def m(self): + raise NotImplementedError + + class Concrete(Base): + + def m(self): + return 42 + + def f(flag): + if flag: + o = Base() + else: + o = Concrete() + return o.m() -def test_simple_meth(): - class Base(object): - def m(self): - raise NotImplementedError - pass # for inspect.getsource() bugs - - class Concrete(Base): - def m(self): - return 42 - pass # for inspect.getsource() bugs - - def f(flag): - if flag: - o = Base() - else: - o = Concrete() - return o.m() + hs = self.hannotate(f, [bool], policy=P_OOPSPEC) - hs = hannotate(f, [bool], policy=P_OOPSPEC_NOVIRTUAL) + def test_simple_meth(self): + class Base(object): + def m(self): + raise NotImplementedError + pass # for inspect.getsource() bugs + + class Concrete(Base): + def m(self): + return 42 + pass # for inspect.getsource() bugs + + def f(flag): + if flag: + o = Base() + else: + o = Concrete() + return o.m() -def test_green_isinstance(): - class Base(object): - pass - class Concrete(Base): - pass + hs = self.hannotate(f, [bool], policy=P_OOPSPEC_NOVIRTUAL) - def f(o): - hint(o, concrete=True) - return isinstance(o, Concrete) - hs = hannotate(f, [Base], policy=P_OOPSPEC_NOVIRTUAL) - assert hs.is_green() + def test_green_isinstance(self): + class Base(object): + pass + class Concrete(Base): + pass + def f(o): + hint(o, concrete=True) + return isinstance(o, Concrete) -def test_cast_pointer_keeps_deepfreeze(): + hs = self.hannotate(f, [Base], policy=P_OOPSPEC_NOVIRTUAL) + assert hs.is_green() - class A(object): - pass - class B(A): - pass - - def getinstance(n): - if n: - return A() - else: - return B() - - def ll_function(n): - a = getinstance(n) - a = hint(a, promote=True) - a = hint(a, deepfreeze=True) - if isinstance(a, B): - return a - return None + def test_cast_pointer_keeps_deepfreeze(self): - hs = hannotate(ll_function, [int], policy=P_NOVIRTUAL) - assert hs.deepfrozen + class A(object): + pass + class B(A): + pass + def getinstance(n): + if n: + return A() + else: + return B() -def test_concrete_fnptr_for_green_call(): + def ll_function(n): + a = getinstance(n) + a = hint(a, promote=True) + a = hint(a, deepfreeze=True) - def h1(n): - return n * 10 - - def h2(n): - return n + 20 - - lst = [h1, h2] - - def ll_function(n, m): - h = hint(lst, deepfreeze=True)[m] - res = h(n) - hint(res, concrete=True) # so 'h' gets green, so 'm' gets green - return m - - hs = hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) - assert hs.is_green() - - -def test_indirect_yellow_call(): - - def h1(n): - return 123 - - def h2(n): - return 456 - - lst = [h1, h2] - - def ll_function(n, m): - h = hint(lst, deepfreeze=True)[m] - return h(n) - - hs = hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) - assert not hs.is_green() - - -def test_indirect_sometimes_residual_pure_red_call(): - def h1(x): - return x-2 - def h2(x): - return x*4 - l = [h1, h2] - def f(n, x): - frozenl = hint(l, deepfreeze=True) - h = frozenl[n&1] - return h(x) - - P = StopAtXPolicy(h1) - P.oopspec = True - P.entrypoint_returns_red = False - hs, hannotator = hannotate(f, [int, int], policy=P, annotator=True) - assert not hs.is_green() - assert isinstance(hs, SomeLLAbstractConstant) - - tsgraph = graphof(hannotator.translator, h2) - hs = hannotator.binding(tsgraph.getargs()[0]) - assert not hs.is_green() - - -def test_indirect_sometimes_residual_red_call(): - class Stuff: - pass - stuff = Stuff() - def h1(x): - stuff.hello = 123 - return x-2 - def h2(x): - return x*4 - l = [h1, h2] - def f(n, x): - frozenl = hint(l, deepfreeze=True) - h = frozenl[n&1] - return h(x) - - P = StopAtXPolicy(h1) - P.oopspec = True - P.entrypoint_returns_red = False - hs, hannotator = hannotate(f, [int, int], policy=P, annotator=True) - assert not hs.is_green() - - tsgraph = graphof(hannotator.translator, h2) - hs = hannotator.binding(tsgraph.getargs()[0]) - assert not hs.is_green() - - -def test_indirect_sometimes_residual_pure_but_fixed_red_call(): - def h1(x): - return x-2 - def h2(x): - return x*4 - l = [h1, h2] - def f(n, x): - frozenl = hint(l, deepfreeze=True) - h = frozenl[n&1] - z = h(x) - hint(z, concrete=True) - return z - - P = StopAtXPolicy(h1) - P.oopspec = True - P.entrypoint_returns_red = False - hs, hannotator = hannotate(f, [int, int], policy=P, annotator=True) - assert hs.is_green() - - #tsgraph = graphof(hannotator.translator, h2) - #hs = hannotator.binding(tsgraph.getargs()[0]) - #assert hs.is_green() - - tsgraph = graphof(hannotator.translator, f) - hs = hannotator.binding(tsgraph.getargs()[0]) - assert hs.is_green() - hs = hannotator.binding(tsgraph.getargs()[1]) - assert hs.is_green() - -def test_ignore_nonjitted_path(): - def f(n): - if we_are_jitted(): - return 5 - else: - return n - hs = hannotate(f, [int]) - assert hs.is_green() - - def g(n): - if not we_are_jitted(): - return n - else: - return 5 - hs = hannotate(g, [int]) - assert hs.is_green() - - def g(n): - if not we_are_jitted(): - return n - else: - return 5 - hs = hannotate(g, [int], backendoptimize=True) - assert hs.is_green() - - -def test_substitute_graph(): - class MetaG: - pass # the details are only used by the timeshifter - - def g(m): - return m * 17 - - def f(n, m): - x = g(n) - y = g(m) - hint(y, concrete=True) - return g(m) - - class MyPolicy(HintAnnotatorPolicy): - entrypoint_returns_red = False - def look_inside_graph(self, graph): - if graph.func is g: - return MetaG # replaces g with a meta-call to metafunc() - else: - return True - - hs, hannotator = hannotate(f, [int, int], policy=MyPolicy(), - annotator=True) - assert hs.is_green() - for graph in hannotator.translator.graphs: - assert 'int_mul' not in flowmodel.summary(graph) - - -def test_cast_ptr_to_int(): - GCS1 = lltype.GcStruct('s1', ('x', lltype.Signed)) - def f(): - p = lltype.malloc(GCS1) - return lltype.cast_ptr_to_int(p) - - hs = hannotate(f, [], policy=P_NOVIRTUAL) - assert not hs.is_green() - -def test_strange_green_result_after_red_switch(): - py.test.skip("is this right?") - class LinkedRules(object): - _immutable_ = True - def __init__(self, data, next=None): - self.data = data - self.next = next - - def find_applicable_rule(self, query): - # self is green, query isn't - # should the result really be green? - while self: - data = self.data - hint(data, concrete=True) - j = 0 - if self.data == query: - return self - self = self.next + if isinstance(a, B): + return a return None - chain = LinkedRules(1, LinkedRules(2, LinkedRules(0))) - def f(x): - rulechain = chain.find_applicable_rule(x) - return rulechain - hs = hannotate(f, [int], policy=P_OOPSPEC_NOVIRTUAL) - assert isinstance(hs, SomeLLAbstractVariable) - -def test_manual_marking_of_pure_functions(): - d = {} - def h1(s): - try: - return d[s] - except KeyError: - d[s] = r = hash(s) - return r - h1._pure_function_ = True - def f(n): - hint(n, concrete=True) - if n == 0: - s = "abc" - else: - s = "123" - a = h1(s) - return a - - P = StopAtXPolicy(h1) - P.oopspec = True - P.entrypoint_returns_red = False - hs = hannotate(f, [int], policy=P) - assert hs.is_green() + hs = self.hannotate(ll_function, [int], policy=P_NOVIRTUAL) + assert hs.deepfrozen + + + def test_concrete_fnptr_for_green_call(self): + + def h1(n): + return n * 10 + + def h2(n): + return n + 20 + + lst = [h1, h2] + + def ll_function(n, m): + h = hint(lst, deepfreeze=True)[m] + res = h(n) + hint(res, concrete=True) # so 'h' gets green, so 'm' gets green + return m + + hs = self.hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) + assert hs.is_green() + + + def test_indirect_yellow_call(self): + + def h1(n): + return 123 + + def h2(n): + return 456 + + lst = [h1, h2] + + def ll_function(n, m): + h = hint(lst, deepfreeze=True)[m] + return h(n) + + hs = self.hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) + assert not hs.is_green() + + + def test_indirect_sometimes_residual_pure_red_call(self): + def h1(x): + return x-2 + def h2(x): + return x*4 + l = [h1, h2] + def f(n, x): + frozenl = hint(l, deepfreeze=True) + h = frozenl[n&1] + return h(x) + + P = StopAtXPolicy(h1) + P.oopspec = True + P.entrypoint_returns_red = False + hs, hannotator = self.hannotate(f, [int, int], policy=P, annotator=True) + assert not hs.is_green() + assert isinstance(hs, SomeLLAbstractConstant) + + tsgraph = graphof(hannotator.translator, h2) + hs = hannotator.binding(tsgraph.getargs()[0]) + assert not hs.is_green() + + + def test_indirect_sometimes_residual_red_call(self): + class Stuff: + pass + stuff = Stuff() + def h1(x): + stuff.hello = 123 + return x-2 + def h2(x): + return x*4 + l = [h1, h2] + def f(n, x): + frozenl = hint(l, deepfreeze=True) + h = frozenl[n&1] + return h(x) + + P = StopAtXPolicy(h1) + P.oopspec = True + P.entrypoint_returns_red = False + hs, hannotator = self.hannotate(f, [int, int], policy=P, annotator=True) + assert not hs.is_green() + + tsgraph = graphof(hannotator.translator, h2) + hs = hannotator.binding(tsgraph.getargs()[0]) + assert not hs.is_green() + + + def test_indirect_sometimes_residual_pure_but_fixed_red_call(self): + def h1(x): + return x-2 + def h2(x): + return x*4 + l = [h1, h2] + def f(n, x): + frozenl = hint(l, deepfreeze=True) + h = frozenl[n&1] + z = h(x) + hint(z, concrete=True) + return z + + P = StopAtXPolicy(h1) + P.oopspec = True + P.entrypoint_returns_red = False + hs, hannotator = self.hannotate(f, [int, int], policy=P, annotator=True) + assert hs.is_green() + + #tsgraph = graphof(hannotator.translator, h2) + #hs = hannotator.binding(tsgraph.getargs()[0]) + #assert hs.is_green() + + tsgraph = graphof(hannotator.translator, f) + hs = hannotator.binding(tsgraph.getargs()[0]) + assert hs.is_green() + hs = hannotator.binding(tsgraph.getargs()[1]) + assert hs.is_green() + + def test_ignore_nonjitted_path(self): + def f(n): + if we_are_jitted(): + return 5 + else: + return n + hs = self.hannotate(f, [int]) + assert hs.is_green() + + def g(n): + if not we_are_jitted(): + return n + else: + return 5 + hs = self.hannotate(g, [int]) + assert hs.is_green() + + def g(n): + if not we_are_jitted(): + return n + else: + return 5 + hs = self.hannotate(g, [int], backendoptimize=True) + assert hs.is_green() + + + def test_substitute_graph(self): + class MetaG: + pass # the details are only used by the timeshifter + + def g(m): + return m * 17 + + def f(n, m): + x = g(n) + y = g(m) + hint(y, concrete=True) + return g(m) + + class MyPolicy(HintAnnotatorPolicy): + entrypoint_returns_red = False + def look_inside_graph(self, graph): + if graph.func is g: + return MetaG # replaces g with a meta-call to metafunc() + else: + return True + + hs, hannotator = self.hannotate(f, [int, int], policy=MyPolicy(), + annotator=True) + assert hs.is_green() + for graph in hannotator.translator.graphs: + assert 'int_mul' not in flowmodel.summary(graph) + + def test_strange_green_result_after_red_switch(self): + py.test.skip("is this right?") + class LinkedRules(object): + _immutable_ = True + def __init__(self, data, next=None): + self.data = data + self.next = next + + def find_applicable_rule(self, query): + # self is green, query isn't + # should the result really be green? + while self: + data = self.data + hint(data, concrete=True) + j = 0 + if self.data == query: + return self + self = self.next + return None + + chain = LinkedRules(1, LinkedRules(2, LinkedRules(0))) + def f(x): + rulechain = chain.find_applicable_rule(x) + return rulechain + hs = self.hannotate(f, [int], policy=P_OOPSPEC_NOVIRTUAL) + assert isinstance(hs, SomeLLAbstractVariable) + + def test_manual_marking_of_pure_functions(self): + d = {} + def h1(s): + try: + return d[s] + except KeyError: + d[s] = r = hash(s) + return r + h1._pure_function_ = True + def f(n): + hint(n, concrete=True) + if n == 0: + s = "abc" + else: + s = "123" + a = h1(s) + return a + + P = StopAtXPolicy(h1) + P.oopspec = True + P.entrypoint_returns_red = False + hs = self.hannotate(f, [int], policy=P) + assert hs.is_green() + +class TestLLType(BaseAnnotatorTest): + type_system = 'lltype' + + malloc = property(lambda self: lltype.malloc) + + def make_struct(self, name, *fields, **kwds): + return lltype.GcStruct(name, *fields, **kwds) + + def annotate_struct(self, S): + return annmodel.SomePtr(lltype.Ptr(S)) + + def test_simple_cast_pointer(self): + GCS1 = lltype.GcStruct('s1', ('x', lltype.Signed)) + GCS2 = lltype.GcStruct('s2', ('sub', GCS1), ('y', lltype.Signed)) + PGCS1 = lltype.Ptr(GCS1) + PGCS2 = lltype.Ptr(GCS2) + def ll1(): + s2 = lltype.malloc(GCS2) + return lltype.cast_pointer(PGCS1, s2) + hs = self.hannotate(ll1, []) + assert isinstance(hs, SomeLLAbstractContainer) + assert hs.concretetype == PGCS1 + def ll1(): + s2 = lltype.malloc(GCS2) + s1 = s2.sub + return lltype.cast_pointer(PGCS2, s1) + hs = self.hannotate(ll1, []) + assert isinstance(hs, SomeLLAbstractContainer) + assert hs.concretetype == PGCS2 + + def test_getarrayitem(self): + A = lltype.GcArray(lltype.Signed, hints={'immutable': True}) + a = lltype.malloc(A, 10) + def ll1(n): + v = a[n] + v = hint(v, concrete=True) + return v + hs, ha = self.hannotate(ll1, [int], annotator=True) + assert hs.eager_concrete + g1 = graphof(ha.translator, ll1) + hs_n = ha.binding(g1.getargs()[0]) + assert hs_n.origins.keys()[0].fixed + + def test_getvarrayitem(self): + A = lltype.GcArray(lltype.Signed, hints={'immutable': True}) + def ll1(n): + a = lltype.malloc(A, 10) + v = a[n] + v = hint(v, concrete=True) + return v + hs, ha = self.hannotate(ll1, [int], annotator=True) + assert hs.eager_concrete + g1 = graphof(ha.translator, ll1) + hs_n = ha.binding(g1.getargs()[0]) + assert hs_n.origins.keys()[0].fixed + + def test_degenerated_merge_substructure(self): + S = lltype.GcStruct('S', ('n', lltype.Signed)) + T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) + + def ll_function(flag): + t = lltype.malloc(T) + t.s.n = 3 + s = lltype.malloc(S) + s.n = 4 + if flag: + s = t.s + return s, t + hs = self.hannotate(ll_function, [bool]) + assert isinstance(hs, SomeLLAbstractContainer) + assert not hs.contentdef.degenerated + assert len(hs.contentdef.fields) == 2 + hs0 = hs.contentdef.fields['item0'].s_value # 's' + assert isinstance(hs0, SomeLLAbstractContainer) + assert hs0.contentdef.degenerated + hs1 = hs.contentdef.fields['item1'].s_value # 't' + assert isinstance(hs1, SomeLLAbstractContainer) + assert hs1.contentdef.degenerated + + def test_degenerated_merge_cross_substructure(self): + py.test.skip("no longer a valid test") + from pypy.rlib import objectmodel + S = lltype.Struct('S', ('n', lltype.Signed)) + T = lltype.GcStruct('T', ('s', S), ('s1', S), ('n', lltype.Float)) + + def ll_function(flag): + t = lltype.malloc(T) + t.s.n = 3 + t.s1.n = 3 + if flag: + s = t.s + else: + s = t.s1 + objectmodel.keepalive_until_here(t) + return s, t + hs = self.hannotate(ll_function, [bool]) + assert isinstance(hs, SomeLLAbstractContainer) + assert not hs.contentdef.degenerated + assert len(hs.contentdef.fields) == 2 + hs0 = hs.contentdef.fields['item0'].s_value # 's' + assert isinstance(hs0, SomeLLAbstractContainer) + assert hs0.contentdef.degenerated + hs1 = hs.contentdef.fields['item1'].s_value # 't' + assert isinstance(hs1, SomeLLAbstractContainer) + assert hs1.contentdef.degenerated + + def test_cast_ptr_to_int(self): + GCS1 = lltype.GcStruct('s1', ('x', lltype.Signed)) + def f(): + p = lltype.malloc(GCS1) + return lltype.cast_ptr_to_int(p) + + hs = self.hannotate(f, [], policy=P_NOVIRTUAL) + assert not hs.is_green() + + +class TestOOType(BaseAnnotatorTest): + type_system = 'ootype' + + malloc = property(lambda self: ootype.new) + + def make_struct(self, name, *fields, **kwds): + fields = dict(fields) + hints = kwds.pop('hints', None) + if hints: + kwds['_hints'] = hints + return ootype.Instance(name, ootype.ROOT, fields, **kwds) + + def annotate_struct(self, S): + return annmodel.SomeOOInstance(S) + + def skip_policy(self): + py.test.skip('fixme? (This policy is not relevant for now)') + + test_simple_list_operations = skip_policy + test_some_more_list_operations = skip_policy + test_make_a_list = skip_policy + test_simple_struct_malloc = skip_policy + test_container_union = skip_policy + test_specialize_deepfreeze_calls = skip_policy + test_deepfreeze_variables = skip_policy + test_cast_pointer_keeps_deepfreeze = skip_policy + test_concrete_fnptr_for_green_call = skip_policy Modified: pypy/branch/lazy-write-barrier/pypy/jit/hintannotator/test/test_toy.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/jit/hintannotator/test/test_toy.py (original) +++ pypy/branch/lazy-write-barrier/pypy/jit/hintannotator/test/test_toy.py Sat Dec 15 13:57:29 2007 @@ -1,20 +1,27 @@ -from pypy.jit.hintannotator.test.test_annotator import hannotate, P_OOPSPEC -from pypy.jit.hintannotator.test.test_annotator import P_OOPSPEC_NOVIRTUAL +from pypy.jit.hintannotator.test.test_annotator import AbstractAnnotatorTest +from pypy.jit.hintannotator.test.test_annotator import P_OOPSPEC, P_OOPSPEC_NOVIRTUAL +class BaseToyTest(AbstractAnnotatorTest): + def test_hannotate_tl(self): + from pypy.jit.tl import tl + self.hannotate(tl.interp, [str, int, int], policy=P_OOPSPEC) -def test_hannotate_tl(): - from pypy.jit.tl import tl - hannotate(tl.interp, [str, int, int], policy=P_OOPSPEC) - -def test_hannotate_tl_novirtual(): - from pypy.jit.tl import tl - hannotate(tl.interp, [str, int, int], policy=P_OOPSPEC_NOVIRTUAL) - -def test_hannotate_tlr_novirtual(): - from pypy.jit.tl import tlr - hannotate(tlr.interpret, [str, int], policy=P_OOPSPEC_NOVIRTUAL) - -def test_hannotate_tlc_novirtual(): - from pypy.jit.tl import tlc - hannotate(tlc.interp_without_call, [str, int, int], - policy=P_OOPSPEC_NOVIRTUAL, backendoptimize=True) + def test_hannotate_tl_novirtual(self): + from pypy.jit.tl import tl + self.hannotate(tl.interp, [str, int, int], policy=P_OOPSPEC_NOVIRTUAL) + + def test_hannotate_tlr_novirtual(self): + from pypy.jit.tl import tlr + self.hannotate(tlr.interpret, [str, int], policy=P_OOPSPEC_NOVIRTUAL) + + def test_hannotate_tlc_novirtual(self): + from pypy.jit.tl import tlc + self.hannotate(tlc.interp_without_call, [str, int, int], + policy=P_OOPSPEC_NOVIRTUAL, backendoptimize=True) + +class TestLLType(BaseToyTest): + type_system = 'lltype' + +## XXX: all tests fail :-( +##class TestOOType(BaseToyTest): +## type_system = 'ootype' Modified: pypy/branch/lazy-write-barrier/pypy/module/__builtin__/__init__.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/module/__builtin__/__init__.py (original) +++ pypy/branch/lazy-write-barrier/pypy/module/__builtin__/__init__.py Sat Dec 15 13:57:29 2007 @@ -60,9 +60,6 @@ 'buffer' : 'app_buffer.buffer', 'reload' : 'app_misc.reload', - 'set' : 'app_sets.set', - 'frozenset' : 'app_sets.frozenset', - '__filestub' : 'app_file_stub.file', } @@ -80,8 +77,8 @@ 'open' : 'state.get(space).w_file', # old-style classes dummy support - '_classobj' : 'space.w_classobj', - '_instance' : 'space.w_instance', + '_classobj' : 'interp_classobj.W_ClassObject', + '_instance' : 'interp_classobj.W_InstanceObject', # default __metaclass__ '__metaclass__' : '(space.w_type)', Modified: pypy/branch/lazy-write-barrier/pypy/module/_random/interp_random.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/module/_random/interp_random.py (original) +++ pypy/branch/lazy-write-barrier/pypy/module/_random/interp_random.py Sat Dec 15 13:57:29 2007 @@ -110,7 +110,7 @@ getrandbits.unwrap_spec = ['self', ObjSpace, int] -W_Random.typedef = TypeDef("W_Random", +W_Random.typedef = TypeDef("Random", __new__ = interp2app(descr_new__), random = interp2app(W_Random.random), seed = interp2app(W_Random.seed), Modified: pypy/branch/lazy-write-barrier/pypy/module/crypt/interp_crypt.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/module/crypt/interp_crypt.py (original) +++ pypy/branch/lazy-write-barrier/pypy/module/crypt/interp_crypt.py Sat Dec 15 13:57:29 2007 @@ -2,8 +2,12 @@ from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.rpython.lltypesystem import rffi, lltype from pypy.translator.tool.cbuild import ExternalCompilationInfo +import sys -eci = ExternalCompilationInfo(libraries=['crypt']) +if sys.platform.startswith('darwin'): + eci = ExternalCompilationInfo() +else: + eci = ExternalCompilationInfo(libraries=['crypt']) c_crypt = rffi.llexternal('crypt', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP, compilation_info=eci, threadsafe=False) Modified: pypy/branch/lazy-write-barrier/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/objspace/std/objspace.py (original) +++ pypy/branch/lazy-write-barrier/pypy/objspace/std/objspace.py Sat Dec 15 13:57:29 2007 @@ -282,15 +282,8 @@ def setup_old_style_classes(self): """NOT_RPYTHON""" # sanity check that this approach is working and is not too late - w_mod, w_dic = self.create_builtin_module('_classobj.py', 'classobj') - w_purify = self.getitem(w_dic, self.wrap('purify')) - w_classobj = self.getitem(w_dic, self.wrap('classobj')) - w_instance = self.getitem(w_dic, self.wrap('instance')) - self.call_function(w_purify) - assert not self.is_true(self.contains(self.builtin.w_dict,self.wrap('_classobj'))),"app-level code has seen dummy old style classes" - assert not self.is_true(self.contains(self.builtin.w_dict,self.wrap('_instance'))),"app-level code has seen dummy old style classes" - self.w_classobj = w_classobj - self.w_instance = w_instance + self.w_classobj = self.getattr(self.builtin, self.wrap('_classobj')) + self.w_instance = self.getattr(self.builtin, self.wrap('_instance')) def create_builtin_module(self, pyname, publicname): """NOT_RPYTHON Modified: pypy/branch/lazy-write-barrier/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/lltypesystem/llmemory.py Sat Dec 15 13:57:29 2007 @@ -61,7 +61,8 @@ assert index.startswith('item') # itemN => N index = int(index[4:]) return parent.getitem(index + self.repeat)._as_ptr() - elif isinstance(A, lltype.FixedSizeArray) and A.OF == self.TYPE: + elif (isinstance(A, lltype.FixedSizeArray) and + array_item_type_match(A.OF, self.TYPE)): # for array of primitives or pointers return lltype.direct_ptradd(firstitemptr, self.repeat) else: @@ -191,7 +192,7 @@ return '< ArrayItemsOffset %r >' % (self.TYPE,) def ref(self, arrayptr): - assert lltype.typeOf(arrayptr).TO == self.TYPE + assert array_type_match(lltype.typeOf(arrayptr).TO, self.TYPE) if isinstance(self.TYPE.OF, lltype.ContainerType): # XXX this doesn't support empty arrays o = arrayptr._obj.getitem(0) @@ -228,7 +229,7 @@ return '< ArrayLengthOffset %r >' % (self.TYPE,) def ref(self, arrayptr): - assert lltype.typeOf(arrayptr).TO == self.TYPE + assert array_type_match(lltype.typeOf(arrayptr).TO, self.TYPE) return lltype._arraylenref._makeptr(arrayptr._obj, arrayptr._solid) @@ -405,6 +406,21 @@ # GCREF is similar to Address but it is GC-aware GCREF = lltype.Ptr(lltype.GcOpaqueType('GCREF')) +# A placeholder for any type that is a GcArray of pointers. +# This can be used in the symbolic offsets above to access such arrays +# in a generic way. +GCARRAY_OF_PTR = lltype.GcArray(GCREF, hints={'placeholder': True}) +gcarrayofptr_lengthoffset = ArrayLengthOffset(GCARRAY_OF_PTR) +gcarrayofptr_itemsoffset = ArrayItemsOffset(GCARRAY_OF_PTR) +gcarrayofptr_singleitemoffset = ItemOffset(GCARRAY_OF_PTR.OF) +def array_type_match(A1, A2): + return A1 == A2 or (A2 == GCARRAY_OF_PTR and + isinstance(A1, lltype.GcArray) and + isinstance(A1.OF, lltype.Ptr) and + not A1._hints.get('nolength')) +def array_item_type_match(T1, T2): + return T1 == T2 or (T2 == GCREF and isinstance(T1, lltype.Ptr)) + class _fakeaccessor(object): def __init__(self, addr): Modified: pypy/branch/lazy-write-barrier/pypy/rpython/lltypesystem/test/test_llmemory.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/lltypesystem/test/test_llmemory.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/lltypesystem/test/test_llmemory.py Sat Dec 15 13:57:29 2007 @@ -524,3 +524,21 @@ assert weakref_deref(lltype.Ptr(S), w) == lltype.nullptr(S) assert weakref_deref(lltype.Ptr(S1), w) == lltype.nullptr(S1) +def test_generic_gcarray_of_ptr(): + S1 = lltype.GcStruct('S1', ('x', lltype.Signed)) + A1 = lltype.GcArray(lltype.Ptr(S1)) + A2 = lltype.GcArray(lltype.Ptr(A1)) + a2 = lltype.malloc(A2, 3) + a2[1] = lltype.malloc(A1, 4) + a2[1][2] = lltype.malloc(S1) + a2[1][2].x = -33 + + adr = cast_ptr_to_adr(a2) + assert (adr + gcarrayofptr_lengthoffset).signed[0] == 3 + adr += gcarrayofptr_itemsoffset + adr += gcarrayofptr_singleitemoffset + adr = adr.address[0] # => a2[1] + assert (adr + gcarrayofptr_lengthoffset).signed[0] == 4 + adr += gcarrayofptr_itemsoffset + 2 * gcarrayofptr_singleitemoffset + adr = adr.address[0] # => s2[1][2] + assert (adr + FieldOffset(S1, 'x')).signed[0] == -33 Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/base.py Sat Dec 15 13:57:29 2007 @@ -8,6 +8,7 @@ needs_zero_gc_pointers = True def set_query_functions(self, is_varsize, has_gcptr_in_varsize, + is_gcarrayofgcptr, getfinalizer, offsets_to_gc_pointers, fixed_size, varsize_item_sizes, @@ -18,6 +19,7 @@ self.getfinalizer = getfinalizer self.is_varsize = is_varsize self.has_gcptr_in_varsize = has_gcptr_in_varsize + self.is_gcarrayofgcptr = is_gcarrayofgcptr self.offsets_to_gc_pointers = offsets_to_gc_pointers self.fixed_size = fixed_size self.varsize_item_sizes = varsize_item_sizes @@ -115,6 +117,15 @@ Typically, 'callback' is a bound method and 'arg' can be None. """ typeid = self.get_type_id(obj) + if self.is_gcarrayofgcptr(typeid): + # a performance shortcut for GcArray(gcptr) + length = (obj + llmemory.gcarrayofptr_lengthoffset).signed[0] + item = obj + llmemory.gcarrayofptr_itemsoffset + while length > 0: + callback(item, arg) + item += llmemory.gcarrayofptr_singleitemoffset + length -= 1 + return offsets = self.offsets_to_gc_pointers(typeid) i = 0 while i < len(offsets): @@ -125,14 +136,13 @@ length = (obj + self.varsize_offset_to_length(typeid)).signed[0] offsets = self.varsize_offsets_to_gcpointers_in_var_part(typeid) itemlength = self.varsize_item_sizes(typeid) - i = 0 - while i < length: + while length > 0: j = 0 while j < len(offsets): callback(item + offsets[j], arg) j += 1 - i += 1 item += itemlength + length -= 1 trace._annspecialcase_ = 'specialize:arg(2)' Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py Sat Dec 15 13:57:29 2007 @@ -91,8 +91,6 @@ print "found %s initializing stores in %s" % (len(result), graph.name) return result -ADDRESS_VOID_FUNC = lltype.FuncType([llmemory.Address], lltype.Void) - class FrameworkGCTransformer(GCTransformer): use_stackless = False root_stack_depth = 163840 @@ -111,75 +109,21 @@ # for regular translation: pick the GC from the config GCClass, GC_PARAMS = choose_gc_from_config(translator.config) - self.FINALIZERTYPE = lltype.Ptr(ADDRESS_VOID_FUNC) - class GCData(object): - # types of the GC information tables - OFFSETS_TO_GC_PTR = lltype.Array(lltype.Signed) - TYPE_INFO = lltype.Struct("type_info", - ("isvarsize", lltype.Bool), - ("gcptrinvarsize", lltype.Bool), - ("finalizer", self.FINALIZERTYPE), - ("fixedsize", lltype.Signed), - ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), - ("varitemsize", lltype.Signed), - ("ofstovar", lltype.Signed), - ("ofstolength", lltype.Signed), - ("varofstoptrs",lltype.Ptr(OFFSETS_TO_GC_PTR)), - ("weakptrofs", lltype.Signed), - ) - TYPE_INFO_TABLE = lltype.Array(TYPE_INFO) - - def q_is_varsize(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].isvarsize - - def q_has_gcptr_in_varsize(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].gcptrinvarsize - - def q_finalizer(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].finalizer - - def q_offsets_to_gc_pointers(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].ofstoptrs - - def q_fixed_size(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].fixedsize - - def q_varsize_item_sizes(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].varitemsize - - def q_varsize_offset_to_variable_part(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].ofstovar - - def q_varsize_offset_to_length(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].ofstolength - - def q_varsize_offsets_to_gcpointers_in_var_part(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].varofstoptrs - - def q_weakpointer_offset(typeid): - ll_assert(typeid > 0, "invalid type_id") - return gcdata.type_info_table[typeid].weakptrofs - self.layoutbuilder = TransformerLayoutBuilder(self) self.get_type_id = self.layoutbuilder.get_type_id - gcdata = GCData() # set up dummy a table, to be overwritten with the real one in finish() - gcdata.type_info_table = lltype.malloc(GCData.TYPE_INFO_TABLE, 0, - immortal=True) + type_info_table = lltype._ptr( + lltype.Ptr(gctypelayout.GCData.TYPE_INFO_TABLE), + "delayed!type_info_table", solid=True) + gcdata = gctypelayout.GCData(type_info_table) + # initialize the following two fields with a random non-NULL address, # to make the annotator happy. The fields are patched in finish() - # to point to a real array (not 'static_roots', another one). - a_random_address = llmemory.cast_ptr_to_adr(gcdata.type_info_table) + # to point to a real array. + foo = lltype.malloc(lltype.FixedSizeArray(llmemory.Address, 1), + immortal=True, zero=True) + a_random_address = llmemory.cast_ptr_to_adr(foo) gcdata.static_root_start = a_random_address # patched in finish() gcdata.static_root_nongcstart = a_random_address # patched in finish() gcdata.static_root_end = a_random_address # patched in finish() @@ -190,6 +134,7 @@ StackRootIterator = self.build_stack_root_iterator() gcdata.gc = GCClass(AddressLinkedList, get_roots=StackRootIterator, **GC_PARAMS) + gcdata.set_query_functions(gcdata.gc) self.num_pushs = 0 self.write_barrier_calls = 0 @@ -197,29 +142,12 @@ # run-time initialization code StackRootIterator.setup_root_stack() gcdata.gc.setup() - gcdata.gc.set_query_functions( - q_is_varsize, - q_has_gcptr_in_varsize, - q_finalizer, - q_offsets_to_gc_pointers, - q_fixed_size, - q_varsize_item_sizes, - q_varsize_offset_to_variable_part, - q_varsize_offset_to_length, - q_varsize_offsets_to_gcpointers_in_var_part, - q_weakpointer_offset) bk = self.translator.annotator.bookkeeper # the point of this little dance is to not annotate - # self.gcdata.type_info_table as a constant. - data_classdef = bk.getuniqueclassdef(GCData) - data_classdef.generalize_attr( - 'type_info_table', - annmodel.SomePtr(lltype.Ptr(GCData.TYPE_INFO_TABLE))) - data_classdef.generalize_attr( - 'static_roots', - annmodel.SomePtr(lltype.Ptr(lltype.Array(llmemory.Address)))) + # self.gcdata.static_root_xyz as constants. XXX is it still needed?? + data_classdef = bk.getuniqueclassdef(gctypelayout.GCData) data_classdef.generalize_attr( 'static_root_start', annmodel.SomeAddress()) @@ -496,9 +424,10 @@ # replace the type_info_table pointer in gcdata -- at this point, # the database is in principle complete, so it has already seen - # the old (empty) array. We need to force it to consider the new - # array now. It's a bit hackish as the old empty array will also - # be generated in the C source, but that's a rather minor problem. + # the delayed pointer. We need to force it to consider the new + # array now. + + self.gcdata.type_info_table._become(table) # XXX because we call inputconst already in replace_malloc, we can't # modify the instance, we have to modify the 'rtyped instance' @@ -508,8 +437,6 @@ self.gcdata) r_gcdata = self.translator.rtyper.getrepr(s_gcdata) ll_instance = rmodel.inputconst(r_gcdata, self.gcdata).value - ll_instance.inst_type_info_table = table - #self.gcdata.type_info_table = table addresses_of_static_ptrs = ( self.layoutbuilder.addresses_of_static_ptrs + @@ -528,7 +455,7 @@ ll_instance.inst_static_root_end = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * (len(ll_static_roots_inside) - additional_ptrs) newgcdependencies = [] - newgcdependencies.append(table) + newgcdependencies.append(self.gcdata.type_info_table) newgcdependencies.append(ll_static_roots_inside) self.write_typeid_list() return newgcdependencies @@ -543,6 +470,7 @@ all.sort() for typeid, TYPE in all: f.write("%s %s\n" % (typeid, TYPE)) + f.close() def transform_graph(self, graph): if self.write_barrier_ptr: @@ -573,7 +501,7 @@ c_type_id = rmodel.inputconst(lltype.Signed, type_id) info = self.layoutbuilder.type_info_list[type_id] - c_size = rmodel.inputconst(lltype.Signed, info["fixedsize"]) + c_size = rmodel.inputconst(lltype.Signed, info.fixedsize) has_finalizer = bool(self.finalizer_funcptr_for_type(TYPE)) c_has_finalizer = rmodel.inputconst(lltype.Bool, has_finalizer) @@ -592,8 +520,8 @@ c_has_finalizer, rmodel.inputconst(lltype.Bool, False)] else: v_length = op.args[-1] - c_ofstolength = rmodel.inputconst(lltype.Signed, info['ofstolength']) - c_varitemsize = rmodel.inputconst(lltype.Signed, info['varitemsize']) + c_ofstolength = rmodel.inputconst(lltype.Signed, info.ofstolength) + c_varitemsize = rmodel.inputconst(lltype.Signed, info.varitemsize) malloc_ptr = self.malloc_varsize_clear_ptr ## if op.opname.startswith('zero'): ## malloc_ptr = self.malloc_varsize_clear_ptr @@ -624,7 +552,7 @@ c_type_id = rmodel.inputconst(lltype.Signed, type_id) info = self.layoutbuilder.type_info_list[type_id] - c_size = rmodel.inputconst(lltype.Signed, info["fixedsize"]) + c_size = rmodel.inputconst(lltype.Signed, info.fixedsize) has_finalizer = bool(self.finalizer_funcptr_for_type(TYPE)) assert not has_finalizer @@ -635,8 +563,8 @@ args = [self.c_const_gc, v_coallocator, c_type_id, c_size] else: v_length = op.args[-1] - c_ofstolength = rmodel.inputconst(lltype.Signed, info['ofstolength']) - c_varitemsize = rmodel.inputconst(lltype.Signed, info['varitemsize']) + c_ofstolength = rmodel.inputconst(lltype.Signed, info.ofstolength) + c_varitemsize = rmodel.inputconst(lltype.Signed, info.varitemsize) malloc_ptr = self.coalloc_varsize_clear_ptr args = [self.c_const_gc, v_coallocator, c_type_id, v_length, c_size, c_varitemsize, c_ofstolength] @@ -698,7 +626,7 @@ c_type_id = rmodel.inputconst(lltype.Signed, type_id) info = self.layoutbuilder.type_info_list[type_id] - c_size = rmodel.inputconst(lltype.Signed, info["fixedsize"]) + c_size = rmodel.inputconst(lltype.Signed, info.fixedsize) malloc_ptr = self.malloc_fixedsize_ptr c_has_finalizer = rmodel.inputconst(lltype.Bool, False) c_has_weakptr = c_can_collect = rmodel.inputconst(lltype.Bool, True) @@ -856,31 +784,9 @@ [llmemory.Address], lltype.Void) else: - fptr = lltype.nullptr(ADDRESS_VOID_FUNC) + fptr = lltype.nullptr(gctypelayout.GCData.FINALIZERTYPE.TO) return fptr - def offsets2table(self, offsets, TYPE): - try: - return self.offsettable_cache[TYPE] - except KeyError: - gcdata = self.transformer.gcdata - cachedarray = lltype.malloc(gcdata.OFFSETS_TO_GC_PTR, - len(offsets), immortal=True) - for i, value in enumerate(offsets): - cachedarray[i] = value - self.offsettable_cache[TYPE] = cachedarray - return cachedarray - - def flatten_table(self): - self.can_add_new_types = False - table = lltype.malloc(self.transformer.gcdata.TYPE_INFO_TABLE, - len(self.type_info_list), immortal=True) - for tableentry, newcontent in zip(table, self.type_info_list): - for key, value in newcontent.items(): - setattr(tableentry, key, value) - self.offsettable_cache = None - return table - def gen_zero_gc_pointers(TYPE, v, llops, previous_steps=None): if previous_steps is None: Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/transform.py Sat Dec 15 13:57:29 2007 @@ -279,7 +279,8 @@ self.need_minimal_transform(graph) if inline: self.graphs_to_inline[graph] = True - return self.mixlevelannotator.graph2delayed(graph) + FUNCTYPE = lltype.FuncType(ll_args, ll_result) + return self.mixlevelannotator.graph2delayed(graph, FUNCTYPE=FUNCTYPE) def inittime_helper(self, ll_helper, ll_args, ll_result, inline=True): ptr = self.annotate_helper(ll_helper, ll_args, ll_result, inline=inline) Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py Sat Dec 15 13:57:29 2007 @@ -1,13 +1,199 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena +from pypy.rlib.debug import ll_assert + + +class GCData(object): + """The GC information tables, and the query functions that the GC + calls to decode their content. The encoding of this information + is done by encode_type_shape(). These two places should be in sync, + obviously, but in principle no other code should depend on the + details of the encoding in TYPE_INFO. + """ + _alloc_flavor_ = 'raw' + + OFFSETS_TO_GC_PTR = lltype.Array(lltype.Signed) + ADDRESS_VOID_FUNC = lltype.FuncType([llmemory.Address], lltype.Void) + FINALIZERTYPE = lltype.Ptr(ADDRESS_VOID_FUNC) + + # structure describing the layout of a typeid + TYPE_INFO = lltype.Struct("type_info", + ("finalizer", FINALIZERTYPE), + ("fixedsize", lltype.Signed), + ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), + ("varitemsize", lltype.Signed), + ("ofstovar", lltype.Signed), + ("ofstolength", lltype.Signed), + ("varofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), + ("weakptrofs", lltype.Signed), + ) + TYPE_INFO_TABLE = lltype.Array(TYPE_INFO) + + def __init__(self, type_info_table): + self.type_info_table = type_info_table + # 'type_info_table' is a list of TYPE_INFO structures when + # running with gcwrapper, or a real TYPE_INFO_TABLE after + # the gctransformer. + + def q_is_varsize(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return (typeid & T_IS_FIXSIZE) == 0 + + def q_has_gcptr_in_varsize(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return (typeid & (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE)) == 0 + + def q_is_gcarrayofgcptr(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return (typeid & + (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE|T_NOT_SIMPLE_GCARRAY)) == 0 + + def q_finalizer(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return self.type_info_table[typeid].finalizer + + def q_offsets_to_gc_pointers(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return self.type_info_table[typeid].ofstoptrs + + def q_fixed_size(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return self.type_info_table[typeid].fixedsize + + def q_varsize_item_sizes(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return self.type_info_table[typeid].varitemsize + + def q_varsize_offset_to_variable_part(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return self.type_info_table[typeid].ofstovar + + def q_varsize_offset_to_length(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return self.type_info_table[typeid].ofstolength + + def q_varsize_offsets_to_gcpointers_in_var_part(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return self.type_info_table[typeid].varofstoptrs + + def q_weakpointer_offset(self, typeid): + ll_assert(typeid > 0, "invalid type_id") + return self.type_info_table[typeid].weakptrofs + + def set_query_functions(self, gc): + gc.set_query_functions( + self.q_is_varsize, + self.q_has_gcptr_in_varsize, + self.q_is_gcarrayofgcptr, + self.q_finalizer, + self.q_offsets_to_gc_pointers, + self.q_fixed_size, + self.q_varsize_item_sizes, + self.q_varsize_offset_to_variable_part, + self.q_varsize_offset_to_length, + self.q_varsize_offsets_to_gcpointers_in_var_part, + self.q_weakpointer_offset) + +# For the q_xxx functions that return flags, we use bit patterns +# in the typeid instead of entries in the type_info_table. The +# following flag combinations are used (the idea being that it's +# very fast on CPUs to check if all flags in a set are all zero): + +# * if T_IS_FIXSIZE is set, the gc object is not var-sized +# * if T_IS_FIXSIZE and T_NO_GCPTR_IN_VARSIZE are both cleared, +# there are gc ptrs in the var-sized part +# * if T_IS_FIXSIZE, T_NO_GCPTR_IN_VARSIZE and T_NOT_SIMPLE_GCARRAY +# are all cleared, the shape is just like GcArray(gcptr) + +T_IS_FIXSIZE = 0x4 +T_NO_GCPTR_IN_VARSIZE = 0x2 +T_NOT_SIMPLE_GCARRAY = 0x1 + +def get_typeid_bitmask(TYPE): + """Return the bits that we would like to be set or cleared in the type_id + corresponding to TYPE. This returns (mask, expected_value), where + the condition is that 'type_id & mask == expected_value'. + """ + if not TYPE._is_varsize(): + return (T_IS_FIXSIZE, T_IS_FIXSIZE) # not var-sized + + if (isinstance(TYPE, lltype.GcArray) + and isinstance(TYPE.OF, lltype.Ptr) + and TYPE.OF.TO._gckind == 'gc'): + # a simple GcArray(gcptr) + return (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE|T_NOT_SIMPLE_GCARRAY, 0) + + if isinstance(TYPE, lltype.Struct): + ARRAY = TYPE._flds[TYPE._arrayfld] + else: + ARRAY = TYPE + assert isinstance(ARRAY, lltype.Array) + if ARRAY.OF != lltype.Void and len(offsets_to_gc_pointers(ARRAY.OF)) > 0: + # var-sized, with gc pointers in the variable part + return (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE|T_NOT_SIMPLE_GCARRAY, + T_NOT_SIMPLE_GCARRAY) + else: + # var-sized, but no gc pointer in the variable part + return (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE, T_NO_GCPTR_IN_VARSIZE) + + +def encode_type_shape(builder, info, TYPE): + """Encode the shape of the TYPE into the TYPE_INFO structure 'info'.""" + offsets = offsets_to_gc_pointers(TYPE) + info.ofstoptrs = builder.offsets2table(offsets, TYPE) + info.finalizer = builder.make_finalizer_funcptr_for_type(TYPE) + info.weakptrofs = weakpointer_offset(TYPE) + if not TYPE._is_varsize(): + #info.isvarsize = False + #info.gcptrinvarsize = False + info.fixedsize = llarena.round_up_for_allocation( + llmemory.sizeof(TYPE)) + info.ofstolength = -1 + # note about round_up_for_allocation(): in the 'info' table + # we put a rounded-up size only for fixed-size objects. For + # varsize ones, the GC must anyway compute the size at run-time + # and round up that result. + else: + #info.isvarsize = True + info.fixedsize = llmemory.sizeof(TYPE, 0) + if isinstance(TYPE, lltype.Struct): + ARRAY = TYPE._flds[TYPE._arrayfld] + ofs1 = llmemory.offsetof(TYPE, TYPE._arrayfld) + info.ofstolength = ofs1 + llmemory.ArrayLengthOffset(ARRAY) + if ARRAY.OF != lltype.Void: + info.ofstovar = ofs1 + llmemory.itemoffsetof(ARRAY, 0) + else: + info.fixedsize = ofs1 + llmemory.sizeof(lltype.Signed) + # XXX we probably don't need isrpystring any more + if ARRAY._hints.get('isrpystring'): + info.fixedsize = llmemory.sizeof(TYPE, 1) + else: + ARRAY = TYPE + info.ofstolength = llmemory.ArrayLengthOffset(ARRAY) + if ARRAY.OF != lltype.Void: + info.ofstovar = llmemory.itemoffsetof(TYPE, 0) + else: + info.fixedsize = (llmemory.ArrayLengthOffset(ARRAY) + + llmemory.sizeof(lltype.Signed)) + assert isinstance(ARRAY, lltype.Array) + if ARRAY.OF != lltype.Void: + offsets = offsets_to_gc_pointers(ARRAY.OF) + else: + offsets = () + info.varofstoptrs = builder.offsets2table(offsets, ARRAY.OF) + info.varitemsize = llmemory.sizeof(ARRAY.OF) + #info.gcptrinvarsize = len(offsets) > 0 + #info.gcarrayofgcptr = (isinstance(TYPE, lltype.GcArray) + # and isinstance(TYPE.OF, lltype.Ptr) + # and TYPE.OF.TO._gckind == 'gc') + +# ____________________________________________________________ class TypeLayoutBuilder(object): can_add_new_types = True def __init__(self): - dummy = {"weakptrofs": -1, - "ofstolength": -1} - self.type_info_list = [dummy] # don't use typeid 0, helps debugging + self.type_info_list = [None] # don't use typeid 0, helps debugging self.id_of_type = {} # {LLTYPE: type_id} self.seen_roots = {} # the following are lists of addresses of gc pointers living inside the @@ -20,6 +206,8 @@ # dictionary address -> list of addresses self.additional_roots_sources = 0 self.finalizer_funcptrs = {} + self.offsettable_cache = {} + self.next_typeid_cache = {} def get_type_id(self, TYPE): try: @@ -27,60 +215,56 @@ except KeyError: assert self.can_add_new_types assert isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)) - # Record the new type_id description as a small dict for now. - # The framework gc transformer will turn it into a - # Struct("type_info") in flatten_table(). - type_id = len(self.type_info_list) + # Record the new type_id description as a TYPE_INFO structure. + # It goes into a list for now, which will be turned into a + # TYPE_INFO_TABLE in flatten_table() by the gc transformer. + + # pick the next type_id with the correct bits set or cleared + mask, expected = get_typeid_bitmask(TYPE) + type_id = self.next_typeid_cache.get((mask, expected), 1) + while True: + if type_id == len(self.type_info_list): + self.type_info_list.append(None) + if (self.type_info_list[type_id] is None and + (type_id & mask) == expected): + break # can use this type_id + else: + type_id += 1 # continue searching + self.next_typeid_cache[mask, expected] = type_id + 1 assert type_id & 0xffff == type_id # make sure it fits into 2 bytes - info = {} - self.type_info_list.append(info) + + # build the TYPE_INFO structure + info = lltype.malloc(GCData.TYPE_INFO, immortal=True, zero=True) + encode_type_shape(self, info, TYPE) + self.type_info_list[type_id] = info self.id_of_type[TYPE] = type_id - offsets = offsets_to_gc_pointers(TYPE) - info["ofstoptrs"] = self.offsets2table(offsets, TYPE) - info["finalizer"] = self.make_finalizer_funcptr_for_type(TYPE) - info["weakptrofs"] = weakpointer_offset(TYPE) - if not TYPE._is_varsize(): - info["isvarsize"] = False - info["gcptrinvarsize"] = False - info["fixedsize"] = llarena.round_up_for_allocation( - llmemory.sizeof(TYPE)) - info["ofstolength"] = -1 - # note about round_up_for_allocation(): in the 'info' table - # we put a rounded-up size only for fixed-size objects. For - # varsize ones, the GC must anyway compute the size at run-time - # and round up that result. - else: - info["isvarsize"] = True - info["fixedsize"] = llmemory.sizeof(TYPE, 0) - if isinstance(TYPE, lltype.Struct): - ARRAY = TYPE._flds[TYPE._arrayfld] - ofs1 = llmemory.offsetof(TYPE, TYPE._arrayfld) - info["ofstolength"] = ofs1 + llmemory.ArrayLengthOffset(ARRAY) - if ARRAY.OF != lltype.Void: - info["ofstovar"] = ofs1 + llmemory.itemoffsetof(ARRAY, 0) - else: - info["fixedsize"] = ofs1 + llmemory.sizeof(lltype.Signed) - if ARRAY._hints.get('isrpystring'): - info["fixedsize"] = llmemory.sizeof(TYPE, 1) - else: - ARRAY = TYPE - info["ofstolength"] = llmemory.ArrayLengthOffset(ARRAY) - if ARRAY.OF != lltype.Void: - info["ofstovar"] = llmemory.itemoffsetof(TYPE, 0) - else: - info["fixedsize"] = llmemory.ArrayLengthOffset(ARRAY) + llmemory.sizeof(lltype.Signed) - assert isinstance(ARRAY, lltype.Array) - if ARRAY.OF != lltype.Void: - offsets = offsets_to_gc_pointers(ARRAY.OF) - else: - offsets = () - info["varofstoptrs"] = self.offsets2table(offsets, ARRAY.OF) - info["varitemsize"] = llmemory.sizeof(ARRAY.OF) - info["gcptrinvarsize"] = len(offsets) > 0 return type_id def offsets2table(self, offsets, TYPE): - return offsets + try: + return self.offsettable_cache[TYPE] + except KeyError: + cachedarray = lltype.malloc(GCData.OFFSETS_TO_GC_PTR, + len(offsets), immortal=True) + for i, value in enumerate(offsets): + cachedarray[i] = value + self.offsettable_cache[TYPE] = cachedarray + return cachedarray + + def flatten_table(self): + self.can_add_new_types = False + self.offsettable_cache = None + table = lltype.malloc(GCData.TYPE_INFO_TABLE, len(self.type_info_list), + immortal=True) + fieldnames = GCData.TYPE_INFO._names + for tableentry, newcontent in zip(table, self.type_info_list): + if newcontent is None: # empty entry + tableentry.weakptrofs = -1 + tableentry.ofstolength = -1 + else: + for name in fieldnames: + setattr(tableentry, name, getattr(newcontent, name)) + return table def finalizer_funcptr_for_type(self, TYPE): if TYPE in self.finalizer_funcptrs: @@ -92,57 +276,8 @@ def make_finalizer_funcptr_for_type(self, TYPE): return None # must be overridden for proper finalizer support - def q_is_varsize(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["isvarsize"] - - def q_has_gcptr_in_varsize(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["gcptrinvarsize"] - - def q_finalizer(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["finalizer"] - - def q_offsets_to_gc_pointers(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["ofstoptrs"] - - def q_fixed_size(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["fixedsize"] - - def q_varsize_item_sizes(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["varitemsize"] - - def q_varsize_offset_to_variable_part(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["ofstovar"] - - def q_varsize_offset_to_length(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["ofstolength"] - - def q_varsize_offsets_to_gcpointers_in_var_part(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["varofstoptrs"] - - def q_weakpointer_offset(self, typeid): - assert typeid > 0 - return self.type_info_list[typeid]["weakptrofs"] - - def get_query_functions(self): - return (self.q_is_varsize, - self.q_has_gcptr_in_varsize, - self.q_finalizer, - self.q_offsets_to_gc_pointers, - self.q_fixed_size, - self.q_varsize_item_sizes, - self.q_varsize_offset_to_variable_part, - self.q_varsize_offset_to_length, - self.q_varsize_offsets_to_gcpointers_in_var_part, - self.q_weakpointer_offset) + def initialize_gc_query_function(self, gc): + return GCData(self.type_info_list).set_query_functions(gc) def consider_constant(self, TYPE, value, gc): if value is not lltype.top_container(value): Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gcwrapper.py Sat Dec 15 13:57:29 2007 @@ -1,5 +1,6 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llheap from pypy.rpython import llinterp +from pypy.rpython.annlowlevel import llhelper from pypy.rpython.memory.support import get_address_linked_list from pypy.rpython.memory import gctypelayout from pypy.objspace.flow.model import Constant @@ -21,7 +22,7 @@ def prepare_graphs(self, flowgraphs): layoutbuilder = DirectRunLayoutBuilder(self.llinterp) self.get_type_id = layoutbuilder.get_type_id - self.gc.set_query_functions(*layoutbuilder.get_query_functions()) + layoutbuilder.initialize_gc_query_function(self.gc) constants = collect_constants(flowgraphs) for obj in constants: @@ -144,7 +145,7 @@ DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0] destrgraph = destrptr._obj.graph else: - return None + return lltype.nullptr(gctypelayout.GCData.FINALIZERTYPE.TO) assert not type_contains_pyobjs(TYPE), "not implemented" def ll_finalizer(addr): @@ -154,7 +155,7 @@ except llinterp.LLException: raise RuntimeError( "a finalizer raised an exception, shouldn't happen") - return ll_finalizer + return llhelper(gctypelayout.GCData.FINALIZERTYPE, ll_finalizer) def collect_constants(graphs): Modified: pypy/branch/lazy-write-barrier/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/ootypesystem/ootype.py Sat Dec 15 13:57:29 2007 @@ -263,11 +263,12 @@ # We try to keep Record as similar to Instance as possible, so backends # can treat them polymorphically, if they choose to do so. - def __init__(self, fields): + def __init__(self, fields, _hints={}): self._fields = frozendict() for name, ITEMTYPE in fields.items(): self._fields[name] = ITEMTYPE, ITEMTYPE._defl() self._null = _null_record(self) + self._hints = frozendict(_hints) def _defl(self): return self._null Modified: pypy/branch/lazy-write-barrier/pypy/rpython/ootypesystem/rbuiltin.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/ootypesystem/rbuiltin.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/ootypesystem/rbuiltin.py Sat Dec 15 13:57:29 2007 @@ -40,6 +40,18 @@ return hop.genop('ooidentityhash', vlist, resulttype = ootype.Signed) +def rtype_ooupcast(hop): + assert isinstance(hop.args_s[0].const, ootype.Instance) + assert isinstance(hop.args_s[1], annmodel.SomeOOInstance) + v_inst = hop.inputarg(hop.args_r[1], arg=1) + return hop.genop('ooupcast', [v_inst], resulttype = hop.r_result.lowleveltype) + +def rtype_oodowncast(hop): + assert isinstance(hop.args_s[0].const, ootype.Instance) + assert isinstance(hop.args_s[1], annmodel.SomeOOInstance) + v_inst = hop.inputarg(hop.args_r[1], arg=1) + return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype) + def rtype_builtin_isinstance(hop): if hop.s_result.is_constant(): return hop.inputconst(ootype.Bool, hop.s_result.const) @@ -98,6 +110,8 @@ BUILTIN_TYPER[ootype.subclassof] = rtype_subclassof BUILTIN_TYPER[ootype.runtimenew] = rtype_runtimenew BUILTIN_TYPER[ootype.ooidentityhash] = rtype_ooidentityhash +BUILTIN_TYPER[ootype.ooupcast] = rtype_ooupcast +BUILTIN_TYPER[ootype.oodowncast] = rtype_oodowncast BUILTIN_TYPER[isinstance] = rtype_builtin_isinstance BUILTIN_TYPER[objectmodel.r_dict] = rtype_r_dict BUILTIN_TYPER[objectmodel.instantiate] = rtype_instantiate Modified: pypy/branch/lazy-write-barrier/pypy/rpython/ootypesystem/test/test_oortype.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/ootypesystem/test/test_oortype.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/ootypesystem/test/test_oortype.py Sat Dec 15 13:57:29 2007 @@ -9,6 +9,7 @@ from pypy.translator.translator import TranslationContext, graphof from pypy.rpython.test.test_llinterp import interpret from pypy.rlib.objectmodel import r_dict +from pypy.tool.error import AnnotatorError from pypy.rpython.ootypesystem import ooregistry # side effects def gengraph(f, args=[], viewBefore=False, viewAfter=False, mangle=True): @@ -300,3 +301,40 @@ res = interpret(oof, [], type_system='ootype') assert res == 42 + +def test_ooupcast(): + A = Instance('A', ootype.ROOT, {}) + B = Instance('B', A, {}) + C = Instance('C', ootype.ROOT) + + def fn(): + b = new(B) + return ooupcast(A, b) + + res = interpret(fn, [], type_system='ootype') + assert typeOf(res) is A + + def fn(): + c = new(C) + return ooupcast(A, c) + + py.test.raises(AnnotatorError, interpret, fn, [], type_system='ootype') + +def test_oodowncast(): + A = Instance('A', ootype.ROOT, {}) + B = Instance('B', A, {}) + C = Instance('C', ootype.ROOT) + + def fn(): + b = new(B) + a = ooupcast(A, b) + return oodowncast(B, a) + + res = interpret(fn, [], type_system='ootype') + assert typeOf(res) is B + + def fn(): + c = new(C) + return oodowncast(A, c) + + py.test.raises(AnnotatorError, interpret, fn, [], type_system='ootype') Modified: pypy/branch/lazy-write-barrier/pypy/rpython/rpbc.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/rpbc.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/rpbc.py Sat Dec 15 13:57:29 2007 @@ -666,7 +666,7 @@ if desc not in self.s_pbc.descriptions: raise TyperError("%r not in %r" % (cls, self)) if self.lowleveltype is Void: - return desc.pyobj + return None return rclass.get_type_repr(self.rtyper).convert_desc(desc) def convert_const(self, cls): Modified: pypy/branch/lazy-write-barrier/pypy/translator/c/database.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/translator/c/database.py (original) +++ pypy/branch/lazy-write-barrier/pypy/translator/c/database.py Sat Dec 15 13:57:29 2007 @@ -164,9 +164,9 @@ self.containerstats[kind] = self.containerstats.get(kind, 0) + 1 self.containerlist.append(node) if self.completed: - assert not node.globalcontainer - # non-global containers are found very late, e.g. _subarrays - # via addresses introduced by the GC transformer + pass # we would like to fail here, but a few containers + # are found very late, e.g. _subarrays via addresses + # introduced by the GC transformer, or the type_info_table return node def get(self, obj): @@ -195,11 +195,15 @@ n = len('delayed!') if len(name) == n: raise - if id(obj) in self.delayedfunctionnames: - return self.delayedfunctionnames[id(obj)][0] - funcname = name[n:] - funcname = self.namespace.uniquename('g_' + funcname) - self.delayedfunctionnames[id(obj)] = funcname, obj + if isinstance(lltype.typeOf(obj).TO, lltype.FuncType): + if id(obj) in self.delayedfunctionnames: + return self.delayedfunctionnames[id(obj)][0] + funcname = name[n:] + funcname = self.namespace.uniquename('g_'+funcname) + self.delayedfunctionnames[id(obj)] = funcname, obj + else: + funcname = None # can't use the name of a + # delayed non-function ptr self.delayedfunctionptrs.append(obj) return funcname # /hack hack hack Modified: pypy/branch/lazy-write-barrier/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/translator/c/test/test_boehm.py (original) +++ pypy/branch/lazy-write-barrier/pypy/translator/c/test/test_boehm.py Sat Dec 15 13:57:29 2007 @@ -98,8 +98,9 @@ assert 0 < res <= 84 def test_id_is_weak(self): - # test that id(obj) does not keep obj alive + # test that compute_unique_id(obj) does not keep obj alive from pypy.rpython.lltypesystem.lloperation import llop + from pypy.rlib.objectmodel import compute_unique_id class State: pass s = State() @@ -113,11 +114,11 @@ pass def run_once(): a = A() - ida = id(a) + ida = compute_unique_id(a) b = B() - idb = id(b) + idb = compute_unique_id(b) c = C() - idc = id(c) + idc = compute_unique_id(c) llop.gc__collect(lltype.Void) llop.gc__collect(lltype.Void) llop.gc__collect(lltype.Void) Modified: pypy/branch/lazy-write-barrier/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/translator/cli/dotnet.py (original) +++ pypy/branch/lazy-write-barrier/pypy/translator/cli/dotnet.py Sat Dec 15 13:57:29 2007 @@ -402,11 +402,13 @@ return OverloadingResolver.lltype_to_annotation(TYPE) def specialize_call(self, hop): - v_obj, v_type = hop.inputargs(*hop.args_r) - if v_type.value is ootype.String or isinstance(v_type.value, (type, types.ClassType)): + TYPE = hop.args_v[1].value + v_obj = hop.inputarg(hop.args_r[0], arg=0) + if TYPE is ootype.String or isinstance(TYPE, (type, types.ClassType)): return hop.genop('oodowncast', [v_obj], hop.r_result.lowleveltype) else: - return hop.genop('cliunbox', [v_obj, v_type], hop.r_result.lowleveltype) + c_type = hop.inputconst(ootype.Void, TYPE) + return hop.genop('cliunbox', [v_obj, c_type], hop.r_result.lowleveltype) native_exc_cache = {} Modified: pypy/branch/lazy-write-barrier/pypy/translator/goal/bench-cronjob.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/translator/goal/bench-cronjob.py (original) +++ pypy/branch/lazy-write-barrier/pypy/translator/goal/bench-cronjob.py Sat Dec 15 13:57:29 2007 @@ -182,10 +182,8 @@ backends = [backend.strip() for backend in """ c c--stackless--_faassen - c--_faassen + c--_faassen--_allworkingmodules c--thread - c--_objspace=taint - c--_allworkingmodules c--gc=marksweep--_faassen c--gc=semispace--_faassen c--gc=generation--_faassen From pedronis at codespeak.net Sat Dec 15 16:34:11 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 15 Dec 2007 16:34:11 +0100 (CET) Subject: [pypy-svn] r49816 - in pypy/dist/pypy/rpython: module/test ootypesystem/test test Message-ID: <20071215153411.47ABC16854F@codespeak.net> Author: pedronis Date: Sat Dec 15 16:34:09 2007 New Revision: 49816 Modified: pypy/dist/pypy/rpython/module/test/test_ll_strtod.py pypy/dist/pypy/rpython/ootypesystem/test/test_oortype.py pypy/dist/pypy/rpython/test/test_llinterp.py pypy/dist/pypy/rpython/test/test_rbuiltin.py pypy/dist/pypy/rpython/test/test_rlist.py pypy/dist/pypy/rpython/test/test_rstr.py pypy/dist/pypy/rpython/test/test_runicode.py Log: don't special case strings/unicode consistently for oo/ll in get_interpreter anymore, in the few places that this was used use hlstr instead. fixes oo tests that started failing. Modified: pypy/dist/pypy/rpython/module/test/test_ll_strtod.py ============================================================================== --- pypy/dist/pypy/rpython/module/test/test_ll_strtod.py (original) +++ pypy/dist/pypy/rpython/module/test/test_ll_strtod.py Sat Dec 15 16:34:09 2007 @@ -11,7 +11,11 @@ assert self.ll_to_string(self.interpret(f, [3.0])) == f(3.0) def test_parts_to_float(self): + from pypy.rpython.annlowlevel import hlstr + def f(a, b, c, d): + a,b,c,d = hlstr(a), hlstr(b), hlstr(c), hlstr(d) + return rarithmetic.parts_to_float(a, b, c, d) data = [ Modified: pypy/dist/pypy/rpython/ootypesystem/test/test_oortype.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/test/test_oortype.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/test/test_oortype.py Sat Dec 15 16:34:09 2007 @@ -194,7 +194,6 @@ assert rtyper.exceptiondata.is_exception_instance(INST) def test_string_annotation(): - py.test.skip("XXX interpret() does special magic about String arguments") def oof(lst): return lst.ll_strlen() Modified: pypy/dist/pypy/rpython/test/test_llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_llinterp.py (original) +++ pypy/dist/pypy/rpython/test/test_llinterp.py Sat Dec 15 16:34:09 2007 @@ -85,10 +85,6 @@ T = typeOf(x) if T == Ptr(PyObject) and someobjects: return object - elif T == Ptr(rstr.STR) or T == ootype.String: - return str - elif T == Ptr(rstr.UNICODE) or T == ootype.Unicode: - return unicode else: return lltype_to_annotation(T) Modified: pypy/dist/pypy/rpython/test/test_rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rbuiltin.py (original) +++ pypy/dist/pypy/rpython/test/test_rbuiltin.py Sat Dec 15 16:34:09 2007 @@ -288,8 +288,11 @@ def test_os_path_exists(self): self._skip_llinterpreter("os.stat()") + from pypy.rpython.annlowlevel import hlstr + import os def f(fn): + fn = hlstr(fn) return os.path.exists(fn) filename = self.string_to_ll(str(py.magic.autopath())) assert self.interpret(f, [filename]) == True @@ -298,8 +301,11 @@ def test_os_isdir(self): self._skip_llinterpreter("os.stat()") + from pypy.rpython.annlowlevel import hlstr + import os def f(fn): + fn = hlstr(fn) return os.path.isdir(fn) assert self.interpret(f, [self.string_to_ll("/")]) == True assert self.interpret(f, [self.string_to_ll(str(py.magic.autopath()))]) == False Modified: pypy/dist/pypy/rpython/test/test_rlist.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rlist.py (original) +++ pypy/dist/pypy/rpython/test/test_rlist.py Sat Dec 15 16:34:09 2007 @@ -1392,8 +1392,10 @@ def test_hints(self): from pypy.rlib.objectmodel import newlist - + from pypy.rpython.annlowlevel import hlstr + def f(z): + z = hlstr(z) x = newlist(sizehint=13) x += z return ''.join(x) Modified: pypy/dist/pypy/rpython/test/test_rstr.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rstr.py (original) +++ pypy/dist/pypy/rpython/test/test_rstr.py Sat Dec 15 16:34:09 2007 @@ -700,15 +700,6 @@ return const('ababa').count(const('aba')) res = self.interpret(fn, []) assert res == 1 - - def test_hlstr(self): - const = self.const - from pypy.rpython.annlowlevel import hlstr - def f(s): - return const("*")+const(hlstr(s))+const("*") == const("*abba*") - - res = self.interpret(f, [self.string_to_ll(const("abba"))]) - assert res def test_getitem_exc(self): const = self.const @@ -781,8 +772,10 @@ assert summary(fgraph) == {} def test_inplace_add(self): + from pypy.rpython.annlowlevel import hlstr const = self.const def f(x, y): + y = const(hlstr(y)) if x > 0: l = [const('a'), const('b')] else: @@ -790,8 +783,17 @@ l += y return const('').join(l) - assert self.ll_to_string(self.interpret(f, [1, self.string_to_ll(const('abc'))])) == 'ababc' + assert self.ll_to_string(self.interpret(f, [1, + self.string_to_ll('abc')])) == 'ababc' + def test_hlstr(self): + const = self.const + from pypy.rpython.annlowlevel import hlstr + def f(s): + return const("*")+const(hlstr(s))+const("*") == const("*abba*") + + res = self.interpret(f, [self.string_to_ll(const("abba"))]) + assert res def FIXME_test_str_to_pystringobj(): def f(n): Modified: pypy/dist/pypy/rpython/test/test_runicode.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_runicode.py (original) +++ pypy/dist/pypy/rpython/test/test_runicode.py Sat Dec 15 16:34:09 2007 @@ -215,19 +215,6 @@ py.test.skip("We should think how to solve this problem") test_rfind_empty_string = test_find_empty_string - - def test_inplace_add(self): - const = self.const - def f(x, y): - if x > 0: - l = [const('a'), const('b')] - else: - l = [const('a')] - l += y - return const('').join(l) - - assert self.ll_to_unicode(self.interpret(f, [1, self.unicode_to_ll(const('abc'))])) == 'ababc' - class TestLLtype(BaseTestRUnicode, LLRtypeMixin): EMPTY_STRING_HASH = -1 From hpk at codespeak.net Sat Dec 15 17:48:43 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 15 Dec 2007 17:48:43 +0100 (CET) Subject: [pypy-svn] r49817 - pypy/extradoc/planning/roadmap Message-ID: <20071215164843.DADC3168553@codespeak.net> Author: hpk Date: Sat Dec 15 17:48:42 2007 New Revision: 49817 Added: pypy/extradoc/planning/roadmap/task_pervasive_benchmarking.txt Log: (hpk, cfbolz): add a task about pervasive benchmarking Added: pypy/extradoc/planning/roadmap/task_pervasive_benchmarking.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_pervasive_benchmarking.txt Sat Dec 15 17:48:42 2007 @@ -0,0 +1,24 @@ + +Pervasive Benchmarking +------------------------ + +status: +- some microbenchmarks, but coverage not consistent + (see pypy/translator/microbench/*) +- some small-to-medium-sized apps as real-world benchmarks +- benchmarks run on one machine (wyvern), no collection of results + +todo: + +- implement more micro- and more large-scale benchmarks +- benchmarks against cpython/pypy/jython/ironpython +- run benchmarks automatically on more servers +- tool for sending benchmark data to a central place +- display benchmarking and progress information centrally + +expected outcome: + +- identification of slow areas of PyPy and for follow up work +- faster PyPy (i.e. some easy-to-do fixing of speed bugs) +- more community participation and contributions in analysing + and helping with speed problems From hpk at codespeak.net Sat Dec 15 17:59:06 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 15 Dec 2007 17:59:06 +0100 (CET) Subject: [pypy-svn] r49818 - pypy/extradoc/planning/roadmap Message-ID: <20071215165906.EBE2A169E7F@codespeak.net> Author: hpk Date: Sat Dec 15 17:59:06 2007 New Revision: 49818 Added: pypy/extradoc/planning/roadmap/readme.txt Log: (cfbolz, hpk) add a suggestion for task decriptions Added: pypy/extradoc/planning/roadmap/readme.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/readme.txt Sat Dec 15 17:59:06 2007 @@ -0,0 +1,21 @@ + + +please use the following structure for task files: + +Task Title +--------------- + +status: + - describe the current status with respect to this task + (so that people not knowing the area yet can understand it) + +todo: + - list steps for completion of this task + +expected outcome: + - describe the effects of completing the task to the project + +dependencies: + - list other tasks that need to be completed before this task is + started + From hpk at codespeak.net Sat Dec 15 18:30:46 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 15 Dec 2007 18:30:46 +0100 (CET) Subject: [pypy-svn] r49819 - pypy/extradoc/planning/roadmap Message-ID: <20071215173046.93CDE169E3C@codespeak.net> Author: hpk Date: Sat Dec 15 18:30:44 2007 New Revision: 49819 Added: pypy/extradoc/planning/roadmap/task_documentation.txt Log: (hpk, cfbolz): draft the documentation task (inspired by the goteborg Q/A discussion) Added: pypy/extradoc/planning/roadmap/task_documentation.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_documentation.txt Sat Dec 15 18:30:44 2007 @@ -0,0 +1,37 @@ +PyPy Documentation +----------------------- + +status: + +- large collection of online pypy/doc/* files, often out of date +- large collection of EU reports (often more accurate than online docs) +- index.txt, getting-started and play1 main entry points for pypy docs +- very hard to maintain the current documentation + +todo: + +- analyze and assess existing documentation +- write 3-5 "books" (with multiple chapters), each by + drafting the outline of a book and discussing it + in the community +- move over parts of the online docs and the EU documents into the + book(s) +- improve support for referencing and inlining source code and (tested) examples + +expected outcome: + +- sorted and much improved user and developer documentation +- easier to maintain the documentation +- possibility to create nice PDFs + + + + +notes: + +titles might be e.g.: + + - "Guide to the Python Standard Interpreter" + - "Building and Programming with RPython" + - "PyPy Architecture and Implementation" + - "Extending and Embedding PyPy's Python Interpreter" From hpk at codespeak.net Sat Dec 15 18:48:49 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 15 Dec 2007 18:48:49 +0100 (CET) Subject: [pypy-svn] r49820 - pypy/extradoc/planning/roadmap Message-ID: <20071215174849.1DD8A169E97@codespeak.net> Author: hpk Date: Sat Dec 15 18:48:47 2007 New Revision: 49820 Added: pypy/extradoc/planning/roadmap/task_pervasive_testing.txt Log: (hpk, cfbolz): draft the pervasive testing task as discussed in gothenburg Added: pypy/extradoc/planning/roadmap/task_pervasive_testing.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_pervasive_testing.txt Sat Dec 15 18:48:47 2007 @@ -0,0 +1,29 @@ +Pervasive Testing +------------------------ + +status: +- automated nightly test runs of pypy tests on 32bit linux machine (wyvern) + +- automated nightly builds on tuatara (PowerPC OSX) for builds and benchmarks + +- nightly run of cpython regression tests on top of several pypy-c's + (running on samuele's linux Desktop machine) + +todo: + +- automated tests on the windows platform + (a dedicated remotely accessible windows machine + which regularly runs the tests) +- allow for all tests (see status above) to easily run on demand + and also from branches +- tool for sending testing data to a central place +- display test results information centrally +- integrate with release tools and process + +expected outcome: + +- easy-to-track and to fix test-regressions +- improved stability and consistency between platforms +- improvements to the release process +- more community participation and contributions in analysing + and helping with testing problems From arigo at codespeak.net Sun Dec 16 11:07:43 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 11:07:43 +0100 (CET) Subject: [pypy-svn] r49824 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20071216100743.91292168473@codespeak.net> Author: arigo Date: Sun Dec 16 11:07:41 2007 New Revision: 49824 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py Log: Don't need to reschedule this array explicitly; the delayed ptr logic does it for us. Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Sun Dec 16 11:07:41 2007 @@ -446,7 +446,6 @@ ll_instance.inst_static_root_end = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(ll_static_roots_inside) newgcdependencies = [] - newgcdependencies.append(self.gcdata.type_info_table) newgcdependencies.append(ll_static_roots_inside) self.write_typeid_list() return newgcdependencies From arigo at codespeak.net Sun Dec 16 11:07:55 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 11:07:55 +0100 (CET) Subject: [pypy-svn] r49825 - in pypy/dist/pypy/rpython/memory: . test Message-ID: <20071216100755.414341684D0@codespeak.net> Author: arigo Date: Sun Dec 16 11:07:54 2007 New Revision: 49825 Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py pypy/dist/pypy/rpython/memory/test/test_gctypelayout.py Log: Fix shallow test failure. Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/gctypelayout.py Sun Dec 16 11:07:54 2007 @@ -272,7 +272,8 @@ return fptr def make_finalizer_funcptr_for_type(self, TYPE): - return None # must be overridden for proper finalizer support + # must be overridden for proper finalizer support + return lltype.nullptr(GCData.ADDRESS_VOID_FUNC) def initialize_gc_query_function(self, gc): return GCData(self.type_info_list).set_query_functions(gc) Modified: pypy/dist/pypy/rpython/memory/test/test_gctypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gctypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gctypelayout.py Sun Dec 16 11:07:54 2007 @@ -1,4 +1,4 @@ -from pypy.rpython.memory.gctypelayout import TypeLayoutBuilder +from pypy.rpython.memory.gctypelayout import TypeLayoutBuilder, GCData from pypy.rpython.memory.gctypelayout import offsets_to_gc_pointers from pypy.rpython.lltypesystem import lltype @@ -36,6 +36,7 @@ for T1, T2 in [(GC_A, GC_S), (GC_A2, GC_S2), (GC_S3, GC_S2)]: tid1 = layoutbuilder.get_type_id(T1) tid2 = layoutbuilder.get_type_id(T2) - lst1 = layoutbuilder.q_varsize_offsets_to_gcpointers_in_var_part(tid1) - lst2 = layoutbuilder.q_offsets_to_gc_pointers(tid2) + gcdata = GCData(layoutbuilder.type_info_list) + lst1 = gcdata.q_varsize_offsets_to_gcpointers_in_var_part(tid1) + lst2 = gcdata.q_offsets_to_gc_pointers(tid2) assert len(lst1) == len(lst2) From arigo at codespeak.net Sun Dec 16 11:12:50 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 11:12:50 +0100 (CET) Subject: [pypy-svn] r49826 - pypy/branch/pypy-more-delayedptr Message-ID: <20071216101250.59BA3168473@codespeak.net> Author: arigo Date: Sun Dec 16 11:12:50 2007 New Revision: 49826 Removed: pypy/branch/pypy-more-delayedptr/ Log: This branch was manually merged. From arigo at codespeak.net Sun Dec 16 11:22:59 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 11:22:59 +0100 (CET) Subject: [pypy-svn] r49827 - pypy/branch/gc-prefetch Message-ID: <20071216102259.BFBDA16846A@codespeak.net> Author: arigo Date: Sun Dec 16 11:22:59 2007 New Revision: 49827 Added: pypy/branch/gc-prefetch/ - copied from r49826, pypy/dist/ Log: A branch for experiments with gcc's __builtin_prefetch(). From arigo at codespeak.net Sun Dec 16 12:28:37 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 12:28:37 +0100 (CET) Subject: [pypy-svn] r49828 - in pypy/branch/gc-prefetch/pypy: config rpython/memory/gc rpython/memory/test Message-ID: <20071216112837.C8FD61684E2@codespeak.net> Author: arigo Date: Sun Dec 16 12:28:35 2007 New Revision: 49828 Added: pypy/branch/gc-prefetch/pypy/rpython/memory/gc/prefetchsemispace.py (contents, props changed) Modified: pypy/branch/gc-prefetch/pypy/config/translationoption.py pypy/branch/gc-prefetch/pypy/rpython/memory/gc/base.py pypy/branch/gc-prefetch/pypy/rpython/memory/test/test_gc.py pypy/branch/gc-prefetch/pypy/rpython/memory/test/test_transformed_gc.py Log: Intermediate check-in (still buggy!). Modified: pypy/branch/gc-prefetch/pypy/config/translationoption.py ============================================================================== --- pypy/branch/gc-prefetch/pypy/config/translationoption.py (original) +++ pypy/branch/gc-prefetch/pypy/config/translationoption.py Sun Dec 16 12:28:35 2007 @@ -41,7 +41,7 @@ requires=[("translation.backend", "llvm")]), ChoiceOption("gc", "Garbage Collection Strategy", ["boehm", "ref", "marksweep", "semispace", "statistics", - "generation", "none"], + "generation", "prefetch", "none"], "ref", requires={ "ref": [("translation.rweakref", False), # XXX ("translation.gctransformer", "ref")], @@ -51,6 +51,7 @@ "marksweep": [("translation.gctransformer", "framework")], "statistics": [("translation.gctransformer", "framework")], "generation": [("translation.gctransformer", "framework")], + "prefetch": [("translation.gctransformer", "framework")], "boehm": [("translation.gctransformer", "boehm")], }, cmdline="--gc"), Modified: pypy/branch/gc-prefetch/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/gc-prefetch/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/gc-prefetch/pypy/rpython/memory/gc/base.py Sun Dec 16 12:28:35 2007 @@ -265,6 +265,12 @@ GC_PARAMS = {'space_size': 8*1024*1024} # XXX adjust from pypy.rpython.memory.gc.semispace import SemiSpaceGC return SemiSpaceGC, GC_PARAMS + elif config.translation.gc == "prefetch": + GC_PARAMS = {'space_size': 8*1024*1024, # XXX adjust + 'prefetch_queue_size': 16} # XXX adjust + from pypy.rpython.memory.gc.prefetchsemispace import \ + PrefetchSemiSpaceGC + return PrefetchSemiSpaceGC, GC_PARAMS elif config.translation.gc == "generation": GC_PARAMS = {'space_size': 8*1024*1024, # XXX adjust 'nursery_size': 896*1024} Added: pypy/branch/gc-prefetch/pypy/rpython/memory/gc/prefetchsemispace.py ============================================================================== --- (empty file) +++ pypy/branch/gc-prefetch/pypy/rpython/memory/gc/prefetchsemispace.py Sun Dec 16 12:28:35 2007 @@ -0,0 +1,90 @@ +from pypy.rpython.memory.gc.semispace import SemiSpaceGC +from pypy.rpython.lltypesystem import lltype, llmemory, rffi +from pypy.rpython.extfunc import register_external +from pypy.rpython.lltypesystem.llmemory import NULL + +def prefetch(addr): + """This function is used to minimize cache-miss latency by moving + data into a cache before it is accessed. You can insert calls to + it into code for which you know addresses of data in memory that + is likely to be accessed soon. + """ + +__builtin_prefetch = rffi.llexternal( + '__builtin_prefetch', + [llmemory.Address, lltype.Signed, lltype.Signed], + lltype.Void, + sandboxsafe=True, _nowrapper=True) + +def llimpl_prefetch(addr): + __builtin_prefetch(addr, 0, 0) +register_external(prefetch, [llmemory.Address], lltype.Void, + 'll_hack.builtin_prefetch', + llimpl=llimpl_prefetch, + llfakeimpl=prefetch, + sandboxsafe=True) + +# ____________________________________________________________ + +# The prefetch_queue is a circular first-in first-out buffer. +# prefetch_queue_next is the index of the next item in prefetch_queue +# that needs to be removed from the queue, processed, and replaced +# by the incoming element. At the beginning, the queue is empty, +# which we represent by filling it with NULLs. + + +class PrefetchSemiSpaceGC(SemiSpaceGC): + + def __init__(self, *args, **kwds): + prefetch_queue_size = kwds.pop('prefetch_queue_size', 4) + SemiSpaceGC.__init__(self, *args, **kwds) + assert prefetch_queue_size & (prefetch_queue_size-1) == 0, ( + "prefetch_queue_size must be a power of 2") + self.prefetch_queue = lltype.malloc( + lltype.FixedSizeArray(llmemory.Address, prefetch_queue_size), + immortal=True) + self.prefetch_queue_mask = prefetch_queue_size - 1 + + def scan_copied(self, scan): + # prepare the prefetch queue + i = self.prefetch_queue_mask + while i >= 0: + self.prefetch_queue[i] = NULL + i -= 1 + self.prefetch_queue_next = 0 + while True: + # scan + while scan < self.free: + curr = scan + self.size_gc_header() + self.trace_and_copy_lazy(curr) + scan += self.size_gc_header() + self.get_size(curr) + # if the prefetch queue is not empty, flush the next item + # (note that this is done by "popping" the most recently + # added item first, instead of in usual first-in first-out + # fashion, so that we know that the queue is completely + # empty as soon as we get a NULL) + i = self.prefetch_queue_next + pointer = self.prefetch_queue[i] + if pointer == NULL: + break # empty queue => done + self.prefetch_queue[i] = NULL + pointer.address[0] = self.copy(pointer.address[0]) + self.prefetch_queue_next = (i - 1) & self.prefetch_queue_mask + return scan + + def trace_and_copy_lazy(self, obj): + self.trace(obj, self._trace_copy_lazy, None) + + def _trace_copy_lazy(self, pointer, ignored): + if pointer.address[0] != NULL: + self.record_pointer_for_tracing(pointer) + + def record_pointer_for_tracing(self, pointer): + prefetch(pointer.address[0]) + i = self.prefetch_queue_next + oldpointer = self.prefetch_queue[i] + self.prefetch_queue[i] = pointer + self.prefetch_queue_next = (i + 1) & self.prefetch_queue_mask + if oldpointer != NULL: + oldpointer.address[0] = self.copy(oldpointer.address[0]) + record_pointer_for_tracing.dont_inline = True Modified: pypy/branch/gc-prefetch/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/gc-prefetch/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/gc-prefetch/pypy/rpython/memory/test/test_gc.py Sun Dec 16 12:28:35 2007 @@ -406,3 +406,7 @@ return 0 res = self.interpret(malloc_a_lot, [], backendopt=True, coalloc=True) assert res == 0 + +class TestPrefetchSemiSpaceGC(GCTest): + from pypy.rpython.memory.gc.prefetchsemispace import PrefetchSemiSpaceGC + GCClass = PrefetchSemiSpaceGC Modified: pypy/branch/gc-prefetch/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/gc-prefetch/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/gc-prefetch/pypy/rpython/memory/test/test_transformed_gc.py Sun Dec 16 12:28:35 2007 @@ -896,3 +896,13 @@ run = self.runner(f, nbargs=0) res = run([]) assert res == 40 * 5 + + +class TestPrefetchSemiSpaceGC(GenericMovingGCTests): + gcname = "prefetch" + + class gcpolicy(gc.FrameworkGcPolicy): + class transformerclass(framework.FrameworkGCTransformer): + from pypy.rpython.memory.gc.prefetchsemispace import PrefetchSemiSpaceGC as GCClass + GC_PARAMS = {'space_size': 2048, 'prefetch_queue_size': 4} + root_stack_depth = 200 From arigo at codespeak.net Sun Dec 16 12:33:05 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 12:33:05 +0100 (CET) Subject: [pypy-svn] r49829 - pypy/branch/gc-prefetch/pypy/rpython/memory/gc Message-ID: <20071216113305.880341684E8@codespeak.net> Author: arigo Date: Sun Dec 16 12:33:05 2007 New Revision: 49829 Modified: pypy/branch/gc-prefetch/pypy/rpython/memory/gc/prefetchsemispace.py Log: Bug fix. Modified: pypy/branch/gc-prefetch/pypy/rpython/memory/gc/prefetchsemispace.py ============================================================================== --- pypy/branch/gc-prefetch/pypy/rpython/memory/gc/prefetchsemispace.py (original) +++ pypy/branch/gc-prefetch/pypy/rpython/memory/gc/prefetchsemispace.py Sun Dec 16 12:33:05 2007 @@ -64,12 +64,13 @@ # fashion, so that we know that the queue is completely # empty as soon as we get a NULL) i = self.prefetch_queue_next + i = (i - 1) & self.prefetch_queue_mask + self.prefetch_queue_next = i pointer = self.prefetch_queue[i] if pointer == NULL: break # empty queue => done self.prefetch_queue[i] = NULL pointer.address[0] = self.copy(pointer.address[0]) - self.prefetch_queue_next = (i - 1) & self.prefetch_queue_mask return scan def trace_and_copy_lazy(self, obj): From arigo at codespeak.net Sun Dec 16 12:40:15 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 12:40:15 +0100 (CET) Subject: [pypy-svn] r49830 - pypy/branch/gc-prefetch/pypy/rpython/memory/gc Message-ID: <20071216114015.106581684F1@codespeak.net> Author: arigo Date: Sun Dec 16 12:40:15 2007 New Revision: 49830 Modified: pypy/branch/gc-prefetch/pypy/rpython/memory/gc/prefetchsemispace.py Log: Simplify the flushing logic. Modified: pypy/branch/gc-prefetch/pypy/rpython/memory/gc/prefetchsemispace.py ============================================================================== --- pypy/branch/gc-prefetch/pypy/rpython/memory/gc/prefetchsemispace.py (original) +++ pypy/branch/gc-prefetch/pypy/rpython/memory/gc/prefetchsemispace.py Sun Dec 16 12:40:15 2007 @@ -52,25 +52,19 @@ self.prefetch_queue[i] = NULL i -= 1 self.prefetch_queue_next = 0 + # scan while True: - # scan - while scan < self.free: - curr = scan + self.size_gc_header() - self.trace_and_copy_lazy(curr) - scan += self.size_gc_header() + self.get_size(curr) - # if the prefetch queue is not empty, flush the next item - # (note that this is done by "popping" the most recently - # added item first, instead of in usual first-in first-out - # fashion, so that we know that the queue is completely - # empty as soon as we get a NULL) - i = self.prefetch_queue_next - i = (i - 1) & self.prefetch_queue_mask - self.prefetch_queue_next = i - pointer = self.prefetch_queue[i] - if pointer == NULL: - break # empty queue => done - self.prefetch_queue[i] = NULL - pointer.address[0] = self.copy(pointer.address[0]) + if scan == self.free: + # flush the remaining items in the prefetch queue + i = self.prefetch_queue_mask + while i >= 0: + self.record_pointer_for_tracing(NULL) + i -= 1 + if scan == self.free: + break # finished + curr = scan + self.size_gc_header() + self.trace_and_copy_lazy(curr) + scan += self.size_gc_header() + self.get_size(curr) return scan def trace_and_copy_lazy(self, obj): @@ -78,10 +72,10 @@ def _trace_copy_lazy(self, pointer, ignored): if pointer.address[0] != NULL: + prefetch(pointer.address[0]) self.record_pointer_for_tracing(pointer) def record_pointer_for_tracing(self, pointer): - prefetch(pointer.address[0]) i = self.prefetch_queue_next oldpointer = self.prefetch_queue[i] self.prefetch_queue[i] = pointer From arigo at codespeak.net Sun Dec 16 12:50:19 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 12:50:19 +0100 (CET) Subject: [pypy-svn] r49831 - pypy/branch/gc-prefetch/pypy/rpython/memory/gc Message-ID: <20071216115019.3E05A168441@codespeak.net> Author: arigo Date: Sun Dec 16 12:50:18 2007 New Revision: 49831 Modified: pypy/branch/gc-prefetch/pypy/rpython/memory/gc/prefetchsemispace.py Log: Temporary usage notes. Modified: pypy/branch/gc-prefetch/pypy/rpython/memory/gc/prefetchsemispace.py ============================================================================== --- pypy/branch/gc-prefetch/pypy/rpython/memory/gc/prefetchsemispace.py (original) +++ pypy/branch/gc-prefetch/pypy/rpython/memory/gc/prefetchsemispace.py Sun Dec 16 12:50:18 2007 @@ -8,6 +8,11 @@ data into a cache before it is accessed. You can insert calls to it into code for which you know addresses of data in memory that is likely to be accessed soon. + + XXX requires gcc. + XXX You also need to tweak the Makefile to set + CFLAGS = -O2 -pthread -march=pentium3m + XXX and recompile. """ __builtin_prefetch = rffi.llexternal( From arigo at codespeak.net Sun Dec 16 12:54:38 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 12:54:38 +0100 (CET) Subject: [pypy-svn] r49832 - pypy/branch/gc-prefetch/pypy/rpython/memory/gc Message-ID: <20071216115438.8D58E168441@codespeak.net> Author: arigo Date: Sun Dec 16 12:54:38 2007 New Revision: 49832 Modified: pypy/branch/gc-prefetch/pypy/rpython/memory/gc/prefetchsemispace.py Log: Reading the gcc docs more carefully. Modified: pypy/branch/gc-prefetch/pypy/rpython/memory/gc/prefetchsemispace.py ============================================================================== --- pypy/branch/gc-prefetch/pypy/rpython/memory/gc/prefetchsemispace.py (original) +++ pypy/branch/gc-prefetch/pypy/rpython/memory/gc/prefetchsemispace.py Sun Dec 16 12:54:38 2007 @@ -22,7 +22,7 @@ sandboxsafe=True, _nowrapper=True) def llimpl_prefetch(addr): - __builtin_prefetch(addr, 0, 0) + __builtin_prefetch(addr, 0, 3) register_external(prefetch, [llmemory.Address], lltype.Void, 'll_hack.builtin_prefetch', llimpl=llimpl_prefetch, From pedronis at codespeak.net Sun Dec 16 14:01:21 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 16 Dec 2007 14:01:21 +0100 (CET) Subject: [pypy-svn] r49834 - pypy/dist/pypy/translator/microbench Message-ID: <20071216130121.C2822168505@codespeak.net> Author: pedronis Date: Sun Dec 16 14:01:20 2007 New Revision: 49834 Modified: pypy/dist/pypy/translator/microbench/microbench.py Log: just because, don't use exec, it should not make a huge difference but even very fractionally benchmarking the compiler is strange Modified: pypy/dist/pypy/translator/microbench/microbench.py ============================================================================== --- pypy/dist/pypy/translator/microbench/microbench.py (original) +++ pypy/dist/pypy/translator/microbench/microbench.py Sun Dec 16 14:01:20 2007 @@ -25,16 +25,17 @@ break else: continue - testcase = microbench + '.' + k + '()' + testcase_name = microbench + '.' + k + '()' + testcase = getattr(globals()[microbench], k) gc.collect() start = time.clock() n = 0 duration = 0.0 while duration < MINIMUM_MICROBENCH_TIME: - exec testcase + testcase() n += 1 duration = time.clock() - start - print '%s took %.2f seconds' % (testcase, duration / float(n)) + print '%s took %.2f seconds' % (testcase_name, duration / float(n)) if __name__ == '__main__': args = sys.argv[1:] From xoraxax at codespeak.net Sun Dec 16 14:38:26 2007 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 16 Dec 2007 14:38:26 +0100 (CET) Subject: [pypy-svn] r49835 - pypy/extradoc/planning/roadmap Message-ID: <20071216133826.5518E168435@codespeak.net> Author: xoraxax Date: Sun Dec 16 14:38:24 2007 New Revision: 49835 Added: pypy/extradoc/planning/roadmap/task_catch_up_with_2_x.txt - copied, changed from r49834, pypy/extradoc/planning/roadmap/readme.txt Modified: pypy/extradoc/planning/roadmap/goal_cpython_replacement.txt Log: Added new task (catch up with cpy 2.5), fixed includes in the goal. Modified: pypy/extradoc/planning/roadmap/goal_cpython_replacement.txt ============================================================================== --- pypy/extradoc/planning/roadmap/goal_cpython_replacement.txt (original) +++ pypy/extradoc/planning/roadmap/goal_cpython_replacement.txt Sun Dec 16 14:38:24 2007 @@ -8,6 +8,8 @@ do the porting of modules and other tasks that are beyond the resources of the core project. +These tasks have no specific ordering currently. (XXX do they?) + .. include:: task_refactor_jit.txt .. include:: task_optimize_jit.txt @@ -18,7 +20,11 @@ .. include:: task_wrapper_generator.txt -.. include:: task_more_modules.txt +.. include:: task_catch_up_with_2_x.txt + +.. include:: task_modules_rffi.txt + +.. include:: task_modules_3rdparty.txt .. include:: task_manual_optimizations.txt From hpk at codespeak.net Sun Dec 16 14:46:22 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sun, 16 Dec 2007 14:46:22 +0100 (CET) Subject: [pypy-svn] r49836 - pypy/extradoc/planning/roadmap Message-ID: <20071216134622.419DE1684F1@codespeak.net> Author: hpk Date: Sun Dec 16 14:46:21 2007 New Revision: 49836 Modified: pypy/extradoc/planning/roadmap/task_pervasive_benchmarking.txt Log: fix ReST Modified: pypy/extradoc/planning/roadmap/task_pervasive_benchmarking.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_pervasive_benchmarking.txt (original) +++ pypy/extradoc/planning/roadmap/task_pervasive_benchmarking.txt Sun Dec 16 14:46:21 2007 @@ -3,8 +3,9 @@ ------------------------ status: + - some microbenchmarks, but coverage not consistent - (see pypy/translator/microbench/*) + (see pypy/translator/microbench) - some small-to-medium-sized apps as real-world benchmarks - benchmarks run on one machine (wyvern), no collection of results From fijal at codespeak.net Sun Dec 16 15:38:46 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 16 Dec 2007 15:38:46 +0100 (CET) Subject: [pypy-svn] r49837 - in pypy/branch/lazy-write-barrier/pypy/rpython/memory: gc gctransform Message-ID: <20071216143846.ECBCD168505@codespeak.net> Author: fijal Date: Sun Dec 16 15:38:45 2007 New Revision: 49837 Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/semispace.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py Log: Check this in not to forget. It contains debug prints and messy code, to clean up Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py Sun Dec 16 15:38:45 2007 @@ -39,6 +39,7 @@ get_roots = get_roots) self.nursery_size = nursery_size assert nursery_size <= space_size // 2 + self.created = 0 def setup(self): SemiSpaceGC.setup(self) @@ -294,6 +295,7 @@ self.remember_young_pointer(addr_struct, newvalue) def append_to_static_roots(self, pointer, arg): + self.created += 1 self.get_roots.append_static_root(pointer) def move_to_static_roots(self, addr_struct): @@ -311,4 +313,5 @@ oldhdr.tid &= ~GCFLAG_NO_YOUNG_PTRS if oldhdr.tid & GCFLAG_NEVER_SET: self.move_to_static_roots(addr_struct) + llop.debug_print(lltype.Void, "new statc root", self.created) remember_young_pointer.dont_inline = True Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/semispace.py Sun Dec 16 15:38:45 2007 @@ -228,11 +228,14 @@ def collect_roots(self): roots = self.get_roots() + counter = 0 while 1: root = roots.pop() + counter += 1 if root == NULL: break root.address[0] = self.copy(root.address[0]) + llop.debug_print(lltype.Void, "collected: ", counter) free_non_gc_object(roots) def copy(self, obj): Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py Sun Dec 16 15:38:45 2007 @@ -17,6 +17,7 @@ from pypy.rpython.memory.gctypelayout import convert_weakref_to, WEAKREFPTR from pypy.rpython.memory.gctransform.log import log from pypy.tool.sourcetools import func_with_new_name +from pypy.rpython.lltypesystem.lloperation import llop import sys @@ -127,6 +128,7 @@ gcdata.static_root_start = a_random_address # patched in finish() gcdata.static_root_nongcstart = a_random_address # patched in finish() gcdata.static_root_end = a_random_address # patched in finish() + gcdata.static_root_real_end = a_random_address self.gcdata = gcdata self.malloc_fnptr_cache = {} @@ -157,6 +159,9 @@ data_classdef.generalize_attr( 'static_root_end', annmodel.SomeAddress()) + data_classdef.generalize_attr( + 'static_root_real_end', + annmodel.SomeAddress()) annhelper = annlowlevel.MixLevelHelperAnnotator(self.translator.rtyper) @@ -380,13 +385,15 @@ self.static_current = gcdata.static_root_start else: self.static_current = gcdata.static_root_nongcstart + self.with_static = with_static def pop(self): - while self.static_current != gcdata.static_root_end: - result = self.static_current - self.static_current += sizeofaddr - if result.address[0].address[0] != llmemory.NULL: - return result.address[0] + if self.with_static: + while self.static_current != gcdata.static_root_end: + result = self.static_current + self.static_current += sizeofaddr + if result.address[0].address[0] != llmemory.NULL: + return result.address[0] while self.stack_current != gcdata.root_stack_base: self.stack_current -= sizeofaddr if self.stack_current.address[0] != llmemory.NULL: @@ -446,13 +453,14 @@ log.info("additional %d potential static roots" % additional_ptrs) ll_static_roots_inside = lltype.malloc(lltype.Array(llmemory.Address), len(addresses_of_static_ptrs) + - additional_ptrs, + int(additional_ptrs), immortal=True) for i in range(len(addresses_of_static_ptrs)): ll_static_roots_inside[i] = addresses_of_static_ptrs[i] ll_instance.inst_static_root_start = llmemory.cast_ptr_to_adr(ll_static_roots_inside) + llmemory.ArrayItemsOffset(lltype.Array(llmemory.Address)) ll_instance.inst_static_root_nongcstart = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(self.layoutbuilder.addresses_of_static_ptrs) ll_instance.inst_static_root_end = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * (len(ll_static_roots_inside) - additional_ptrs) + ll_instance.inst_static_root_real_end = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(ll_static_roots_inside) newgcdependencies = [] newgcdependencies.append(self.gcdata.type_info_table) From arigo at codespeak.net Sun Dec 16 15:52:22 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 15:52:22 +0100 (CET) Subject: [pypy-svn] r49838 - pypy/dist/pypy/interpreter Message-ID: <20071216145222.741FE1684F7@codespeak.net> Author: arigo Date: Sun Dec 16 15:52:22 2007 New Revision: 49838 Modified: pypy/dist/pypy/interpreter/baseobjspace.py Log: Yay! Finally found out how an ExecutionContext can show up during translation. It's caused by helper init-time code (e.g. in module/*/__init__.py) that runs after space._freeze_() has been called, and causes indirectly a call to space.getexecutioncontext(). Fixed this issue by storing the ExecutionContext needed during early translation at some other location that is never seen by the translator. The space.threadlocals object that is frozen is thus always pristine. Modified: pypy/dist/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/dist/pypy/interpreter/baseobjspace.py (original) +++ pypy/dist/pypy/interpreter/baseobjspace.py Sun Dec 16 15:52:22 2007 @@ -6,6 +6,7 @@ from pypy.rlib.jit import hint from pypy.tool.cache import Cache from pypy.tool.uid import HUGEVAL_BYTES +from pypy.rlib.objectmodel import we_are_translated import os, sys __all__ = ['ObjSpace', 'OperationError', 'Wrappable', 'W_Root'] @@ -397,6 +398,22 @@ def getexecutioncontext(self): "Return what we consider to be the active execution context." + # Important: the annotator must not see a prebuilt ExecutionContext + # for reasons related to the specialization of the framestack attribute + # so we make sure that the threadlocals never *have* an + # ExecutionContext during translation. + if self.config.translating and not we_are_translated(): + assert self.threadlocals.getvalue() is None, ( + "threadlocals got an ExecutionContext during translation!") + try: + return self._ec_during_translation + except AttributeError: + ec = self.createexecutioncontext() + self._ec_during_translation = ec + return ec + # normal case follows. The 'thread' module installs a real + # thread-local object in self.threadlocals, so this builds + # and caches a new ec in each thread. ec = self.threadlocals.getvalue() if ec is None: ec = self.createexecutioncontext() @@ -404,10 +421,6 @@ return ec def _freeze_(self): - # Important: the annotator must not see a prebuilt ExecutionContext - # for reasons related to the specialization of the framestack attribute - # so we make sure there is no executioncontext at freeze-time - self.threadlocals.setvalue(None) return True def createexecutioncontext(self): From fijal at codespeak.net Sun Dec 16 16:11:35 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 16 Dec 2007 16:11:35 +0100 (CET) Subject: [pypy-svn] r49839 - in pypy/branch/lazy-write-barrier/pypy/rpython/memory: . gc gctransform Message-ID: <20071216151135.2D1DB1684FB@codespeak.net> Author: fijal Date: Sun Dec 16 16:11:35 2007 New Revision: 49839 Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/base.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/semispace.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py Log: * Keep stackroots non-gc first, iterate backwards * Cleanup gctypelayout in order to get rid of hacks * Few renames. Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/base.py Sun Dec 16 16:11:35 2007 @@ -6,6 +6,7 @@ moving_gc = False needs_write_barrier = False needs_zero_gc_pointers = True + prebuilt_gc_objects_are_static_roots = True def set_query_functions(self, is_varsize, has_gcptr_in_varsize, is_gcarrayofgcptr, Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py Sun Dec 16 16:11:35 2007 @@ -15,7 +15,7 @@ # The following flag is set for static roots which are not on the list # of static roots yet, but will appear with write barrier -GCFLAG_NEVER_SET = 1 << (GCFLAGSHIFT+2) +GCFLAG_NO_HEAP_PTRS = 1 << (GCFLAGSHIFT+2) DEBUG_PRINT = False @@ -27,6 +27,7 @@ """ inline_simple_malloc = True needs_write_barrier = True + prebuilt_gc_objects_are_static_roots = True def __init__(self, AddressLinkedList, nursery_size=128, @@ -141,7 +142,7 @@ SemiSpaceGC.init_gc_object(self, addr, typeid, flags) def init_gc_object_immortal(self, addr, typeid, - flags=GCFLAG_NO_YOUNG_PTRS|GCFLAG_NEVER_SET): + flags=GCFLAG_NO_YOUNG_PTRS|GCFLAG_NO_HEAP_PTRS): SemiSpaceGC.init_gc_object_immortal(self, addr, typeid, flags) def semispace_collect(self, size_changing=False): @@ -300,7 +301,7 @@ def move_to_static_roots(self, addr_struct): objhdr = self.header(addr_struct) - objhdr.tid &= ~GCFLAG_NEVER_SET + objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS self.trace(addr_struct, self.append_to_static_roots, None) def remember_young_pointer(self, addr_struct, addr): @@ -311,7 +312,6 @@ oldhdr.forw = self.old_objects_pointing_to_young self.old_objects_pointing_to_young = addr_struct oldhdr.tid &= ~GCFLAG_NO_YOUNG_PTRS - if oldhdr.tid & GCFLAG_NEVER_SET: + if oldhdr.tid & GCFLAG_NO_HEAP_PTRS: self.move_to_static_roots(addr_struct) - llop.debug_print(lltype.Void, "new statc root", self.created) remember_young_pointer.dont_inline = True Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/semispace.py Sun Dec 16 16:11:35 2007 @@ -235,7 +235,6 @@ if root == NULL: break root.address[0] = self.copy(root.address[0]) - llop.debug_print(lltype.Void, "collected: ", counter) free_non_gc_object(roots) def copy(self, obj): Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py Sun Dec 16 16:11:35 2007 @@ -126,9 +126,8 @@ immortal=True, zero=True) a_random_address = llmemory.cast_ptr_to_adr(foo) gcdata.static_root_start = a_random_address # patched in finish() - gcdata.static_root_nongcstart = a_random_address # patched in finish() + gcdata.static_root_nongcend = a_random_address # patched in finish() gcdata.static_root_end = a_random_address # patched in finish() - gcdata.static_root_real_end = a_random_address self.gcdata = gcdata self.malloc_fnptr_cache = {} @@ -154,14 +153,11 @@ 'static_root_start', annmodel.SomeAddress()) data_classdef.generalize_attr( - 'static_root_nongcstart', + 'static_root_nongcend', annmodel.SomeAddress()) data_classdef.generalize_attr( 'static_root_end', annmodel.SomeAddress()) - data_classdef.generalize_attr( - 'static_root_real_end', - annmodel.SomeAddress()) annhelper = annlowlevel.MixLevelHelperAnnotator(self.translator.rtyper) @@ -382,18 +378,16 @@ def __init__(self, with_static=True): self.stack_current = gcdata.root_stack_top if with_static: - self.static_current = gcdata.static_root_start + self.static_current = gcdata.static_root_end - sizeofaddr else: - self.static_current = gcdata.static_root_nongcstart - self.with_static = with_static + self.static_current = gcdata.static_root_nongcend - sizeofaddr def pop(self): - if self.with_static: - while self.static_current != gcdata.static_root_end: - result = self.static_current - self.static_current += sizeofaddr - if result.address[0].address[0] != llmemory.NULL: - return result.address[0] + while self.static_current != gcdata.static_root_start: + result = self.static_current + self.static_current -= sizeofaddr + if result.address[0].address[0] != llmemory.NULL: + return result.address[0] while self.stack_current != gcdata.root_stack_base: self.stack_current -= sizeofaddr if self.stack_current.address[0] != llmemory.NULL: @@ -446,21 +440,20 @@ ll_instance = rmodel.inputconst(r_gcdata, self.gcdata).value addresses_of_static_ptrs = ( - self.layoutbuilder.addresses_of_static_ptrs + - self.layoutbuilder.addresses_of_static_ptrs_in_nongc) + self.layoutbuilder.addresses_of_static_ptrs_in_nongc + + self.layoutbuilder.addresses_of_static_ptrs) log.info("found %s static roots" % (len(addresses_of_static_ptrs), )) additional_ptrs = self.layoutbuilder.additional_roots_sources log.info("additional %d potential static roots" % additional_ptrs) ll_static_roots_inside = lltype.malloc(lltype.Array(llmemory.Address), len(addresses_of_static_ptrs) + - int(additional_ptrs), + additional_ptrs, immortal=True) for i in range(len(addresses_of_static_ptrs)): ll_static_roots_inside[i] = addresses_of_static_ptrs[i] - ll_instance.inst_static_root_start = llmemory.cast_ptr_to_adr(ll_static_roots_inside) + llmemory.ArrayItemsOffset(lltype.Array(llmemory.Address)) - ll_instance.inst_static_root_nongcstart = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(self.layoutbuilder.addresses_of_static_ptrs) - ll_instance.inst_static_root_end = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * (len(ll_static_roots_inside) - additional_ptrs) - ll_instance.inst_static_root_real_end = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(ll_static_roots_inside) + ll_instance.inst_static_root_start = llmemory.cast_ptr_to_adr(ll_static_roots_inside) + llmemory.ArrayItemsOffset(lltype.Array(llmemory.Address)) + llmemory.sizeof(llmemory.Address) * (-1) + ll_instance.inst_static_root_nongcend = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(self.layoutbuilder.addresses_of_static_ptrs_in_nongc) + ll_instance.inst_static_root_end = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(addresses_of_static_ptrs) newgcdependencies = [] newgcdependencies.append(self.gcdata.type_info_table) Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py Sun Dec 16 16:11:35 2007 @@ -286,12 +286,7 @@ return self.seen_roots[id(value)] = True - # XXX hack, a lot of gengc details here - from pypy.rpython.memory.gc.generation import GenerationGC - if isinstance(gc, GenerationGC): - gen_gc = True - else: - gen_gc = False + lazy_write_barrier = gc.prebuilt_gc_objects_are_static_roots if isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)): typeid = self.get_type_id(TYPE) @@ -305,15 +300,15 @@ # they could be changed later to point to GC heap objects. adr = llmemory.cast_ptr_to_adr(value._as_ptr()) if TYPE._gckind == "gc": - if gen_gc: - for a in gc_pointers_inside(value, adr): + if lazy_write_barrier: + for a in gc_pointers_inside(value, adr, mutable_only=False): self.additional_roots_sources += 1 return else: appendto = self.addresses_of_static_ptrs else: appendto = self.addresses_of_static_ptrs_in_nongc - for a in mutable_gc_pointers_inside(value, adr): + for a in gc_pointers_inside(value, adr): appendto.append(a) # ____________________________________________________________ @@ -350,51 +345,32 @@ def weakpointer_offset(TYPE): if TYPE == WEAKREF: return llmemory.offsetof(WEAKREF, "weakptr") - return -1 - -def gc_pointers_inside(v, adr): - t = lltype.typeOf(v) - if isinstance(t, lltype.Struct): - for n, t2 in t._flds.iteritems(): - if isinstance(t2, lltype.Ptr) and t2.TO._gckind == 'gc': - yield adr + llmemory.offsetof(t, n) - elif isinstance(t2, (lltype.Array, lltype.Struct)): - for a in gc_pointers_inside(getattr(v, n), - adr + llmemory.offsetof(t, n)): - yield a - elif isinstance(t, lltype.Array): - if isinstance(t.OF, lltype.Ptr) and t.OF.TO._gckind == 'gc': - for i in range(len(v.items)): - yield adr + llmemory.itemoffsetof(t, i) - elif isinstance(t.OF, lltype.Struct): - for i in range(len(v.items)): - for a in gc_pointers_inside(v.items[i], - adr + llmemory.itemoffsetof(t, i)): - yield a - + return -1 -def mutable_gc_pointers_inside(v, adr): +def gc_pointers_inside(v, adr, mutable_only=True): t = lltype.typeOf(v) if isinstance(t, lltype.Struct): - if t._hints.get('immutable'): + if mutable_only and t._hints.get('immutable'): return for n, t2 in t._flds.iteritems(): if isinstance(t2, lltype.Ptr) and t2.TO._gckind == 'gc': yield adr + llmemory.offsetof(t, n) elif isinstance(t2, (lltype.Array, lltype.Struct)): - for a in mutable_gc_pointers_inside(getattr(v, n), - adr + llmemory.offsetof(t, n)): + for a in gc_pointers_inside(getattr(v, n), + adr + llmemory.offsetof(t, n), + mutable_only): yield a elif isinstance(t, lltype.Array): - if t._hints.get('immutable'): + if mutable_only and t._hints.get('immutable'): return if isinstance(t.OF, lltype.Ptr) and t.OF.TO._gckind == 'gc': for i in range(len(v.items)): yield adr + llmemory.itemoffsetof(t, i) elif isinstance(t.OF, lltype.Struct): for i in range(len(v.items)): - for a in mutable_gc_pointers_inside(v.items[i], - adr + llmemory.itemoffsetof(t, i)): + for a in gc_pointers_inside(v.items[i], + adr + llmemory.itemoffsetof(t, i), + mutable_only): yield a def zero_gc_pointers(p): From arigo at codespeak.net Sun Dec 16 16:21:49 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 16:21:49 +0100 (CET) Subject: [pypy-svn] r49840 - pypy/branch/lazy-write-barrier/pypy/rpython/memory Message-ID: <20071216152149.225471684E8@codespeak.net> Author: arigo Date: Sun Dec 16 16:21:48 2007 New Revision: 49840 Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py Log: IMHO a saner default value for the mutable_only keyword argument. Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py Sun Dec 16 16:21:48 2007 @@ -301,14 +301,14 @@ adr = llmemory.cast_ptr_to_adr(value._as_ptr()) if TYPE._gckind == "gc": if lazy_write_barrier: - for a in gc_pointers_inside(value, adr, mutable_only=False): + for a in gc_pointers_inside(value, adr): self.additional_roots_sources += 1 return else: appendto = self.addresses_of_static_ptrs else: appendto = self.addresses_of_static_ptrs_in_nongc - for a in gc_pointers_inside(value, adr): + for a in gc_pointers_inside(value, adr, mutable_only=True): appendto.append(a) # ____________________________________________________________ @@ -347,7 +347,7 @@ return llmemory.offsetof(WEAKREF, "weakptr") return -1 -def gc_pointers_inside(v, adr, mutable_only=True): +def gc_pointers_inside(v, adr, mutable_only=False): t = lltype.typeOf(v) if isinstance(t, lltype.Struct): if mutable_only and t._hints.get('immutable'): From arigo at codespeak.net Sun Dec 16 16:26:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 16:26:09 +0100 (CET) Subject: [pypy-svn] r49841 - in pypy/branch/lazy-write-barrier/pypy/rpython/memory: . gc Message-ID: <20071216152609.6B8D11684E8@codespeak.net> Author: arigo Date: Sun Dec 16 16:26:08 2007 New Revision: 49841 Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py Log: Some confusion about the meaning of "prebuilt_gc_objects_are_static_roots". Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py Sun Dec 16 16:26:08 2007 @@ -27,7 +27,7 @@ """ inline_simple_malloc = True needs_write_barrier = True - prebuilt_gc_objects_are_static_roots = True + prebuilt_gc_objects_are_static_roots = False def __init__(self, AddressLinkedList, nursery_size=128, Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py Sun Dec 16 16:26:08 2007 @@ -286,8 +286,6 @@ return self.seen_roots[id(value)] = True - lazy_write_barrier = gc.prebuilt_gc_objects_are_static_roots - if isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)): typeid = self.get_type_id(TYPE) hdr = gc.gcheaderbuilder.new_header(value) @@ -300,7 +298,7 @@ # they could be changed later to point to GC heap objects. adr = llmemory.cast_ptr_to_adr(value._as_ptr()) if TYPE._gckind == "gc": - if lazy_write_barrier: + if not gc.prebuilt_gc_objects_are_static_roots: for a in gc_pointers_inside(value, adr): self.additional_roots_sources += 1 return From arigo at codespeak.net Sun Dec 16 16:34:25 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 16:34:25 +0100 (CET) Subject: [pypy-svn] r49842 - pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform Message-ID: <20071216153425.6CCE91684E8@codespeak.net> Author: arigo Date: Sun Dec 16 16:34:24 2007 New Revision: 49842 Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py Log: Avoid taking the address of element "-1" of the array - not sure how fakeaddresses handle that. Instead, backwards iteration is done by having 'current' point not at the next element to return, but just after it, as usual. Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py Sun Dec 16 16:34:24 2007 @@ -378,16 +378,16 @@ def __init__(self, with_static=True): self.stack_current = gcdata.root_stack_top if with_static: - self.static_current = gcdata.static_root_end - sizeofaddr + self.static_current = gcdata.static_root_end else: - self.static_current = gcdata.static_root_nongcend - sizeofaddr + self.static_current = gcdata.static_root_nongcend def pop(self): while self.static_current != gcdata.static_root_start: - result = self.static_current self.static_current -= sizeofaddr - if result.address[0].address[0] != llmemory.NULL: - return result.address[0] + result = self.static_current.address[0] + if result.address[0] != llmemory.NULL: + return result while self.stack_current != gcdata.root_stack_base: self.stack_current -= sizeofaddr if self.stack_current.address[0] != llmemory.NULL: @@ -451,7 +451,7 @@ immortal=True) for i in range(len(addresses_of_static_ptrs)): ll_static_roots_inside[i] = addresses_of_static_ptrs[i] - ll_instance.inst_static_root_start = llmemory.cast_ptr_to_adr(ll_static_roots_inside) + llmemory.ArrayItemsOffset(lltype.Array(llmemory.Address)) + llmemory.sizeof(llmemory.Address) * (-1) + ll_instance.inst_static_root_start = llmemory.cast_ptr_to_adr(ll_static_roots_inside) + llmemory.ArrayItemsOffset(lltype.Array(llmemory.Address)) ll_instance.inst_static_root_nongcend = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(self.layoutbuilder.addresses_of_static_ptrs_in_nongc) ll_instance.inst_static_root_end = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(addresses_of_static_ptrs) From arigo at codespeak.net Sun Dec 16 16:56:53 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 16:56:53 +0100 (CET) Subject: [pypy-svn] r49843 - pypy/extradoc/planning/roadmap Message-ID: <20071216155653.9F3A41684F1@codespeak.net> Author: arigo Date: Sun Dec 16 16:56:52 2007 New Revision: 49843 Modified: pypy/extradoc/planning/roadmap/task_cpython_api.txt pypy/extradoc/planning/roadmap/task_ctypes.txt pypy/extradoc/planning/roadmap/task_gui_support.txt pypy/extradoc/planning/roadmap/task_manual_optimizations.txt pypy/extradoc/planning/roadmap/task_modules_rffi.txt pypy/extradoc/planning/roadmap/task_multi_platform.txt pypy/extradoc/planning/roadmap/task_separate_compilation.txt pypy/extradoc/planning/roadmap/task_wrapper_generator.txt Log: A quick review of the tasks, completing a few texts. Modified: pypy/extradoc/planning/roadmap/task_cpython_api.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_cpython_api.txt (original) +++ pypy/extradoc/planning/roadmap/task_cpython_api.txt Sun Dec 16 16:56:52 2007 @@ -5,3 +5,5 @@ would require writing a lot of new functions (for the CPython API), though. Also it is not clear how large the expected speed impact would be (because of e.g. pinning). + +We need to think more about this before we can be more precise. Modified: pypy/extradoc/planning/roadmap/task_ctypes.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_ctypes.txt (original) +++ pypy/extradoc/planning/roadmap/task_ctypes.txt Sun Dec 16 16:56:52 2007 @@ -5,3 +5,6 @@ the ctypes module on top of already existing _ffi wrapper. This would allow to access quite a bit of libraries which use ctypes already (like SDL wrapper). + +This is probably a medium-sized task. The details of the ctypes semantics +are messy and can only be found by poking around the implementation. Modified: pypy/extradoc/planning/roadmap/task_gui_support.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_gui_support.txt (original) +++ pypy/extradoc/planning/roadmap/task_gui_support.txt Sun Dec 16 16:56:52 2007 @@ -5,7 +5,7 @@ traction is support for one of the multi-platform GUI toolkits. The choice is probably between gtk (in use by olpc and others), Qt (written in c++, using SIP to get a cpython package) and wxWindows, with Tkinter as a -possible third. Someone with more understanding of the Python +possible fourth. Someone with more understanding of the Python community than us should make the pick. Which one would bring more users to PyPy? Would any one of them prompt proponents of other GUI toolkits to make their own ports? Modified: pypy/extradoc/planning/roadmap/task_manual_optimizations.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_manual_optimizations.txt (original) +++ pypy/extradoc/planning/roadmap/task_manual_optimizations.txt Sun Dec 16 16:56:52 2007 @@ -4,6 +4,7 @@ There are still some hand optimizations of the Python interpreter that are well worth doing, since they can be expected to yield significant performance improvements. For instance, some of our mircobenchmarks -show particularly poor performance in some areas. The use of better -algorithms should improve our performance numbers significantly for -some of the benchmarks. +show particularly poor performance in some areas. + +The amount of effort required to get what kind of speed-up for +what kind of application remains unknown at this point in time. Modified: pypy/extradoc/planning/roadmap/task_modules_rffi.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_modules_rffi.txt (original) +++ pypy/extradoc/planning/roadmap/task_modules_rffi.txt Sun Dec 16 16:56:52 2007 @@ -10,9 +10,12 @@ ========================================== Some modules are implemented in Python at application level. For -preformance reasons, they should be implemented at interpreter level. +preformance reasons, many of them should eventually be implemented +at interpreter level. -XXX Which ones? +These modules are: binascii, cPickle, cStringIO, cmath, +collections, datetime, functional, functools(incomplete), imp, +itertools. Write new modules ================= Modified: pypy/extradoc/planning/roadmap/task_multi_platform.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_multi_platform.txt (original) +++ pypy/extradoc/planning/roadmap/task_multi_platform.txt Sun Dec 16 16:56:52 2007 @@ -2,9 +2,10 @@ ====================== In order to be a viable alternative to CPython, PyPy needs to be -tested and run cleanly on more platforms than it currently does. Amd64 -with Linux is probably the most urgent one, but there are probably -others that have some problems. +tested and run cleanly on more platforms than it currently does +(Linux/IA32, OS/X, and Windows). Amd64 with Linux is probably +the most urgent one, but there are probably others that have +some problems. It probably makes sense to put this item off until we are closer to having a production ready system for the i386. We also have a possible Modified: pypy/extradoc/planning/roadmap/task_separate_compilation.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_separate_compilation.txt (original) +++ pypy/extradoc/planning/roadmap/task_separate_compilation.txt Sun Dec 16 16:56:52 2007 @@ -4,4 +4,5 @@ To make porting third party modules reasonable, it should not be necessary to recompile all of PyPy every time you want to integrate a new extension -module. This requires supporting separate compilation. +module. This requires supporting separate compilation in our translation +tool chain. Modified: pypy/extradoc/planning/roadmap/task_wrapper_generator.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_wrapper_generator.txt (original) +++ pypy/extradoc/planning/roadmap/task_wrapper_generator.txt Sun Dec 16 16:56:52 2007 @@ -1,8 +1,9 @@ Support a tool like Swig or Boost ================================= -To make it simpler to integrate C extensions with PyPy, we should support -at least one tool that is already in use in the Python community. +To make it simpler to integrate C libraries with PyPy, we should support +at least one of the wrapper generators that are already in use in the +Python community. Google uses Swig heavily, so in a scenario where Google pays for some development, it would be natural to do Swig. Once PyPy gets traction From arigo at codespeak.net Sun Dec 16 17:38:33 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 17:38:33 +0100 (CET) Subject: [pypy-svn] r49845 - pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc Message-ID: <20071216163833.4B3A81684FB@codespeak.net> Author: arigo Date: Sun Dec 16 17:38:31 2007 New Revision: 49845 Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py Log: Clean-ups. Modified: pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py Sun Dec 16 17:38:31 2007 @@ -1,4 +1,4 @@ -import sys, os +import sys from pypy.rpython.memory.gc.semispace import SemiSpaceGC, GCFLAGSHIFT, \ GCFLAG_IMMORTAL from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage @@ -40,7 +40,6 @@ get_roots = get_roots) self.nursery_size = nursery_size assert nursery_size <= space_size // 2 - self.created = 0 def setup(self): SemiSpaceGC.setup(self) @@ -296,7 +295,6 @@ self.remember_young_pointer(addr_struct, newvalue) def append_to_static_roots(self, pointer, arg): - self.created += 1 self.get_roots.append_static_root(pointer) def move_to_static_roots(self, addr_struct): From arigo at codespeak.net Sun Dec 16 17:39:17 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 17:39:17 +0100 (CET) Subject: [pypy-svn] r49846 - pypy/dist/pypy/rpython/memory Message-ID: <20071216163917.E65611684FB@codespeak.net> Author: arigo Date: Sun Dec 16 17:39:17 2007 New Revision: 49846 Added: pypy/dist/pypy/rpython/memory/gcwrapper.py.merge.tmp - copied, changed from r49844, pypy/dist/pypy/rpython/memory/gcwrapper.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/lazy-write-barrier/pypy/rpython/memory/gcwrapper.py revisions 49724 to 49844: ------------------------------------------------------------------------ r49815 | fijal | 2007-12-15 13:57:29 +0100 (Sat, 15 Dec 2007) | 2 lines Merge dist -> branch ------------------------------------------------------------------------ r49790 | arigo | 2007-12-14 16:51:03 +0100 (Fri, 14 Dec 2007) | 3 lines * Adapt gcwrapper to the new world. * Surely GCFLAG_NEVER_SET should be set on immortal objects? ------------------------------------------------------------------------ r49777 | fijal | 2007-12-14 13:15:57 +0100 (Fri, 14 Dec 2007) | 2 lines Create a branch from 49724 to experiment with lazy write barrier ------------------------------------------------------------------------ From arigo at codespeak.net Sun Dec 16 17:39:20 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 17:39:20 +0100 (CET) Subject: [pypy-svn] r49847 - pypy/dist/pypy/rpython/memory Message-ID: <20071216163920.0068516850A@codespeak.net> Author: arigo Date: Sun Dec 16 17:39:19 2007 New Revision: 49847 Added: pypy/dist/pypy/rpython/memory/gctypelayout.py.merge.tmp - copied, changed from r49844, pypy/dist/pypy/rpython/memory/gctypelayout.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctypelayout.py revisions 49724 to 49844: ------------------------------------------------------------------------ r49841 | arigo | 2007-12-16 16:26:08 +0100 (Sun, 16 Dec 2007) | 2 lines Some confusion about the meaning of "prebuilt_gc_objects_are_static_roots". ------------------------------------------------------------------------ r49840 | arigo | 2007-12-16 16:21:48 +0100 (Sun, 16 Dec 2007) | 2 lines IMHO a saner default value for the mutable_only keyword argument. ------------------------------------------------------------------------ r49839 | fijal | 2007-12-16 16:11:35 +0100 (Sun, 16 Dec 2007) | 4 lines * Keep stackroots non-gc first, iterate backwards * Cleanup gctypelayout in order to get rid of hacks * Few renames. ------------------------------------------------------------------------ r49815 | fijal | 2007-12-15 13:57:29 +0100 (Sat, 15 Dec 2007) | 2 lines Merge dist -> branch ------------------------------------------------------------------------ r49792 | arigo | 2007-12-14 16:54:44 +0100 (Fri, 14 Dec 2007) | 2 lines Don't overwrite the flags provided by GenerationGC.init_gc_object_immortal(). ------------------------------------------------------------------------ r49788 | fijal | 2007-12-14 16:36:30 +0100 (Fri, 14 Dec 2007) | 2 lines Don't check in pdb ------------------------------------------------------------------------ r49787 | fijal | 2007-12-14 16:35:17 +0100 (Fri, 14 Dec 2007) | 2 lines Another evil hack in the gctypelayout (C-c C-v programming) ------------------------------------------------------------------------ r49781 | fijal | 2007-12-14 14:05:19 +0100 (Fri, 14 Dec 2007) | 2 lines Fix calculation ------------------------------------------------------------------------ r49780 | fijal | 2007-12-14 13:55:46 +0100 (Fri, 14 Dec 2007) | 2 lines inprogress checkin of lazy write barrier approach ------------------------------------------------------------------------ r49777 | fijal | 2007-12-14 13:15:57 +0100 (Fri, 14 Dec 2007) | 2 lines Create a branch from 49724 to experiment with lazy write barrier ------------------------------------------------------------------------ From arigo at codespeak.net Sun Dec 16 17:41:46 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 17:41:46 +0100 (CET) Subject: [pypy-svn] r49848 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20071216164146.08D8916843A@codespeak.net> Author: arigo Date: Sun Dec 16 17:41:46 2007 New Revision: 49848 Added: pypy/dist/pypy/rpython/memory/gctransform/framework.py.merge.tmp - copied, changed from r49844, pypy/dist/pypy/rpython/memory/gctransform/framework.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/lazy-write-barrier/pypy/rpython/memory/gctransform/framework.py revisions 49724 to 49844: ------------------------------------------------------------------------ r49842 | arigo | 2007-12-16 16:34:24 +0100 (Sun, 16 Dec 2007) | 4 lines Avoid taking the address of element "-1" of the array - not sure how fakeaddresses handle that. Instead, backwards iteration is done by having 'current' point not at the next element to return, but just after it, as usual. ------------------------------------------------------------------------ r49839 | fijal | 2007-12-16 16:11:35 +0100 (Sun, 16 Dec 2007) | 4 lines * Keep stackroots non-gc first, iterate backwards * Cleanup gctypelayout in order to get rid of hacks * Few renames. ------------------------------------------------------------------------ r49837 | fijal | 2007-12-16 15:38:45 +0100 (Sun, 16 Dec 2007) | 3 lines Check this in not to forget. It contains debug prints and messy code, to clean up ------------------------------------------------------------------------ r49815 | fijal | 2007-12-15 13:57:29 +0100 (Sat, 15 Dec 2007) | 2 lines Merge dist -> branch ------------------------------------------------------------------------ r49787 | fijal | 2007-12-14 16:35:17 +0100 (Fri, 14 Dec 2007) | 2 lines Another evil hack in the gctypelayout (C-c C-v programming) ------------------------------------------------------------------------ r49780 | fijal | 2007-12-14 13:55:46 +0100 (Fri, 14 Dec 2007) | 2 lines inprogress checkin of lazy write barrier approach ------------------------------------------------------------------------ r49777 | fijal | 2007-12-14 13:15:57 +0100 (Fri, 14 Dec 2007) | 2 lines Create a branch from 49724 to experiment with lazy write barrier ------------------------------------------------------------------------ From arigo at codespeak.net Sun Dec 16 17:41:49 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 17:41:49 +0100 (CET) Subject: [pypy-svn] r49849 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20071216164149.EEDD91684F4@codespeak.net> Author: arigo Date: Sun Dec 16 17:41:49 2007 New Revision: 49849 Added: pypy/dist/pypy/rpython/memory/gc/base.py.merge.tmp - copied, changed from r49844, pypy/dist/pypy/rpython/memory/gc/base.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/base.py revisions 49724 to 49844: ------------------------------------------------------------------------ r49839 | fijal | 2007-12-16 16:11:35 +0100 (Sun, 16 Dec 2007) | 4 lines * Keep stackroots non-gc first, iterate backwards * Cleanup gctypelayout in order to get rid of hacks * Few renames. ------------------------------------------------------------------------ r49815 | fijal | 2007-12-15 13:57:29 +0100 (Sat, 15 Dec 2007) | 2 lines Merge dist -> branch ------------------------------------------------------------------------ r49777 | fijal | 2007-12-14 13:15:57 +0100 (Fri, 14 Dec 2007) | 2 lines Create a branch from 49724 to experiment with lazy write barrier ------------------------------------------------------------------------ From arigo at codespeak.net Sun Dec 16 17:43:16 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 17:43:16 +0100 (CET) Subject: [pypy-svn] r49850 - in pypy/dist/pypy/rpython/memory: . gc gctransform test Message-ID: <20071216164316.DD1FB1684F7@codespeak.net> Author: arigo Date: Sun Dec 16 17:43:16 2007 New Revision: 49850 Added: pypy/dist/pypy/rpython/memory/gc/base.py - copied unchanged from r49849, pypy/dist/pypy/rpython/memory/gc/base.py.merge.tmp pypy/dist/pypy/rpython/memory/gc/generation.py - copied unchanged from r49849, pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/gc/marksweep.py - copied unchanged from r49849, pypy/branch/lazy-write-barrier/pypy/rpython/memory/gc/marksweep.py pypy/dist/pypy/rpython/memory/gctransform/framework.py - copied unchanged from r49849, pypy/dist/pypy/rpython/memory/gctransform/framework.py.merge.tmp pypy/dist/pypy/rpython/memory/gctypelayout.py - copied unchanged from r49849, pypy/dist/pypy/rpython/memory/gctypelayout.py.merge.tmp pypy/dist/pypy/rpython/memory/gcwrapper.py - copied unchanged from r49849, pypy/dist/pypy/rpython/memory/gcwrapper.py.merge.tmp pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py - copied unchanged from r49849, pypy/branch/lazy-write-barrier/pypy/rpython/memory/test/test_transformed_gc.py Removed: pypy/dist/pypy/rpython/memory/gc/base.py.merge.tmp pypy/dist/pypy/rpython/memory/gctransform/framework.py.merge.tmp pypy/dist/pypy/rpython/memory/gctypelayout.py.merge.tmp pypy/dist/pypy/rpython/memory/gcwrapper.py.merge.tmp Log: Merge of the lazy-write-barrier branch by fijal. With the generation gc, prebuilt gc objects are now only recorded as potential static roots when a gc pointer is written into them at run-time. This reduces the number of static roots a lot, although it only gives marginal speed improvements (there seem to be not that many full collects during a regular pypy-c execution). From arigo at codespeak.net Sun Dec 16 17:46:13 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 17:46:13 +0100 (CET) Subject: [pypy-svn] r49851 - pypy/dist/pypy/interpreter Message-ID: <20071216164613.A86911684F4@codespeak.net> Author: arigo Date: Sun Dec 16 17:46:11 2007 New Revision: 49851 Modified: pypy/dist/pypy/interpreter/executioncontext.py pypy/dist/pypy/interpreter/pyopcode.py Log: Tweak inlining in the main interpreter loop. Gives good speed-ups. Modified: pypy/dist/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/dist/pypy/interpreter/executioncontext.py (original) +++ pypy/dist/pypy/interpreter/executioncontext.py Sun Dec 16 17:46:11 2007 @@ -99,24 +99,22 @@ def bytecode_trace(self, frame): "Trace function called before each bytecode." - # XXX there should be some flag here which checks whether - # this should be really invoked. We spend roughly 0.5% time - # here when not doing anything - # First, call yield_thread() before each Nth bytecode, - # as selected by sys.setcheckinterval() - ticker = self.ticker - if ticker <= 0: + # this is split into a fast path and a slower path that is + # not invoked every time bytecode_trace() is. + ticker = self.ticker - 1 + self.ticker = ticker + if ticker < 0 or frame.w_f_trace is not None: + self._do_bytecode_trace(frame) + bytecode_trace.always_inline = True + + def _do_bytecode_trace(self, frame): + if self.ticker < 0: Action.perform_actions(self.space.pending_actions) Action.perform_actions(self.pending_actions) - ticker = self.space.sys.checkinterval - self.ticker = ticker - 1 + self.ticker = self.space.sys.checkinterval if frame.w_f_trace is None or self.is_tracing: return - self._do_bytecode_trace(frame) - - - def _do_bytecode_trace(self, frame): - code = getattr(frame, 'pycode') + code = frame.pycode if frame.instr_lb <= frame.last_instr < frame.instr_ub: if frame.last_instr <= frame.instr_prev: # We jumped backwards in the same line. Modified: pypy/dist/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/dist/pypy/interpreter/pyopcode.py (original) +++ pypy/dist/pypy/interpreter/pyopcode.py Sun Dec 16 17:46:11 2007 @@ -283,11 +283,15 @@ # access a local variable directly w_value = f.fastlocals_w[varindex] if w_value is None: - varname = f.getlocalvarname(varindex) - message = "local variable '%s' referenced before assignment" % varname - raise OperationError(f.space.w_UnboundLocalError, f.space.wrap(message)) + f._load_fast_failed(varindex) f.pushvalue(w_value) + def _load_fast_failed(f, varindex): + varname = f.getlocalvarname(varindex) + message = "local variable '%s' referenced before assignment" % varname + raise OperationError(f.space.w_UnboundLocalError, f.space.wrap(message)) + _load_fast_failed.dont_inline = True + def LOAD_CONST(f, constindex, *ignored): w_const = f.getconstant_w(constindex) f.pushvalue(w_const) @@ -639,14 +643,20 @@ # not in the globals, now look in the built-ins w_value = f.get_builtin().getdictvalue(f.space, w_varname) if w_value is None: - varname = f.space.str_w(w_varname) - message = "global name '%s' is not defined" % varname - raise OperationError(f.space.w_NameError, - f.space.wrap(message)) + f._load_global_failed(w_varname) return w_value + _load_global.always_inline = True + + def _load_global_failed(f, w_varname): + varname = f.space.str_w(w_varname) + message = "global name '%s' is not defined" % varname + raise OperationError(f.space.w_NameError, + f.space.wrap(message)) + _load_global_failed.dont_inline = True def LOAD_GLOBAL(f, nameindex, *ignored): f.pushvalue(f._load_global(f.getname_w(nameindex))) + LOAD_GLOBAL.always_inline = True def DELETE_FAST(f, varindex, *ignored): if f.fastlocals_w[varindex] is None: @@ -678,6 +688,7 @@ w_obj = f.popvalue() w_value = f.space.getattr(w_obj, w_attributename) f.pushvalue(w_value) + LOAD_ATTR.always_inline = True def cmp_lt(f, w_1, w_2): return f.space.lt(w_1, w_2) def cmp_le(f, w_1, w_2): return f.space.le(w_1, w_2) From arigo at codespeak.net Sun Dec 16 18:10:38 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 18:10:38 +0100 (CET) Subject: [pypy-svn] r49852 - pypy/dist/pypy/interpreter Message-ID: <20071216171038.87EC016850E@codespeak.net> Author: arigo Date: Sun Dec 16 18:10:38 2007 New Revision: 49852 Modified: pypy/dist/pypy/interpreter/executioncontext.py pypy/dist/pypy/interpreter/pyopcode.py Log: * It's not "always_inline", it's "_always_inline_". * Should specify it on LOAD_FAST too. Modified: pypy/dist/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/dist/pypy/interpreter/executioncontext.py (original) +++ pypy/dist/pypy/interpreter/executioncontext.py Sun Dec 16 18:10:38 2007 @@ -105,7 +105,7 @@ self.ticker = ticker if ticker < 0 or frame.w_f_trace is not None: self._do_bytecode_trace(frame) - bytecode_trace.always_inline = True + bytecode_trace._always_inline_ = True def _do_bytecode_trace(self, frame): if self.ticker < 0: Modified: pypy/dist/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/dist/pypy/interpreter/pyopcode.py (original) +++ pypy/dist/pypy/interpreter/pyopcode.py Sun Dec 16 18:10:38 2007 @@ -285,6 +285,7 @@ if w_value is None: f._load_fast_failed(varindex) f.pushvalue(w_value) + LOAD_FAST._always_inline_ = True def _load_fast_failed(f, varindex): varname = f.getlocalvarname(varindex) @@ -645,7 +646,7 @@ if w_value is None: f._load_global_failed(w_varname) return w_value - _load_global.always_inline = True + _load_global._always_inline_ = True def _load_global_failed(f, w_varname): varname = f.space.str_w(w_varname) @@ -656,7 +657,7 @@ def LOAD_GLOBAL(f, nameindex, *ignored): f.pushvalue(f._load_global(f.getname_w(nameindex))) - LOAD_GLOBAL.always_inline = True + LOAD_GLOBAL._always_inline_ = True def DELETE_FAST(f, varindex, *ignored): if f.fastlocals_w[varindex] is None: @@ -688,7 +689,7 @@ w_obj = f.popvalue() w_value = f.space.getattr(w_obj, w_attributename) f.pushvalue(w_value) - LOAD_ATTR.always_inline = True + LOAD_ATTR._always_inline_ = True def cmp_lt(f, w_1, w_2): return f.space.lt(w_1, w_2) def cmp_le(f, w_1, w_2): return f.space.le(w_1, w_2) From arigo at codespeak.net Sun Dec 16 18:14:16 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 16 Dec 2007 18:14:16 +0100 (CET) Subject: [pypy-svn] r49853 - in pypy/dist/pypy: interpreter module/thread/test rpython rpython/memory rpython/memory/gc rpython/microbench translator/backendopt translator/c/test translator/goal translator/stackless Message-ID: <20071216171416.2930316850E@codespeak.net> Author: arigo Date: Sun Dec 16 18:14:15 2007 New Revision: 49853 Modified: pypy/dist/pypy/interpreter/pyopcode.py pypy/dist/pypy/module/thread/test/test_ll_thread.py pypy/dist/pypy/module/thread/test/test_lock.py pypy/dist/pypy/rpython/extfunc.py pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/gc/marksweep.py pypy/dist/pypy/rpython/memory/gc/semispace.py pypy/dist/pypy/rpython/memory/support.py pypy/dist/pypy/rpython/microbench/microbench.py pypy/dist/pypy/translator/backendopt/inline.py pypy/dist/pypy/translator/c/test/test_newgc.py pypy/dist/pypy/translator/goal/targetlbench.py pypy/dist/pypy/translator/stackless/code.py Log: Rename the 'dont_inline' flag to '_dont_inline_' for better consistency. Modified: pypy/dist/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/dist/pypy/interpreter/pyopcode.py (original) +++ pypy/dist/pypy/interpreter/pyopcode.py Sun Dec 16 18:14:15 2007 @@ -291,7 +291,7 @@ varname = f.getlocalvarname(varindex) message = "local variable '%s' referenced before assignment" % varname raise OperationError(f.space.w_UnboundLocalError, f.space.wrap(message)) - _load_fast_failed.dont_inline = True + _load_fast_failed._dont_inline_ = True def LOAD_CONST(f, constindex, *ignored): w_const = f.getconstant_w(constindex) @@ -653,7 +653,7 @@ message = "global name '%s' is not defined" % varname raise OperationError(f.space.w_NameError, f.space.wrap(message)) - _load_global_failed.dont_inline = True + _load_global_failed._dont_inline_ = True def LOAD_GLOBAL(f, nameindex, *ignored): f.pushvalue(f._load_global(f.getname_w(nameindex))) Modified: pypy/dist/pypy/module/thread/test/test_ll_thread.py ============================================================================== --- pypy/dist/pypy/module/thread/test/test_ll_thread.py (original) +++ pypy/dist/pypy/module/thread/test/test_ll_thread.py Sun Dec 16 18:14:15 2007 @@ -63,7 +63,7 @@ y = Y() y.z = Z(i) start_new_thread(Y.bootstrap, (y,)) - g.dont_inline = True + g._dont_inline_ = True def f(): main_ident = get_ident() Modified: pypy/dist/pypy/module/thread/test/test_lock.py ============================================================================== --- pypy/dist/pypy/module/thread/test/test_lock.py (original) +++ pypy/dist/pypy/module/thread/test/test_lock.py Sun Dec 16 18:14:15 2007 @@ -41,7 +41,7 @@ ok3 = l.acquire(False) res = ok1 and not ok2 and ok3 return res - g.dont_inline = True + g._dont_inline_ = True def f(): res = g() # the lock must have been freed by now - we use refcounting Modified: pypy/dist/pypy/rpython/extfunc.py ============================================================================== --- pypy/dist/pypy/rpython/extfunc.py (original) +++ pypy/dist/pypy/rpython/extfunc.py Sun Dec 16 18:14:15 2007 @@ -192,7 +192,7 @@ return original_impl(*args) impl = func_with_new_name(ll_wrapper, name + '_wrapper') if rtyper.annotator.translator.config.translation.sandbox: - impl.dont_inline = True + impl._dont_inline_ = True # store some attributes to the 'impl' function, where # the eventual call to rtyper.getcallable() will find them # and transfer them to the final lltype.functionptr(). Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Sun Dec 16 18:14:15 2007 @@ -312,4 +312,4 @@ oldhdr.tid &= ~GCFLAG_NO_YOUNG_PTRS if oldhdr.tid & GCFLAG_NO_HEAP_PTRS: self.move_to_static_roots(addr_struct) - remember_young_pointer.dont_inline = True + remember_young_pointer._dont_inline_ = True Modified: pypy/dist/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/dist/pypy/rpython/memory/gc/marksweep.py Sun Dec 16 18:14:15 2007 @@ -102,7 +102,7 @@ #llop.debug_print(lltype.Void, 'malloc typeid', typeid, # '->', llmemory.cast_adr_to_int(result)) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) - malloc_fixedsize.dont_inline = True + malloc_fixedsize._dont_inline_ = True def malloc_fixedsize_clear(self, typeid, size, can_collect, has_finalizer=False, contains_weakptr=False): @@ -136,7 +136,7 @@ #llop.debug_print(lltype.Void, 'malloc typeid', typeid, # '->', llmemory.cast_adr_to_int(result)) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) - malloc_fixedsize_clear.dont_inline = True + malloc_fixedsize_clear._dont_inline_ = True def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, can_collect, has_finalizer=False): @@ -171,7 +171,7 @@ # 'typeid', typeid, # '->', llmemory.cast_adr_to_int(result)) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) - malloc_varsize.dont_inline = True + malloc_varsize._dont_inline_ = True def malloc_varsize_clear(self, typeid, length, size, itemsize, offset_to_length, can_collect, @@ -208,7 +208,7 @@ # 'typeid', typeid, # '->', llmemory.cast_adr_to_int(result)) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) - malloc_varsize_clear.dont_inline = True + malloc_varsize_clear._dont_inline_ = True def collect(self): # 1. mark from the roots, and also the objects that objects-with-del Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Sun Dec 16 18:14:15 2007 @@ -107,7 +107,7 @@ if not self.try_obtain_free_space(needed): raise memoryError return self.free - obtain_free_space.dont_inline = True + obtain_free_space._dont_inline_ = True def try_obtain_free_space(self, needed): # XXX for bonus points do big objects differently Modified: pypy/dist/pypy/rpython/memory/support.py ============================================================================== --- pypy/dist/pypy/rpython/memory/support.py (original) +++ pypy/dist/pypy/rpython/memory/support.py Sun Dec 16 18:14:15 2007 @@ -47,14 +47,14 @@ new.length = 0 self.chunk = new return new - enlarge.dont_inline = True + enlarge._dont_inline_ = True def shrink(self): old = self.chunk self.chunk = old.previous unused_chunks.put(old) return self.chunk - shrink.dont_inline = True + shrink._dont_inline_ = True def append(self, addr): if addr == llmemory.NULL: Modified: pypy/dist/pypy/rpython/microbench/microbench.py ============================================================================== --- pypy/dist/pypy/rpython/microbench/microbench.py (original) +++ pypy/dist/pypy/rpython/microbench/microbench.py Sun Dec 16 18:14:15 2007 @@ -11,7 +11,7 @@ class MetaBench(type): def __new__(self, cls_name, bases, cls_dict): loop = cls_dict['loop'] - loop.dont_inline = True + loop._dont_inline_ = True myglob = { 'init': cls_dict['init'], 'loop': loop, Modified: pypy/dist/pypy/translator/backendopt/inline.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/inline.py (original) +++ pypy/dist/pypy/translator/backendopt/inline.py Sun Dec 16 18:14:15 2007 @@ -634,7 +634,7 @@ graph = getattr(funcobj, 'graph', None) if graph is not None: if getattr(getattr(funcobj, '_callable', None), - 'dont_inline', False): + '_dont_inline_', False): continue result.append((parentgraph, graph)) if op.opname == "oosend": @@ -666,7 +666,7 @@ graph = getattr(funcobj, 'graph', None) if graph is not None: if getattr(getattr(funcobj, '_callable', None), - 'dont_inline', False): + '_dont_inline_', False): continue if candidate(graph): tag = Constant('inline', Void) Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Sun Dec 16 18:14:15 2007 @@ -298,7 +298,7 @@ a = A() a.b = g(1) return a - make.dont_inline = True + make._dont_inline_ = True def f(): a = make() llop.gc__collect(lltype.Void) @@ -316,7 +316,7 @@ pass def g(x): # cause a collect llop.gc__collect(lltype.Void) - g.dont_inline = True + g._dont_inline_ = True global_a = A() global_a.b = B() global_a.b.a = A() @@ -330,7 +330,7 @@ g(1) b0 = a.b b0.c = b.c = 42 - make.dont_inline = True + make._dont_inline_ = True def f(): make() llop.gc__collect(lltype.Void) @@ -356,7 +356,7 @@ for i in range(1000): prepare(B(), -1) # probably overwrites collected memory return a.value - g.dont_inline = True + g._dont_inline_ = True def f(): b = B() prepare(b, 123) @@ -420,7 +420,7 @@ a.x = None def make(): a.x = A(42) - make.dont_inline = True + make._dont_inline_ = True def f(): make() llop.gc__collect(lltype.Void) @@ -481,12 +481,12 @@ a = lltype.malloc(A) a.value = -n * 7 return lltype.cast_opaque_ptr(lltype.Ptr(O), a) - gethidden.dont_inline = True + gethidden._dont_inline_ = True def reveal(o): return lltype.cast_opaque_ptr(lltype.Ptr(A), o) def overwrite(a, i): a.value = i - overwrite.dont_inline = True + overwrite._dont_inline_ = True def f(): o = gethidden(10) llop.gc__collect(lltype.Void) Modified: pypy/dist/pypy/translator/goal/targetlbench.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetlbench.py (original) +++ pypy/dist/pypy/translator/goal/targetlbench.py Sun Dec 16 18:14:15 2007 @@ -16,7 +16,7 @@ for i in range(x): a.append(i) return 0 -longername.dont_inline = True +longername._dont_inline_ = True def entry_point(argv): size = int(argv[1]) Modified: pypy/dist/pypy/translator/stackless/code.py ============================================================================== --- pypy/dist/pypy/translator/stackless/code.py (original) +++ pypy/dist/pypy/translator/stackless/code.py Sun Dec 16 18:14:15 2007 @@ -310,7 +310,7 @@ fn = llmemory.cast_adr_to_ptr(fnaddr, FUNCTYPE) return fn() call_function_retval_xyz.stackless_explicit = True - call_function_retval_xyz.dont_inline = True + call_function_retval_xyz._dont_inline_ = True fnname = 'call_function_retval_' + typename fn = func_with_new_name(call_function_retval_xyz, fnname) globals()[fnname] = fn From cfbolz at codespeak.net Mon Dec 17 00:17:08 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 17 Dec 2007 00:17:08 +0100 (CET) Subject: [pypy-svn] r49859 - in pypy/dist/pypy/module/__builtin__: . test Message-ID: <20071216231708.182D316847B@codespeak.net> Author: cfbolz Date: Mon Dec 17 00:17:08 2007 New Revision: 49859 Modified: pypy/dist/pypy/module/__builtin__/interp_classobj.py pypy/dist/pypy/module/__builtin__/test/test_classobj.py Log: fix bug in __new__ Modified: pypy/dist/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/dist/pypy/module/__builtin__/interp_classobj.py Mon Dec 17 00:17:08 2007 @@ -264,7 +264,7 @@ raise OperationError( space.w_TypeError, space.wrap("instance() first arg must be class")) - if w_dict is None: + if space.is_w(w_dict, space.w_None): w_dict = space.newdict() elif not space.is_true(space.isinstance(w_dict, space.w_dict)): raise OperationError( Modified: pypy/dist/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/dist/pypy/module/__builtin__/test/test_classobj.py Mon Dec 17 00:17:08 2007 @@ -574,6 +574,10 @@ class A: b = 1 a = A() + a = type(a).__new__(type(a), A) + assert a.b == 1 + a = type(a).__new__(type(a), A, None) + assert a.b == 1 a = type(a).__new__(type(a), A, {'c': 2}) assert a.b == 1 assert a.c == 2 From jacob at codespeak.net Mon Dec 17 01:31:00 2007 From: jacob at codespeak.net (jacob at codespeak.net) Date: Mon, 17 Dec 2007 01:31:00 +0100 (CET) Subject: [pypy-svn] r49860 - pypy/dist/lib-python/modified-2.4.1 Message-ID: <20071217003100.9EAAB168450@codespeak.net> Author: jacob Date: Mon Dec 17 01:30:59 2007 New Revision: 49860 Modified: pypy/dist/lib-python/modified-2.4.1/tarfile.py Log: I assume that the radix should be 10 here. Not having a radix is an error in any case. Modified: pypy/dist/lib-python/modified-2.4.1/tarfile.py ============================================================================== --- pypy/dist/lib-python/modified-2.4.1/tarfile.py (original) +++ pypy/dist/lib-python/modified-2.4.1/tarfile.py Mon Dec 17 01:30:59 2007 @@ -343,7 +343,7 @@ -self.zlib.MAX_WBITS, self.zlib.DEF_MEM_LEVEL, 0) - timestamp = struct.pack(" Author: arigo Date: Mon Dec 17 10:10:11 2007 New Revision: 49861 Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: Fix test and add comment. Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/gctypelayout.py Mon Dec 17 10:10:11 2007 @@ -203,6 +203,9 @@ self.addresses_of_static_ptrs = [] # this lists contains pointers in raw Structs and Arrays self.addresses_of_static_ptrs_in_nongc = [] + # if not gc.prebuilt_gc_objects_are_static_roots, then + # additional_roots_sources counts the number of locations + # within prebuilt GC objects that are of type Ptr(Gc) self.additional_roots_sources = 0 self.finalizer_funcptrs = {} self.offsettable_cache = {} Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Mon Dec 17 10:10:11 2007 @@ -890,7 +890,17 @@ run, transformer = self.runner(f, nbargs=2, transformer=True) run([1, 4]) assert len(transformer.layoutbuilder.addresses_of_static_ptrs) == 0 - assert transformer.layoutbuilder.additional_roots_sources == 5 + assert transformer.layoutbuilder.additional_roots_sources >= 4 + # NB. Remember that additional_roots_sources does not count + # the number of prebuilt GC objects, but the number of locations + # within prebuilt GC objects that are of type Ptr(Gc). + # At the moment we get additional_roots_sources == 6: + # * all[0] + # * all[1] + # * parent.sub + # * parent2.sub + # * the GcArray pointer from gc.wr_to_objects_with_id + # * the GcArray pointer from gc.object_id_dict. class TestGenerationalNoFullCollectGC(GCTest): # test that nursery is doing its job and that no full collection From arigo at codespeak.net Mon Dec 17 10:29:25 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 17 Dec 2007 10:29:25 +0100 (CET) Subject: [pypy-svn] r49862 - pypy/dist/pypy/objspace/std Message-ID: <20071217092925.8860A168457@codespeak.net> Author: arigo Date: Mon Dec 17 10:29:24 2007 New Revision: 49862 Modified: pypy/dist/pypy/objspace/std/objspace.py Log: Fix nonsensical inheritence from flowcontext.PyFrame. This killed the CALL_METHOD optimization entirely! Modified: pypy/dist/pypy/objspace/std/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/objspace.py (original) +++ pypy/dist/pypy/objspace/std/objspace.py Mon Dec 17 10:29:24 2007 @@ -16,7 +16,6 @@ from pypy.rlib.rarithmetic import base_int from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.jit import hint, we_are_jitted -from pypy.objspace.flow.flowcontext import PyFrame as FlowPyFrame import sys import os import __builtin__ @@ -59,7 +58,7 @@ # Import all the object types and implementations self.model = StdTypeModel(self.config) - class StdObjSpaceFrame(FlowPyFrame): + class StdObjSpaceFrame(pyframe.PyFrame): if self.config.objspace.std.optimized_int_add: if self.config.objspace.std.withsmallint: def BINARY_ADD(f, oparg, *ignored): From arigo at codespeak.net Mon Dec 17 11:02:37 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 17 Dec 2007 11:02:37 +0100 (CET) Subject: [pypy-svn] r49863 - in pypy/dist/pypy: interpreter interpreter/test module/__builtin__ objspace objspace/flow objspace/std objspace/std/test Message-ID: <20071217100237.65CEB168460@codespeak.net> Author: arigo Date: Mon Dec 17 11:02:36 2007 New Revision: 49863 Added: pypy/dist/pypy/objspace/std/callmethod.py - copied, changed from r49862, pypy/dist/pypy/interpreter/callmethod.py pypy/dist/pypy/objspace/std/test/test_callmethod.py - copied, changed from r49862, pypy/dist/pypy/interpreter/test/test_callmethod.py Removed: pypy/dist/pypy/interpreter/callmethod.py pypy/dist/pypy/interpreter/test/test_callmethod.py Modified: pypy/dist/pypy/interpreter/baseobjspace.py pypy/dist/pypy/interpreter/pyopcode.py pypy/dist/pypy/module/__builtin__/descriptor.py pypy/dist/pypy/objspace/descroperation.py pypy/dist/pypy/objspace/flow/flowcontext.py pypy/dist/pypy/objspace/std/objspace.py Log: Use a nicer approach for the fallback of the CALL_METHOD opcode, based on the one used for CALL_LIKELY_BUILTIN. This required moving callmethod to the stdobjspace, which makes some sense anyway. Modified: pypy/dist/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/dist/pypy/interpreter/baseobjspace.py (original) +++ pypy/dist/pypy/interpreter/baseobjspace.py Mon Dec 17 11:02:36 2007 @@ -181,10 +181,8 @@ config = get_pypy_config(translating=False) self.config = config - # import extra modules for side-effects, possibly based on config + # import extra modules for side-effects import pypy.interpreter.nestedscope # register *_DEREF bytecodes - if self.config.objspace.opcodes.CALL_METHOD: - import pypy.interpreter.callmethod # register *_METHOD bytecodes self.interned_strings = {} self.pending_actions = [] Modified: pypy/dist/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/dist/pypy/interpreter/pyopcode.py (original) +++ pypy/dist/pypy/interpreter/pyopcode.py Mon Dec 17 11:02:36 2007 @@ -931,6 +931,25 @@ f.dropvalues(nargs) f.pushvalue(w_result) + def LOOKUP_METHOD(f, nameindex, *ignored): + # overridden by faster version in the standard object space. + space = f.space + w_obj = f.popvalue() + w_name = f.getname_w(nameindex) + w_value = space.getattr(w_obj, w_name) + f.pushvalue(w_value) + #f.pushvalue(None) + + def CALL_METHOD(f, nargs, *ignored): + # overridden by faster version in the standard object space. + # 'nargs' is the argument count excluding the implicit 'self' + w_callable = f.peekvalue(nargs) + try: + w_result = f.space.call_valuestack(w_callable, nargs, f) + finally: + f.dropvalues(nargs + 1) + f.pushvalue(w_result) + ## def EXTENDED_ARG(f, oparg, *ignored): ## opcode = f.nextop() ## oparg = oparg<<16 | f.nextarg() Modified: pypy/dist/pypy/module/__builtin__/descriptor.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/descriptor.py (original) +++ pypy/dist/pypy/module/__builtin__/descriptor.py Mon Dec 17 11:02:36 2007 @@ -4,7 +4,7 @@ Arguments from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError -from pypy.interpreter.callmethod import object_getattribute +from pypy.objspace.descroperation import object_getattribute from pypy.interpreter.function import StaticMethod, ClassMethod from pypy.interpreter.typedef import GetSetProperty, descr_get_dict, \ descr_set_dict, interp_attrproperty_w Modified: pypy/dist/pypy/objspace/descroperation.py ============================================================================== --- pypy/dist/pypy/objspace/descroperation.py (original) +++ pypy/dist/pypy/objspace/descroperation.py Mon Dec 17 11:02:36 2007 @@ -6,6 +6,13 @@ from pypy.interpreter.typedef import default_identity_hash from pypy.tool.sourcetools import compile2, func_with_new_name +def object_getattribute(space): + "Utility that returns the app-level descriptor object.__getattribute__." + w_src, w_getattribute = space.lookup_in_type_where(space.w_object, + '__getattribute__') + return w_getattribute +object_getattribute._annspecialcase_ = 'specialize:memo' + def raiseattrerror(space, w_obj, name, w_descr=None): w_type = space.type(w_obj) typename = w_type.getname(space, '?') Modified: pypy/dist/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/dist/pypy/objspace/flow/flowcontext.py (original) +++ pypy/dist/pypy/objspace/flow/flowcontext.py Mon Dec 17 11:02:36 2007 @@ -19,24 +19,6 @@ self.block = block self.currentstate = currentstate -class PyFrame(pyframe.PyFrame): - def LOOKUP_METHOD(f, nameindex, *ignored): - space = f.space - w_obj = f.popvalue() - w_name = f.getname_w(nameindex) - w_value = space.getattr(w_obj, w_name) - f.pushvalue(w_value) - #f.pushvalue(None) - - def CALL_METHOD(f, nargs, *ignored): - # 'nargs' is the argument count excluding the implicit 'self' - w_callable = f.peekvalue(nargs) - try: - w_result = f.space.call_valuestack(w_callable, nargs, f) - finally: - f.dropvalues(nargs + 1) - f.pushvalue(w_result) - class SpamBlock(Block): # make slots optional, for debugging if hasattr(Block, '__slots__'): @@ -236,7 +218,7 @@ # create an empty frame suitable for the code object # while ignoring any operation like the creation of the locals dict self.recorder = [] - frame = PyFrame(self.space, self.code, + frame = pyframe.PyFrame(self.space, self.code, self.w_globals, self.closure) frame.last_instr = 0 return frame Modified: pypy/dist/pypy/objspace/std/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/objspace.py (original) +++ pypy/dist/pypy/objspace/std/objspace.py Mon Dec 17 11:02:36 2007 @@ -134,6 +134,12 @@ f.dropvalues(nargs) f.pushvalue(w_result) + if self.config.objspace.opcodes.CALL_METHOD: + # def LOOKUP_METHOD(...): + from pypy.objspace.std.callmethod import LOOKUP_METHOD + # def CALL_METHOD(...): + from pypy.objspace.std.callmethod import CALL_METHOD + if self.config.objspace.std.logspaceoptypes: _space_op_types = [] for name, func in pyframe.PyFrame.__dict__.iteritems(): @@ -606,6 +612,13 @@ else: self.setitem(w_obj, w_key, w_value) + def call_method(self, w_obj, methname, *arg_w): + if self.config.objspace.opcodes.CALL_METHOD: + from pypy.objspace.std.callmethod import call_method_opt + return call_method_opt(self, w_obj, methname, *arg_w) + else: + return ObjSpace.call_method(self, w_obj, methname, *arg_w) + # support for the deprecated __getslice__, __setslice__, __delslice__ def getslice(self, w_obj, w_start, w_stop): From antocuni at codespeak.net Mon Dec 17 13:53:21 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 17 Dec 2007 13:53:21 +0100 (CET) Subject: [pypy-svn] r49864 - pypy/extradoc/planning/roadmap Message-ID: <20071217125321.A3ABA168433@codespeak.net> Author: antocuni Date: Mon Dec 17 13:53:20 2007 New Revision: 49864 Modified: pypy/extradoc/planning/roadmap/goal_cpython_replacement.txt Log: remove duplicate include Modified: pypy/extradoc/planning/roadmap/goal_cpython_replacement.txt ============================================================================== --- pypy/extradoc/planning/roadmap/goal_cpython_replacement.txt (original) +++ pypy/extradoc/planning/roadmap/goal_cpython_replacement.txt Mon Dec 17 13:53:20 2007 @@ -34,5 +34,4 @@ .. include:: task_gui_support.txt -.. include:: task_modules_3rdparty.txt From cfbolz at codespeak.net Mon Dec 17 14:33:57 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 17 Dec 2007 14:33:57 +0100 (CET) Subject: [pypy-svn] r49865 - in pypy/dist/pypy/module/__builtin__: . test Message-ID: <20071217133357.77D5C168480@codespeak.net> Author: cfbolz Date: Mon Dec 17 14:33:56 2007 New Revision: 49865 Modified: pypy/dist/pypy/module/__builtin__/interp_classobj.py pypy/dist/pypy/module/__builtin__/test/test_classobj.py Log: fix failures in test_class.py Modified: pypy/dist/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/dist/pypy/module/__builtin__/interp_classobj.py Mon Dec 17 14:33:56 2007 @@ -212,10 +212,15 @@ def make_binary_returning_notimplemented_instance_method(name): def binaryop(self, space, w_other): - w_meth = self.getattr(space, space.wrap(name), False) - if w_meth is None: - return space.w_NotImplemented - return space.call_function(w_meth, w_other) + try: + w_meth = self.getattr(space, space.wrap(name), False) + except OperationError, e: + if e.match(space, space.w_AttributeError): + return space.w_NotImplemented + else: + if w_meth is None: + return space.w_NotImplemented + return space.call_function(w_meth, w_other) return binaryop def make_binary_instance_method(name): @@ -321,13 +326,7 @@ w_descr_get = space.lookup(w_value, '__get__') if w_descr_get is None: return w_value - try: - return space.call_function(w_descr_get, w_value, self, self.w_class) - except OperationError, e: - if exc or not e.match(space, space.w_AttributeError): - raise - return None - + return space.call_function(w_descr_get, w_value, self, self.w_class) def descr_getattribute(self, space, w_attr): #import pdb; pdb.set_trace() Modified: pypy/dist/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/dist/pypy/module/__builtin__/test/test_classobj.py Mon Dec 17 14:33:56 2007 @@ -610,9 +610,15 @@ class E: __eq__ = property(booh) + __iadd__ = property(booh) + e = E() + raises(TypeError, "e += 1") # does not crash E() == E() + class I: + __init__ = property(booh) + raises(AttributeError, I) def test_multiple_inheritance_more(self): l = [] From antocuni at codespeak.net Mon Dec 17 14:35:55 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 17 Dec 2007 14:35:55 +0100 (CET) Subject: [pypy-svn] r49866 - pypy/extradoc/planning/roadmap Message-ID: <20071217133555.DB7B4168454@codespeak.net> Author: antocuni Date: Mon Dec 17 14:35:55 2007 New Revision: 49866 Modified: pypy/extradoc/planning/roadmap/goal_cpython_replacement.txt pypy/extradoc/planning/roadmap/task_pervasive_benchmarking.txt Log: - try to give a sensible order to the various tasks - add task_ctypes to the cpython_goal Modified: pypy/extradoc/planning/roadmap/goal_cpython_replacement.txt ============================================================================== --- pypy/extradoc/planning/roadmap/goal_cpython_replacement.txt (original) +++ pypy/extradoc/planning/roadmap/goal_cpython_replacement.txt Mon Dec 17 14:35:55 2007 @@ -8,30 +8,31 @@ do the porting of modules and other tasks that are beyond the resources of the core project. -These tasks have no specific ordering currently. (XXX do they?) +These tasks are loosley ordered by categories (speed, extra features, +external modules) (XXX: have a more structured outline?) .. include:: task_refactor_jit.txt .. include:: task_optimize_jit.txt -.. include:: task_separate_compilation.txt - -.. include:: task_cpython_api.txt +.. include:: task_manual_optimizations.txt -.. include:: task_wrapper_generator.txt +.. include:: task_multi_platform.txt .. include:: task_catch_up_with_2_x.txt -.. include:: task_modules_rffi.txt +.. include:: task_parser_cleanup.txt -.. include:: task_modules_3rdparty.txt +.. include:: task_separate_compilation.txt -.. include:: task_manual_optimizations.txt +.. include:: task_modules_rffi.txt -.. include:: task_parser_cleanup.txt +.. include:: task_ctypes.txt -.. include:: task_multi_platform.txt +.. include:: task_wrapper_generator.txt -.. include:: task_gui_support.txt +.. include:: task_cpython_api.txt +.. include:: task_modules_3rdparty.txt +.. include:: task_gui_support.txt Modified: pypy/extradoc/planning/roadmap/task_pervasive_benchmarking.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_pervasive_benchmarking.txt (original) +++ pypy/extradoc/planning/roadmap/task_pervasive_benchmarking.txt Mon Dec 17 14:35:55 2007 @@ -7,7 +7,7 @@ - some microbenchmarks, but coverage not consistent (see pypy/translator/microbench) - some small-to-medium-sized apps as real-world benchmarks -- benchmarks run on one machine (wyvern), no collection of results +- benchmarks run on one machine (tuatara), no collection of results todo: From arigo at codespeak.net Mon Dec 17 15:02:07 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 17 Dec 2007 15:02:07 +0100 (CET) Subject: [pypy-svn] r49869 - pypy/extradoc/planning/roadmap Message-ID: <20071217140207.295C2168477@codespeak.net> Author: arigo Date: Mon Dec 17 15:02:05 2007 New Revision: 49869 Modified: pypy/extradoc/planning/roadmap/task_modules_rffi.txt Log: The array module of PyPy pretends to be an extension module, but that's a lie - it's only app-level, maybe geninterp'ed. Modified: pypy/extradoc/planning/roadmap/task_modules_rffi.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_modules_rffi.txt (original) +++ pypy/extradoc/planning/roadmap/task_modules_rffi.txt Mon Dec 17 15:02:05 2007 @@ -13,7 +13,7 @@ preformance reasons, many of them should eventually be implemented at interpreter level. -These modules are: binascii, cPickle, cStringIO, cmath, +These modules are: array, binascii, cPickle, cStringIO, cmath, collections, datetime, functional, functools(incomplete), imp, itertools. From arigo at codespeak.net Mon Dec 17 15:07:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 17 Dec 2007 15:07:09 +0100 (CET) Subject: [pypy-svn] r49870 - pypy/dist/pypy/module/__builtin__ Message-ID: <20071217140709.B8DB11684BF@codespeak.net> Author: arigo Date: Mon Dec 17 15:07:09 2007 New Revision: 49870 Removed: pypy/dist/pypy/module/__builtin__/app_complex.py Modified: pypy/dist/pypy/module/__builtin__/__init__.py Log: Remove the app-level implementation of complex numbers, for now. There is W_ComplexObject in the std objspace that is always used. Modified: pypy/dist/pypy/module/__builtin__/__init__.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/__init__.py (original) +++ pypy/dist/pypy/module/__builtin__/__init__.py Mon Dec 17 15:07:09 2007 @@ -55,8 +55,6 @@ 'vars' : 'app_inspect.vars', 'dir' : 'app_inspect.dir', - 'complex' : 'app_complex.complex', - 'buffer' : 'app_buffer.buffer', 'reload' : 'app_misc.reload', From arigo at codespeak.net Mon Dec 17 15:08:08 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 17 Dec 2007 15:08:08 +0100 (CET) Subject: [pypy-svn] r49871 - pypy/dist/pypy/module/__builtin__/test Message-ID: <20071217140808.8B8FA1684BF@codespeak.net> Author: arigo Date: Mon Dec 17 15:08:08 2007 New Revision: 49871 Removed: pypy/dist/pypy/module/__builtin__/test/test_complexobject.py Log: Forgot to remove the test file too. (There is also one in the stdobjspace.) From arigo at codespeak.net Mon Dec 17 15:11:47 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 17 Dec 2007 15:11:47 +0100 (CET) Subject: [pypy-svn] r49872 - pypy/dist/pypy/config Message-ID: <20071217141147.34A181684BF@codespeak.net> Author: arigo Date: Mon Dec 17 15:11:46 2007 New Revision: 49872 Modified: pypy/dist/pypy/config/pypyoption.py Log: Time to enable optimized_int_add again in faassen builds: it speeds up a Boehm faassen by 7.5% on pystone. Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Mon Dec 17 15:11:46 2007 @@ -262,7 +262,7 @@ ("objspace.std.withmethodcache", True), # ("objspace.std.withfastslice", True), ("objspace.std.withprebuiltchar", True), -# ("objspace.std.optimized_int_add", True), + ("objspace.std.optimized_int_add", True), ], cmdline="--allopts --faassen", negation=False), From antocuni at codespeak.net Mon Dec 17 16:16:25 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 17 Dec 2007 16:16:25 +0100 (CET) Subject: [pypy-svn] r49873 - pypy/extradoc/planning/roadmap Message-ID: <20071217151625.EC8AD1684CC@codespeak.net> Author: antocuni Date: Mon Dec 17 16:16:24 2007 New Revision: 49873 Added: pypy/extradoc/planning/roadmap/goal_jython_ironpython_replacement.txt (contents, props changed) pypy/extradoc/planning/roadmap/task_bytecode_compiler.txt (contents, props changed) pypy/extradoc/planning/roadmap/task_external_objects.txt (contents, props changed) pypy/extradoc/planning/roadmap/task_integration_vm.txt (contents, props changed) pypy/extradoc/planning/roadmap/task_jit_ootype.txt (contents, props changed) pypy/extradoc/planning/roadmap/task_oothreading.txt (contents, props changed) pypy/extradoc/planning/roadmap/task_stdlib_modules.txt (contents, props changed) Modified: pypy/extradoc/planning/roadmap/task_catch_up_with_2_x.txt pypy/extradoc/planning/roadmap/task_documentation.txt pypy/extradoc/planning/roadmap/task_pervasive_benchmarking.txt pypy/extradoc/planning/roadmap/task_pervasive_testing.txt pypy/extradoc/planning/roadmap/task_separate_compilation.txt pypy/extradoc/planning/roadmap/task_wrapper_generator.txt Log: - add a goal a the relative tasks about being a competitor to jython and ironpython - use a consistent character to underline task titles Added: pypy/extradoc/planning/roadmap/goal_jython_ironpython_replacement.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/goal_jython_ironpython_replacement.txt Mon Dec 17 16:16:24 2007 @@ -0,0 +1,27 @@ +This is a roadmap for the specific target of + +"Making PyPy a viable replacement for Jython and/or IronPython" + +Currently, the biggest advantage of pypy-jvm and pypy-cli over Jython +and IronPython is compliance to CPython, which is already better than +their counterparts. However, a lot of work has still to be done before +they can be used in a production environment. + +Speed-wise, pypy-jvm is already slightly faster than Jython; however, +pypy-cli is about 6x-10x slower than IronPython. + +.. include:: task_external_objects.txt + +.. include:: task_integration_vm.txt + +.. include:: task_manual_optimizations.txt + +.. include:: task_jit_ootype.txt + +.. include:: task_bytecode_compiler.txt + +.. include:: task_oothreading.txt + +.. include:: task_separate_compilation.txt + +.. include:: task_stdlib_modules.txt Added: pypy/extradoc/planning/roadmap/task_bytecode_compiler.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_bytecode_compiler.txt Mon Dec 17 16:16:24 2007 @@ -0,0 +1,25 @@ +Compile Python sources into JVM/.NET bytecode +============================================= + +Status: + + - pypy-jvm and pypy-cli interprets python bytecode + +Todo: + + - make pypy-jvm and pypy-cli emitting native bytecode instead of + interpreting python bytecode; + + - the plan is to reuse part of the JIT machinery to automatically + convert the Python interpreter into a compiler + +Expected outcome: + + - compilation to native bytecode should give a speedup around 2/2.5x; + + - emitting native bytecode is needed for doing some tasks that are + possible with Jython/IronPython (i.e., writing applets) + +Dependencies: + + - `Port the JIT to ootype`_ Modified: pypy/extradoc/planning/roadmap/task_catch_up_with_2_x.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_catch_up_with_2_x.txt (original) +++ pypy/extradoc/planning/roadmap/task_catch_up_with_2_x.txt Mon Dec 17 16:16:24 2007 @@ -1,5 +1,5 @@ Catch up with CPython 2.5/2.6 ------------------------------ +============================= status: - PyPy already implements a few features of 2.5 (PEP 309, PEP 338, PEP 341, PEP 357, Modified: pypy/extradoc/planning/roadmap/task_documentation.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_documentation.txt (original) +++ pypy/extradoc/planning/roadmap/task_documentation.txt Mon Dec 17 16:16:24 2007 @@ -1,5 +1,5 @@ PyPy Documentation ------------------------ +================== status: Added: pypy/extradoc/planning/roadmap/task_external_objects.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_external_objects.txt Mon Dec 17 16:16:24 2007 @@ -0,0 +1,24 @@ +Refactor RPython external objects interface +=========================================== + +Status: + + - RPython programs compiled with gencli can make limited use of .NET + classes; it is possible to test them without translation by using + Python for .NET. + + - genjvm does not support external objects at all + +Todo: + + - refactor the code in gencli and make it shareable with genjvm + + - add support for external objects to genjvm; the idea is to make + programs testable by using JPype_ or something similar + +Expected outcome: + + - ability to use JVM classes from RPython programs + +.. _JPype: http://jpype.sourceforge.net/ + Added: pypy/extradoc/planning/roadmap/task_integration_vm.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_integration_vm.txt Mon Dec 17 16:16:24 2007 @@ -0,0 +1,30 @@ +Integration with the hosting virtual machine +============================================ + +Status: + + - pypy-cli can access and use .NET classes, through the clr module, + but in a limited way. Advanced features such as delegates and + inheritance from .NET classes are not supported yet. + + - pypy-jvm cannot access JVM classes. + +Todo: + + - improve the clr module + + - refactor it so that most of the code can be shared between pypy-cli + and pypy-jvm + + - create the equivalent for the JVM + + +Expected outcome: + + - ability to access to JVM and .NET classes as Jython and IronPython + users expect + + +Dependencies: + + - `Refactor RPython external objects interface`_ Added: pypy/extradoc/planning/roadmap/task_jit_ootype.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_jit_ootype.txt Mon Dec 17 16:16:24 2007 @@ -0,0 +1,16 @@ +Port the JIT to ootype +====================== + +Status: + + - The JIT works only for low level backends + +Todo: + + - port the hintannotator/timeshifter to ootype + + - add new JIT backends for the JVM and/or the CLI + +Expected outcome: + + - pypy-jvm-jit and pypy-cli-jit Added: pypy/extradoc/planning/roadmap/task_oothreading.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_oothreading.txt Mon Dec 17 16:16:24 2007 @@ -0,0 +1,19 @@ +Make pypy-jvm and pypy-cli multithreaded +======================================== + +Status: + + - pypy-jvm and pypy-cli don't support multiple threads + +Todo: + + - add the relevant locks to the Python interpreter in order to make + pypy-jvm and pypy-cli thread-safe; + + - this task would be probably also useful for implementing a GIL-free + threading model for low level backends; + +Expected outcome: + + - ability to use use free threading in pypy-jvm and pypy-cli + Modified: pypy/extradoc/planning/roadmap/task_pervasive_benchmarking.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_pervasive_benchmarking.txt (original) +++ pypy/extradoc/planning/roadmap/task_pervasive_benchmarking.txt Mon Dec 17 16:16:24 2007 @@ -1,6 +1,6 @@ Pervasive Benchmarking ------------------------- +====================== status: Modified: pypy/extradoc/planning/roadmap/task_pervasive_testing.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_pervasive_testing.txt (original) +++ pypy/extradoc/planning/roadmap/task_pervasive_testing.txt Mon Dec 17 16:16:24 2007 @@ -1,5 +1,5 @@ Pervasive Testing ------------------------- +================= status: - automated nightly test runs of pypy tests on 32bit linux machine (wyvern) Modified: pypy/extradoc/planning/roadmap/task_separate_compilation.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_separate_compilation.txt (original) +++ pypy/extradoc/planning/roadmap/task_separate_compilation.txt Mon Dec 17 16:16:24 2007 @@ -1,4 +1,3 @@ - Separate compilation ==================== Added: pypy/extradoc/planning/roadmap/task_stdlib_modules.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/roadmap/task_stdlib_modules.txt Mon Dec 17 16:16:24 2007 @@ -0,0 +1,34 @@ +Implementing missing standard modules +===================================== + +Status: + + - A lot of standard modules such as ``socket`` don't work on top of + pypy-jvm and pypy-cli + +Todo: + + - Implement relevant modules using the API offered by the hosting + virtual machine (either at app-level or interp-level). E.g., + implement ``socket`` on top of JVM/.NET sockets. + + - Maybe it would be possible to reuse existing modules from Jython_ + and FePy_. + +Expected outcome: + + - Having a better coverage of the Python standard library + +Dependencies: + + - `Separate compilation`_ + + - `Refactor RPython external objects interface`_ + + - `Integration with the hosting virtual machine`_ + + +.. _Jython: http://www.jython.org/ + +.. _FePy: http://fepy.sourceforge.net/ + Modified: pypy/extradoc/planning/roadmap/task_wrapper_generator.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_wrapper_generator.txt (original) +++ pypy/extradoc/planning/roadmap/task_wrapper_generator.txt Mon Dec 17 16:16:24 2007 @@ -4,9 +4,3 @@ To make it simpler to integrate C libraries with PyPy, we should support at least one of the wrapper generators that are already in use in the Python community. - -Google uses Swig heavily, so in a scenario where Google pays for some -development, it would be natural to do Swig. Once PyPy gets traction -we expect proponents of other tools to make their own ports. - - From antocuni at codespeak.net Mon Dec 17 16:29:25 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 17 Dec 2007 16:29:25 +0100 (CET) Subject: [pypy-svn] r49874 - pypy/dist/pypy/translator/cli/test Message-ID: <20071217152925.F2A151684C0@codespeak.net> Author: antocuni Date: Mon Dec 17 16:29:24 2007 New Revision: 49874 Modified: pypy/dist/pypy/translator/cli/test/test_string.py pypy/dist/pypy/translator/cli/test/test_unicode.py Log: skip these tests Modified: pypy/dist/pypy/translator/cli/test/test_string.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_string.py (original) +++ pypy/dist/pypy/translator/cli/test/test_string.py Mon Dec 17 16:29:24 2007 @@ -21,6 +21,8 @@ def test_hlstr(self): py.test.skip("CLI tests can't have string as input arguments") + test_inplace_add = test_hlstr + def test_getitem_exc(self): py.test.skip('fixme!') Modified: pypy/dist/pypy/translator/cli/test/test_unicode.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_unicode.py (original) +++ pypy/dist/pypy/translator/cli/test/test_unicode.py Mon Dec 17 16:29:24 2007 @@ -18,3 +18,6 @@ def test_getitem_exc(self): py.test.skip('fixme!') + + def test_inplace_add(self): + py.test.skip("CLI tests can't have string as input arguments") From cfbolz at codespeak.net Mon Dec 17 16:58:33 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 17 Dec 2007 16:58:33 +0100 (CET) Subject: [pypy-svn] r49875 - in pypy/branch/applevel-ctypes/pypy/lib: _ctypes ctypes Message-ID: <20071217155833.D194F1684D7@codespeak.net> Author: cfbolz Date: Mon Dec 17 16:58:33 2007 New Revision: 49875 Added: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/dummy.py (contents, props changed) Modified: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/__init__.py pypy/branch/applevel-ctypes/pypy/lib/ctypes/__init__.py Log: add enough dummies to _ctypes that ctypes imports. had to disable one safety check in ctypes though. Modified: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/__init__.py ============================================================================== --- pypy/branch/applevel-ctypes/pypy/lib/_ctypes/__init__.py (original) +++ pypy/branch/applevel-ctypes/pypy/lib/_ctypes/__init__.py Mon Dec 17 16:58:33 2007 @@ -0,0 +1,14 @@ +from _ctypes.dummy import Union, Structure, Array, _Pointer, CFuncPtr +from _ctypes.dummy import ArgumentError, dlopen, sizeof, byref, addressof +from _ctypes.dummy import alignment, resize, _SimpleCData +from _ctypes.dummy import _memmove_addr, _memset_addr, _string_at_addr +from _ctypes.dummy import _cast_addr + + + +__version__ = '1.0.2' +#XXX platform dependant? +RTLD_LOCAL = 0 +RTLD_GLOBAL = 256 +FUNCFLAG_CDECL = 1 +FUNCFLAG_PYTHONAPI = 4 Added: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/dummy.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/_ctypes/dummy.py Mon Dec 17 16:58:33 2007 @@ -0,0 +1,54 @@ +class UnionType(type): + pass + +class Union(object): + __metaclass__ = UnionType + +class Structure(type): + pass + +class Array(type): + pass + +class PointerType(type): + pass + +class _Pointer(object): + __metaclass__ = PointerType + + +class SimpleType(type): + def __mul__(self, other): + pass + +class _SimpleCData(object): + __metaclass__ = SimpleType + def from_param(self, *args, **kwargs): + pass + +class CFuncPtrType(type): + pass + +class CFuncPtr(object): + __metaclass__ = CFuncPtrType + def __init__(self, func): + pass + +class ArgumentError(Exception): + pass + + +def dummyfunc(*args, **kwargs): + return None + +dlopen = dummyfunc +sizeof = dummyfunc +byref = dummyfunc +addressof = dummyfunc +alignment = dummyfunc +resize = dummyfunc +_memmove_addr = dummyfunc +_memset_addr = dummyfunc +_string_at_addr = dummyfunc +_cast_addr = dummyfunc + Modified: pypy/branch/applevel-ctypes/pypy/lib/ctypes/__init__.py ============================================================================== --- pypy/branch/applevel-ctypes/pypy/lib/ctypes/__init__.py (original) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/__init__.py Mon Dec 17 16:58:33 2007 @@ -136,6 +136,7 @@ def _check_size(typ, typecode=None): # Check if sizeof(ctypes_type) against struct.calcsize. This # should protect somewhat against a misconfigured libffi. + return # XXXXXXXXX from struct import calcsize if typecode is None: # Most _type_ codes are the same as used in struct From cfbolz at codespeak.net Mon Dec 17 17:00:30 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 17 Dec 2007 17:00:30 +0100 (CET) Subject: [pypy-svn] r49876 - in pypy/branch/applevel-ctypes/pypy/lib: _ctypes _ctypes/test ctypes ctypes/macholib ctypes/test Message-ID: <20071217160030.F01641684D7@codespeak.net> Author: cfbolz Date: Mon Dec 17 17:00:28 2007 New Revision: 49876 Modified: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/ (props changed) pypy/branch/applevel-ctypes/pypy/lib/_ctypes/test/ (props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/ (props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/macholib/ (props changed) pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/ (props changed) Log: fixeol From cfbolz at codespeak.net Mon Dec 17 18:36:55 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 17 Dec 2007 18:36:55 +0100 (CET) Subject: [pypy-svn] r49877 - pypy/branch/applevel-ctypes/pypy/lib/ctypes/test Message-ID: <20071217173655.4C754168459@codespeak.net> Author: cfbolz Date: Mon Dec 17 18:36:54 2007 New Revision: 49877 Removed: pypy/branch/applevel-ctypes/pypy/lib/ctypes/test/test_integers.py Log: kill empty test file From cfbolz at codespeak.net Mon Dec 17 22:30:10 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 17 Dec 2007 22:30:10 +0100 (CET) Subject: [pypy-svn] r49878 - pypy/branch/applevel-ctypes/pypy/lib/_ctypes Message-ID: <20071217213010.480DB168527@codespeak.net> Author: cfbolz Date: Mon Dec 17 22:30:08 2007 New Revision: 49878 Added: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/primitive.py (contents, props changed) Modified: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/__init__.py pypy/branch/applevel-ctypes/pypy/lib/_ctypes/dummy.py Log: trying to get some of the ctypes' test_number tests to pass Modified: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/__init__.py ============================================================================== --- pypy/branch/applevel-ctypes/pypy/lib/_ctypes/__init__.py (original) +++ pypy/branch/applevel-ctypes/pypy/lib/_ctypes/__init__.py Mon Dec 17 22:30:08 2007 @@ -1,9 +1,10 @@ from _ctypes.dummy import Union, Structure, Array, _Pointer, CFuncPtr from _ctypes.dummy import ArgumentError, dlopen, sizeof, byref, addressof -from _ctypes.dummy import alignment, resize, _SimpleCData +from _ctypes.dummy import alignment, resize from _ctypes.dummy import _memmove_addr, _memset_addr, _string_at_addr from _ctypes.dummy import _cast_addr +from _ctypes.primitive import _SimpleCData __version__ = '1.0.2' Modified: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/dummy.py ============================================================================== --- pypy/branch/applevel-ctypes/pypy/lib/_ctypes/dummy.py (original) +++ pypy/branch/applevel-ctypes/pypy/lib/_ctypes/dummy.py Mon Dec 17 22:30:08 2007 @@ -17,15 +17,6 @@ __metaclass__ = PointerType -class SimpleType(type): - def __mul__(self, other): - pass - -class _SimpleCData(object): - __metaclass__ = SimpleType - def from_param(self, *args, **kwargs): - pass - class CFuncPtrType(type): pass Added: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/primitive.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/_ctypes/primitive.py Mon Dec 17 22:30:08 2007 @@ -0,0 +1,105 @@ +SIMPLE_TYPE_CHARS = "cbBhHiIlLdfuzZqQPXOv" +# type converters +def convert_intlike(val): + if not isinstance(val, (int, long)): + raise TypeError("int expected, got %s" % (type(val), )) + return val + +def convert_floatlike(val): + if not isinstance(val, (float, int, long)): + raise TypeError("float expected, got %s" % (type(val), )) + return float(val) + +def convert_char(val): + if not isinstance(val, str) or not len(val) == 1: + raise TypeError("one character string expected") + return val + +def convert_nothing(val): + return val + + +TP_TO_CONVERTER = { + 'c': convert_char, + 'b': convert_intlike, + 'B': convert_intlike, + 'h': convert_intlike, + 'H': convert_intlike, + 'i': convert_intlike, + 'I': convert_intlike, + 'l': convert_intlike, + 'L': convert_intlike, + 'q': convert_intlike, + 'Q': convert_intlike, + 'f': convert_floatlike, + 'd': convert_floatlike, + 'P': convert_nothing, #XXX + # not part of struct + 'O': convert_nothing, + 'z': convert_nothing, #XXX +} + + +class NULL(object): + pass +NULL = NULL() + +TP_TO_DEFAULT = { + 'c': '\x00', + 'b': 0, + 'B': 0, + 'h': 0, + 'H': 0, + 'i': 0, + 'I': 0, + 'l': 0, + 'L': 0, + 'q': 0, + 'Q': 0, + 'f': 0.0, + 'd': 0.0, + 'P': None, + # not part of struct + 'O': NULL, + 'z': None, +} + +DEFAULT_VALUE = object() + +class SimpleType(type): + def __new__(self, name, bases, dct): + tp = dct['_type_'] + if (not isinstance(tp, str) or + not len(tp) == 1 or + tp not in SIMPLE_TYPE_CHARS): + raise ValueError('%s is not a type character' % (tp)) + default = TP_TO_DEFAULT[tp] + converter = TP_TO_CONVERTER[tp] + def __init__(self, value=DEFAULT_VALUE): + self._value = default + if value is not DEFAULT_VALUE: + self.value = value + dct['__init__'] = __init__ + dct['_converter'] = staticmethod(TP_TO_CONVERTER[tp]) + result = type.__new__(self, name, bases, dct) + return result + + def __mul__(self, other): + pass + +class _SimpleCData(object): + __metaclass__ = SimpleType + _type_ = 'i' + def from_param(self, *args, **kwargs): + pass + + def _getvalue(self): + return self._value + + def _setvalue(self, val): + self._value = self._converter(val) + + def __repr__(self): + return "%s(%s)" % (type(self).__name__, self.value) + value = property(_getvalue, _setvalue) + From jacob at codespeak.net Mon Dec 17 22:38:52 2007 From: jacob at codespeak.net (jacob at codespeak.net) Date: Mon, 17 Dec 2007 22:38:52 +0100 (CET) Subject: [pypy-svn] r49879 - pypy/dist/lib-python/modified-2.4.1 Message-ID: <20071217213852.5C7F91684C8@codespeak.net> Author: jacob Date: Mon Dec 17 22:38:51 2007 New Revision: 49879 Modified: pypy/dist/lib-python/modified-2.4.1/tarfile.py Log: Fixed problem with code modified by Anders Lehman earlier. PyPy does not copy the behaviour of CPython 2.4 for int() and long() whith a second (base) argument, ignoring trailng null characters. Tarfile relied on this. Python 2.5 behaves like PyPy. This fix works with both versions. Modified: pypy/dist/lib-python/modified-2.4.1/tarfile.py ============================================================================== --- pypy/dist/lib-python/modified-2.4.1/tarfile.py (original) +++ pypy/dist/lib-python/modified-2.4.1/tarfile.py Mon Dec 17 22:38:51 2007 @@ -65,23 +65,25 @@ # from tarfile import * __all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError"] -builtin_int = int -builtin_long = long +# Calls to long() and int() with a string and a second (base) argument +# relied on an undocumented behaviour of the conversion. Trailing null +# characters were ignored (but were not ignored if the second argument was +# missing). The behaviour changed in Python2.5, so that it is always an +# error to have trailing null characters. This code is compatible with +# both Python 2.4 and 2.5. -def long(string, base): - return builtin_long(int(string, base)) +def to_long(string, base=None): + return long(to_int(string, base)) -def int(string, base=None): - print "My int func", repr(string) +def to_int(string, base=None): if base == None: - return builtin_int(string) + return int(string) try: end = string.index('\x00') - except ValueError: - return builtin_int(string, base) - res = builtin_int(string[:end],base) - print oct(res) - return res + except ValueError, AttributeError: + return int(string, base) + return int(string[:end], base) + #--------------------------------------------------------- # tar constants @@ -343,7 +345,7 @@ -self.zlib.MAX_WBITS, self.zlib.DEF_MEM_LEVEL, 0) - timestamp = struct.pack(" lastpos: @@ -1717,7 +1719,7 @@ pos += 24 isextended = ord(buf[482]) - origsize = int(buf[483:495], 8) + origsize = to_int(buf[483:495], 8) # If the isextended flag is given, # there are extra headers to process. @@ -1727,8 +1729,8 @@ pos = 0 for i in xrange(21): try: - offset = int(buf[pos:pos + 12], 8) - numbytes = int(buf[pos + 12:pos + 24], 8) + offset = to_int(buf[pos:pos + 12], 8) + numbytes = to_int(buf[pos + 12:pos + 24], 8) except ValueError: break if offset > lastpos: From exarkun at codespeak.net Tue Dec 18 00:02:38 2007 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Tue, 18 Dec 2007 00:02:38 +0100 (CET) Subject: [pypy-svn] r49881 - pypy/dist/pypy/translator/goal/test2 Message-ID: <20071217230238.EE3D4168527@codespeak.net> Author: exarkun Date: Tue Dec 18 00:02:38 2007 New Revision: 49881 Modified: pypy/dist/pypy/translator/goal/test2/test_app_main.py Log: skip the pexpect-using app_main tests if the available version of pexpect is older than 2.1 Versions of pexpect older than 2.1 seem to have problems with py.io which leads to AttributeErrors. Modified: pypy/dist/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/dist/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/dist/pypy/translator/goal/test2/test_app_main.py Tue Dec 18 00:02:38 2007 @@ -63,6 +63,21 @@ import pexpect except ImportError, e: py.test.skip(str(e)) + else: + # Version is of the style "0.999" or "2.1". Older versions of + # pexpect try to get the fileno of stdin, which generally won't + # work with py.test (due to sys.stdin being a DontReadFromInput + # instance). + version = map(int, pexpect.__version__.split('.')) + + # I only tested 0.999 and 2.1. The former does not work, the + # latter does. Feel free to refine this measurement. + # -exarkun, 17/12/2007 + if version < [2, 1]: + py.test.skip( + "pexpect version too old, requires 2.1 or newer: %r" % ( + pexpect.__version__,)) + kwds.setdefault('timeout', 10) print 'SPAWN:', args, kwds child = pexpect.spawn(*args, **kwds) From antocuni at codespeak.net Tue Dec 18 10:35:09 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 18 Dec 2007 10:35:09 +0100 (CET) Subject: [pypy-svn] r49882 - in pypy/dist/pypy/jit/hintannotator: . test Message-ID: <20071218093509.7DEC816853A@codespeak.net> Author: antocuni Date: Tue Dec 18 10:35:08 2007 New Revision: 49882 Modified: pypy/dist/pypy/jit/hintannotator/model.py pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Log: a new test and the corresponding fix Modified: pypy/dist/pypy/jit/hintannotator/model.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/model.py (original) +++ pypy/dist/pypy/jit/hintannotator/model.py Tue Dec 18 10:35:08 2007 @@ -429,7 +429,10 @@ def oosend(hs_v1, hs_name, *args_hs): RESTYPE = getbookkeeper().current_op_concretetype() - return SomeLLAbstractVariable(RESTYPE) + if RESTYPE is not ootype.Void: + return SomeLLAbstractVariable(RESTYPE) + else: + return # XXX: is it right? class __extend__(SomeLLAbstractConstant): Modified: pypy/dist/pypy/jit/hintannotator/test/test_annotator.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/test/test_annotator.py (original) +++ pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Tue Dec 18 10:35:08 2007 @@ -1019,3 +1019,13 @@ test_deepfreeze_variables = skip_policy test_cast_pointer_keeps_deepfreeze = skip_policy test_concrete_fnptr_for_green_call = skip_policy + + def test_void_oosend(self): + class Foo: + def bar(self): + pass + + def fn(): + f = Foo() + f.bar() + hs = self.hannotate(fn, [], policy=P_OOPSPEC_NOVIRTUAL) From cfbolz at codespeak.net Tue Dec 18 11:41:42 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 18 Dec 2007 11:41:42 +0100 (CET) Subject: [pypy-svn] r49883 - in pypy/branch/applevel-ctypes/pypy/lib: _ctypes _ctypes/test ctypes Message-ID: <20071218104142.A5074168548@codespeak.net> Author: cfbolz Date: Tue Dec 18 11:41:42 2007 New Revision: 49883 Added: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/dll.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/_ctypes/function.py (contents, props changed) pypy/branch/applevel-ctypes/pypy/lib/_ctypes/test/test_dll.py (contents, props changed) Modified: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/__init__.py pypy/branch/applevel-ctypes/pypy/lib/_ctypes/dummy.py pypy/branch/applevel-ctypes/pypy/lib/ctypes/__init__.py Log: whack whack whack until math.sin can be called Modified: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/__init__.py ============================================================================== --- pypy/branch/applevel-ctypes/pypy/lib/_ctypes/__init__.py (original) +++ pypy/branch/applevel-ctypes/pypy/lib/_ctypes/__init__.py Tue Dec 18 11:41:42 2007 @@ -1,10 +1,12 @@ -from _ctypes.dummy import Union, Structure, Array, _Pointer, CFuncPtr -from _ctypes.dummy import ArgumentError, dlopen, sizeof, byref, addressof +from _ctypes.dummy import Union, Structure, Array, _Pointer +from _ctypes.dummy import ArgumentError, sizeof, byref, addressof from _ctypes.dummy import alignment, resize from _ctypes.dummy import _memmove_addr, _memset_addr, _string_at_addr from _ctypes.dummy import _cast_addr from _ctypes.primitive import _SimpleCData +from _ctypes.function import CFuncPtr +from _ctypes.dll import dlopen __version__ = '1.0.2' Added: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/dll.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/_ctypes/dll.py Tue Dec 18 11:41:42 2007 @@ -0,0 +1,7 @@ +import _ffi + +def dlopen(name, mode): + # XXX mode is ignored + if name is None: + return None # XXX seems to mean the cpython lib + return _ffi.CDLL(name) Modified: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/dummy.py ============================================================================== --- pypy/branch/applevel-ctypes/pypy/lib/_ctypes/dummy.py (original) +++ pypy/branch/applevel-ctypes/pypy/lib/_ctypes/dummy.py Tue Dec 18 11:41:42 2007 @@ -17,14 +17,6 @@ __metaclass__ = PointerType -class CFuncPtrType(type): - pass - -class CFuncPtr(object): - __metaclass__ = CFuncPtrType - def __init__(self, func): - pass - class ArgumentError(Exception): pass @@ -32,7 +24,6 @@ def dummyfunc(*args, **kwargs): return None -dlopen = dummyfunc sizeof = dummyfunc byref = dummyfunc addressof = dummyfunc Added: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/function.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/_ctypes/function.py Tue Dec 18 11:41:42 2007 @@ -0,0 +1,41 @@ +import _ffi + +class CFuncPtrType(type): + pass + +class CFuncPtr(object): + __metaclass__ = CFuncPtrType + argtypes = None + _argtypes = None + restype = None + _restype = None + def __init__(self, stuff): + if isinstance(stuff, tuple): + print stuff + name, dll = stuff + self.name = name + self.dll = dll + self._funcptr = None + else: + self.name = None + self.dll = None + + def __call__(self, *args): + assert self.argtypes is not None #XXX for now + assert self.restype is not None #XXX for now + if len(args) != len(self.argtypes): + raise TypeError("%s takes %s arguments, given %s" % (self.name, + len(self.argtypes), len(args))) + return self._getfuncptr()(*args) + + def _getfuncptr(self): + if self._funcptr is not None: + if (self.argtypes is self._argtypes + and self.restype is self._restype): + return self._funcptr + argtps = [argtype._type_ for argtype in self.argtypes] + restp = self.restype._type_ + self._funcptr = funcptr = self.dll._handle.ptr(self.name, argtps, restp) + self._argtypes = self.argtypes + self._restype = self.restype + return funcptr Added: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/test/test_dll.py ============================================================================== --- (empty file) +++ pypy/branch/applevel-ctypes/pypy/lib/_ctypes/test/test_dll.py Tue Dec 18 11:41:42 2007 @@ -0,0 +1,8 @@ +import ctypes + +def test_dll_simple(): + dll = ctypes.cdll.LoadLibrary("libm.so.6") + sin = dll.sin + sin.argtypes = [ctypes.c_double] + sin.restype = ctypes.c_double + assert sin(0) == 0 Modified: pypy/branch/applevel-ctypes/pypy/lib/ctypes/__init__.py ============================================================================== --- pypy/branch/applevel-ctypes/pypy/lib/ctypes/__init__.py (original) +++ pypy/branch/applevel-ctypes/pypy/lib/ctypes/__init__.py Tue Dec 18 11:41:42 2007 @@ -343,9 +343,9 @@ self._handle = handle def __repr__(self): - return "<%s '%s', handle %x at %x>" % \ + return "<%s '%s', handle %s at %x>" % \ (self.__class__.__name__, self._name, - (self._handle & (_sys.maxint*2 + 1)), + (self._handle), id(self) & (_sys.maxint*2 + 1)) def __getattr__(self, name): From cfbolz at codespeak.net Tue Dec 18 12:20:57 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 18 Dec 2007 12:20:57 +0100 (CET) Subject: [pypy-svn] r49884 - in pypy/dist/pypy/module/_codecs: . test Message-ID: <20071218112057.C265416853F@codespeak.net> Author: cfbolz Date: Tue Dec 18 12:20:57 2007 New Revision: 49884 Modified: pypy/dist/pypy/module/_codecs/interp_codecs.py pypy/dist/pypy/module/_codecs/test/test_codecs.py Log: fix an rpython assertion error Modified: pypy/dist/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/dist/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/dist/pypy/module/_codecs/interp_codecs.py Tue Dec 18 12:20:57 2007 @@ -28,16 +28,14 @@ space.wrap(endpos), space.wrap(reason)) w_res = space.call_function(w_errorhandler, w_exc) - try: - w_replace, w_newpos = space.unpacktuple(w_res, 2) - except OperationError, e: - if not e.match(space, space.w_TypeError): - raise + if (not space.is_true(space.isinstance(w_res, space.w_tuple)) + or space.int_w(space.len(w_res)) != 2): raise OperationError( space.w_TypeError, space.wrap("encoding error handler must return " "(unicode, int) tuple, not %s" % ( space.str_w(space.repr(w_res))))) + w_replace, w_newpos = space.unpacktuple(w_res, 2) newpos = space.int_w(w_newpos) if (newpos < 0): newpos = len(input) + newpos Modified: pypy/dist/pypy/module/_codecs/test/test_codecs.py ============================================================================== --- pypy/dist/pypy/module/_codecs/test/test_codecs.py (original) +++ pypy/dist/pypy/module/_codecs/test/test_codecs.py Tue Dec 18 12:20:57 2007 @@ -281,13 +281,19 @@ def test_cpytest_decode(self): import codecs - print 1 assert codecs.decode('\xe4\xf6\xfc', 'latin-1') == u'\xe4\xf6\xfc' - print 2 raises(TypeError, codecs.decode) - print 3 assert codecs.decode('abc') == u'abc' - print 4 raises(UnicodeDecodeError, codecs.decode, '\xff', 'ascii') - + def test_bad_errorhandler_return(self): + import codecs + def baddecodereturn1(exc): + return 42 + codecs.register_error("test.baddecodereturn1", baddecodereturn1) + raises(TypeError, "\xff".decode, "ascii", "test.baddecodereturn1") + raises(TypeError, "\\".decode, "unicode-escape", "test.baddecodereturn1") + raises(TypeError, "\\x0".decode, "unicode-escape", "test.baddecodereturn1") + raises(TypeError, "\\x0y".decode, "unicode-escape", "test.baddecodereturn1") + raises(TypeError, "\\Uffffeeee".decode, "unicode-escape", "test.baddecodereturn1") + raises(TypeError, "\\uyyyy".decode, "raw-unicode-escape", "test.baddecodereturn1") From antocuni at codespeak.net Tue Dec 18 14:18:47 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 18 Dec 2007 14:18:47 +0100 (CET) Subject: [pypy-svn] r49885 - in pypy/dist/pypy/jit/hintannotator: . test Message-ID: <20071218131847.09486168543@codespeak.net> Author: antocuni Date: Tue Dec 18 14:18:47 2007 New Revision: 49885 Modified: pypy/dist/pypy/jit/hintannotator/model.py pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Log: one more test for oosend (when equivalent to direct_call) Modified: pypy/dist/pypy/jit/hintannotator/model.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/model.py (original) +++ pypy/dist/pypy/jit/hintannotator/model.py Tue Dec 18 14:18:47 2007 @@ -30,6 +30,7 @@ oois subclassof instanceof + oostring """.split() BINARY_OPERATIONS = """int_add int_sub int_mul int_mod int_and int_rshift @@ -104,6 +105,8 @@ v_callable = self.spaceop.args[0] retdeps = greenorigindependencies.setdefault(self, []) retdeps.append(v_callable) + elif self.spaceop.opname == 'oosend': + args = self.spaceop.args[1:] else: raise AssertionError(self.spaceop.opname) @@ -468,21 +471,25 @@ # normal call if not hasattr(fnobj, 'graph'): raise NotImplementedError("XXX call to externals or primitives") - if not bookkeeper.annotator.policy.look_inside_graph(fnobj.graph): - return cannot_follow_call(bookkeeper, fnobj.graph, args_hs, - lltype.typeOf(fnobj).RESULT) + + return hs_f1._call_single_graph(fnobj.graph, lltype.typeOf(fnobj).RESULT, *args_hs) + + def _call_single_graph(hs_f1, graph, RESULT, *args_hs): + bookkeeper = getbookkeeper() + if not bookkeeper.annotator.policy.look_inside_graph(graph): + return cannot_follow_call(bookkeeper, graph, args_hs, RESULT) # recursive call from the entry point to itself: ignore them and # just hope the annotations are correct - if (bookkeeper.getdesc(fnobj.graph)._cache.get(None, None) is + if (bookkeeper.getdesc(graph)._cache.get(None, None) is bookkeeper.annotator.translator.graphs[0]): - return variableoftype(lltype.typeOf(fnobj).RESULT) + return variableoftype(RESULT) myorigin = bookkeeper.myorigin() myorigin.__class__ = CallOpOriginFlags # thud fixed = myorigin.read_fixed() tsgraphs_accum = [] - hs_res = bookkeeper.graph_call(fnobj.graph, fixed, args_hs, + hs_res = bookkeeper.graph_call(graph, fixed, args_hs, tsgraphs_accum) myorigin.any_called_graph = tsgraphs_accum[0] @@ -503,6 +510,7 @@ def oosend(hs_c1, hs_name, *args_hs): TYPE = hs_c1.concretetype name = hs_name.const + _, meth = TYPE._lookup(name) graph_list = TYPE._lookup_graphs(name) if not graph_list: # it's a method of a BuiltinType @@ -515,7 +523,14 @@ myorigin = origin) # if hs_c1.is_constant(): ... return hs_res - #import pdb;pdb.set_trace() + elif len(graph_list) == 1: + # like a direct_call + graph = graph_list.pop() + METH = lltype.typeOf(meth) + return hs_c1._call_single_graph(graph, METH.RESULT, hs_c1, *args_hs) # prepend hs_c1 to the args + else: + # like an indirect_call + XXX fixme def getfield(hs_c1, hs_fieldname): S = hs_c1.concretetype.TO Modified: pypy/dist/pypy/jit/hintannotator/test/test_annotator.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/test/test_annotator.py (original) +++ pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Tue Dec 18 14:18:47 2007 @@ -323,6 +323,18 @@ assert hs.concretetype == lltype.Signed assert len(hs.origins) == 5 + def test_simple_method_call(self): + class A: + def ll2(self, x, y, z): + return x + (y + 42) + obj = A() + def ll1(x, y, z): + return obj.ll2(x, y - z, x + y + z) + hs = self.hannotate(ll1, [int, int, int], policy=P_NOVIRTUAL) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + assert len(hs.origins) == 5 + def test_simple_list_operations(self): def ll_function(x, y, index): l = [x] @@ -693,7 +705,6 @@ hs = self.hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) assert not hs.is_green() - def test_indirect_sometimes_residual_pure_red_call(self): def h1(x): return x-2 From arigo at codespeak.net Tue Dec 18 14:22:35 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 Dec 2007 14:22:35 +0100 (CET) Subject: [pypy-svn] r49886 - in pypy/dist/pypy/rpython/memory: gc gctransform Message-ID: <20071218132235.62E90168543@codespeak.net> Author: arigo Date: Tue Dec 18 14:22:29 2007 New Revision: 49886 Modified: pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/gc/semispace.py pypy/dist/pypy/rpython/memory/gctransform/framework.py Log: It seems useful to inline the common calls to malloc_varsize() too. Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Tue Dec 18 14:22:29 2007 @@ -26,6 +26,7 @@ list, chained to each other via their 'forw' header field. """ inline_simple_malloc = True + inline_simple_malloc_varsize = True needs_write_barrier = True prebuilt_gc_objects_are_static_roots = False Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Tue Dec 18 14:22:29 2007 @@ -21,6 +21,7 @@ class SemiSpaceGC(MovingGCBase): _alloc_flavor_ = "raw" inline_simple_malloc = True + inline_simple_malloc_varsize = True needs_zero_gc_pointers = False HDR = lltype.Struct('header', ('forw', llmemory.Address), Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Tue Dec 18 14:22:29 2007 @@ -257,6 +257,28 @@ else: self.malloc_fast_ptr = None + # in some GCs we can also inline the common case of + # malloc_varsize(typeid, length, (3 constant sizes), True, False) + if getattr(GCClass, 'inline_simple_malloc_varsize', False): + # make a copy of this function so that it gets annotated + # independently and the constants are folded inside + malloc_varsize_clear_fast = func_with_new_name( + GCClass.malloc_varsize_clear.im_func, + "malloc_varsize_clear_fast") + s_False = annmodel.SomeBool(); s_False.const = False + s_True = annmodel.SomeBool(); s_True .const = True + self.malloc_varsize_clear_fast_ptr = getfn( + malloc_varsize_clear_fast, + [s_gc, annmodel.SomeInteger(nonneg=True), + annmodel.SomeInteger(nonneg=True), + annmodel.SomeInteger(nonneg=True), + annmodel.SomeInteger(nonneg=True), + annmodel.SomeInteger(nonneg=True), + s_True, s_False], s_gcref, + inline = True) + else: + self.malloc_varsize_clear_fast_ptr = None + if GCClass.moving_gc: self.id_ptr = getfn(GCClass.id.im_func, [s_gc, s_gcref], annmodel.SomeInteger(), @@ -521,11 +543,11 @@ v_length = op.args[-1] c_ofstolength = rmodel.inputconst(lltype.Signed, info.ofstolength) c_varitemsize = rmodel.inputconst(lltype.Signed, info.varitemsize) - malloc_ptr = self.malloc_varsize_clear_ptr -## if op.opname.startswith('zero'): -## malloc_ptr = self.malloc_varsize_clear_ptr -## else: -## malloc_ptr = self.malloc_varsize_clear_ptr + if (self.malloc_varsize_clear_fast_ptr is not None and + c_can_collect.value and not c_has_finalizer.value): + malloc_ptr = self.malloc_varsize_clear_fast_ptr + else: + malloc_ptr = self.malloc_varsize_clear_ptr args = [self.c_const_gc, c_type_id, v_length, c_size, c_varitemsize, c_ofstolength, c_can_collect, c_has_finalizer] From arigo at codespeak.net Tue Dec 18 14:44:32 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 Dec 2007 14:44:32 +0100 (CET) Subject: [pypy-svn] r49887 - pypy/dist/pypy/objspace/std Message-ID: <20071218134432.78E94168547@codespeak.net> Author: arigo Date: Tue Dec 18 14:44:30 2007 New Revision: 49887 Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py pypy/dist/pypy/objspace/std/objspace.py Log: StrDictImplement.get() is called so often that I strongly suspect this to be a useful performance hack. (will check) Modified: pypy/dist/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/dictmultiobject.py Tue Dec 18 14:44:30 2007 @@ -457,6 +457,10 @@ def get(self, w_lookup): space = self.space + # -- This is called extremely often. Hack for performance -- + if type(w_lookup) is space.StringObjectCls: + return self.content.get(w_lookup.unwrap(space), None) + # -- End of performance hack -- w_lookup_type = space.type(w_lookup) if space.is_w(w_lookup_type, space.w_str): return self.content.get(space.str_w(w_lookup), None) Modified: pypy/dist/pypy/objspace/std/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/objspace.py (original) +++ pypy/dist/pypy/objspace/std/objspace.py Tue Dec 18 14:44:30 2007 @@ -190,6 +190,15 @@ else: from pypy.objspace.std import dictobject self.DictObjectCls = dictobject.W_DictObject + assert self.DictObjectCls in self.model.typeorder + + if not self.config.objspace.std.withrope: + from pypy.objspace.std import stringobject + self.StringObjectCls = stringobject.W_StringObject + else: + from pypy.objspace.std import ropeobject + self.StringObjectCls = ropeobject.W_RopeObject + assert self.StringObjectCls in self.model.typeorder # install all the MultiMethods into the space instance for name, mm in self.MM.__dict__.items(): From arigo at codespeak.net Tue Dec 18 15:21:14 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 Dec 2007 15:21:14 +0100 (CET) Subject: [pypy-svn] r49888 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20071218142114.155DA1684FF@codespeak.net> Author: arigo Date: Tue Dec 18 15:21:14 2007 New Revision: 49888 Modified: pypy/dist/pypy/translator/backendopt/constfold.py pypy/dist/pypy/translator/backendopt/test/test_constfold.py Log: In constfold: For the case where link1.target contains only a switch, rewire link1 to go directly to the correct exit based on a constant switch value. This is a situation that occurs typically after inlining. Modified: pypy/dist/pypy/translator/backendopt/constfold.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/constfold.py (original) +++ pypy/dist/pypy/translator/backendopt/constfold.py Tue Dec 18 15:21:14 2007 @@ -130,8 +130,44 @@ else: constants[v2] = v1 +def rewire_link_for_known_exitswitch(link1, llexitvalue): + # For the case where link1.target contains only a switch, rewire link1 + # to go directly to the correct exit based on a constant switch value. + # This is a situation that occurs typically after inlining; see + # test_fold_exitswitch_along_one_path. + block = link1.target + if block.exits[-1].exitcase == "default": + defaultexit = block.exits[-1] + nondefaultexits = block.exits[:-1] + else: + defaultexit = None + nondefaultexits = block.exits + for nextlink in nondefaultexits: + if nextlink.llexitcase == llexitvalue: + break # found -- the result is in 'nextlink' + else: + if defaultexit is None: + return # exit case not found! just ignore the problem here + nextlink = defaultexit + blockmapping = dict(zip(block.inputargs, link1.args)) + newargs = [] + for v in nextlink.args: + if isinstance(v, Variable): + v = blockmapping[v] + newargs.append(v) + link1.target = nextlink.target + link1.args = newargs + def prepare_constant_fold_link(link, constants, splitblocks): block = link.target + if not block.operations: + # when the target block has no operation, there is nothing we can do + # except trying to fold an exitswitch + if block.exitswitch is not None and block.exitswitch in constants: + llexitvalue = constants[block.exitswitch].value + rewire_link_for_known_exitswitch(link, llexitvalue) + return + folded_count = fold_op_list(block.operations, constants, exit_early=True) n = len(block.operations) Modified: pypy/dist/pypy/translator/backendopt/test/test_constfold.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_constfold.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_constfold.py Tue Dec 18 15:21:14 2007 @@ -245,3 +245,31 @@ assert summary(graph) == {'debug_print': 1} constant_fold_graph(graph) assert summary(graph) == {'debug_print': 1} + + +def test_fold_exitswitch_along_one_path(): + def g(n): + if n == 42: + return 5 + else: + return n+1 + def fn(n): + if g(n) == 5: + return 100 + else: + return 0 + + graph, t = get_graph(fn, [int]) + from pypy.translator.backendopt import removenoops, inline + inline.auto_inline_graphs(t, [graph], threshold=999) + constant_fold_graph(graph) + removenoops.remove_same_as(graph) + if conftest.option.view: + t.view() + # check that the graph starts with a condition (which should be 'n==42') + # and that if this condition is true, it goes directly to 'return 100'. + assert len(graph.startblock.exits) == 2 + assert graph.startblock.exits[1].exitcase == True + assert graph.startblock.exits[1].target is graph.returnblock + check_graph(graph, [10], 0, t) + check_graph(graph, [42], 100, t) From arigo at codespeak.net Tue Dec 18 15:47:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 Dec 2007 15:47:09 +0100 (CET) Subject: [pypy-svn] r49889 - in pypy/dist/pypy/translator/backendopt: . test Message-ID: <20071218144709.5A3D516856B@codespeak.net> Author: arigo Date: Tue Dec 18 15:47:08 2007 New Revision: 49889 Modified: pypy/dist/pypy/translator/backendopt/constfold.py pypy/dist/pypy/translator/backendopt/test/test_constfold.py Log: Another case for constfold: after an exitswitch on a variable, we know in all links the value of that variable. Two tests showing when this can be useful (the second one is a common occurrence in lltypesystem/rdict). Modified: pypy/dist/pypy/translator/backendopt/constfold.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/constfold.py (original) +++ pypy/dist/pypy/translator/backendopt/constfold.py Tue Dec 18 15:47:08 2007 @@ -234,11 +234,22 @@ def constant_diffuse(graph): + count = 0 + # after 'exitswitch vexit', replace 'vexit' with the corresponding constant + # if it also appears on the outgoing links + for block in graph.iterblocks(): + vexit = block.exitswitch + if isinstance(vexit, Variable): + for link in block.exits: + if vexit in link.args: + remap = {vexit: Constant(link.llexitcase, + vexit.concretetype)} + link.args = [remap.get(v, v) for v in link.args] + count += 1 # if the same constants appear at the same positions in all links # into a block remove them from the links, remove the corresponding # input variables and introduce equivalent same_as at the beginning # of the block then try to fold the block further - count = 0 for block, links in mkentrymap(graph).iteritems(): if block is graph.startblock: continue Modified: pypy/dist/pypy/translator/backendopt/test/test_constfold.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_constfold.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_constfold.py Tue Dec 18 15:47:08 2007 @@ -273,3 +273,47 @@ assert graph.startblock.exits[1].target is graph.returnblock check_graph(graph, [10], 0, t) check_graph(graph, [42], 100, t) + +def test_knownswitch_after_exitswitch(): + def fn(n): + cond = n > 10 + if cond: + return cond + 5 + else: + return cond + 17 + + graph, t = get_graph(fn, [int]) + from pypy.translator.backendopt import removenoops + removenoops.remove_same_as(graph) + constant_fold_graph(graph) + if conftest.option.view: + t.view() + assert summary(graph) == {'int_gt': 1} + check_graph(graph, [2], 17, t) + check_graph(graph, [42], 6, t) + +def test_coalesce_exitswitchs(): + def g(n): + return n > 5 and n < 20 + def fn(n): + if g(n): + return 100 + else: + return 0 + + graph, t = get_graph(fn, [int]) + from pypy.translator.backendopt import removenoops, inline + inline.auto_inline_graphs(t, [graph], threshold=999) + removenoops.remove_same_as(graph) + constant_fold_graph(graph) + if conftest.option.view: + t.view() + # check that the graph starts with a condition (which should be 'n > 5') + # and that if this condition is false, it goes directly to 'return 0'. + assert summary(graph) == {'int_gt': 1, 'int_lt': 1} + assert len(graph.startblock.exits) == 2 + assert graph.startblock.exits[0].exitcase == False + assert graph.startblock.exits[0].target is graph.returnblock + check_graph(graph, [2], 0, t) + check_graph(graph, [10], 100, t) + check_graph(graph, [42], 0, t) From arigo at codespeak.net Tue Dec 18 15:55:40 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 Dec 2007 15:55:40 +0100 (CET) Subject: [pypy-svn] r49890 - pypy/dist/pypy/config Message-ID: <20071218145540.835371684F4@codespeak.net> Author: arigo Date: Tue Dec 18 15:55:39 2007 New Revision: 49890 Modified: pypy/dist/pypy/config/pypyoption.py Log: Document why we need weakrefs here. Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Tue Dec 18 15:55:39 2007 @@ -185,6 +185,7 @@ "version type objects when changing them", cmdline=None, default=False, + # weakrefs needed, because of get_subclasses() requires=[("translation.rweakref", True)]), BoolOption("withshadowtracking", From arigo at codespeak.net Tue Dec 18 16:01:35 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 Dec 2007 16:01:35 +0100 (CET) Subject: [pypy-svn] r49891 - in pypy/dist/pypy/translator: c locality Message-ID: <20071218150135.A3C1916856A@codespeak.net> Author: arigo Date: Tue Dec 18 16:01:29 2007 New Revision: 49891 Removed: pypy/dist/pypy/translator/locality/ Modified: pypy/dist/pypy/translator/c/genc.py Log: This experiment proved to be not useful. Move it to the svn limbo land. Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Tue Dec 18 16:01:29 2007 @@ -14,7 +14,6 @@ from pypy.rpython.lltypesystem import lltype from pypy.tool.udir import udir from pypy.tool import isolate -from pypy.translator.locality.calltree import CallTree from pypy.translator.c.support import log, c_string_constant from pypy.rpython.typesystem import getfunctionptr from pypy.translator.c import gc @@ -396,11 +395,6 @@ self.funcnodes = funcnodes self.othernodes = othernodes self.path = path - # disabled this for a while, does worsen things -# graph = CallTree(self.funcnodes, self.database) -# graph.simulate() -# graph.optimize() -# self.funcnodes = graph.ordered_funcnodes() def uniquecname(self, name): assert name.endswith('.c') From arigo at codespeak.net Tue Dec 18 16:22:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 Dec 2007 16:22:09 +0100 (CET) Subject: [pypy-svn] r49892 - pypy/dist/pypy/translator/c Message-ID: <20071218152209.53CC516842E@codespeak.net> Author: arigo Date: Tue Dec 18 16:22:08 2007 New Revision: 49892 Modified: pypy/dist/pypy/translator/c/funcgen.py Log: We generate switch { } statements in which break; is never useful :-) Modified: pypy/dist/pypy/translator/c/funcgen.py ============================================================================== --- pypy/dist/pypy/translator/c/funcgen.py (original) +++ pypy/dist/pypy/translator/c/funcgen.py Tue Dec 18 16:22:08 2007 @@ -285,7 +285,7 @@ yield 'case %s:' % self.db.get(link.llexitcase) for op in self.gen_link(link): yield '\t' + op - yield 'break;' + # 'break;' not needed, as gen_link ends in a 'goto' # Emit default case yield 'default:' From arigo at codespeak.net Tue Dec 18 18:05:11 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 Dec 2007 18:05:11 +0100 (CET) Subject: [pypy-svn] r49894 - pypy/dist/pypy/jit/hintannotator Message-ID: <20071218170511.C9CCC168539@codespeak.net> Author: arigo Date: Tue Dec 18 18:05:10 2007 New Revision: 49894 Modified: pypy/dist/pypy/jit/hintannotator/model.py Log: Fix SyntaxError. Modified: pypy/dist/pypy/jit/hintannotator/model.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/model.py (original) +++ pypy/dist/pypy/jit/hintannotator/model.py Tue Dec 18 18:05:10 2007 @@ -530,7 +530,7 @@ return hs_c1._call_single_graph(graph, METH.RESULT, hs_c1, *args_hs) # prepend hs_c1 to the args else: # like an indirect_call - XXX fixme + XXX-fixme def getfield(hs_c1, hs_fieldname): S = hs_c1.concretetype.TO From arigo at codespeak.net Tue Dec 18 18:39:21 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 Dec 2007 18:39:21 +0100 (CET) Subject: [pypy-svn] r49896 - in pypy/dist/pypy/translator/llvm: . module test Message-ID: <20071218173921.6ECB2168559@codespeak.net> Author: arigo Date: Tue Dec 18 18:39:20 2007 New Revision: 49896 Added: pypy/dist/pypy/translator/llvm/test/test_new_gc.py Modified: pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/gc.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/module/genexterns.c pypy/dist/pypy/translator/llvm/test/test_symbolic.py pypy/dist/pypy/translator/llvm/typedefnode.py Log: Some small progress in genllvm towards supporting the framework GCs. Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Tue Dec 18 18:39:20 2007 @@ -12,7 +12,7 @@ from pypy.translator.llvm.arraynode import ArrayNode, StrArrayNode, \ VoidArrayNode, ArrayNoLengthNode, DebugStrNode -from pypy.rpython.lltypesystem import lltype, llmemory, rffi +from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi from pypy.objspace.flow.model import Constant, Variable from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic from pypy.rlib.objectmodel import CDefinedIntSymbolic @@ -143,76 +143,50 @@ for type_ in types: self.prepare_type(type_) - def prepare_constant(self, type_, value): - if isinstance(type_, lltype.Primitive): - #log.prepareconstant(value, "(is primitive)") - if type_ is llmemory.Address: + def prepare_constant(self, ct, value): + # always add type (it is safe) + self.prepare_type(ct) + + if isinstance(ct, lltype.Primitive): + # special cases for address + if ct is llmemory.Address: # prepare the constant data which this address references - assert isinstance(value, llmemory.fakeaddress) - if value: - self.prepare_constant(lltype.typeOf(value.ptr), value.ptr) - return + fakedaddress = value + if fakedaddress: + ptrvalue = fakedaddress.ptr + ct = lltype.typeOf(ptrvalue) + self.prepare_constant(ct, ptrvalue) + ##elif ct is llmemory.WeakGcAddress: + ## return # XXX sometime soon - if isinstance(type_, lltype.Ptr) and isinstance(value._obj, int): + else: + if isinstance(value, llmemory.AddressOffset): + self.prepare_offset(value) return - - if isinstance(type_, lltype.Ptr): - type_ = type_.TO - value = value._obj - - #log.prepareconstant("preparing ptr", value) + if isinstance(ct, lltype.Ptr): + ptrvalue = value + ct = ct.TO + value = ptrvalue._obj # we dont need a node for nulls if value is None: return + # we dont need a node for tagged pointers + if isinstance(value, int): + return # we can share data via pointers + assert isinstance(ct, lltype.ContainerType) if value not in self.obj2node: - self.addpending(value, self.create_constant_node(type_, value)) - - # always add type (it is safe) - self.prepare_type(type_) + self.addpending(value, self.create_constant_node(ct, value)) def prepare_arg_value(self, const_or_var): """if const_or_var is not already in a dictionary self.obj2node, the appropriate node gets constructed and gets added to self._pendingsetup and to self.obj2node""" if isinstance(const_or_var, Constant): - ct = const_or_var.concretetype - if isinstance(ct, lltype.Primitive): - # special cases for address - if ct is llmemory.Address: - fakedaddress = const_or_var.value - if fakedaddress: - ptrvalue = fakedaddress.ptr - ct = lltype.typeOf(ptrvalue) - else: - return -## elif ct is llmemory.WeakGcAddress: -## return # XXX sometime soon - - else: - if isinstance(const_or_var.value, llmemory.AddressOffset): - self.prepare_offset(const_or_var.value) - return - else: - assert isinstance(ct, lltype.Ptr), "Preparation of non primitive and non pointer" - ptrvalue = const_or_var.value - - value = ptrvalue._obj - - if isinstance(value, int): - return - - # Only prepare root values at this point - if isinstance(ct, lltype.Array) or isinstance(ct, lltype.Struct): - p, c = lltype.parentlink(value) - if p is None: - #log.prepareargvalue("skipping preparing non root", value) - return - - if value is not None and value not in self.obj2node: - self.addpending(value, self.create_constant_node(ct.TO, value)) + self.prepare_constant(const_or_var.concretetype, + const_or_var.value) else: assert isinstance(const_or_var, Variable) @@ -226,7 +200,11 @@ if isinstance(offset, llmemory.CompositeOffset): for value in offset.offsets: self.prepare_offset(value) - else: + elif isinstance(offset, llarena.RoundedUpForAllocation): + print '<<<<<<<<<<<<<<<', offset.basesize + #import pdb; pdb.set_trace() + self.prepare_offset(offset.basesize) + elif hasattr(offset, 'TYPE'): self.prepare_type(offset.TYPE) def setup_all(self): @@ -621,6 +599,11 @@ return repr def repr_offset(self, value): + if isinstance(value, llarena.RoundedUpForAllocation): + # XXX not supported when used in a CompositeOffset + r_basesize = self.repr_offset(value.basesize) + return "((%s + 7) & ~ 7)" + from_, indices, to = self.get_offset(value, []) # void array special cases Modified: pypy/dist/pypy/translator/llvm/gc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/gc.py (original) +++ pypy/dist/pypy/translator/llvm/gc.py Tue Dec 18 18:39:20 2007 @@ -36,7 +36,8 @@ def op_collect(self, codewriter, opr): raise Exception, 'GcPolicy should not be used directly' - def new(db, gcpolicy=None): + def new(db, config): + gcpolicy = config.translation.gctransformer if gcpolicy == 'boehm': gcpolicy = BoehmGcPolicy(db) elif gcpolicy == 'ref': @@ -150,3 +151,7 @@ def gc_libraries(self): return ['pthread'] + + def get_real_weakref_type(self): + from pypy.rpython.memory.gctransform import framework + return framework.WEAKREF Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Tue Dec 18 18:39:20 2007 @@ -120,7 +120,7 @@ c_db = cbuild.generate_graphs_for_llinterp() self.db = Database(self, self.translator) - self.db.gcpolicy = GcPolicy.new(self.db, self.config.translation.gc) + self.db.gcpolicy = GcPolicy.new(self.db, self.config) # get entry point entry_point = self.get_entry_point(func) Modified: pypy/dist/pypy/translator/llvm/module/genexterns.c ============================================================================== --- pypy/dist/pypy/translator/llvm/module/genexterns.c (original) +++ pypy/dist/pypy/translator/llvm/module/genexterns.c Tue Dec 18 18:39:20 2007 @@ -20,6 +20,15 @@ memset(ptr, 0, size); } +/* alignment for arena-based garbage collectors: the following line + enforces an alignment of 8. This number 8 is also hard-coded in + database.py:repr_offset(). */ +#define MEMORY_ALIGNMENT 8 +long ROUND_UP_FOR_ALLOCATION(long x) { + return (x + (MEMORY_ALIGNMENT-1)) & ~(MEMORY_ALIGNMENT-1); +} + + char *RPython_StartupCode() { // is there any garbage collection / memory management initialisation __GC_STARTUP_CODE__ Added: pypy/dist/pypy/translator/llvm/test/test_new_gc.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/llvm/test/test_new_gc.py Tue Dec 18 18:39:20 2007 @@ -0,0 +1,16 @@ +import sys + +import py +from pypy.translator.llvm.test.runtest import * + + +def test_1(): + py.test.skip("in-progress") + def fn(n): + d = {} + for i in range(n): + d[i] = str(i) + return d[n//2] + + mod, f = compile_test(fn, [int], gcpolicy="semispace") + assert f(5000) == fn(5000) Modified: pypy/dist/pypy/translator/llvm/test/test_symbolic.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_symbolic.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_symbolic.py Tue Dec 18 18:39:20 2007 @@ -92,7 +92,7 @@ assert res == 51 def test_computed_int_symbolic(): - py.test.skip("'GCHeaderOffset' object has no attribute 'TYPE'") + py.test.skip("XXX compile_llvm() fails to link") too_early = True def compute_fn(): assert not too_early Modified: pypy/dist/pypy/translator/llvm/typedefnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/typedefnode.py (original) +++ pypy/dist/pypy/translator/llvm/typedefnode.py Tue Dec 18 18:39:20 2007 @@ -1,4 +1,4 @@ -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, llmemory from pypy.translator.llvm.node import Node class TypeDefNode(Node): @@ -151,7 +151,11 @@ return ExtOpaqueTypeNode(db, TYPE) else: return OpaqueTypeNode(db, TYPE) - + + elif TYPE == llmemory.WeakRef: + REALT = db.gcpolicy.get_real_weakref_type() + return create_typedef_node(db, REALT) + else: assert False, "create_typedef_node %s %s" % (TYPE, type(TYPE)) From antocuni at codespeak.net Tue Dec 18 18:58:12 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 18 Dec 2007 18:58:12 +0100 (CET) Subject: [pypy-svn] r49898 - in pypy/dist/pypy: jit/hintannotator jit/hintannotator/test translator Message-ID: <20071218175812.41651168567@codespeak.net> Author: antocuni Date: Tue Dec 18 18:58:11 2007 New Revision: 49898 Modified: pypy/dist/pypy/jit/hintannotator/model.py pypy/dist/pypy/jit/hintannotator/test/test_annotator.py pypy/dist/pypy/translator/simplify.py Log: - add some new tests that calls methods instead of functions, to make them test oosend in ootype - make these tests passing Modified: pypy/dist/pypy/jit/hintannotator/model.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/model.py (original) +++ pypy/dist/pypy/jit/hintannotator/model.py Tue Dec 18 18:58:11 2007 @@ -3,7 +3,7 @@ from pypy.jit.hintannotator.bookkeeper import getbookkeeper from pypy.rpython.lltypesystem import lltype, lloperation from pypy.rpython.ootypesystem import ootype -from pypy.translator.simplify import get_funcobj +from pypy.translator.simplify import get_funcobj, get_functype UNARY_OPERATIONS = """same_as hint getfield setfield getsubstruct getarraysize getinteriorfield getinteriorarraysize setinteriorfield @@ -107,6 +107,13 @@ retdeps.append(v_callable) elif self.spaceop.opname == 'oosend': args = self.spaceop.args[1:] + methname = self.spaceop.args[0].value + TYPE = self.spaceop.args[1].concretetype + graphs = TYPE._lookup_graphs(methname) + if len(graphs) > 1: + v_self = self.spaceop.args[1] + retdeps = greenorigindependencies.setdefault(self, []) + retdeps.append(v_self) else: raise AssertionError(self.spaceop.opname) @@ -174,7 +181,6 @@ c.__dict__.update(self.__dict__) return c - class SomeLLAbstractConstant(SomeLLAbstractValue): " color: dont know yet.. " @@ -409,9 +415,13 @@ args_hs = args_hs[:-1] assert hs_graph_list.is_constant() graph_list = hs_graph_list.const + FUNC = get_functype(hs_v1.concretetype) + return hs_v1._call_multiple_graphs(graph_list, FUNC.RESULT, *args_hs) + + def _call_multiple_graphs(hs_v1, graph_list, RESULT, *args_hs): if graph_list is None: # cannot follow indirect calls to unknown targets - return variableoftype(hs_v1.concretetype.TO.RESULT) + return variableoftype(RESULT) bookkeeper = getbookkeeper() myorigin = bookkeeper.myorigin() @@ -430,12 +440,9 @@ # function return annmodel.unionof(hs_res, bookkeeper.current_op_binding()) - def oosend(hs_v1, hs_name, *args_hs): + def oosend(hs_v1, hs_name, *args_hs): RESTYPE = getbookkeeper().current_op_concretetype() - if RESTYPE is not ootype.Void: - return SomeLLAbstractVariable(RESTYPE) - else: - return # XXX: is it right? + return variableoftype(RESTYPE) class __extend__(SomeLLAbstractConstant): @@ -494,14 +501,8 @@ myorigin.any_called_graph = tsgraphs_accum[0] if isinstance(hs_res, SomeLLAbstractConstant): - hs_res.myorigin = myorigin + hs_res.myorigin = myorigin -## elif fnobj.graph.name.startswith('ll_stritem'): -## if isinstance(hs_res, SomeLLAbstractVariable): -## print hs_res -## import pdb; pdb.set_trace() - - # we need to make sure that hs_res does not become temporarily less # general as a result of calling another specialized version of the # function @@ -511,13 +512,14 @@ TYPE = hs_c1.concretetype name = hs_name.const _, meth = TYPE._lookup(name) + METH = lltype.typeOf(meth) graph_list = TYPE._lookup_graphs(name) if not graph_list: # it's a method of a BuiltinType bk = getbookkeeper() origin = bk.myorigin() d = setadd(hs_c1.origins, origin) - RESTYPE = bk.current_op_concretetype() + RESTYPE = bk.current_op_concretetype() hs_res = SomeLLAbstractConstant(RESTYPE, d, eager_concrete = hs_c1.eager_concrete, myorigin = origin) @@ -526,11 +528,10 @@ elif len(graph_list) == 1: # like a direct_call graph = graph_list.pop() - METH = lltype.typeOf(meth) return hs_c1._call_single_graph(graph, METH.RESULT, hs_c1, *args_hs) # prepend hs_c1 to the args else: # like an indirect_call - XXX-fixme + return hs_c1._call_multiple_graphs(graph_list, METH.RESULT, hs_c1, *args_hs) def getfield(hs_c1, hs_fieldname): S = hs_c1.concretetype.TO Modified: pypy/dist/pypy/jit/hintannotator/test/test_annotator.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/test/test_annotator.py (original) +++ pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Tue Dec 18 18:58:11 2007 @@ -335,6 +335,18 @@ assert hs.concretetype == lltype.Signed assert len(hs.origins) == 5 + def test_simple_method_call_var(self): + class A: + def ll2(self, x, y, z): + return x + (y + 42) + def ll1(x, y, z): + obj = A() + return obj.ll2(x, y - z, x + y + z) + hs = self.hannotate(ll1, [int, int, int], policy=P_NOVIRTUAL) + assert isinstance(hs, SomeLLAbstractConstant) + assert hs.concretetype == lltype.Signed + assert len(hs.origins) == 5 + def test_simple_list_operations(self): def ll_function(x, y, index): l = [x] @@ -705,6 +717,23 @@ hs = self.hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) assert not hs.is_green() + def test_indirect_method_yellow_call(self): + class A: + def h1(self, n): + return 123 + + class B(A): + def h1(self, n): + return 456 + + lst = [A(), B()] + + def ll_function(n, m): + obj = hint(lst, deepfreeze=True)[m] + return obj.h1(n) + hs = self.hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) + assert not hs.is_green() + def test_indirect_sometimes_residual_pure_red_call(self): def h1(x): return x-2 @@ -1040,3 +1069,6 @@ f = Foo() f.bar() hs = self.hannotate(fn, [], policy=P_OOPSPEC_NOVIRTUAL) + + def test_simple_method_call_var(self): + py.test.skip('fixme!') Modified: pypy/dist/pypy/translator/simplify.py ============================================================================== --- pypy/dist/pypy/translator/simplify.py (original) +++ pypy/dist/pypy/translator/simplify.py Tue Dec 18 18:58:11 2007 @@ -10,7 +10,8 @@ from pypy.objspace.flow.model import Variable, Constant, Block, Link from pypy.objspace.flow.model import c_last_exception from pypy.objspace.flow.model import checkgraph, traverse, mkentrymap -from pypy.rpython.lltypesystem import lloperation +from pypy.rpython.lltypesystem import lloperation, lltype +from pypy.rpython.ootypesystem import ootype def get_funcobj(func): """ @@ -21,6 +22,12 @@ else: return func # ootypesystem +def get_functype(TYPE): + if isinstance(TYPE, lltype.Ptr): + return TYPE.TO + elif isinstance(TYPE, ootype.StaticMethod): + return TYPE + assert False def get_graph(arg, translator): from pypy.translator.translator import graphof From arigo at codespeak.net Tue Dec 18 19:57:16 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 18 Dec 2007 19:57:16 +0100 (CET) Subject: [pypy-svn] r49899 - pypy/dist/pypy/translator/llvm Message-ID: <20071218185716.CC3B0168556@codespeak.net> Author: arigo Date: Tue Dec 18 19:57:15 2007 New Revision: 49899 Modified: pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/funcnode.py pypy/dist/pypy/translator/llvm/gc.py pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/modwrapper.py pypy/dist/pypy/translator/llvm/node.py pypy/dist/pypy/translator/llvm/opwriter.py Log: More progress. Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Tue Dec 18 19:57:15 2007 @@ -36,6 +36,7 @@ def __init__(self, genllvm, translator): self.genllvm = genllvm self.translator = translator + self.gctransformer = None self.obj2node = {} self._pendingsetup = [] self._tmpcount = 1 @@ -180,7 +181,7 @@ if value not in self.obj2node: self.addpending(value, self.create_constant_node(ct, value)) - def prepare_arg_value(self, const_or_var): + def prepare_arg(self, const_or_var): """if const_or_var is not already in a dictionary self.obj2node, the appropriate node gets constructed and gets added to self._pendingsetup and to self.obj2node""" @@ -189,27 +190,22 @@ const_or_var.value) else: assert isinstance(const_or_var, Variable) + self.prepare_type(const_or_var.concretetype) - def prepare_arg(self, const_or_var): - #log.prepare(const_or_var) - self.prepare_type(const_or_var.concretetype) - self.prepare_arg_value(const_or_var) - def prepare_offset(self, offset): if isinstance(offset, llmemory.CompositeOffset): for value in offset.offsets: self.prepare_offset(value) elif isinstance(offset, llarena.RoundedUpForAllocation): - print '<<<<<<<<<<<<<<<', offset.basesize - #import pdb; pdb.set_trace() self.prepare_offset(offset.basesize) elif hasattr(offset, 'TYPE'): self.prepare_type(offset.TYPE) def setup_all(self): + self.gcpolicy.setup() while self._pendingsetup: - node = self._pendingsetup.pop(0) + node = self._pendingsetup.pop() #log.settingup(node) node.setup() @@ -602,7 +598,10 @@ if isinstance(value, llarena.RoundedUpForAllocation): # XXX not supported when used in a CompositeOffset r_basesize = self.repr_offset(value.basesize) - return "((%s + 7) & ~ 7)" + # XXX XXX XXX we hit an llvm assertion with the following + # expression! for now let's not align anything :-( + #return "and(i32 add(i32 %s, i32 7), i32 -8)" % r_basesize + return r_basesize from_, indices, to = self.get_offset(value, []) Modified: pypy/dist/pypy/translator/llvm/funcnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/funcnode.py (original) +++ pypy/dist/pypy/translator/llvm/funcnode.py Tue Dec 18 19:57:15 2007 @@ -1,4 +1,4 @@ -from pypy.objspace.flow.model import Block, Constant, Link +from pypy.objspace.flow.model import Block, Constant, Link, copygraph, Variable from pypy.objspace.flow.model import mkentrymap, c_last_exception from pypy.rpython.lltypesystem import lltype from pypy.translator.llvm.node import FuncNode @@ -40,14 +40,45 @@ # ______________________________________________________________________ # main entry points from genllvm - def post_setup_transform(self): - remove_double_links(self.db.translator.annotator, self.graph) - no_links_to_startblock(self.graph) + def patch_graph(self): + graph = self.graph + if self.db.gctransformer: + # inline the GC helpers (malloc, write_barrier) into + # a copy of the graph + graph = copygraph(graph, shallow=True) + self.db.gctransformer.inline_helpers(graph) + # the 'gc_reload_possibly_moved' operations make the graph not + # really SSA. Fix them now. + for block in graph.iterblocks(): + rename = {} + for op in list(block.operations): + if rename: + op.args = [rename.get(v, v) for v in op.args] + if op.opname == 'gc_reload_possibly_moved': + v_newaddr, v_targetvar = op.args + assert isinstance(v_targetvar.concretetype, lltype.Ptr) + v_newptr = Variable() + v_newptr.concretetype = v_targetvar.concretetype + op.opname = 'cast_adr_to_ptr' + op.args = [v_newaddr] + op.result = v_newptr + rename[v_targetvar] = v_newptr + if rename: + block.exitswitch = rename.get(block.exitswitch, + block.exitswitch) + for link in block.exits: + link.args = [rename.get(v, v) for v in link.args] + # fix special cases that llvm can't handle + remove_double_links(self.db.translator.annotator, graph) + no_links_to_startblock(graph) + return graph def writedecl(self, codewriter): codewriter.declare(self.getdecl()) def writeimpl(self, codewriter): + self.oldgraph = self.graph + self.graph = self.patch_graph() graph = self.graph log.writeimpl(graph.name) codewriter.openfunc(self.getdecl()) @@ -69,6 +100,8 @@ codewriter._indent('call void @abort()') codewriter._indent('unreachable') codewriter.closefunc() + self.graph = self.oldgraph + del self.oldgraph # ______________________________________________________________________ # writing helpers for entry points Modified: pypy/dist/pypy/translator/llvm/gc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/gc.py (original) +++ pypy/dist/pypy/translator/llvm/gc.py Tue Dec 18 19:57:15 2007 @@ -11,6 +11,9 @@ n_malloced = 0 def __init__(self, db): raise Exception, 'GcPolicy should not be used directly' + + def setup(self): + pass def genextern_code(self): return '' @@ -143,9 +146,15 @@ def __init__(self, db): self.db = db + def setup(self): + c_fnptr = self.db.gctransformer.frameworkgc_setup_ptr + self.db.prepare_arg(c_fnptr) + def genextern_code(self): - r = '' - r += '#define __GC_STARTUP_CODE__\n' + fnptr = self.db.gctransformer.frameworkgc_setup_ptr.value + fnnode = self.db.obj2node[fnptr._obj] + r = 'void %s(void); /* forward declaration */\n' % (fnnode.name[1:],) + r += '#define __GC_STARTUP_CODE__ %s();\n' % (fnnode.name[1:],) r += '#define __GC_SETUP_CODE__\n' return r Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Tue Dec 18 19:57:15 2007 @@ -121,6 +121,7 @@ self.db = Database(self, self.translator) self.db.gcpolicy = GcPolicy.new(self.db, self.config) + self.db.gctransformer = c_db.gctransformer # get entry point entry_point = self.get_entry_point(func) @@ -138,9 +139,6 @@ self.translator.rtyper.specialize_more_blocks() self.db.setup_all() self._checkpoint('setup_all externs') - - for node in self.db.getnodes(): - node.post_setup_transform() self._print_node_stats() @@ -168,7 +166,7 @@ bk = self.translator.annotator.bookkeeper ptr = getfunctionptr(bk.getdesc(func).getuniquegraph()) c = inputconst(lltype.typeOf(ptr), ptr) - self.db.prepare_arg_value(c) + self.db.prepare_arg(c) # ensure unqiue entry node name for testing entry_node = self.db.obj2node[c.value._obj] Modified: pypy/dist/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/dist/pypy/translator/llvm/modwrapper.py (original) +++ pypy/dist/pypy/translator/llvm/modwrapper.py Tue Dec 18 19:57:15 2007 @@ -22,9 +22,10 @@ rpyexc_fetch_type.argtypes = [] rpyexc_fetch_type.restype = ctypes.c_void_p -GC_get_heap_size_wrapper = _c.GC_get_heap_size -GC_get_heap_size_wrapper.argtypes = [] -GC_get_heap_size_wrapper.restype = ctypes.c_int +if hasattr(_c, 'GC_get_heap_size'): + GC_get_heap_size_wrapper = _c.GC_get_heap_size + GC_get_heap_size_wrapper.argtypes = [] + GC_get_heap_size_wrapper.restype = ctypes.c_int startup_code = _c.ctypes_RPython_StartupCode startup_code.argtypes = [] Modified: pypy/dist/pypy/translator/llvm/node.py ============================================================================== --- pypy/dist/pypy/translator/llvm/node.py (original) +++ pypy/dist/pypy/translator/llvm/node.py Tue Dec 18 19:57:15 2007 @@ -29,9 +29,6 @@ def setup(self): pass - def post_setup_transform(self): - pass - def writesetupcode(self, codewriter): " pre entry-point setup " pass Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Tue Dec 18 19:57:15 2007 @@ -591,5 +591,5 @@ def debug_llinterpcall(self, opr): self.codewriter.call(None, "void", "@abort", [], []) # cheat llvm - self.codewriter.cast(opr.retref, opr.rettype, 'null', opr.rettype) + self.codewriter.cast(opr.retref, opr.rettype, 'undef', opr.rettype) From rxe at codespeak.net Tue Dec 18 21:11:24 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 18 Dec 2007 21:11:24 +0100 (CET) Subject: [pypy-svn] r49900 - pypy/dist/pypy/translator/tool Message-ID: <20071218201124.877CD16856E@codespeak.net> Author: rxe Date: Tue Dec 18 21:11:22 2007 New Revision: 49900 Modified: pypy/dist/pypy/translator/tool/cbuild.py Log: run the same optimization flags we do for genllvm. feel free to revert this to give genllvm back its glory :-) Modified: pypy/dist/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/dist/pypy/translator/tool/cbuild.py (original) +++ pypy/dist/pypy/translator/tool/cbuild.py Tue Dec 18 21:11:22 2007 @@ -367,8 +367,8 @@ self.libraries.append('m') if 'pthread' not in self.libraries: self.libraries.append('pthread') - self.compile_extra += ['-O2', '-pthread'] - self.link_extra += ['-pthread'] + self.compile_extra += ['-O3', '-fomit-frame-pointer', '-pthread'] + self.link_extra += ['-O3', '-pthread'] if sys.platform == 'win32': self.link_extra += ['/DEBUG'] # generate .pdb file if sys.platform == 'darwin': From cfbolz at codespeak.net Tue Dec 18 21:22:48 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 18 Dec 2007 21:22:48 +0100 (CET) Subject: [pypy-svn] r49901 - in pypy/branch/applevel-ctypes/pypy/module/_ffi: . test Message-ID: <20071218202248.6873B16856C@codespeak.net> Author: cfbolz Date: Tue Dec 18 21:22:47 2007 New Revision: 49901 Modified: pypy/branch/applevel-ctypes/pypy/module/_ffi/__init__.py pypy/branch/applevel-ctypes/pypy/module/_ffi/app_ffi.py pypy/branch/applevel-ctypes/pypy/module/_ffi/interp_ffi.py pypy/branch/applevel-ctypes/pypy/module/_ffi/structure.py pypy/branch/applevel-ctypes/pypy/module/_ffi/test/test__ffi.py Log: refactor _ffi structure handling: don't recompute the offsets every time a structure instances is created (!), only do it every time a new structure type is created. Modified: pypy/branch/applevel-ctypes/pypy/module/_ffi/__init__.py ============================================================================== --- pypy/branch/applevel-ctypes/pypy/module/_ffi/__init__.py (original) +++ pypy/branch/applevel-ctypes/pypy/module/_ffi/__init__.py Tue Dec 18 21:22:47 2007 @@ -12,6 +12,7 @@ interpleveldefs = { 'CDLL' : 'interp_ffi.W_CDLL', 'FuncPtr' : 'interp_ffi.W_FuncPtr', + 'Structure' : 'structure.W_Structure', 'StructureInstance' : 'structure.W_StructureInstance', 'ArrayInstance' : 'array.W_ArrayInstance', '_get_type' : 'interp_ffi._w_get_type', @@ -19,7 +20,5 @@ } appleveldefs = { - 'Structure' : 'app_ffi.Structure', 'Array' : 'app_ffi.Array', - #'StructureInstance' : 'app_ffi.StructureInstance', } Modified: pypy/branch/applevel-ctypes/pypy/module/_ffi/app_ffi.py ============================================================================== --- pypy/branch/applevel-ctypes/pypy/module/_ffi/app_ffi.py (original) +++ pypy/branch/applevel-ctypes/pypy/module/_ffi/app_ffi.py Tue Dec 18 21:22:47 2007 @@ -1,24 +1,5 @@ # NOT_RPYTHON -class Structure(object): - def check_fields(self, fields): - import _ffi - for name, letter in fields: - _ffi._get_type(letter) - - def __init__(self, fields): - self.check_fields(fields) - self.fields = fields - - def __call__(self, *args, **kwds): - from _ffi import StructureInstance - if args: - if len(args) > 1: - raise TypeError("Can give at most one non-keyword argument") - if kwds: - raise TypeError("Keyword arguments not allowed when passing address argument") - return StructureInstance(self, args[0], None) - return StructureInstance(self, None, kwds) class Array(object): def __init__(self, of): Modified: pypy/branch/applevel-ctypes/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/branch/applevel-ctypes/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/branch/applevel-ctypes/pypy/module/_ffi/interp_ffi.py Tue Dec 18 21:22:47 2007 @@ -124,14 +124,14 @@ __new__ = interp2app(descr_new_cdll), ptr = interp2app(W_CDLL.ptr), __doc__ = """ C Dynamically loaded library -use CDLL(libname) to create handle to C library (argument is processed the -same way as dlopen processes it). On such library call: +use CDLL(libname) to create a handle to a C library (the argument is processed +the same way as dlopen processes it). On such a library you can call: lib.ptr(func_name, argtype_list, restype) where argtype_list is a list of single characters and restype is a single -character. Character meanings are more or less the same as in struct module, -except that s has trailing \x00 added, while p is considered a raw buffer. -""" +character. The character meanings are more or less the same as in the struct +module, except that s has trailing \x00 added, while p is considered a raw +buffer.""" ) def pack_pointer(space, add_arg, argdesc, w_arg, push_func): Modified: pypy/branch/applevel-ctypes/pypy/module/_ffi/structure.py ============================================================================== --- pypy/branch/applevel-ctypes/pypy/module/_ffi/structure.py (original) +++ pypy/branch/applevel-ctypes/pypy/module/_ffi/structure.py Tue Dec 18 21:22:47 2007 @@ -3,9 +3,9 @@ to app-level with apropriate interface """ -from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable,\ - Arguments -from pypy.interpreter.gateway import interp2app +from pypy.interpreter.baseobjspace import W_Root, Wrappable +from pypy.interpreter.gateway import interp2app, ObjSpace +from pypy.interpreter.argument import Arguments from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.rpython.lltypesystem import lltype, rffi from pypy.interpreter.error import OperationError, wrap_oserror @@ -13,7 +13,7 @@ # the other one is in rlib/libffi, we should refactor it to reuse the same # logic, I'll not touch it by now, and refactor it later from pypy.module.struct.nativefmttable import native_fmttable as struct_native_fmttable -from pypy.module._ffi.interp_ffi import wrap_value, unwrap_value, _get_type +from pypy.module._ffi.interp_ffi import wrap_value, unwrap_value, _get_type, TYPEMAP native_fmttable = {} for key, value in struct_native_fmttable.items(): @@ -45,55 +45,95 @@ size += field_desc['size'] return size, pos + +class W_Structure(Wrappable): + def __init__(self, space, w_fields): + fields = unpack_fields(space, w_fields) + name_to_offset = {} + for i in range(len(fields)): + name, letter = fields[i] + if letter not in TYPEMAP: + raise OperationError(space.w_ValueError, space.wrap( + "Unkown type letter %s" % (letter,))) + if name in name_to_offset: + raise OperationError(space.w_ValueError, space.wrap( + "duplicate field name %s" % (name, ))) + name_to_offset[name] = i + size, pos = size_and_pos(fields) + self.size = size + self.ll_positions = pos + self.fields = fields + self.name_to_offset = name_to_offset + + def descr_call(self, space, __args__): + args_w, kwargs_w = __args__.unpack() + if args_w: + if len(args_w) > 1: + raise OperationError( + space.w_TypeError, + space.wrap("Can give at most one non-keyword argument")) + if kwargs_w: + raise OperationError( + space.w_TypeError, + space.wrap("Keyword arguments not allowed when passing address argument")) + return W_StructureInstance(space, self, args_w[0], None) + return W_StructureInstance(space, self, None, kwargs_w) + +def descr_new_structure(space, w_type, w_fields): + return W_Structure(space, w_fields) + +W_Structure.typedef = TypeDef( + 'Structure', + __new__ = interp2app(descr_new_structure), + __call__ = interp2app(W_Structure.descr_call, + unwrap_spec=['self', ObjSpace, Arguments]), +) +W_Structure.typedef.acceptable_as_base_class = False + def push_field(self, num, value): - ptr = rffi.ptradd(self.ll_buffer, self.ll_positions[num]) + ptr = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[num]) TP = lltype.typeOf(value) T = lltype.Ptr(rffi.CArray(TP)) rffi.cast(T, ptr)[0] = value push_field._annspecialcase_ = 'specialize:argtype(2)' def cast_pos(self, i, ll_t): - pos = rffi.ptradd(self.ll_buffer, self.ll_positions[i]) + pos = rffi.ptradd(self.ll_buffer, self.shape.ll_positions[i]) TP = lltype.Ptr(rffi.CArray(ll_t)) return rffi.cast(TP, pos)[0] cast_pos._annspecialcase_ = 'specialize:arg(2)' class W_StructureInstance(Wrappable): - def __init__(self, space, w_shape, w_address, w_fieldinits): + def __init__(self, space, shape, w_address, fieldinits_w): self.free_afterwards = False - w_fields = space.getattr(w_shape, space.wrap('fields')) - fields = unpack_fields(space, w_fields) - size, pos = size_and_pos(fields) - self.fields = fields - if space.is_true(w_address): + self.shape = shape + if w_address is not None: self.ll_buffer = rffi.cast(rffi.VOIDP, space.int_w(w_address)) else: self.free_afterwards = True - self.ll_buffer = lltype.malloc(rffi.VOIDP.TO, size, flavor='raw', + self.ll_buffer = lltype.malloc(rffi.VOIDP.TO, shape.size, flavor='raw', zero=True) - self.ll_positions = pos - if space.is_true(w_fieldinits): - for w_field in space.unpackiterable(w_fieldinits): - w_value = space.getitem(w_fieldinits, w_field) - self.setattr(space, space.str_w(w_field), w_value) + if fieldinits_w: + for field, w_value in fieldinits_w.iteritems(): + self.setattr(space, field, w_value) + + def getindex(self, space, attr): + try: + return self.shape.name_to_offset[attr] + except KeyError: + raise OperationError(space.w_AttributeError, space.wrap( + "C Structure has no attribute %s" % attr)) def getattr(self, space, attr): - if attr.startswith('tm'): - pass - for i in range(len(self.fields)): - name, c = self.fields[i] - if name == attr: - return wrap_value(space, cast_pos, self, i, c) - raise OperationError(space.w_AttributeError, space.wrap( - "C Structure has no attribute %s" % attr)) + i = self.getindex(space, attr) + _, c = self.shape.fields[i] + return wrap_value(space, cast_pos, self, i, c) getattr.unwrap_spec = ['self', ObjSpace, str] def setattr(self, space, attr, w_value): - for i in range(len(self.fields)): - name, c = self.fields[i] - if name == attr: - unwrap_value(space, push_field, self, i, c, w_value, None) - return + i = self.getindex(space, attr) + _, c = self.shape.fields[i] + unwrap_value(space, push_field, self, i, c, w_value, None) setattr.unwrap_spec = ['self', ObjSpace, str, W_Root] def __del__(self): @@ -113,3 +153,5 @@ __setattr__ = interp2app(W_StructureInstance.setattr), buffer = GetSetProperty(W_StructureInstance.getbuffer), ) +W_StructureInstance.typedef.acceptable_as_base_class = False + Modified: pypy/branch/applevel-ctypes/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- pypy/branch/applevel-ctypes/pypy/module/_ffi/test/test__ffi.py (original) +++ pypy/branch/applevel-ctypes/pypy/module/_ffi/test/test__ffi.py Tue Dec 18 21:22:47 2007 @@ -258,9 +258,7 @@ raises(ValueError, "lib.ptr('get_char', ['x'], None)") raises(ValueError, "lib.ptr('get_char', [], 'x')") raises(ValueError, "_ffi.Structure(['x1', 'xx'])") - S = _ffi.Structure([('x1', 'i')]) - S.fields[0] = ('x1', 'xx') - raises(ValueError, "S()") + raises(ValueError, _ffi.Structure, [('x1', 'xx')]) raises(ValueError, "_ffi.Array('xx')") A = _ffi.Array('i') A.of = 'xx' @@ -305,3 +303,14 @@ _ffi.CallbackPtr(compare, ['i', 'i'], 'i')) res = [ll_to_sort[i] for i in range(len(to_sort))] assert res == sorted(to_sort) + + def test_setattr_struct(self): + import _ffi + X = _ffi.Structure([('value1', 'i'), ('value2', 'i')]) + x = X(value1=1, value2=2) + assert x.value1 == 1 + assert x.value2 == 2 + x.value1 = 3 + assert x.value1 == 3 + raises(AttributeError, "x.foo") + raises(AttributeError, "x.foo = 1") From rxe at codespeak.net Tue Dec 18 21:45:58 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Tue, 18 Dec 2007 21:45:58 +0100 (CET) Subject: [pypy-svn] r49902 - pypy/dist/pypy/translator/tool Message-ID: <20071218204558.61B171684EC@codespeak.net> Author: rxe Date: Tue Dec 18 21:45:57 2007 New Revision: 49902 Modified: pypy/dist/pypy/translator/tool/cbuild.py Log: -O flag doesnt do link time optimizations. add -O3 and -fomit-frame-pointer for darwin also Modified: pypy/dist/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/dist/pypy/translator/tool/cbuild.py (original) +++ pypy/dist/pypy/translator/tool/cbuild.py Tue Dec 18 21:45:57 2007 @@ -368,7 +368,7 @@ if 'pthread' not in self.libraries: self.libraries.append('pthread') self.compile_extra += ['-O3', '-fomit-frame-pointer', '-pthread'] - self.link_extra += ['-O3', '-pthread'] + self.link_extra += ['-pthread'] if sys.platform == 'win32': self.link_extra += ['/DEBUG'] # generate .pdb file if sys.platform == 'darwin': @@ -380,7 +380,7 @@ if s + 'lib' not in self.library_dirs and \ os.path.exists(s + 'lib'): self.library_dirs.append(s + 'lib') - self.compile_extra += ['-O2'] + self.compile_extra += ['-O3', '-fomit-frame-pointer'] if outputfilename is None: self.outputfilename = py.path.local(cfilenames[0]).new(ext=ext) From arigo at codespeak.net Wed Dec 19 11:42:45 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 Dec 2007 11:42:45 +0100 (CET) Subject: [pypy-svn] r49925 - pypy/dist/pypy/translator/microbench Message-ID: <20071219104245.C20E01684EC@codespeak.net> Author: arigo Date: Wed Dec 19 11:42:44 2007 New Revision: 49925 Modified: pypy/dist/pypy/translator/microbench/microbench.py Log: For now, tweak the microbench runner so that the global data for all test_* modules are not all loaded in the process at the same time. This allows us to add new test_* modules with large global data (e.g. test_unicode) without affecting the overall process' memory usage. This is what caused test_list_extend's performance to suddenly drop. Modified: pypy/dist/pypy/translator/microbench/microbench.py ============================================================================== --- pypy/dist/pypy/translator/microbench/microbench.py (original) +++ pypy/dist/pypy/translator/microbench/microbench.py Wed Dec 19 11:42:44 2007 @@ -6,19 +6,25 @@ import os, time, sys, gc +try: + this_dir = os.path.dirname(__file__) +except NameError: + this_dir = os.path.dirname(sys.argv[0]) + microbenches = [] for fname in os.listdir('.'): if not fname.startswith('test_') or not fname.endswith('.py'): continue microbench = fname[:-3] - exec 'import ' + microbench microbenches.append(microbench) def run(test_cases): MINIMUM_MICROBENCH_TIME = 1.0 for microbench in microbenches: - for k in [s for s in globals()[microbench].__dict__ if s.startswith('test_')] : + testmoddict = {} + execfile(os.path.join(this_dir, microbench + '.py'), testmoddict) + for k in [s for s in testmoddict if s.startswith('test_')] : if test_cases: for tc in test_cases: if k.startswith(tc): @@ -26,7 +32,7 @@ else: continue testcase_name = microbench + '.' + k + '()' - testcase = getattr(globals()[microbench], k) + testcase = testmoddict[k] gc.collect() start = time.clock() n = 0 From arigo at codespeak.net Wed Dec 19 17:48:51 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 Dec 2007 17:48:51 +0100 (CET) Subject: [pypy-svn] r49936 - pypy/dist/pypy/objspace/std/test Message-ID: <20071219164851.A98E416856C@codespeak.net> Author: arigo Date: Wed Dec 19 17:48:50 2007 New Revision: 49936 Modified: pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py Log: Fix test. Ideally we should test both the shortcut and the regular path in StrDictImplementation but it's not obvious how to do the former with the FakeSpace. Modified: pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/dist/pypy/objspace/std/test/test_dictmultiobject.py Wed Dec 19 17:48:50 2007 @@ -84,6 +84,7 @@ w_StopIteration = StopIteration w_None = None + StringObjectCls = None # xxx untested: shortcut in StrDictImpl.getitem class TestDictImplementation: def setup_method(self,method): From antocuni at codespeak.net Wed Dec 19 17:53:27 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 19 Dec 2007 17:53:27 +0100 (CET) Subject: [pypy-svn] r49937 - pypy/dist/pypy/doc Message-ID: <20071219165327.68E2916856C@codespeak.net> Author: antocuni Date: Wed Dec 19 17:53:26 2007 New Revision: 49937 Modified: pypy/dist/pypy/doc/geninterp.txt Log: remove references to the dead _classobj.py Modified: pypy/dist/pypy/doc/geninterp.txt ============================================================================== --- pypy/dist/pypy/doc/geninterp.txt (original) +++ pypy/dist/pypy/doc/geninterp.txt Wed Dec 19 17:53:26 2007 @@ -19,15 +19,6 @@ .. _`application-level`: coding-guide.html#app-preferable .. _exceptions: ../../pypy/lib/_exceptions.py -.. _oldstyle: ../../pypy/lib/_classobj.py - -Examples are exceptions_ and oldstyle_ classes. They are -needed in a very early phase of bootstrapping StdObjspace, but -for simplicity, they are written as RPythonic application -level code. This implies that the interpreter must be quite -completely initialized to execute this code, which is -impossible in the early phase, where we have neither -exceptions implemented nor classes available. Solution ++++++++ @@ -178,10 +169,9 @@ +++++++++++++++++++++++++++++++++++ .. _`_exceptions.py`: ../../pypy/lib/_exceptions.py -.. _`_classobj.py`: ../../pypy/lib/_classobj.py Code written in application space can consist of complete files -to be translated (`_exceptions.py`_, `_classobj.py`_), or they +to be translated (e.g. `_exceptions.py`_), or they can be tiny snippets scattered all over a source file, similar to our example from above. From arigo at codespeak.net Wed Dec 19 18:01:37 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 Dec 2007 18:01:37 +0100 (CET) Subject: [pypy-svn] r49938 - pypy/dist/pypy/doc Message-ID: <20071219170137.6802816857A@codespeak.net> Author: arigo Date: Wed Dec 19 18:01:36 2007 New Revision: 49938 Modified: pypy/dist/pypy/doc/geninterp.txt Log: This paragraph can stay, with minor fixes to remove _classobj. Modified: pypy/dist/pypy/doc/geninterp.txt ============================================================================== --- pypy/dist/pypy/doc/geninterp.txt (original) +++ pypy/dist/pypy/doc/geninterp.txt Wed Dec 19 18:01:36 2007 @@ -20,6 +20,14 @@ .. _`application-level`: coding-guide.html#app-preferable .. _exceptions: ../../pypy/lib/_exceptions.py +An example is exceptions_. They are +needed in a very early phase of bootstrapping StdObjspace, but +for simplicity, they are written as RPythonic application +level code. This implies that the interpreter must be quite +completely initialized to execute this code, which is +impossible in the early phase, where we have neither +exceptions implemented nor classes available. + Solution ++++++++ From arigo at codespeak.net Wed Dec 19 18:01:44 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 Dec 2007 18:01:44 +0100 (CET) Subject: [pypy-svn] r49939 - pypy/dist/pypy/doc Message-ID: <20071219170144.CC45116857B@codespeak.net> Author: arigo Date: Wed Dec 19 18:01:44 2007 New Revision: 49939 Modified: pypy/dist/pypy/doc/_ref.txt Log: Regenerate _ref.txt. Modified: pypy/dist/pypy/doc/_ref.txt ============================================================================== --- pypy/dist/pypy/doc/_ref.txt (original) +++ pypy/dist/pypy/doc/_ref.txt Wed Dec 19 18:01:44 2007 @@ -56,15 +56,10 @@ .. _`pypy/lib/stackless.py`: ../../pypy/lib/stackless.py .. _`pypy/lib/test2`: ../../pypy/lib/test2 .. _`module/`: -.. _`pypy/module`: -.. _`pypy/module/`: ../../pypy/module +.. _`pypy/module`: ../../pypy/module .. _`pypy/module/__builtin__/__init__.py`: ../../pypy/module/__builtin__/__init__.py -.. _`pypy/module/_demo`: ../../pypy/module/_demo -.. _`pypy/module/_sre`: ../../pypy/module/_sre -.. _`pypy/module/_sre/interp_sre.py`: ../../pypy/module/_sre/interp_sre.py .. _`pypy/module/_stackless/test/test_clonable.py`: ../../pypy/module/_stackless/test/test_clonable.py .. _`pypy/module/_stackless/test/test_composable_coroutine.py`: ../../pypy/module/_stackless/test/test_composable_coroutine.py -.. _`pypy/module/readline`: ../../pypy/module/readline .. _`objspace/`: .. _`pypy/objspace`: ../../pypy/objspace .. _`objspace/dump.py`: ../../pypy/objspace/dump.py @@ -83,14 +78,10 @@ .. _`pypy/rpython`: .. _`pypy/rpython/`: .. _`rpython/`: ../../pypy/rpython -.. _`pypy/rpython/controllerentry.py`: ../../pypy/rpython/controllerentry.py .. _`rpython/lltypesystem/`: ../../pypy/rpython/lltypesystem .. _`pypy/rpython/lltypesystem/lltype.py`: .. _`rpython/lltypesystem/lltype.py`: ../../pypy/rpython/lltypesystem/lltype.py .. _`rpython/memory/`: ../../pypy/rpython/memory -.. _`pypy/rpython/module/`: ../../pypy/rpython/module -.. _`pypy/rpython/module/ll_os.py`: ../../pypy/rpython/module/ll_os.py -.. _`pypy/rpython/module/test`: ../../pypy/rpython/module/test .. _`rpython/ootypesystem/`: ../../pypy/rpython/ootypesystem .. _`rpython/ootypesystem/ootype.py`: ../../pypy/rpython/ootypesystem/ootype.py .. _`rpython/rint.py`: ../../pypy/rpython/rint.py @@ -106,10 +97,6 @@ .. _`translator/`: ../../pypy/translator .. _`translator/backendopt/`: ../../pypy/translator/backendopt .. _`translator/c/`: ../../pypy/translator/c -.. _`pypy/translator/c/extfunc.py`: ../../pypy/translator/c/extfunc.py -.. _`pypy/translator/c/src/`: ../../pypy/translator/c/src -.. _`pypy/translator/c/src/ll_os.h`: ../../pypy/translator/c/src/ll_os.h -.. _`pypy/translator/c/test/test_extfunc.py`: ../../pypy/translator/c/test/test_extfunc.py .. _`translator/cli/`: ../../pypy/translator/cli .. _`translator/goal/`: ../../pypy/translator/goal .. _`pypy/translator/goal/targetnopstandalone.py`: ../../pypy/translator/goal/targetnopstandalone.py From arigo at codespeak.net Wed Dec 19 18:05:32 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 Dec 2007 18:05:32 +0100 (CET) Subject: [pypy-svn] r49940 - pypy/dist/pypy/translator/backendopt/test Message-ID: <20071219170532.8CAEE16857E@codespeak.net> Author: arigo Date: Wed Dec 19 18:05:31 2007 New Revision: 49940 Modified: pypy/dist/pypy/translator/backendopt/test/test_canraise.py Log: This test makes no sense on ootype, currently. Modified: pypy/dist/pypy/translator/backendopt/test/test_canraise.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_canraise.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_canraise.py Wed Dec 19 18:05:31 2007 @@ -1,3 +1,4 @@ +import py from pypy.translator.translator import TranslationContext, graphof from pypy.translator.simplify import get_funcobj from pypy.translator.backendopt.canraise import RaiseAnalyzer @@ -180,6 +181,5 @@ class TestOOType(OORtypeMixin, BaseTestCanRaise): - pass - - + def test_can_raise_recursive(self): + py.test.skip("ootype: no explicit stack checks raising RuntimeError") From arigo at codespeak.net Wed Dec 19 18:42:39 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 19 Dec 2007 18:42:39 +0100 (CET) Subject: [pypy-svn] r49941 - pypy/dist/pypy/translator/llvm Message-ID: <20071219174239.C1E1F168553@codespeak.net> Author: arigo Date: Wed Dec 19 18:42:38 2007 New Revision: 49941 Modified: pypy/dist/pypy/translator/llvm/database.py Log: Works with a HEAD revision of llvm. Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Wed Dec 19 18:42:38 2007 @@ -598,10 +598,9 @@ if isinstance(value, llarena.RoundedUpForAllocation): # XXX not supported when used in a CompositeOffset r_basesize = self.repr_offset(value.basesize) - # XXX XXX XXX we hit an llvm assertion with the following - # expression! for now let's not align anything :-( - #return "and(i32 add(i32 %s, i32 7), i32 -8)" % r_basesize - return r_basesize + # Note that the following expression is known to crash 'llc'; + # you may need to upgrade llvm. + return "and(i32 add(i32 %s, i32 7), i32 -8)" % r_basesize from_, indices, to = self.get_offset(value, []) From antocuni at codespeak.net Wed Dec 19 19:16:19 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 19 Dec 2007 19:16:19 +0100 (CET) Subject: [pypy-svn] r49942 - pypy/dist/pypy/translator/jvm Message-ID: <20071219181619.4A5B3168576@codespeak.net> Author: antocuni Date: Wed Dec 19 19:16:18 2007 New Revision: 49942 Modified: pypy/dist/pypy/translator/jvm/generator.py Log: dump the state of the program in case jtype is None; this should help to fix the inconsistently failing test_weakref_simple Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Wed Dec 19 19:16:18 2007 @@ -1,3 +1,9 @@ +try: + import pycrash + mypycrash = pycrash.PyCrash({'AppName': 'genjvm'}) +except ImportError: + mypycrash = None + from pypy.objspace.flow import model as flowmodel from pypy.translator.oosupport.metavm import Generator from pypy.translator.oosupport.treebuilder import SubOperation @@ -1379,6 +1385,13 @@ '.implements ' + jinterface.descriptor.int_class_name() + '\n') def add_field(self, fobj): + try: + fobj.jtype.descriptor + except AttributeError: + if mypycrash is not None: + mypycrash.forceDump() + mypycrash.saveToFile("/tmp/test_jvm_weakref.pycrash") + kw = ['public'] if fobj.is_static: kw.append('static') self.curclass.out('.field %s %s %s\n' % ( From rxe at codespeak.net Wed Dec 19 22:45:37 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Wed, 19 Dec 2007 22:45:37 +0100 (CET) Subject: [pypy-svn] r49943 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071219214537.7C1A3168578@codespeak.net> Author: rxe Date: Wed Dec 19 22:45:35 2007 New Revision: 49943 Modified: pypy/dist/pypy/translator/llvm/genllvm.py pypy/dist/pypy/translator/llvm/test/test_standalone.py Log: catch exception in entrypoint Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Wed Dec 19 22:45:35 2007 @@ -25,21 +25,29 @@ def augment_entrypoint(translator, entrypoint): bk = translator.annotator.bookkeeper graph_entrypoint = bk.getdesc(entrypoint).getuniquegraph() - s_result = translator.annotator.binding(graph_entrypoint.getreturnvar()) get_argc = rffi.llexternal('_pypy_getargc', [], rffi.INT) get_argv = rffi.llexternal('_pypy_getargv', [], rffi.CCHARPP) + import os + def new_entrypoint(): argc = get_argc() argv = get_argv() args = [rffi.charp2str(argv[i]) for i in range(argc)] - return entrypoint(args) + result = 255 + try: + result = entrypoint(args) + except Exception, exc: + os.write(2, 'DEBUG: An uncaught exception was raised in entrypoint: ' + str(exc) + '\n') + + return result + entrypoint._annenforceargs_ = [s_list_of_strings] mixlevelannotator = MixLevelHelperAnnotator(translator.rtyper) - - graph = mixlevelannotator.getgraph(new_entrypoint, [], s_result) + res = annmodel.lltype_to_annotation(lltype.Signed) + graph = mixlevelannotator.getgraph(new_entrypoint, [], res) mixlevelannotator.finish() mixlevelannotator.backend_optimize() @@ -116,9 +124,8 @@ # XXX please dont ask! from pypy.translator.c.genc import CStandaloneBuilder cbuild = CStandaloneBuilder(self.translator, func, config=self.config) - #cbuild.stackless = self.stackless c_db = cbuild.generate_graphs_for_llinterp() - + self.db = Database(self, self.translator) self.db.gcpolicy = GcPolicy.new(self.db, self.config) self.db.gctransformer = c_db.gctransformer @@ -135,9 +142,6 @@ # set up externs nodes self.setup_externs(c_db, self.db) - - self.translator.rtyper.specialize_more_blocks() - self.db.setup_all() self._checkpoint('setup_all externs') self._print_node_stats() @@ -175,7 +179,7 @@ self.function_count[name] += 1 Node.nodename_count[name] = self.function_count[name] + 1 name += '_%d' % self.function_count[name] - entry_node.name = name + entry_node.name = name else: self.function_count[name] = 1 Modified: pypy/dist/pypy/translator/llvm/test/test_standalone.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_standalone.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_standalone.py Wed Dec 19 22:45:35 2007 @@ -86,3 +86,19 @@ exe_name = 'test_strtod' compile_standalone(entry_point, exe_name=exe_name) data = cmdexec(exe_name, '3.13e1') + +def test_exception_leaking(): + def entry_point(argv): + if len(argv) > 5: + raise ValueError + print 'ok' + return 0 + + exe_name = 'test_exception_leaking' + compile_standalone(entry_point, exe_name=exe_name) + data = cmdexec(exe_name, 'abc', 'def') + assert data.startswith('ok') + try: + data = cmdexec(exe_name, 'abc', 'def', 'abc', 'def', 'abc', 'def') + except py.process.cmdexec.Error, exc: + assert exc.err.startswith('DEBUG') From arigo at codespeak.net Thu Dec 20 10:25:25 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 Dec 2007 10:25:25 +0100 (CET) Subject: [pypy-svn] r49946 - pypy/dist/pypy/translator/llvm Message-ID: <20071220092525.73C39169E3E@codespeak.net> Author: arigo Date: Thu Dec 20 10:25:20 2007 New Revision: 49946 Modified: pypy/dist/pypy/translator/llvm/genllvm.py Log: * Restore two lines that may have been deleted by accident, causing many tests to fail. * Use 1 instead of 255 as the Unix process exit code in case of exception. Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Thu Dec 20 10:25:20 2007 @@ -36,7 +36,7 @@ argv = get_argv() args = [rffi.charp2str(argv[i]) for i in range(argc)] - result = 255 + result = 1 try: result = entrypoint(args) except Exception, exc: @@ -142,6 +142,9 @@ # set up externs nodes self.setup_externs(c_db, self.db) + + self.translator.rtyper.specialize_more_blocks() + self.db.setup_all() self._checkpoint('setup_all externs') self._print_node_stats() From antocuni at codespeak.net Thu Dec 20 10:32:58 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 20 Dec 2007 10:32:58 +0100 (CET) Subject: [pypy-svn] r49947 - in pypy/dist/pypy/jit/hintannotator: . test Message-ID: <20071220093258.41BA9169E3C@codespeak.net> Author: antocuni Date: Thu Dec 20 10:32:56 2007 New Revision: 49947 Modified: pypy/dist/pypy/jit/hintannotator/model.py pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Log: yai! One more test passes :-) Modified: pypy/dist/pypy/jit/hintannotator/model.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/model.py (original) +++ pypy/dist/pypy/jit/hintannotator/model.py Thu Dec 20 10:32:56 2007 @@ -518,12 +518,19 @@ # it's a method of a BuiltinType bk = getbookkeeper() origin = bk.myorigin() - d = setadd(hs_c1.origins, origin) + d = hs_c1.origins.copy() + eager_concrete = hs_c1.eager_concrete + for hs_arg in args_hs: + d.update(hs_arg.origins) + eager_concrete = eager_concrete or hs_arg.eager_concrete + d.update({origin: True}) + RESTYPE = bk.current_op_concretetype() hs_res = SomeLLAbstractConstant(RESTYPE, d, - eager_concrete = hs_c1.eager_concrete, + eager_concrete = eager_concrete, myorigin = origin) - # if hs_c1.is_constant(): ... + # if hs_c1.is_constant(): # and hs_arg.is_constat() for all args_hs + # XXX # constfold here? return hs_res elif len(graph_list) == 1: # like a direct_call Modified: pypy/dist/pypy/jit/hintannotator/test/test_annotator.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/test/test_annotator.py (original) +++ pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Thu Dec 20 10:32:56 2007 @@ -75,7 +75,6 @@ assert len(hs.origins) == 4 assert hs.concretetype == lltype.Signed - def test_simple_hint_result(self): def ll_function(cond, x,y): if cond: @@ -699,6 +698,22 @@ hs = self.hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) assert hs.is_green() + def test_concrete_fnptr_for_green_method_call(self): + class A: + def h(self, n): + return n*10 + class B(A): + def h(self, n): + return n+20 + lst = [A(), B()] + def ll_function(n, m): + obj = hint(lst, deepfreeze=True)[m] + res = obj.h(n) + hint(res, concrete=True) # so 'obj' and 'h' get green, so 'm' gets green + return m + + hs = self.hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) + assert hs.is_green() def test_indirect_yellow_call(self): From rxe at codespeak.net Thu Dec 20 14:50:34 2007 From: rxe at codespeak.net (rxe at codespeak.net) Date: Thu, 20 Dec 2007 14:50:34 +0100 (CET) Subject: [pypy-svn] r49951 - pypy/dist/pypy/translator/llvm Message-ID: <20071220135034.3F126169E42@codespeak.net> Author: rxe Date: Thu Dec 20 14:50:32 2007 New Revision: 49951 Modified: pypy/dist/pypy/translator/llvm/genllvm.py Log: oops - only meant to chop this line. rechop Modified: pypy/dist/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/dist/pypy/translator/llvm/genllvm.py (original) +++ pypy/dist/pypy/translator/llvm/genllvm.py Thu Dec 20 14:50:32 2007 @@ -143,7 +143,6 @@ # set up externs nodes self.setup_externs(c_db, self.db) - self.translator.rtyper.specialize_more_blocks() self.db.setup_all() self._checkpoint('setup_all externs') From antocuni at codespeak.net Thu Dec 20 18:35:45 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 20 Dec 2007 18:35:45 +0100 (CET) Subject: [pypy-svn] r49959 - pypy/dist/pypy/jit/hintannotator/test Message-ID: <20071220173545.AA149169EB8@codespeak.net> Author: antocuni Date: Thu Dec 20 18:35:45 2007 New Revision: 49959 Modified: pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Log: try hard not to duplicate code that tests indirect_call and oosend; one more test passes Modified: pypy/dist/pypy/jit/hintannotator/test/test_annotator.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/test/test_annotator.py (original) +++ pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Thu Dec 20 18:35:45 2007 @@ -20,6 +20,38 @@ P_NOVIRTUAL = HintAnnotatorPolicy(novirtualcontainer=True, entrypoint_returns_red=False) + +def setup_for_indirect_call(h, g): + def call(fn, x): + return fn(x) + return call, [h, g] + +def setup_for_oosend(h, g): + def call(obj, x): + return obj.call(x) + + class A: + def call(self, x): + return h(x) + class B(A): + def call(self, x): + return g(x) + + return call, [A(), B()] + +def test_setup_for_indirect_call_oosend(): + def h(x): + return x+1 + def g(x): + return x*2 + call, lst = setup_for_indirect_call(h, g) + assert call(lst[0], 41) == 42 + assert call(lst[1], 41) == 82 + + call, lst = setup_for_oosend(h, g) + assert call(lst[0], 41) == 42 + assert call(lst[1], 41) == 82 + class AbstractAnnotatorTest: type_system = None @@ -715,50 +747,32 @@ hs = self.hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) assert hs.is_green() - def test_indirect_yellow_call(self): - + def test_indirect_yellow_call(self, setup=setup_for_indirect_call): def h1(n): return 123 - def h2(n): return 456 - - lst = [h1, h2] - + call, lst = setup(h1, h2) def ll_function(n, m): h = hint(lst, deepfreeze=True)[m] - return h(n) + return call(h, n) hs = self.hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) assert not hs.is_green() - def test_indirect_method_yellow_call(self): - class A: - def h1(self, n): - return 123 - - class B(A): - def h1(self, n): - return 456 - - lst = [A(), B()] + def test_indirect_yellow_call_oosend(self): + self.test_indirect_yellow_call(setup_for_oosend) - def ll_function(n, m): - obj = hint(lst, deepfreeze=True)[m] - return obj.h1(n) - hs = self.hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) - assert not hs.is_green() - - def test_indirect_sometimes_residual_pure_red_call(self): + def test_indirect_sometimes_residual_pure_red_call(self, setup=setup_for_indirect_call): def h1(x): return x-2 def h2(x): return x*4 - l = [h1, h2] + call, lst = setup(h1, h2) def f(n, x): - frozenl = hint(l, deepfreeze=True) + frozenl = hint(lst, deepfreeze=True) h = frozenl[n&1] - return h(x) + return call(h, x) P = StopAtXPolicy(h1) P.oopspec = True @@ -771,6 +785,8 @@ hs = hannotator.binding(tsgraph.getargs()[0]) assert not hs.is_green() + def test_indirect_sometimes_residual_pure_red_call_oosend(self): + self.test_indirect_sometimes_residual_pure_red_call(setup_for_oosend) def test_indirect_sometimes_residual_red_call(self): class Stuff: From antocuni at codespeak.net Thu Dec 20 18:43:28 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 20 Dec 2007 18:43:28 +0100 (CET) Subject: [pypy-svn] r49960 - pypy/dist/pypy/jit/hintannotator/test Message-ID: <20071220174328.7BCE7169E8A@codespeak.net> Author: antocuni Date: Thu Dec 20 18:43:28 2007 New Revision: 49960 Modified: pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Log: two more tests pass out of the box :-) Modified: pypy/dist/pypy/jit/hintannotator/test/test_annotator.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/test/test_annotator.py (original) +++ pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Thu Dec 20 18:43:28 2007 @@ -788,7 +788,7 @@ def test_indirect_sometimes_residual_pure_red_call_oosend(self): self.test_indirect_sometimes_residual_pure_red_call(setup_for_oosend) - def test_indirect_sometimes_residual_red_call(self): + def test_indirect_sometimes_residual_red_call(self, setup=setup_for_indirect_call): class Stuff: pass stuff = Stuff() @@ -797,11 +797,11 @@ return x-2 def h2(x): return x*4 - l = [h1, h2] + call, lst = setup(h1, h2) def f(n, x): - frozenl = hint(l, deepfreeze=True) + frozenl = hint(lst, deepfreeze=True) h = frozenl[n&1] - return h(x) + return call(h, x) P = StopAtXPolicy(h1) P.oopspec = True @@ -813,17 +813,19 @@ hs = hannotator.binding(tsgraph.getargs()[0]) assert not hs.is_green() + def test_indirect_sometimes_residual_red_call_oosend(self): + self.test_indirect_sometimes_residual_red_call(setup_for_oosend) - def test_indirect_sometimes_residual_pure_but_fixed_red_call(self): + def test_indirect_sometimes_residual_pure_but_fixed_red_call(self, setup=setup_for_indirect_call): def h1(x): return x-2 def h2(x): return x*4 - l = [h1, h2] + call, lst = setup(h1, h2) def f(n, x): - frozenl = hint(l, deepfreeze=True) + frozenl = hint(lst, deepfreeze=True) h = frozenl[n&1] - z = h(x) + z = call(h, x) hint(z, concrete=True) return z @@ -843,6 +845,9 @@ hs = hannotator.binding(tsgraph.getargs()[1]) assert hs.is_green() + def test_indirect_sometimes_residual_pure_but_fixed_red_call_oosend(self): + self.test_indirect_sometimes_residual_pure_but_fixed_red_call(setup_for_oosend) + def test_ignore_nonjitted_path(self): def f(n): if we_are_jitted(): From antocuni at codespeak.net Thu Dec 20 18:50:47 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 20 Dec 2007 18:50:47 +0100 (CET) Subject: [pypy-svn] r49962 - pypy/dist/pypy/jit/hintannotator/test Message-ID: <20071220175047.E7794169EAE@codespeak.net> Author: antocuni Date: Thu Dec 20 18:50:47 2007 New Revision: 49962 Modified: pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Log: merge two almost identical tests, one more test passes Modified: pypy/dist/pypy/jit/hintannotator/test/test_annotator.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/test/test_annotator.py (original) +++ pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Thu Dec 20 18:50:47 2007 @@ -711,41 +711,24 @@ assert hs.deepfrozen - def test_concrete_fnptr_for_green_call(self): - + def test_concrete_fnptr_for_green_call(self, setup=setup_for_indirect_call): def h1(n): return n * 10 - def h2(n): return n + 20 - - lst = [h1, h2] + call, lst = setup(h1, h2) def ll_function(n, m): h = hint(lst, deepfreeze=True)[m] - res = h(n) + res = call(h, n) hint(res, concrete=True) # so 'h' gets green, so 'm' gets green return m hs = self.hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) assert hs.is_green() - def test_concrete_fnptr_for_green_method_call(self): - class A: - def h(self, n): - return n*10 - class B(A): - def h(self, n): - return n+20 - lst = [A(), B()] - def ll_function(n, m): - obj = hint(lst, deepfreeze=True)[m] - res = obj.h(n) - hint(res, concrete=True) # so 'obj' and 'h' get green, so 'm' gets green - return m - - hs = self.hannotate(ll_function, [int, int], policy=P_NOVIRTUAL) - assert hs.is_green() + def test_concrete_fnptr_for_green_call_oosend(self): + self.test_concrete_fnptr_for_green_call(setup_for_oosend) def test_indirect_yellow_call(self, setup=setup_for_indirect_call): def h1(n): @@ -1094,7 +1077,6 @@ test_specialize_deepfreeze_calls = skip_policy test_deepfreeze_variables = skip_policy test_cast_pointer_keeps_deepfreeze = skip_policy - test_concrete_fnptr_for_green_call = skip_policy def test_void_oosend(self): class Foo: From arigo at codespeak.net Thu Dec 20 19:50:20 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 Dec 2007 19:50:20 +0100 (CET) Subject: [pypy-svn] r49966 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071220185020.76CBE169E76@codespeak.net> Author: arigo Date: Thu Dec 20 19:50:20 2007 New Revision: 49966 Modified: pypy/dist/pypy/translator/llvm/arraynode.py pypy/dist/pypy/translator/llvm/codewriter.py pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/gc.py pypy/dist/pypy/translator/llvm/opwriter.py pypy/dist/pypy/translator/llvm/structnode.py pypy/dist/pypy/translator/llvm/test/test_symbolic.py pypy/dist/pypy/translator/llvm/typedefnode.py Log: A lot of efforts to insert the GC header fields in front of the corresponding structures. This is a mess because we have to change all the 'getelementptr', which needs field indices. Modified: pypy/dist/pypy/translator/llvm/arraynode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/arraynode.py (original) +++ pypy/dist/pypy/translator/llvm/arraynode.py Thu Dec 20 19:50:20 2007 @@ -35,6 +35,8 @@ self.make_name(name) def setup(self): + for value in self.db.gcpolicy.gcheader_initdata(self.value): + self.db.prepare_constant(lltype.typeOf(value), value) for item in self.value.items: self.db.prepare_constant(self.arraytype, item) @@ -60,22 +62,24 @@ def get_typerepr(self): arraylen = self.get_arrayvalue()[0] - typeval = self.db.repr_type(self.arraytype) - return "{ %s, [%s x %s] }" % (self.db.get_machine_word(), - arraylen, typeval) + typedefnode = self.db.obj2node[lltype.typeOf(self.value)] + return typedefnode.get_typerepr(arraylen) def constantvalue(self): physicallen, arrayrepr = self.get_arrayvalue() typeval = self.db.repr_type(self.arraytype) # first length is logical, second is physical - value = "%s %s, [%s x %s]\n\t%s" % (self.db.get_machine_word(), + result = "" + for value in self.db.gcpolicy.gcheader_initdata(self.value): + result += "%s, " % (self.db.repr_constant(value)[1],) + result += "%s %s, [%s x %s]\n\t%s" % (self.db.get_machine_word(), self.get_length(), physicallen, typeval, arrayrepr) - return "%s {%s}" % (self.get_typerepr(), value) + return "%s {%s}" % (self.get_typerepr(), result) class ArrayNoLengthNode(ArrayNode): def get_typerepr(self): @@ -85,7 +89,6 @@ def constantvalue(self): physicallen, arrayrepr = self.get_arrayvalue() - typeval = self.db.repr_type(self.arraytype) s = "%s %s" % (self.get_typerepr(), arrayrepr) return s @@ -113,6 +116,9 @@ r = 'c"%s"' % "".join(s) return item_length, r +class StrArrayNoLengthNode(StrArrayNode, ArrayNoLengthNode): + pass + class VoidArrayNode(ConstantNode): __slots__ = "db value".split() prefix = '@voidarrayinstance' @@ -124,10 +130,23 @@ name = '' #str(value).split()[1] self.make_name(name) + def setup(self): + for value in self.db.gcpolicy.gcheader_initdata(self.value): + self.db.prepare_constant(lltype.typeOf(value), value) + def get_typerepr(self): - return '[%s x i8]' % self.get_length() + typedefnode = self.db.obj2node[lltype.typeOf(self.value)] + return typedefnode.get_typerepr() + + def get_length(self): + """ returns logical length of array """ + items = self.value.items + return len(items) def constantvalue(self): - return "{ %s } {%s %s}" % (self.db.get_machine_word(), - self.db.get_machine_word(), - len(self.value.items)) + result = "" + for value in self.db.gcpolicy.gcheader_initdata(self.value): + result += "%s %s, " % self.db.repr_constant(value) + result += "%s %s" % (self.db.get_machine_word(), + self.get_length()) + return "%s { %s }" % (self.get_typerepr(), result) Modified: pypy/dist/pypy/translator/llvm/codewriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/codewriter.py (original) +++ pypy/dist/pypy/translator/llvm/codewriter.py Thu Dec 20 19:50:20 2007 @@ -46,6 +46,7 @@ self._append(" %s:" % name) def globalinstance(self, name, typeandata, linkage=None): + assert not typeandata.startswith('i8') if linkage is None: linkage = self.linkage self._append("%s = %sglobal %s" % (name, linkage, typeandata)) Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Thu Dec 20 19:50:20 2007 @@ -3,14 +3,15 @@ from pypy.translator.llvm.log import log from pypy.translator.llvm.typedefnode import create_typedef_node +from pypy.translator.llvm.typedefnode import getindexhelper from pypy.translator.llvm.funcnode import FuncImplNode from pypy.translator.llvm.extfuncnode import ExternalFuncNode from pypy.translator.llvm.opaquenode import OpaqueNode, ExtOpaqueNode from pypy.translator.llvm.structnode import StructNode, StructVarsizeNode, \ - getindexhelper, FixedSizeArrayNode + FixedSizeArrayNode from pypy.translator.llvm.arraynode import ArrayNode, StrArrayNode, \ - VoidArrayNode, ArrayNoLengthNode, DebugStrNode + VoidArrayNode, ArrayNoLengthNode, StrArrayNoLengthNode, DebugStrNode from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi from pypy.objspace.flow.model import Constant, Variable @@ -95,7 +96,10 @@ elif isinstance(type_, lltype.Array): if type_.OF is lltype.Char: - node = StrArrayNode(self, value) + if type_._hints.get("nolength", False): + node = StrArrayNoLengthNode(self, value) + else: + node = StrArrayNode(self, value) elif type_.OF is lltype.Void: node = VoidArrayNode(self, value) else: @@ -240,35 +244,11 @@ indices = [("i32", 0)] for _, ii in children: - name = None - - # this is because FixedSizeArray can sometimes be accessed like an - # Array and then sometimes a Struct + typedefnode = self.obj2node[TYPE] if isinstance(ii, str): - name = ii - assert name in list(TYPE._names) - fieldnames = TYPE._names_without_voids() - indexref = fieldnames.index(name) - else: - indexref = ii - - if isinstance(TYPE, lltype.FixedSizeArray): - indices.append(("i32", indexref)) - TYPE = TYPE.OF - - elif isinstance(TYPE, lltype.Array): - if not TYPE._hints.get("nolength", False): - indices.append(("i32", 1)) - indices.append(("i32", indexref)) - TYPE = TYPE.OF - - elif isinstance(TYPE, lltype.Struct): - assert name is not None - TYPE = getattr(TYPE, name) - indices.append(("i32", indexref)) - + TYPE = typedefnode.fieldname_to_getelementptr(indices, ii) else: - raise Exception("unsupported type: %s" % TYPE) + TYPE = typedefnode.indexref_to_getelementptr(indices, ii) indices_str = ', '.join ([('%s %s' % (x,y)) for x, y in indices]) ref = "getelementptr(%s* %s, %s)" % ( @@ -604,19 +584,6 @@ from_, indices, to = self.get_offset(value, []) - # void array special cases - if isinstance(from_, lltype.Array) and from_.OF is lltype.Void: - assert not isinstance(value, (llmemory.FieldOffset, llmemory.ItemOffset)) - if isinstance(value, llmemory.ArrayLengthOffset): - pass # ok cases! - elif isinstance(value, llmemory.ArrayItemsOffset): - to = from_ - indices = [(self.database.get_machine_word(), 1)] - else: - s = value.offsets[0] - isinstance(value, llmemory.CompositeOffset) - return self.repr_offset(s) - if from_ is lltype.Void: assert isinstance(value, llmemory.ItemOffset) return "0" @@ -649,7 +616,7 @@ # jumps to a field position in a struct from_ = value.TYPE - pos = getindexhelper(value.fldname, value.TYPE) + pos = getindexhelper(self.database, value.fldname, value.TYPE) indices.append((word, pos)) to = getattr(value.TYPE, value.fldname) @@ -662,7 +629,9 @@ # jumps to the place where the array length is stored from_ = value.TYPE # or assert isinstance(value.TYPE, lltype.Array) - indices.append((word, 0)) + typedefnode = self.database.obj2node[value.TYPE] + indexref = typedefnode.indexref_for_length() + indices.append((word, indexref)) to = lltype.Signed elif isinstance(value, llmemory.ArrayItemsOffset): @@ -671,11 +640,19 @@ pass else: indices.append((word, 0)) - + + if value.TYPE.OF is lltype.Void: + # skip over the whole structure in order to get to the + # (not-really-existent) array part + return self.get_offset(llmemory.ItemOffset(value.TYPE), + indices) + # jumps to the beginning of array area from_ = value.TYPE if not isinstance(value.TYPE, lltype.FixedSizeArray) and not value.TYPE._hints.get("nolength", False): - indices.append((word, 1)) + typedefnode = self.database.obj2node[value.TYPE] + indexref = typedefnode.indexref_for_items() + indices.append((word, indexref)) indices.append((word, 0)) # go to the 1st item if isinstance(value.TYPE, lltype.FixedSizeArray): indices.append((word, 0)) # go to the 1st item @@ -685,7 +662,9 @@ elif isinstance(value, llmemory.CompositeOffset): from_, indices, to = self.get_offset(value.offsets[0], indices) for item in value.offsets[1:]: - _, indices, to = self.get_offset(item, indices) + _, indices, to1 = self.get_offset(item, indices) + if to1 is not lltype.Void: + to = to1 else: raise Exception("unsupported offset") Modified: pypy/dist/pypy/translator/llvm/gc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/gc.py (original) +++ pypy/dist/pypy/translator/llvm/gc.py Thu Dec 20 19:50:20 2007 @@ -1,5 +1,6 @@ import sys from pypy.rpython.lltypesystem.rstr import STR +from pypy.rpython.lltypesystem import lltype from pypy.translator.c import gc from pypy.translator.llvm.log import log @@ -17,10 +18,16 @@ def genextern_code(self): return '' + + def gcheader_definition(self, TYPE): + return [] + + def gcheader_initdata(self, container): + return [] def gc_libraries(self): return [] - + def get_count(self, inc=False): if inc: self.n_malloced = self.n_malloced + 1 @@ -158,9 +165,33 @@ r += '#define __GC_SETUP_CODE__\n' return r + def gcheader_definition(self, TYPE): + if needs_gcheader(TYPE): + return self.db.gctransformer.gc_fields() + else: + return [] + + def gcheader_initdata(self, container): + if needs_gcheader(container._TYPE): + o = lltype.top_container(container) + return self.db.gctransformer.gc_field_values_for(o) + else: + return [] + def gc_libraries(self): return ['pthread'] def get_real_weakref_type(self): from pypy.rpython.memory.gctransform import framework return framework.WEAKREF + + +def needs_gcheader(T): + if not isinstance(T, lltype.ContainerType): + return False + if T._gckind != 'gc': + return False + if isinstance(T, lltype.GcStruct): + if T._first_struct() != (None, None): + return False # gcheader already in the first field + return True Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Thu Dec 20 19:50:20 2007 @@ -104,7 +104,9 @@ if not ARRAYTYPE._hints.get("nolength", False): # skip the length field indices.append((self.word, 0)) - indices.append((self.word, 1)) + typedefnode = self.db.obj2node[ARRAYTYPE] + indexref = typedefnode.indexref_for_items() + indices.append((self.word, indexref)) else: assert isinstance(ARRAYTYPE, lltype.FixedSizeArray) indices.append((self.word, 0)) @@ -326,34 +328,15 @@ else: indices = [("i32", 0)] for arg in args: - name = None - # this is because FixedSizeArray can sometimes be accessed like an - # Array and then sometimes a Struct + typedefnode = self.db.obj2node[TYPE] if arg.concretetype is lltype.Void: + # access via a field name name = arg.value - assert name in list(TYPE._names) - fieldnames = TYPE._names_without_voids() - indexref = fieldnames.index(name) + TYPE = typedefnode.fieldname_to_getelementptr(indices, name) else: + # access via an array index indexref = self.db.repr_arg(arg) - - if isinstance(TYPE, lltype.FixedSizeArray): - indices.append(("i32", indexref)) - TYPE = TYPE.OF - - elif isinstance(TYPE, lltype.Array): - if not TYPE._hints.get("nolength", False): - indices.append(("i32", 1)) - indices.append(("i32", indexref)) - TYPE = TYPE.OF - - elif isinstance(TYPE, lltype.Struct): - assert name is not None - TYPE = getattr(TYPE, name) - indices.append(("i32", indexref)) - - else: - raise Exception("unsupported type: %s" % TYPE) + TYPE = typedefnode.indexref_to_getelementptr(indices, indexref) return TYPE, indices @@ -405,7 +388,9 @@ if isinstance(TYPE, lltype.Array): assert not TYPE._hints.get("nolength", False) # gets the length - indices.append(("i32", 0)) + typedefnode = self.db.obj2node[TYPE] + indexref = typedefnode.indexref_for_length() + indices.append(("i32", indexref)) lengthref = self._tmp() self.codewriter.getelementptr(lengthref, opr.argtypes[0], opr.argrefs[0], indices, getptr=False) else: @@ -417,11 +402,12 @@ getarraysize = getinteriorarraysize def direct_fieldptr(self, opr): - from pypy.translator.llvm.structnode import getindexhelper + from pypy.translator.llvm.typedefnode import getindexhelper op = opr.op assert opr.rettype != "void" - index = getindexhelper(op.args[1].value, + index = getindexhelper(self.db, + op.args[1].value, op.args[0].concretetype.TO) assert index != -1 tmpvar = self._tmp() Modified: pypy/dist/pypy/translator/llvm/structnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/structnode.py (original) +++ pypy/dist/pypy/translator/llvm/structnode.py Thu Dec 20 19:50:20 2007 @@ -1,16 +1,7 @@ from pypy.translator.llvm.node import ConstantNode +from pypy.translator.llvm.gc import needs_gcheader from pypy.rpython.lltypesystem import lltype -def getindexhelper(name, struct): - assert name in list(struct._names) - - fieldnames = struct._names_without_voids() - try: - index = fieldnames.index(name) - except ValueError: - index = -1 - return index - class StructNode(ConstantNode): __slots__ = "db value structtype _get_types".split() @@ -26,21 +17,24 @@ self.make_name(name) def _compute_types(self): - return [(name, self.structtype._flds[name]) - for name in self.structtype._names_without_voids()] + result = list(self.db.gcpolicy.gcheader_definition(self.structtype)) + for name in self.structtype._names_without_voids(): + result.append((name, self.structtype._flds[name])) + return result def _getvalues(self): - values = [] - for name, T in self._get_types: - value = getattr(self.value, name) - values.append(self.db.repr_constant(value)[1]) + values = list(self.db.gcpolicy.gcheader_initdata(self.value)) + for name in self.structtype._names_without_voids(): + values.append(getattr(self.value, name)) return values - + + def _getvaluesrepr(self): + values = self._getvalues() + return [self.db.repr_constant(value)[1] for value in values] + def setup(self): - for name, T in self._get_types: - assert T is not lltype.Void - value = getattr(self.value, name) - self.db.prepare_constant(T, value) + for value in self._getvalues(): + self.db.prepare_constant(lltype.typeOf(value), value) p, c = lltype.parentlink(self.value) if p is not None: @@ -51,7 +45,7 @@ def constantvalue(self): """ Returns the constant representation for this node. """ - values = self._getvalues() + values = self._getvaluesrepr() if len(values) > 3: all_values = ",\n\t".join(values) return "%s {\n\t%s }" % (self.get_typerepr(), all_values) @@ -69,7 +63,7 @@ def constantvalue(self): """ Returns the constant representation for this node. """ - values = self._getvalues() + values = self._getvaluesrepr() all_values = ",\n ".join(values) return "%s [\n %s\n ]\n" % (self.get_typerepr(), all_values) @@ -84,14 +78,6 @@ class StructVarsizeNode(StructNode): prefix = '@sv_inst_' - def _getvalues(self): - values = [] - for name, T in self._get_types[:-1]: - value = getattr(self.value, name) - values.append(self.db.repr_constant(value)[1]) - values.append(self._get_lastnoderepr()) - return values - def _get_lastnode_helper(self): lastname, LASTT = self._get_types[-1] assert isinstance(LASTT, lltype.Array) or ( @@ -102,16 +88,13 @@ def _get_lastnode(self): return self._get_lastnode_helper()[0] - def _get_lastnoderepr(self): - return self._get_lastnode_helper()[1] - def setup(self): super(StructVarsizeNode, self).setup() def get_typerepr(self): try: return self._get_typerepr_cache - except: + except AttributeError: # last type is a special case and need to be worked out recursively types = self._get_types[:-1] types_repr = [self.db.repr_type(T) for name, T in types] Modified: pypy/dist/pypy/translator/llvm/test/test_symbolic.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_symbolic.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_symbolic.py Thu Dec 20 19:50:20 2007 @@ -167,7 +167,6 @@ def test_itemoffset_void(): A = lltype.GcArray(lltype.Void) s = llmemory.sizeof(A, 1) - s += llmemory.sizeof(lltype.Signed) def f(): return s fn = compile_function(f, []) Modified: pypy/dist/pypy/translator/llvm/typedefnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/typedefnode.py (original) +++ pypy/dist/pypy/translator/llvm/typedefnode.py Thu Dec 20 19:50:20 2007 @@ -1,6 +1,18 @@ from pypy.rpython.lltypesystem import lltype, llmemory from pypy.translator.llvm.node import Node +def getindexhelper(db, name, struct): + assert name in list(struct._names) + + fieldnames = struct._names_without_voids() + try: + index = fieldnames.index(name) + except ValueError: + index = -1 + else: + index += len(db.gcpolicy.gcheader_definition(struct)) + return index + class TypeDefNode(Node): __slots__ = "".split() @@ -30,15 +42,38 @@ def setup(self): self.db.prepare_type(self.ARRAY.OF) + for name, F in self.db.gcpolicy.gcheader_definition(self.ARRAY): + self.db.prepare_type(F) - def writetypedef(self, codewriter): + def get_typerepr(self, arraylen=0): + gchdr = self.db.gcpolicy.gcheader_definition(self.ARRAY) if self.ARRAY._hints.get("nolength", False): - codewriter.typedef(self.ref, - "%s" % self.db.repr_type(self.ARRAY.OF)) + assert len(gchdr) == 0 + return "%s" % self.db.repr_type(self.ARRAY.OF) else: - codewriter.typedef(self.ref, - "{ %s, [0 x %s] }" % (self.db.get_machine_word(), - self.db.repr_type(self.ARRAY.OF))) + fields = [self.db.repr_type(F) for name, F in gchdr] + fields.append(self.db.get_machine_word()) + fields.append("[%d x %s]" % (arraylen, + self.db.repr_type(self.ARRAY.OF),)) + return "{ %s }" % ", ".join(fields) + + def writetypedef(self, codewriter): + codewriter.typedef(self.ref, self.get_typerepr()) + + def indexref_to_getelementptr(self, indices, indexref): + TYPE = self.ARRAY + if not TYPE._hints.get("nolength", False): + indices.append(("i32", self.indexref_for_items())) + indices.append(("i32", indexref)) + return TYPE.OF + + def indexref_for_length(self): + gchdr = self.db.gcpolicy.gcheader_definition(self.ARRAY) + return len(gchdr) + 0 + + def indexref_for_items(self): + gchdr = self.db.gcpolicy.gcheader_definition(self.ARRAY) + return len(gchdr) + 1 class VoidArrayTypeNode(TypeDefNode): " void arrays dont have any real elements " @@ -53,8 +88,23 @@ self.ARRAY = ARRAY self.make_name() + def setup(self): + for name, F in self.db.gcpolicy.gcheader_definition(self.ARRAY): + self.db.prepare_type(F) + + def get_typerepr(self, arraylen=0): + assert not self.ARRAY._hints.get("nolength", False), "XXX" + gchdr = self.db.gcpolicy.gcheader_definition(self.ARRAY) + fields = [self.db.repr_type(F) for name, F in gchdr] + fields.append(self.db.get_machine_word()) + return "{ %s }" % ", ".join(fields) + def writetypedef(self, codewriter): - codewriter.typedef(self.ref, "{ %s }" % self.db.get_machine_word()) + codewriter.typedef(self.ref, self.get_typerepr()) + + def indexref_for_length(self): + gchdr = self.db.gcpolicy.gcheader_definition(self.ARRAY) + return len(gchdr) + 0 class StructTypeNode(TypeDefNode): __slots__ = "db STRUCT".split() @@ -69,9 +119,13 @@ self.make_name(name) def _fields(self): - return [getattr(self.STRUCT, name) - for name in self.STRUCT._names_without_voids()] - + types = [] + for name, T in self.db.gcpolicy.gcheader_definition(self.STRUCT): + types.append(T) + for name in self.STRUCT._names_without_voids(): + types.append(getattr(self.STRUCT, name)) + return types + def setup(self): for F in self._fields(): self.db.prepare_type(F) @@ -81,10 +135,17 @@ codewriter.typedef(self.ref, "{ %s }" % ", ".join(fields_types)) + def fieldname_to_getelementptr(self, indices, name): + TYPE = self.STRUCT + indexref = getindexhelper(self.db, name, TYPE) + indices.append(("i32", indexref)) + return getattr(TYPE, name) + class FixedSizeArrayTypeNode(StructTypeNode): prefix = '%fixarray_' def setup(self): + assert self.STRUCT._gckind != 'gc' FIELDS = self._fields() if FIELDS: self.db.prepare_type(FIELDS[0]) @@ -94,6 +155,12 @@ "[%s x %s]" % (self.STRUCT.length, self.db.repr_type(self.STRUCT.OF))) + def indexref_to_getelementptr(self, indices, indexref): + TYPE = self.STRUCT + assert TYPE._gckind != 'gc' + indices.append(("i32", indexref)) + return TYPE.OF + class FuncTypeNode(TypeDefNode): __slots__ = "db T".split() prefix = '%functiontype' From antocuni at codespeak.net Thu Dec 20 21:30:06 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 20 Dec 2007 21:30:06 +0100 (CET) Subject: [pypy-svn] r49968 - in pypy/dist/pypy: jit/hintannotator jit/hintannotator/test rpython/ootypesystem Message-ID: <20071220203006.CE2EA169E91@codespeak.net> Author: antocuni Date: Thu Dec 20 21:30:04 2007 New Revision: 49968 Modified: pypy/dist/pypy/jit/hintannotator/model.py pypy/dist/pypy/jit/hintannotator/test/test_annotator.py pypy/dist/pypy/rpython/ootypesystem/ootype.py Log: - disable oopspec in the policy for ootype tests - add two new tests about nonfrozen lists and strings, and make them passing - introduce the concept of "immutable types" for ootype; they are treated as they were always deepfrozen Modified: pypy/dist/pypy/jit/hintannotator/model.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/model.py (original) +++ pypy/dist/pypy/jit/hintannotator/model.py Thu Dec 20 21:30:04 2007 @@ -515,30 +515,16 @@ METH = lltype.typeOf(meth) graph_list = TYPE._lookup_graphs(name) if not graph_list: - # it's a method of a BuiltinType + # it's a graphless method of a BuiltinADTType bk = getbookkeeper() - origin = bk.myorigin() - d = hs_c1.origins.copy() - eager_concrete = hs_c1.eager_concrete - for hs_arg in args_hs: - d.update(hs_arg.origins) - eager_concrete = eager_concrete or hs_arg.eager_concrete - d.update({origin: True}) - - RESTYPE = bk.current_op_concretetype() - hs_res = SomeLLAbstractConstant(RESTYPE, d, - eager_concrete = eager_concrete, - myorigin = origin) - # if hs_c1.is_constant(): # and hs_arg.is_constat() for all args_hs - # XXX # constfold here? - return hs_res + return handle_highlevel_operation_novirtual(bk, True, TYPE.immutable, hs_c1, *args_hs) elif len(graph_list) == 1: # like a direct_call graph = graph_list.pop() return hs_c1._call_single_graph(graph, METH.RESULT, hs_c1, *args_hs) # prepend hs_c1 to the args else: # like an indirect_call - return hs_c1._call_multiple_graphs(graph_list, METH.RESULT, hs_c1, *args_hs) + return hs_c1._call_multiple_graphs(graph_list, METH.RESULT, hs_c1, *args_hs) # prepend hs_c1 to the args def getfield(hs_c1, hs_fieldname): S = hs_c1.concretetype.TO @@ -731,6 +717,22 @@ # ____________________________________________________________ +def handle_highlevel_operation_novirtual(bookkeeper, ismethod, immutable, *args_hs): + RESULT = bookkeeper.current_op_concretetype() + if ismethod and (immutable or args_hs[0].deepfrozen): + for hs_v in args_hs: + if not isinstance(hs_v, SomeLLAbstractConstant): + break + else: + myorigin = bookkeeper.myorigin() + d = newset({myorigin: True}, *[hs_c.origins + for hs_c in args_hs]) + return SomeLLAbstractConstant(RESULT, d, + eager_concrete = False, # probably + myorigin = myorigin) + return variableoftype(RESULT) + + def handle_highlevel_operation(bookkeeper, ll_func, *args_hs): # parse the oopspec and fill in the arguments operation_name, args = ll_func.oopspec.split('(', 1) @@ -751,19 +753,8 @@ if bookkeeper.annotator.policy.novirtualcontainer: # "blue variables" disabled, we just return a red var all the time. # Exception: an operation on a frozen container is constant-foldable. - RESULT = bookkeeper.current_op_concretetype() - if '.' in operation_name and args_hs[0].deepfrozen: - for hs_v in args_hs: - if not isinstance(hs_v, SomeLLAbstractConstant): - break - else: - myorigin = bookkeeper.myorigin() - d = newset({myorigin: True}, *[hs_c.origins - for hs_c in args_hs]) - return SomeLLAbstractConstant(RESULT, d, - eager_concrete = False, # probably - myorigin = myorigin) - return variableoftype(RESULT) + ismethod = '.' in operation_name + return handle_highlevel_operation_novirtual(bookkeeper, ismethod, False, *args_hs) # --- the code below is not used any more except by test_annotator.py --- if operation_name == 'newlist': Modified: pypy/dist/pypy/jit/hintannotator/test/test_annotator.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/test/test_annotator.py (original) +++ pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Thu Dec 20 21:30:04 2007 @@ -71,6 +71,7 @@ graph1 = graphof(t, func) # build hint annotator types + policy = self.fixpolicy(policy) hannotator = HintAnnotator(base_translator=t, policy=policy) hs = hannotator.build_types(graph1, [SomeLLAbstractConstant(v.concretetype, {OriginFlags(): True}) @@ -84,6 +85,9 @@ else: return hs + def fixpolicy(self, policy): + return policy + class BaseAnnotatorTest(AbstractAnnotatorTest): @@ -415,6 +419,25 @@ hs = self.hannotate(ll_function, [int], policy=P_OOPSPEC_NOVIRTUAL) assert hs.is_green() + def test_nonfrozen_list(self): + lst = [5, 7, 9] + def ll_function(x): + mylist = hint(lst, concrete=True) + hint(x, concrete=True) + z = mylist[x] + return z + hs = self.hannotate(ll_function, [int], policy=P_OOPSPEC_NOVIRTUAL) + assert isinstance(hs, SomeLLAbstractVariable) + + def test_nonfrozen_string(self): + s = 'foobar' + def ll_function(x): + z = s[x] + hint(z, concrete=True) + return z + hs = self.hannotate(ll_function, [int], policy=P_NOVIRTUAL) + assert hs.is_green() + def test_prebuilt_structure(self): S = self.make_struct('S', ('n', lltype.Signed)) s = self.malloc(S) @@ -1053,9 +1076,14 @@ class TestOOType(BaseAnnotatorTest): type_system = 'ootype' - malloc = property(lambda self: ootype.new) + def fixpolicy(self, policy): + import copy + newpolicy = copy.copy(policy) + newpolicy.oopspec = False + return newpolicy + def make_struct(self, name, *fields, **kwds): fields = dict(fields) hints = kwds.pop('hints', None) @@ -1069,11 +1097,14 @@ def skip_policy(self): py.test.skip('fixme? (This policy is not relevant for now)') + # these tests fail because ootype doesn't support SomeLLAbstractContainer test_simple_list_operations = skip_policy test_some_more_list_operations = skip_policy test_make_a_list = skip_policy test_simple_struct_malloc = skip_policy test_container_union = skip_policy + + # these tests fail because of deepfreeze test_specialize_deepfreeze_calls = skip_policy test_deepfreeze_variables = skip_policy test_cast_pointer_keeps_deepfreeze = skip_policy Modified: pypy/dist/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/ootype.py Thu Dec 20 21:30:04 2007 @@ -204,6 +204,7 @@ return graphs + class SpecializableType(OOType): def _specialize_type(self, TYPE, generic_types): if isinstance(TYPE, SpecializableType): @@ -300,6 +301,8 @@ class BuiltinADTType(BuiltinType): + immutable = False # conservative + def _setup_methods(self, generic_types, can_raise=[]): methods = {} for name, meth in self._GENERIC_METHODS.iteritems(): @@ -325,6 +328,8 @@ class AbstractString(BuiltinADTType): + immutable = True + def __init__(self): self._null = _null_string(self) From arigo at codespeak.net Thu Dec 20 21:36:47 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 20 Dec 2007 21:36:47 +0100 (CET) Subject: [pypy-svn] r49969 - in pypy/dist/pypy/rpython: lltypesystem memory Message-ID: <20071220203647.E2882169E91@codespeak.net> Author: arigo Date: Thu Dec 20 21:36:47 2007 New Revision: 49969 Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py pypy/dist/pypy/rpython/memory/gctypelayout.py Log: Experimentally simplify the offsets generated for the GC. I didn't run all tests everywhere after this change, but the new offsets seem sane, so new failures probably show other code's limitations (and I will fix them then). Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py Thu Dec 20 21:36:47 2007 @@ -299,7 +299,10 @@ offsetof._annspecialcase_ = 'specialize:memo' def itemoffsetof(TYPE, n=0): - return ArrayItemsOffset(TYPE) + ItemOffset(TYPE.OF) * n + result = ArrayItemsOffset(TYPE) + if n != 0: + result += ItemOffset(TYPE.OF) * n + return result itemoffsetof._annspecialcase_ = 'specialize:memo' # ------------------------------------------------------------- Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/gctypelayout.py Thu Dec 20 21:36:47 2007 @@ -159,21 +159,14 @@ ARRAY = TYPE._flds[TYPE._arrayfld] ofs1 = llmemory.offsetof(TYPE, TYPE._arrayfld) info.ofstolength = ofs1 + llmemory.ArrayLengthOffset(ARRAY) - if ARRAY.OF != lltype.Void: - info.ofstovar = ofs1 + llmemory.itemoffsetof(ARRAY, 0) - else: - info.fixedsize = ofs1 + llmemory.sizeof(lltype.Signed) + info.ofstovar = ofs1 + llmemory.itemoffsetof(ARRAY, 0) # XXX we probably don't need isrpystring any more if ARRAY._hints.get('isrpystring'): info.fixedsize = llmemory.sizeof(TYPE, 1) else: ARRAY = TYPE info.ofstolength = llmemory.ArrayLengthOffset(ARRAY) - if ARRAY.OF != lltype.Void: - info.ofstovar = llmemory.itemoffsetof(TYPE, 0) - else: - info.fixedsize = (llmemory.ArrayLengthOffset(ARRAY) + - llmemory.sizeof(lltype.Signed)) + info.ofstovar = llmemory.itemoffsetof(TYPE, 0) assert isinstance(ARRAY, lltype.Array) if ARRAY.OF != lltype.Void: offsets = offsets_to_gc_pointers(ARRAY.OF) From antocuni at codespeak.net Thu Dec 20 21:41:59 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 20 Dec 2007 21:41:59 +0100 (CET) Subject: [pypy-svn] r49970 - in pypy/dist/pypy/jit/hintannotator: . test Message-ID: <20071220204159.C6C15168578@codespeak.net> Author: antocuni Date: Thu Dec 20 21:41:59 2007 New Revision: 49970 Modified: pypy/dist/pypy/jit/hintannotator/model.py pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Log: propagate deepfrozen also when calling methods Modified: pypy/dist/pypy/jit/hintannotator/model.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/model.py (original) +++ pypy/dist/pypy/jit/hintannotator/model.py Thu Dec 20 21:41:59 2007 @@ -730,7 +730,7 @@ return SomeLLAbstractConstant(RESULT, d, eager_concrete = False, # probably myorigin = myorigin) - return variableoftype(RESULT) + return variableoftype(RESULT, deepfrozen=args_hs[0].deepfrozen) def handle_highlevel_operation(bookkeeper, ll_func, *args_hs): Modified: pypy/dist/pypy/jit/hintannotator/test/test_annotator.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/test/test_annotator.py (original) +++ pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Thu Dec 20 21:41:59 2007 @@ -1106,7 +1106,6 @@ # these tests fail because of deepfreeze test_specialize_deepfreeze_calls = skip_policy - test_deepfreeze_variables = skip_policy test_cast_pointer_keeps_deepfreeze = skip_policy def test_void_oosend(self): From antocuni at codespeak.net Thu Dec 20 21:50:56 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 20 Dec 2007 21:50:56 +0100 (CET) Subject: [pypy-svn] r49971 - pypy/dist/pypy/jit/hintannotator Message-ID: <20071220205056.A7949169EA8@codespeak.net> Author: antocuni Date: Thu Dec 20 21:50:55 2007 New Revision: 49971 Modified: pypy/dist/pypy/jit/hintannotator/model.py Log: (arigo) propagate deepfrozen only when it's a method, else the first argument it's not a "self", i.e. no special role Modified: pypy/dist/pypy/jit/hintannotator/model.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/model.py (original) +++ pypy/dist/pypy/jit/hintannotator/model.py Thu Dec 20 21:50:55 2007 @@ -730,7 +730,8 @@ return SomeLLAbstractConstant(RESULT, d, eager_concrete = False, # probably myorigin = myorigin) - return variableoftype(RESULT, deepfrozen=args_hs[0].deepfrozen) + deepfrozen = ismethod and args_hs[0].deepfrozen # if self is deepfrozen, the result is it too + return variableoftype(RESULT, deepfrozen=deepfrozen) def handle_highlevel_operation(bookkeeper, ll_func, *args_hs): From antocuni at codespeak.net Fri Dec 21 10:46:22 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 21 Dec 2007 10:46:22 +0100 (CET) Subject: [pypy-svn] r49973 - in pypy/dist/pypy/jit/hintannotator: . test Message-ID: <20071221094622.D9407168435@codespeak.net> Author: antocuni Date: Fri Dec 21 10:46:21 2007 New Revision: 49973 Modified: pypy/dist/pypy/jit/hintannotator/bookkeeper.py pypy/dist/pypy/jit/hintannotator/model.py pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Log: deepfreeze oodowncast, ooupcast, and null constant pointers Modified: pypy/dist/pypy/jit/hintannotator/bookkeeper.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/bookkeeper.py (original) +++ pypy/dist/pypy/jit/hintannotator/bookkeeper.py Fri Dec 21 10:46:21 2007 @@ -6,6 +6,7 @@ from pypy.objspace.flow.model import Variable, Block, Link, FunctionGraph from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem import lltype, lloperation +from pypy.rpython.ootypesystem import ootype from pypy.tool.algo.unionfind import UnionFind from pypy.translator.backendopt import graphanalyze from pypy.translator.unsimplify import copyvar @@ -291,7 +292,10 @@ res = hintmodel.SomeLLAbstractConstant(const.concretetype, {}) res.const = const.value # we want null pointers to be deepfrozen! - if isinstance(const.concretetype, lltype.Ptr): + if isinstance(const.concretetype, (lltype.Ptr, + ootype.Instance, + ootype.BuiltinType, + ootype.StaticMethod)): if not const.value: res.deepfrozen = True return res Modified: pypy/dist/pypy/jit/hintannotator/model.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/model.py (original) +++ pypy/dist/pypy/jit/hintannotator/model.py Fri Dec 21 10:46:21 2007 @@ -410,6 +410,9 @@ RESTYPE = getbookkeeper().current_op_concretetype() return SomeLLAbstractVariable(RESTYPE, hs_v1.deepfrozen) + ooupcast = cast_pointer + oodowncast = cast_pointer + def indirect_call(hs_v1, *args_hs): hs_graph_list = args_hs[-1] args_hs = args_hs[:-1] @@ -566,6 +569,8 @@ myorigin = origin, deepfrozen = hs_c1.deepfrozen) + ooupcast = cast_pointer + oodowncast = cast_pointer class __extend__(SomeLLAbstractContainer): Modified: pypy/dist/pypy/jit/hintannotator/test/test_annotator.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/test/test_annotator.py (original) +++ pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Fri Dec 21 10:46:21 2007 @@ -1106,7 +1106,6 @@ # these tests fail because of deepfreeze test_specialize_deepfreeze_calls = skip_policy - test_cast_pointer_keeps_deepfreeze = skip_policy def test_void_oosend(self): class Foo: From arigo at codespeak.net Fri Dec 21 11:01:45 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Dec 2007 11:01:45 +0100 (CET) Subject: [pypy-svn] r49975 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071221100145.60CB9168437@codespeak.net> Author: arigo Date: Fri Dec 21 11:01:44 2007 New Revision: 49975 Modified: pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/test/test_bigtest.py pypy/dist/pypy/translator/llvm/test/test_new_gc.py pypy/dist/pypy/translator/llvm/test/test_symbolic.py Log: Fix failing tests. Improve test_new_gc (still skipped though). Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Fri Dec 21 11:01:44 2007 @@ -584,9 +584,9 @@ from_, indices, to = self.get_offset(value, []) - if from_ is lltype.Void: - assert isinstance(value, llmemory.ItemOffset) + if from_ is lltype.Void or not indices: return "0" + assert to is not lltype.Void r = self.database.repr_type indices_as_str = ", ".join("%s %s" % (w, i) for w, i in indices) @@ -605,9 +605,10 @@ # skips over a fixed size item (eg array access) from_ = value.TYPE - lasttype, lastvalue = indices[-1] - assert lasttype == word - indices[-1] = (word, lastvalue + value.repeat) + if from_ is not lltype.Void: + lasttype, lastvalue = indices[-1] + assert lasttype == word + indices[-1] = (word, lastvalue + value.repeat) to = value.TYPE elif isinstance(value, llmemory.FieldOffset): Modified: pypy/dist/pypy/translator/llvm/test/test_bigtest.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_bigtest.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_bigtest.py Fri Dec 21 11:01:44 2007 @@ -7,7 +7,7 @@ result, start, end = entry_point(1000) assert result - assert end - start > 0 and end - start < 5.0 + assert end - start > 0 and end - start < 15.0 def test_rpystone(): from pypy.translator.test.rpystone import pystones as entry_point Modified: pypy/dist/pypy/translator/llvm/test/test_new_gc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_new_gc.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_new_gc.py Fri Dec 21 11:01:44 2007 @@ -2,7 +2,36 @@ import py from pypy.translator.llvm.test.runtest import * +from pypy.rpython.lltypesystem import lltype, llmemory, llarena +def test_gc_offsets(): + py.test.skip("in-progress") + STRUCT = lltype.GcStruct('S1', ('x', lltype.Signed)) + ARRAY = lltype.GcArray(lltype.Signed) + s1 = llarena.round_up_for_allocation(llmemory.sizeof(STRUCT)) + s2 = llmemory.offsetof(STRUCT, 'x') + s3 = llmemory.ArrayLengthOffset(ARRAY) + s4 = llmemory.sizeof(ARRAY, 0) + s5 = llmemory.ArrayItemsOffset(ARRAY) + def fn(): + return (s1 * 100000000 + + s2 * 1000000 + + s3 * 10000 + + s4 * 100 + + s5) + mod, f = compile_test(fn, [], gcpolicy="semispace") + res = f() + i1 = (res // 100000000) % 100 + i2 = (res // 1000000) % 100 + i3 = (res // 10000) % 100 + i4 = (res // 100) % 100 + i5 = (res // 1) % 100 + assert i1 % 8 == 0 + assert 12 <= i1 <= 24 + assert 8 <= i2 <= i1 - 4 + assert 8 <= i3 <= 16 + assert i4 == i5 + assert i3 + 4 <= i5 def test_1(): py.test.skip("in-progress") Modified: pypy/dist/pypy/translator/llvm/test/test_symbolic.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_symbolic.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_symbolic.py Fri Dec 21 11:01:44 2007 @@ -23,12 +23,14 @@ def test_sizeof_array_with_no_length(): A = lltype.GcArray(lltype.Signed, hints={'nolength': True}) + B = lltype.Array(lltype.Signed, hints={'nolength': True}) a = lltype.malloc(A, 5, zero=True) arraysize = llmemory.itemoffsetof(A, 10) signedsize = llmemory.sizeof(lltype.Signed) + b_items = llmemory.ArrayItemsOffset(B) def f(): - return a[0] + arraysize-signedsize*10 + return (a[0] + arraysize-signedsize*10) * 1000 + b_items fn = compile_function(f, []) res = fn() assert res == 0 From arigo at codespeak.net Fri Dec 21 11:12:44 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Dec 2007 11:12:44 +0100 (CET) Subject: [pypy-svn] r49976 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20071221101244.35C0A1684E1@codespeak.net> Author: arigo Date: Fri Dec 21 11:12:43 2007 New Revision: 49976 Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py Log: Humpf. Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/transform.py Fri Dec 21 11:12:43 2007 @@ -416,7 +416,7 @@ except OverflowError: raise MemoryError() return tot_size - _ll_compute_size._inline_ = True + _ll_compute_size._always_inline_ = True def _ll_malloc_varsize_no_length(length, size, itemsize): tot_size = _ll_compute_size(length, size, itemsize) From arigo at codespeak.net Fri Dec 21 11:44:20 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Dec 2007 11:44:20 +0100 (CET) Subject: [pypy-svn] r49977 - in pypy/dist/pypy: rpython rpython/memory/gctransform translator/backendopt translator/c Message-ID: <20071221104420.7F3251684C7@codespeak.net> Author: arigo Date: Fri Dec 21 11:44:18 2007 New Revision: 49977 Modified: pypy/dist/pypy/rpython/annlowlevel.py pypy/dist/pypy/rpython/memory/gctransform/transform.py pypy/dist/pypy/translator/backendopt/all.py pypy/dist/pypy/translator/c/funcgen.py Log: backend_optimize() was not called on some of the helpers introduced by the GC. Fixing this and the naive way annlowlevel finds which new graphs it must backend- optimize. Add a sanity-check that backendopts are not applied on graphs already seen by the backend. Modified: pypy/dist/pypy/rpython/annlowlevel.py ============================================================================== --- pypy/dist/pypy/rpython/annlowlevel.py (original) +++ pypy/dist/pypy/rpython/annlowlevel.py Fri Dec 21 11:44:18 2007 @@ -142,7 +142,7 @@ self.delayedreprs = {} self.delayedconsts = [] self.delayedfuncs = [] - self.original_graph_count = len(rtyper.annotator.translator.graphs) + self.newgraphs = {} def getgraph(self, ll_function, args_s, s_result): # get the graph of the mix-level helper ll_function and prepare it for @@ -233,6 +233,8 @@ rtyper = self.rtyper ann = rtyper.annotator bk = ann.bookkeeper + translator = ann.translator + original_graph_count = len(translator.graphs) for ll_function, graph, args_s, s_result in self.pending: # mark the return block as already annotated, because the return var # annotation was forced in getgraph() above. This prevents temporary @@ -241,6 +243,7 @@ ann.annotated[graph.returnblock] = graph s_function = bk.immutablevalue(ll_function) bk.emulate_pbc_call(graph, s_function, args_s) + self.newgraphs[graph] = True ann.complete_helpers(self.policy) for ll_function, graph, args_s, s_result in self.pending: s_real_result = ann.binding(graph.getreturnvar()) @@ -250,9 +253,13 @@ " found by annotating: %r" % (graph, s_result, s_real_result)) del self.pending[:] + for graph in translator.graphs[original_graph_count:]: + self.newgraphs[graph] = True def finish_rtype(self): rtyper = self.rtyper + translator = rtyper.annotator.translator + original_graph_count = len(translator.graphs) rtyper.type_system.perform_normalizations(rtyper) for r in self.delayedreprs: r.set_setup_delayed(False) @@ -261,6 +268,7 @@ p._become(repr.convert_const(obj)) rtyper.call_all_setups() for p, graph in self.delayedfuncs: + self.newgraphs[graph] = True real_p = rtyper.getcallable(graph) REAL = lltype.typeOf(real_p).TO FUNCTYPE = lltype.typeOf(p).TO @@ -272,13 +280,14 @@ self.delayedreprs.clear() del self.delayedconsts[:] del self.delayedfuncs[:] + for graph in translator.graphs[original_graph_count:]: + self.newgraphs[graph] = True def backend_optimize(self, **flags): # only optimize the newly created graphs from pypy.translator.backendopt.all import backend_optimizations translator = self.rtyper.annotator.translator - newgraphs = translator.graphs[self.original_graph_count:] - self.original_graph_count = len(translator.graphs) + newgraphs = self.newgraphs.keys() backend_optimizations(translator, newgraphs, secondary=True, **flags) # ____________________________________________________________ Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/transform.py Fri Dec 21 11:44:18 2007 @@ -292,6 +292,7 @@ self.finished_helpers = True if self.translator is not None: self.mixlevelannotator.finish_rtype() + self.mixlevelannotator.backend_optimize() def finish_tables(self): pass Modified: pypy/dist/pypy/translator/backendopt/all.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/all.py (original) +++ pypy/dist/pypy/translator/backendopt/all.py Fri Dec 21 11:44:18 2007 @@ -42,6 +42,8 @@ if graphs is None: graphs = translator.graphs + for graph in graphs: + assert not hasattr(graph, '_seen_by_the_backend') if config.print_statistics: print "before optimizations:" Modified: pypy/dist/pypy/translator/c/funcgen.py ============================================================================== --- pypy/dist/pypy/translator/c/funcgen.py (original) +++ pypy/dist/pypy/translator/c/funcgen.py Fri Dec 21 11:44:18 2007 @@ -37,6 +37,7 @@ oldgraph""".split() def __init__(self, graph, db, exception_policy=None, functionname=None): + graph._seen_by_the_backend = True self.graph = graph self.db = db self.gcpolicy = db.gcpolicy From arigo at codespeak.net Fri Dec 21 11:52:18 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Dec 2007 11:52:18 +0100 (CET) Subject: [pypy-svn] r49978 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20071221105218.3F417168503@codespeak.net> Author: arigo Date: Fri Dec 21 11:52:17 2007 New Revision: 49978 Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py Log: Sorry, this breaks all tests. Need to investigate more... Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/transform.py Fri Dec 21 11:52:17 2007 @@ -292,7 +292,7 @@ self.finished_helpers = True if self.translator is not None: self.mixlevelannotator.finish_rtype() - self.mixlevelannotator.backend_optimize() + #self.mixlevelannotator.backend_optimize() def finish_tables(self): pass From arigo at codespeak.net Fri Dec 21 12:01:30 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Dec 2007 12:01:30 +0100 (CET) Subject: [pypy-svn] r49979 - in pypy/dist/pypy/rpython: . memory/gctransform Message-ID: <20071221110130.802DF16851E@codespeak.net> Author: arigo Date: Fri Dec 21 12:01:30 2007 New Revision: 49979 Modified: pypy/dist/pypy/rpython/annlowlevel.py pypy/dist/pypy/rpython/memory/gctransform/transform.py Log: Fix. Modified: pypy/dist/pypy/rpython/annlowlevel.py ============================================================================== --- pypy/dist/pypy/rpython/annlowlevel.py (original) +++ pypy/dist/pypy/rpython/annlowlevel.py Fri Dec 21 12:01:30 2007 @@ -289,6 +289,7 @@ translator = self.rtyper.annotator.translator newgraphs = self.newgraphs.keys() backend_optimizations(translator, newgraphs, secondary=True, **flags) + self.newgraphs.clear() # ____________________________________________________________ Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/transform.py Fri Dec 21 12:01:30 2007 @@ -292,7 +292,7 @@ self.finished_helpers = True if self.translator is not None: self.mixlevelannotator.finish_rtype() - #self.mixlevelannotator.backend_optimize() + self.mixlevelannotator.backend_optimize() def finish_tables(self): pass From arigo at codespeak.net Fri Dec 21 14:26:41 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Dec 2007 14:26:41 +0100 (CET) Subject: [pypy-svn] r49980 - pypy/dist/pypy/translator/c Message-ID: <20071221132641.E68C2168528@codespeak.net> Author: arigo Date: Fri Dec 21 14:26:39 2007 New Revision: 49980 Modified: pypy/dist/pypy/translator/c/database.py Log: Log the database "finish" phases. Modified: pypy/dist/pypy/translator/c/database.py ============================================================================== --- pypy/dist/pypy/translator/c/database.py (original) +++ pypy/dist/pypy/translator/c/database.py Fri Dec 21 14:26:39 2007 @@ -287,11 +287,14 @@ # list: finish_callbacks = [] if self.gctransformer: - finish_callbacks.append(self.gctransformer.finish_helpers) + finish_callbacks.append(('GC transformer: finished helpers', + self.gctransformer.finish_helpers)) if self.stacklesstransformer: - finish_callbacks.append(self.stacklesstransformer.finish) + finish_callbacks.append(('Stackless transformer: finished', + self.stacklesstransformer.finish)) if self.gctransformer: - finish_callbacks.append(self.gctransformer.finish_tables) + finish_callbacks.append(('GC transformer: finished tables', + self.gctransformer.finish_tables)) def add_dependencies(newdependencies): for value in newdependencies: @@ -335,8 +338,9 @@ continue # progress - follow all dependencies again if finish_callbacks: - finish = finish_callbacks.pop(0) + logmsg, finish = finish_callbacks.pop(0) newdependencies = finish() + log.database(logmsg) if newdependencies: add_dependencies(newdependencies) continue # progress - follow all dependencies again @@ -347,6 +351,7 @@ self.completed = True if show_progress: dump() + log.database("Completed") def globalcontainers(self): for node in self.containerlist: From arigo at codespeak.net Fri Dec 21 14:27:34 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Dec 2007 14:27:34 +0100 (CET) Subject: [pypy-svn] r49981 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20071221132734.AD9CA16851E@codespeak.net> Author: arigo Date: Fri Dec 21 14:27:34 2007 New Revision: 49981 Modified: pypy/dist/pypy/rpython/memory/gctransform/boehm.py pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gctransform/refcounting.py pypy/dist/pypy/rpython/memory/gctransform/transform.py Log: Make sure the database sees all the finalizers early enough. (fixes test_newgc.py: TestUsingStacklessFramework().test_framework_finalizer) Modified: pypy/dist/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/boehm.py Fri Dec 21 14:27:34 2007 @@ -108,7 +108,7 @@ " v = cast_adr_to_ptr(addr, PTR_TYPE)\n" "%s\n")%(static_body,) exec src in d - fptr = self.annotate_helper(d['ll_finalizer'], [llmemory.Address], lltype.Void) + fptr = self.annotate_finalizer(d['ll_finalizer'], [llmemory.Address], lltype.Void) elif destrptr: EXC_INSTANCE_TYPE = self.translator.rtyper.exceptiondata.lltype_of_exception_value def ll_finalizer(addr): @@ -116,7 +116,7 @@ v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) ll_call_destructor(destrptr, v) llop.gc_restore_exception(lltype.Void, exc_instance) - fptr = self.annotate_helper(ll_finalizer, [llmemory.Address], lltype.Void) + fptr = self.annotate_finalizer(ll_finalizer, [llmemory.Address], lltype.Void) else: fptr = lltype.nullptr(self.FINALIZER_PTR.TO) Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Fri Dec 21 14:27:34 2007 @@ -801,9 +801,9 @@ def ll_finalizer(addr): v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) ll_call_destructor(destrptr, v) - fptr = self.transformer.annotate_helper(ll_finalizer, - [llmemory.Address], - lltype.Void) + fptr = self.transformer.annotate_finalizer(ll_finalizer, + [llmemory.Address], + lltype.Void) else: fptr = lltype.nullptr(gctypelayout.GCData.FINALIZERTYPE.TO) return fptr Modified: pypy/dist/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/refcounting.py Fri Dec 21 14:27:34 2007 @@ -247,7 +247,7 @@ 'HDRPTR':lltype.Ptr(self.HDR)} exec src in d this = d['ll_deallocator'] - fptr = self.annotate_helper(this, [llmemory.Address], lltype.Void) + fptr = self.annotate_finalizer(this, [llmemory.Address], lltype.Void) self.static_deallocator_funcptrs[TYPE] = fptr for p in find_gc_ptrs_in_type(TYPE): self.static_deallocation_funcptr_for_type(p.TO) Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/transform.py Fri Dec 21 14:27:34 2007 @@ -111,6 +111,7 @@ self.lltype_to_classdef = translator.rtyper.lltype_to_classdef_mapping() self.graphs_to_inline = {} self.graph_dependencies = {} + self.ll_finalizers_ptrs = [] if self.MinimalGCTransformer: self.minimalgctransformer = self.MinimalGCTransformer(self) else: @@ -286,6 +287,11 @@ ptr = self.annotate_helper(ll_helper, ll_args, ll_result, inline=inline) return Constant(ptr, lltype.typeOf(ptr)) + def annotate_finalizer(self, ll_finalizer, ll_args, ll_result): + fptr = self.annotate_helper(ll_finalizer, ll_args, ll_result) + self.ll_finalizers_ptrs.append(fptr) + return fptr + def finish_helpers(self): if self.translator is not None: self.mixlevelannotator.finish_annotate() @@ -293,6 +299,11 @@ if self.translator is not None: self.mixlevelannotator.finish_rtype() self.mixlevelannotator.backend_optimize() + # Make sure that the database also sees all finalizers now. + # XXX we need to think more about the interaction with stackless... + # It is likely that the finalizers need special support there + newgcdependencies = self.ll_finalizers_ptrs + return newgcdependencies def finish_tables(self): pass From cfbolz at codespeak.net Fri Dec 21 14:55:47 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 21 Dec 2007 14:55:47 +0100 (CET) Subject: [pypy-svn] r49982 - pypy/dist/pypy/lang/prolog/interpreter Message-ID: <20071221135547.4C0591684FD@codespeak.net> Author: cfbolz Date: Fri Dec 21 14:55:46 2007 New Revision: 49982 Added: pypy/dist/pypy/lang/prolog/interpreter/eclipseprologparser.py (contents, props changed) pypy/dist/pypy/lang/prolog/interpreter/targetparserstandalone.py (contents, props changed) Log: some hacks to make a prolog parser that Jens can use to make a Prolog editor in Eclipse. Extremely advanced technology. Added: pypy/dist/pypy/lang/prolog/interpreter/eclipseprologparser.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/prolog/interpreter/eclipseprologparser.py Fri Dec 21 14:55:46 2007 @@ -0,0 +1,300 @@ +import autopath +from pypy.lang.prolog.interpreter.parsing import OrderTransformer, make_default_operations, unescape +from pypy.rlib.parsing.tree import Nonterminal, Symbol, RPythonVisitor + +class ASTTermBuilder(RPythonVisitor): + + def __init__(self): + self.varname_to_var = {} + + def build(self, s): + "NOT_RPYTHON" + if isinstance(s, list): + return self.build_many(s) + return self.build_query(s) + + def build_many(self, trees): + ot = OrderTransformer() + facts = [] + for tree in trees: + s = ot.transform(tree) + facts.append(self.build_fact(s)) + return facts + + def build_query(self, s): + ot = OrderTransformer() + s = ot.transform(s) + return self.visit(s.children[0]) + + def build_fact(self, node): + self.varname_to_var = {} + return self.visit(node.children[0]) + + def visit(self, node): + node = self.find_first_interesting(node) + return self.dispatch(node) + + def general_nonterminal_visit(self, node): + children = [] + name = "" + token = None + for child in node.children: + if isinstance(child, Symbol): + name = self.general_symbol_visit(child).name + token = self.make_token(child) + else: + children.append(child) + children = [self.visit(child) for child in children] + if len(children) == 1 and (name == "-" or name == "+"): + if name == "-": + factor = -1 + else: + factor = 1 + child = children[0] + if isinstance(child, Number): + child.value *= factor + return child + if isinstance(child, Float): + child.value *= factor + return child + result = Term() + result.setup(token, children, name) + return result + + def build_list(self, node): + result = [] + while node is not None: + node = self._build_list(node, result) + return result + + def _build_list(self, node, result): + node = self.find_first_interesting(node) + if isinstance(node, Nonterminal): + child = node.children[1] + if (isinstance(child, Symbol) and + node.children[1].additional_info == ","): + element = self.visit(node.children[0]) + result.append(element) + return node.children[2] + result.append(self.visit(node)) + + def find_first_interesting(self, node): + if isinstance(node, Nonterminal) and len(node.children) == 1: + return self.find_first_interesting(node.children[0]) + return node + + def general_symbol_visit(self, node): + if node.additional_info.startswith("'"): + end = len(node.additional_info) - 1 + assert end >= 0 + name = unescape(node.additional_info[1:end]) + else: + name = node.additional_info + result = Atom() + result.setup(self.make_token(node), name) + return result + + def visit_VAR(self, node): + varname = node.additional_info + result = Var() + result.setup(self.make_token(node), varname) + return result + + def visit_NUMBER(self, node): + s = node.additional_info + try: + result = Number() + result.setup(self.make_token(node), int(s)) + return result + except ValueError: + result = Float() + result.setup(self.make_token(node), float(s)) + return result + + def visit_complexterm(self, node): + name = self.general_symbol_visit(node.children[0]).name + children = self.build_list(node.children[2]) + result = Term() + result.setup(self.make_token(node.children[1]), children, name) + return result + + def visit_expr(self, node): + if node.children[0].additional_info == '-': + result = self.visit(node.children[1]) + if isinstance(result, Number): + result.value = -result.value + elif isinstance(result, Float): + result.value = -result.value + return self.visit(node.children[1]) + + def visit_listexpr(self, node): + node = node.children[1] + if len(node.children) == 1: + l = self.build_list(node) + start = Atom() + start.setup(None, "[]") + else: + l = self.build_list(node.children[0]) + start = self.visit(node.children[2]) + l.reverse() + curr = start + for elt in l: + curr = Term() + curr.setup(None, [elt, curr], ".") + return curr + + def make_token(self, node): + token = Token() + source_pos = node.token.source_pos + token.setup(node.token.source, source_pos.i, source_pos.lineno, + source_pos.columnno) + return token + + +class Token(object): + def __init__(self): + pass + + def setup(self, text, startpos, line, column): + self.text = text + self.startpos = startpos + self.line = line + self.column = column + +class Node(object): + def __init__(self): + self.children = [None] # trick the annotator + + def setup(self, token, children=None): + self.token = token + if children is not None: + self.children = children + else: + self.children = [] + + def num_children(self): + return len(self.children) + + def get_child(self, i): + return self.children[i] + +class Atom(Node): + def __init__(self): + pass + def setup(self, token, name): + Node.setup(self, token) + self.name = name + +class Number(Node): + def __init__(self): + pass + def setup(self, token, value): + Node.setup(self, token) + self.value = value + +class Float(Node): + def __init__(self): + pass + def setup(self, token, value): + Node.setup(self, token) + self.value = value + +class Var(Node): + def __init__(self): + pass + def setup(self, token, varname): + Node.setup(self, token) + self.varname = varname + +class Term(Node): + def __init__(self): + pass + def setup(self, token, children, name): + Node.setup(self, token, children) + self.name = name + + +class Lines(object): + def __init__(self): + self.parser = None + self.operations = make_default_operations() + self.terms = [] + + def num_terms(self): + return len(self.terms) + + def get_term(self, i): + return self.terms[i] + +class ParseError(Exception): + def __init__(self, pos, reason): + self.pos = pos + self.reason = reason + self.args = (pos, reason) + + +def _build_and_run(lines, tree): + from pypy.lang.prolog.interpreter.parsing import TermBuilder + builder = ASTTermBuilder() + term = builder.build_query(tree) + if (isinstance(term, Term) and term.name == ":-" and + len(term.children) == 1): + child = term.get_child(0) + if isinstance(child, Term) and child.name == "op": + if len(child.children) != 3: + raise ParseError(child.token.startpos, "expecting three arguments") + precedence = child.children[0] + form = child.children[1] + name = child.children[2] + if not isinstance(precedence, Number): + raise ParseError(precedence.token.startpos, "first argument to op should be number") + if not isinstance(form, Atom): + raise ParseError(precedence.token.startpos, "second argument to op should be atom") + if not isinstance(name, Atom): + raise ParseError(precedence.token.startpos, "third argument to op should be atom") + parser = impl_op(lines.operations, precedence.value, form.name, name.name) + lines.parser = parser + + lines.terms.append(term) + return lines.parser + +def parse(s): + from pypy.lang.prolog.interpreter.parsing import parse_file + lines = Lines() + trees = parse_file(s, lines.parser, _build_and_run, lines) + return lines + + +def impl_op(operations, precedence, typ, name): + from pypy.lang.prolog.interpreter import parsing + precedence_to_ops = {} + for prec, allops in operations: + precedence_to_ops[prec] = allops + for form, ops in allops: + try: + index = ops.index(name) + del ops[index] + except ValueError: + pass + if precedence != 0: + if precedence in precedence_to_ops: + allops = precedence_to_ops[precedence] + for form, ops in allops: + if form == typ: + ops.append(name) + break + else: + allops.append((typ, [name])) + else: + for i in range(len(operations)): + (prec, allops) = operations[i] + if precedence > prec: + operations.insert(i, (precedence, [(typ, [name])])) + break + else: + operations.append((precedence, [(typ, [name])])) + return parsing.make_parser_at_runtime(operations) + + +if __name__ == '__main__': + x = parse(":- op(900, xfx, hello). a hello b.") Added: pypy/dist/pypy/lang/prolog/interpreter/targetparserstandalone.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/prolog/interpreter/targetparserstandalone.py Fri Dec 21 14:55:46 2007 @@ -0,0 +1,10 @@ +from pypy.lang.prolog.interpreter.eclipseprologparser import parse + +def entry_point(args): + if len(args) > 1: + t = parse(args[1]) + return 0 + return -1 + +def target(driver, args): + return entry_point, None From arigo at codespeak.net Fri Dec 21 14:56:03 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Dec 2007 14:56:03 +0100 (CET) Subject: [pypy-svn] r49983 - in pypy/dist/pypy/translator/llvm: . test Message-ID: <20071221135603.998F61684FD@codespeak.net> Author: arigo Date: Fri Dec 21 14:55:57 2007 New Revision: 49983 Modified: pypy/dist/pypy/translator/llvm/opwriter.py pypy/dist/pypy/translator/llvm/test/test_lladdresses.py Log: Support for cast_int_to_adr and back. Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Fri Dec 21 14:55:57 2007 @@ -133,8 +133,10 @@ self.shiftop(opr) elif op.opname.startswith('cast_') or op.opname.startswith('truncate_'): - if op.opname == "cast_ptr_to_int": + if op.opname in ("cast_ptr_to_int", "cast_adr_to_int"): self.cast_ptr_to_int(opr) + elif op.opname in ("cast_int_to_ptr", "cast_int_to_adr"): + self.cast_int_to_ptr(opr) else: self.cast_primitive(opr) else: @@ -265,7 +267,11 @@ def cast_ptr_to_int(self, opr): self.codewriter.cast(opr.retref, opr.argtypes[0], opr.argrefs[0], opr.rettype, 'ptrtoint') - + + def cast_int_to_ptr(self, opr): + self.codewriter.cast(opr.retref, opr.argtypes[0], + opr.argrefs[0], opr.rettype, 'inttoptr') + def int_is_true(self, opr): self.codewriter.binaryop("icmp ne", opr.retref, opr.argtypes[0], opr.argrefs[0], "0") Modified: pypy/dist/pypy/translator/llvm/test/test_lladdresses.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_lladdresses.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_lladdresses.py Fri Dec 21 14:55:57 2007 @@ -154,5 +154,17 @@ return s1.x + s1.y fn = compile_function(f, []) - assert f() == 579 + assert fn() == 579 +def test_cast_to_int(): + S1 = lltype.GcStruct("S1", ("x", lltype.Signed), ("y", lltype.Signed)) + s1 = lltype.malloc(S1, zero=True) + def f(): + a1 = llmemory.cast_ptr_to_adr(s1) + i1 = llmemory.cast_adr_to_int(a1) + a2 = llmemory.cast_int_to_adr(i1) + s2 = llmemory.cast_adr_to_ptr(a2, lltype.Ptr(S1)) + return int(s1 == s2) + + fn = compile_function(f, []) + assert fn() == 1 From arigo at codespeak.net Fri Dec 21 15:05:35 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Dec 2007 15:05:35 +0100 (CET) Subject: [pypy-svn] r49984 - pypy/dist/pypy/config Message-ID: <20071221140535.824261684FD@codespeak.net> Author: arigo Date: Fri Dec 21 15:05:35 2007 New Revision: 49984 Modified: pypy/dist/pypy/config/translationoption.py Log: Experimentally at least, we can now use the framework gc with llvm. Modified: pypy/dist/pypy/config/translationoption.py ============================================================================== --- pypy/dist/pypy/config/translationoption.py (original) +++ pypy/dist/pypy/config/translationoption.py Fri Dec 21 15:05:35 2007 @@ -28,9 +28,10 @@ requires={ "c": [("translation.type_system", "lltype")], "llvm": [("translation.type_system", "lltype"), - ("translation.gc", "boehm"), + #("translation.gc", "boehm"), ("translation.backendopt.raisingop2direct_call", True), - ("translation.rweakref", False)], + #("translation.rweakref", False), + ], "cli": [("translation.type_system", "ootype")], "jvm": [("translation.type_system", "ootype")], "js": [("translation.type_system", "ootype")], From arigo at codespeak.net Fri Dec 21 16:26:45 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Dec 2007 16:26:45 +0100 (CET) Subject: [pypy-svn] r49985 - in pypy/dist/pypy: rpython rpython/lltypesystem translator/llvm/test Message-ID: <20071221152645.1E903168522@codespeak.net> Author: arigo Date: Fri Dec 21 16:26:43 2007 New Revision: 49985 Modified: pypy/dist/pypy/rpython/extfunc.py pypy/dist/pypy/rpython/lltypesystem/llarena.py pypy/dist/pypy/translator/llvm/test/test_new_gc.py Log: Fighting the nice wrappers that get inserted in the middle of GC code and that allocate stuff themselves. Modified: pypy/dist/pypy/rpython/extfunc.py ============================================================================== --- pypy/dist/pypy/rpython/extfunc.py (original) +++ pypy/dist/pypy/rpython/extfunc.py Fri Dec 21 16:26:43 2007 @@ -181,16 +181,25 @@ if hasattr(self, fake_method_name): # If we have both an {ll,oo}impl and a {ll,oo}fakeimpl, # we need a wrapper that selects the proper one and calls it - from pypy.rlib.objectmodel import running_on_llinterp - from pypy.rlib.debug import llinterpcall from pypy.tool.sourcetools import func_with_new_name - original_impl = impl - def ll_wrapper(*args): - if running_on_llinterp: - return llinterpcall(s_result, fakeimpl, *args) - else: - return original_impl(*args) - impl = func_with_new_name(ll_wrapper, name + '_wrapper') + # Using '*args' is delicate because this wrapper is also + # created for init-time functions like llarena.arena_malloc + # which are called before the GC is fully initialized + args = ', '.join(['arg%d' % i for i in range(len(args_ll))]) + d = {'original_impl': impl, + 's_result': s_result, + 'fakeimpl': fakeimpl, + } + exec py.code.compile(""" + from pypy.rlib.objectmodel import running_on_llinterp + from pypy.rlib.debug import llinterpcall + def ll_wrapper(%s): + if running_on_llinterp: + return llinterpcall(s_result, fakeimpl, %s) + else: + return original_impl(%s) + """ % (args, args, args)) in d + impl = func_with_new_name(d['ll_wrapper'], name + '_wrapper') if rtyper.annotator.translator.config.translation.sandbox: impl._dont_inline_ = True # store some attributes to the 'impl' function, where Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llarena.py Fri Dec 21 16:26:43 2007 @@ -360,7 +360,8 @@ llimpl_round_up_for_allocation = rffi.llexternal('ROUND_UP_FOR_ALLOCATION', [rffi.INT], rffi.INT, - sandboxsafe=True) + sandboxsafe=True, + _nowrapper=True) register_external(round_up_for_allocation, [int], int, 'll_arena.round_up_for_allocation', llimpl=llimpl_round_up_for_allocation, Modified: pypy/dist/pypy/translator/llvm/test/test_new_gc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_new_gc.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_new_gc.py Fri Dec 21 16:26:43 2007 @@ -5,7 +5,6 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena def test_gc_offsets(): - py.test.skip("in-progress") STRUCT = lltype.GcStruct('S1', ('x', lltype.Signed)) ARRAY = lltype.GcArray(lltype.Signed) s1 = llarena.round_up_for_allocation(llmemory.sizeof(STRUCT)) @@ -34,12 +33,11 @@ assert i3 + 4 <= i5 def test_1(): - py.test.skip("in-progress") def fn(n): d = {} for i in range(n): d[i] = str(i) - return d[n//2] + return int(d[n//2]) mod, f = compile_test(fn, [int], gcpolicy="semispace") assert f(5000) == fn(5000) From arigo at codespeak.net Fri Dec 21 16:33:39 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Dec 2007 16:33:39 +0100 (CET) Subject: [pypy-svn] r49986 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071221153339.C9E6616852A@codespeak.net> Author: arigo Date: Fri Dec 21 16:33:39 2007 New Revision: 49986 Modified: pypy/dist/pypy/translator/llvm/test/test_new_gc.py Log: test_weakref passes out of the box. Modified: pypy/dist/pypy/translator/llvm/test/test_new_gc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_new_gc.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_new_gc.py Fri Dec 21 16:33:39 2007 @@ -41,3 +41,40 @@ mod, f = compile_test(fn, [int], gcpolicy="semispace") assert f(5000) == fn(5000) + +def test_weakref(): + import weakref + from pypy.rlib import rgc + + class A: + pass + + keepalive = [] + def fn(): + n = 7000 + weakrefs = [] + a = None + for i in range(n): + if i & 1 == 0: + a = A() + a.index = i + assert a is not None + weakrefs.append(weakref.ref(a)) + if i % 7 == 6: + keepalive.append(a) + rgc.collect() + count_free = 0 + for i in range(n): + a = weakrefs[i]() + if i % 7 == 6: + assert a is not None + if a is not None: + assert a.index == i & ~1 + else: + count_free += 1 + return count_free + + mod, f = compile_test(fn, [], gcpolicy="semispace") + res = f() + # more than half of them should have been freed, ideally up to 6000 + assert 3500 <= res <= 6000 From arigo at codespeak.net Fri Dec 21 16:50:59 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Dec 2007 16:50:59 +0100 (CET) Subject: [pypy-svn] r49987 - in pypy/dist/pypy/translator: c llvm llvm/test Message-ID: <20071221155059.2F170168518@codespeak.net> Author: arigo Date: Fri Dec 21 16:50:58 2007 New Revision: 49987 Modified: pypy/dist/pypy/translator/c/node.py pypy/dist/pypy/translator/llvm/database.py pypy/dist/pypy/translator/llvm/test/test_new_gc.py Log: Support for prebuilt weakrefs. Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Fri Dec 21 16:50:58 2007 @@ -908,6 +908,7 @@ ptarget = obj._dereference() wrapper = db.gcpolicy.convert_weakref_to(ptarget) container = wrapper._obj + obj._converted_weakref = container # hack for genllvm :-/ return db.getcontainernode(container, _dont_write_c_code=False) Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Fri Dec 21 16:50:58 2007 @@ -114,6 +114,14 @@ else: node = OpaqueNode(self, value) + elif type_ is llmemory.WeakRef: + # XXX this uses a hack in translator.c.node.weakrefnode_factory() + # because we need to obtain not just *a* conversion of the weakref + # by the gcpolicy, but *the same* one as was already registered + # in the genc database and seen by the gctransformer + value = value._converted_weakref + return self.create_constant_node(lltype.typeOf(value), value) + assert node is not None, "%s not supported" % (type_) return node @@ -161,8 +169,6 @@ ptrvalue = fakedaddress.ptr ct = lltype.typeOf(ptrvalue) self.prepare_constant(ct, ptrvalue) - ##elif ct is llmemory.WeakGcAddress: - ## return # XXX sometime soon else: if isinstance(value, llmemory.AddressOffset): @@ -389,7 +395,6 @@ lltype.UnsignedLongLong: "i64", lltype.SignedLongLong: "i64", llmemory.Address: "i8*", - #XXX llmemory.WeakGcAddress: "i8*", } # 32 bit platform @@ -418,7 +423,6 @@ lltype.Bool : self.repr_bool, lltype.Void : self.repr_void, llmemory.Address : self.repr_address, - #llmemory.WeakGcAddress : self.repr_weakgcaddress, } try: @@ -536,11 +540,6 @@ res = "bitcast(%s to i8*)" % (ref,) return res - def repr_weakgcaddress(self, type_, value): - assert isinstance(value, llmemory.fakeweakaddress) - log.WARNING("XXX weakgcaddress completely ignored...") - return 'null' - def repr_signed(self, type_, value): if isinstance(value, Symbolic): return self.repr_symbolic(type_, value) Modified: pypy/dist/pypy/translator/llvm/test/test_new_gc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_new_gc.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_new_gc.py Fri Dec 21 16:50:58 2007 @@ -78,3 +78,27 @@ res = f() # more than half of them should have been freed, ideally up to 6000 assert 3500 <= res <= 6000 + +def test_prebuilt_weakref(): + import weakref + from pypy.rlib import rgc + class A: + pass + a = A() + a.hello = 42 + refs = [weakref.ref(a), weakref.ref(A())] + rgc.collect() + def fn(): + result = 0 + for i in range(2): + a = refs[i]() + rgc.collect() + if a is None: + result += (i+1) + else: + result += a.hello * (i+1) + return result + + mod, f = compile_test(fn, [], gcpolicy="semispace") + res = f() + assert res == fn() From arigo at codespeak.net Fri Dec 21 16:51:29 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Dec 2007 16:51:29 +0100 (CET) Subject: [pypy-svn] r49988 - pypy/dist/pypy/translator/llvm/test Message-ID: <20071221155129.95AFB168518@codespeak.net> Author: arigo Date: Fri Dec 21 16:51:29 2007 New Revision: 49988 Added: pypy/dist/pypy/translator/llvm/test/test_newgc.py - copied unchanged from r49987, pypy/dist/pypy/translator/llvm/test/test_new_gc.py Removed: pypy/dist/pypy/translator/llvm/test/test_new_gc.py Log: Rename, as a minor attempt at consistency with c/test/test_newgc.py. From arigo at codespeak.net Fri Dec 21 17:19:01 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Dec 2007 17:19:01 +0100 (CET) Subject: [pypy-svn] r49989 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20071221161901.106DA168522@codespeak.net> Author: arigo Date: Fri Dec 21 17:19:01 2007 New Revision: 49989 Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py Log: Comment this out now to reduce verbosity. Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Fri Dec 21 17:19:01 2007 @@ -88,8 +88,8 @@ mallocvars = {target.inputargs[index]: True} mallocnum += 1 find_in_block(target, mallocvars) - if result: - print "found %s initializing stores in %s" % (len(result), graph.name) + #if result: + # print "found %s initializing stores in %s" % (len(result), graph.name) return result class FrameworkGCTransformer(GCTransformer): From arigo at codespeak.net Fri Dec 21 17:55:05 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Dec 2007 17:55:05 +0100 (CET) Subject: [pypy-svn] r49990 - pypy/dist/pypy/translator/llvm Message-ID: <20071221165505.736F7168522@codespeak.net> Author: arigo Date: Fri Dec 21 17:55:04 2007 New Revision: 49990 Modified: pypy/dist/pypy/translator/llvm/gc.py Log: Fix this and try again. Modified: pypy/dist/pypy/translator/llvm/gc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/gc.py (original) +++ pypy/dist/pypy/translator/llvm/gc.py Fri Dec 21 17:55:04 2007 @@ -46,6 +46,9 @@ def op_collect(self, codewriter, opr): raise Exception, 'GcPolicy should not be used directly' + def op_set_max_heap_size(self, codewriter, opr): + pass + def new(db, config): gcpolicy = config.translation.gctransformer if gcpolicy == 'boehm': @@ -129,9 +132,6 @@ [targetvar, 0, size, boundary_size]) - def op_set_max_heap_size(self, codewriter, opr): - pass - def op__collect(self, codewriter, opr): codewriter.call(opr.retref, opr.rettype, "@pypy_gc__collect", opr.argtypes, opr.argrefs) From arigo at codespeak.net Fri Dec 21 19:05:29 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 21 Dec 2007 19:05:29 +0100 (CET) Subject: [pypy-svn] r49991 - in pypy/dist/pypy: config translator/llvm translator/llvm/module translator/llvm/test Message-ID: <20071221180529.0A99016842B@codespeak.net> Author: arigo Date: Fri Dec 21 19:05:28 2007 New Revision: 49991 Modified: pypy/dist/pypy/config/translationoption.py pypy/dist/pypy/translator/llvm/gc.py pypy/dist/pypy/translator/llvm/module/boehm.h pypy/dist/pypy/translator/llvm/opwriter.py pypy/dist/pypy/translator/llvm/test/test_newgc.py Log: Support for weakrefs in llvm with Boehm too. Modified: pypy/dist/pypy/config/translationoption.py ============================================================================== --- pypy/dist/pypy/config/translationoption.py (original) +++ pypy/dist/pypy/config/translationoption.py Fri Dec 21 19:05:28 2007 @@ -28,9 +28,7 @@ requires={ "c": [("translation.type_system", "lltype")], "llvm": [("translation.type_system", "lltype"), - #("translation.gc", "boehm"), ("translation.backendopt.raisingop2direct_call", True), - #("translation.rweakref", False), ], "cli": [("translation.type_system", "ootype")], "jvm": [("translation.type_system", "ootype")], Modified: pypy/dist/pypy/translator/llvm/gc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/gc.py (original) +++ pypy/dist/pypy/translator/llvm/gc.py Fri Dec 21 19:05:28 2007 @@ -109,6 +109,10 @@ def gc_libraries(self): return ['gc', 'pthread'] + def get_real_weakref_type(self): + from pypy.rpython.memory.gctransform import boehm + return boehm.WEAKLINK + def _zeromalloc(self, codewriter, targetvar, size=1, atomic=False, exc_flag=False): """ assumes malloc of word size """ Modified: pypy/dist/pypy/translator/llvm/module/boehm.h ============================================================================== --- pypy/dist/pypy/translator/llvm/module/boehm.h (original) +++ pypy/dist/pypy/translator/llvm/module/boehm.h Fri Dec 21 19:05:28 2007 @@ -35,6 +35,15 @@ GC_REGISTER_FINALIZER(whatever, (GC_finalization_proc)proc, NULL, NULL, NULL); } +void pypy_disappearing_link(void *link, void *obj) { + if (GC_base(obj) == NULL) + ; /* 'obj' is probably a prebuilt object - it makes no */ + /* sense to register it then, and it crashes Boehm in */ + /* quite obscure ways */ + else + GC_GENERAL_REGISTER_DISAPPEARING_LINK(link, obj); +} + extern GC_all_interior_pointers; // startup specific code for boehm Modified: pypy/dist/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/dist/pypy/translator/llvm/opwriter.py (original) +++ pypy/dist/pypy/translator/llvm/opwriter.py Fri Dec 21 19:05:28 2007 @@ -323,6 +323,9 @@ self.codewriter.cast(tmpvar, opr.argtypes[1], opr.argrefs[1], 'i8 *') self.codewriter.call(None, 'void', '@pypy_register_finalizer', ['i8 *', 'i8 *'], [opr.argrefs[0], tmpvar]) + def boehm_disappearing_link(self, opr): + self.codewriter.call(None, 'void', '@pypy_disappearing_link', ['i8 *', 'i8 *'], [opr.argrefs[0], opr.argrefs[1]]) + def call_boehm_gc_alloc(self, opr): word = self.db.get_machine_word() self.codewriter.call(opr.retref, 'i8*', '@pypy_malloc', Modified: pypy/dist/pypy/translator/llvm/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_newgc.py Fri Dec 21 19:05:28 2007 @@ -42,63 +42,72 @@ mod, f = compile_test(fn, [int], gcpolicy="semispace") assert f(5000) == fn(5000) -def test_weakref(): - import weakref - from pypy.rlib import rgc - - class A: - pass - - keepalive = [] - def fn(): - n = 7000 - weakrefs = [] - a = None - for i in range(n): - if i & 1 == 0: - a = A() - a.index = i - assert a is not None - weakrefs.append(weakref.ref(a)) - if i % 7 == 6: - keepalive.append(a) - rgc.collect() - count_free = 0 - for i in range(n): - a = weakrefs[i]() - if i % 7 == 6: +class BaseTestGC(object): + + def test_weakref(self): + import weakref + from pypy.rlib import rgc + + class A: + pass + + keepalive = [] + def fn(): + n = 7000 + weakrefs = [] + a = None + for i in range(n): + if i & 1 == 0: + a = A() + a.index = i assert a is not None - if a is not None: - assert a.index == i & ~1 - else: - count_free += 1 - return count_free - - mod, f = compile_test(fn, [], gcpolicy="semispace") - res = f() - # more than half of them should have been freed, ideally up to 6000 - assert 3500 <= res <= 6000 - -def test_prebuilt_weakref(): - import weakref - from pypy.rlib import rgc - class A: - pass - a = A() - a.hello = 42 - refs = [weakref.ref(a), weakref.ref(A())] - rgc.collect() - def fn(): - result = 0 - for i in range(2): - a = refs[i]() + weakrefs.append(weakref.ref(a)) + if i % 7 == 6: + keepalive.append(a) rgc.collect() - if a is None: - result += (i+1) - else: - result += a.hello * (i+1) - return result - - mod, f = compile_test(fn, [], gcpolicy="semispace") - res = f() - assert res == fn() + count_free = 0 + for i in range(n): + a = weakrefs[i]() + if i % 7 == 6: + assert a is not None + if a is not None: + assert a.index == i & ~1 + else: + count_free += 1 + return count_free + + mod, f = compile_test(fn, [], gcpolicy=self.gcpolicy) + res = f() + # more than half of them should have been freed, ideally up to 6000 + assert 3500 <= res <= 6000 + + def test_prebuilt_weakref(self): + import weakref + from pypy.rlib import rgc + class A: + pass + a = A() + a.hello = 42 + refs = [weakref.ref(a), weakref.ref(A())] + rgc.collect() + def fn(): + result = 0 + for i in range(2): + a = refs[i]() + rgc.collect() + if a is None: + result += (i+1) + else: + result += a.hello * (i+1) + return result + + mod, f = compile_test(fn, [], gcpolicy=self.gcpolicy) + res = f() + assert res == fn() + + +class TestBoehmGC(BaseTestGC): + gcpolicy = "boehm" + +class TestFrameworkGC(BaseTestGC): + gcpolicy = "semispace" From antocuni at codespeak.net Fri Dec 21 22:15:43 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 21 Dec 2007 22:15:43 +0100 (CET) Subject: [pypy-svn] r49992 - in pypy/dist/pypy/jit/hintannotator: . test Message-ID: <20071221211543.CE3A616851E@codespeak.net> Author: antocuni Date: Fri Dec 21 22:15:43 2007 New Revision: 49992 Modified: pypy/dist/pypy/jit/hintannotator/model.py pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Log: argh! We need to propagate deepfrozen in each of the two branches, not only in one. One more test passes Modified: pypy/dist/pypy/jit/hintannotator/model.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/model.py (original) +++ pypy/dist/pypy/jit/hintannotator/model.py Fri Dec 21 22:15:43 2007 @@ -724,6 +724,7 @@ def handle_highlevel_operation_novirtual(bookkeeper, ismethod, immutable, *args_hs): RESULT = bookkeeper.current_op_concretetype() + deepfrozen = ismethod and args_hs[0].deepfrozen # if self is deepfrozen, the result is it too if ismethod and (immutable or args_hs[0].deepfrozen): for hs_v in args_hs: if not isinstance(hs_v, SomeLLAbstractConstant): @@ -734,8 +735,8 @@ for hs_c in args_hs]) return SomeLLAbstractConstant(RESULT, d, eager_concrete = False, # probably - myorigin = myorigin) - deepfrozen = ismethod and args_hs[0].deepfrozen # if self is deepfrozen, the result is it too + myorigin = myorigin, + deepfrozen=deepfrozen) return variableoftype(RESULT, deepfrozen=deepfrozen) Modified: pypy/dist/pypy/jit/hintannotator/test/test_annotator.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/test/test_annotator.py (original) +++ pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Fri Dec 21 22:15:43 2007 @@ -1104,9 +1104,6 @@ test_simple_struct_malloc = skip_policy test_container_union = skip_policy - # these tests fail because of deepfreeze - test_specialize_deepfreeze_calls = skip_policy - def test_void_oosend(self): class Foo: def bar(self): From antocuni at codespeak.net Fri Dec 21 23:51:59 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 21 Dec 2007 23:51:59 +0100 (CET) Subject: [pypy-svn] r49993 - in pypy/dist/pypy/rpython/ootypesystem: . test Message-ID: <20071221225159.2455D1684DB@codespeak.net> Author: antocuni Date: Fri Dec 21 23:51:58 2007 New Revision: 49993 Modified: pypy/dist/pypy/rpython/ootypesystem/ootype.py pypy/dist/pypy/rpython/ootypesystem/test/test_ootype.py Log: we can't assume that there is alway a graph attached to the method Modified: pypy/dist/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/ootype.py Fri Dec 21 23:51:58 2007 @@ -198,7 +198,8 @@ def _lookup_graphs(self, meth_name): _, meth = self._lookup(meth_name) graphs = set() - graphs.add(meth.graph) # we assume there is always a graph + if not getattr(meth, 'abstract', False): + graphs.add(meth.graph) for SUBTYPE in self._subclasses: graphs.update(SUBTYPE._lookup_graphs(meth_name)) return graphs Modified: pypy/dist/pypy/rpython/ootypesystem/test/test_ootype.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/test/test_ootype.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/test/test_ootype.py Fri Dec 21 23:51:58 2007 @@ -508,4 +508,30 @@ assert len(TYPE_B._lookup_graphs('ofoo')) == 1 assert len(TYPE_A._lookup_graphs('obar')) == 1 assert len(TYPE_B._lookup_graphs('obar')) == 1 - + +def test_lookup_graphs_abstract(): + from pypy.translator.translator import TranslationContext, graphof + class A: + pass + class B(A): + def foo(self): + pass + class C(A): + def foo(self): + pass + + def fn(flag): + obj = flag and B() or C() + obj.foo() + return obj + + t = TranslationContext() + t.buildannotator().build_types(fn, [int]) + t.buildrtyper(type_system='ootype').specialize() + graph = graphof(t, fn) + TYPE_A = graph.getreturnvar().concretetype + TYPE_B = TYPE_A._subclasses[0] + TYPE_C = TYPE_A._subclasses[1] + assert len(TYPE_A._lookup_graphs('ofoo')) == 2 + assert len(TYPE_B._lookup_graphs('ofoo')) == 1 + assert len(TYPE_C._lookup_graphs('ofoo')) == 1 From arigo at codespeak.net Sat Dec 22 09:11:48 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Dec 2007 09:11:48 +0100 (CET) Subject: [pypy-svn] r49995 - in pypy/dist/pypy/translator: . backendopt/test Message-ID: <20071222081148.A51FC1684E6@codespeak.net> Author: arigo Date: Sat Dec 22 09:11:47 2007 New Revision: 49995 Modified: pypy/dist/pypy/translator/backendopt/test/test_inline.py pypy/dist/pypy/translator/simplify.py Log: How hard do we want to enforce the condition that indirect_call() is never done with a Constant target? Modified: pypy/dist/pypy/translator/backendopt/test/test_inline.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_inline.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_inline.py Sat Dec 22 09:11:47 2007 @@ -718,3 +718,16 @@ eval_func, t = self.check_auto_inlining(fn5, [], checkvirtual=True) res = eval_func([]) assert res == 42 + + def test_indirect_call_becomes_direct(self): + def h1(n): + return n+1 + def h2(n): + return n+2 + def g(myfunc, n): + return myfunc(n*5) + def f(x, y): + return g(h1, x) + g(h2, y) + eval_func = self.check_inline(g, f, [int, int]) + res = eval_func([10, 173]) + assert res == f(10, 173) Modified: pypy/dist/pypy/translator/simplify.py ============================================================================== --- pypy/dist/pypy/translator/simplify.py (original) +++ pypy/dist/pypy/translator/simplify.py Sat Dec 22 09:11:47 2007 @@ -366,7 +366,12 @@ def rename_op(op): args = [rename(a) for a in op.args] op = SpaceOperation(op.opname, args, rename(op.result), op.offset) - #op = SpaceOperation(op.opname, args, rename(op.result)) + # special case... + if op.opname == 'indirect_call': + if isinstance(op.args[0], Constant): + assert isinstance(op.args[-1], Constant) + del op.args[-1] + op.opname = 'direct_call' return op for op in link.target.operations: link.prevblock.operations.append(rename_op(op)) From arigo at codespeak.net Sat Dec 22 09:16:39 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Dec 2007 09:16:39 +0100 (CET) Subject: [pypy-svn] r49996 - in pypy/dist/pypy/rpython/memory/gctransform: . test Message-ID: <20071222081639.2D2211684E1@codespeak.net> Author: arigo Date: Sat Dec 22 09:16:38 2007 New Revision: 49996 Modified: pypy/dist/pypy/rpython/memory/gctransform/refcounting.py pypy/dist/pypy/rpython/memory/gctransform/test/test_refcounting.py pypy/dist/pypy/rpython/memory/gctransform/transform.py Log: Tests are a bit too precise... they won't work after backendopt. Modified: pypy/dist/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/refcounting.py Sat Dec 22 09:16:38 2007 @@ -115,8 +115,8 @@ self.dynamic_deallocator_funcptrs = {} self.queryptr2dynamic_deallocator_funcptr = {} - def finish_helpers(self): - GCTransformer.finish_helpers(self) + def finish_helpers(self, **kwds): + GCTransformer.finish_helpers(self, **kwds) from pypy.translator.backendopt.malloc import remove_mallocs seen = {} graphs = [] Modified: pypy/dist/pypy/rpython/memory/gctransform/test/test_refcounting.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/test/test_refcounting.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/test/test_refcounting.py Sat Dec 22 09:16:38 2007 @@ -127,7 +127,7 @@ transformer = cls(t) fptr = getattr(transformer, attr)(TYPE) transformer.transform_graph(graphof(t, f)) - transformer.finish() + transformer.finish(backendopt=False) if conftest.option.view: t.view() if fptr: Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/transform.py Sat Dec 22 09:16:38 2007 @@ -292,13 +292,14 @@ self.ll_finalizers_ptrs.append(fptr) return fptr - def finish_helpers(self): + def finish_helpers(self, backendopt=True): if self.translator is not None: self.mixlevelannotator.finish_annotate() self.finished_helpers = True if self.translator is not None: self.mixlevelannotator.finish_rtype() - self.mixlevelannotator.backend_optimize() + if backendopt: + self.mixlevelannotator.backend_optimize() # Make sure that the database also sees all finalizers now. # XXX we need to think more about the interaction with stackless... # It is likely that the finalizers need special support there @@ -308,8 +309,8 @@ def finish_tables(self): pass - def finish(self): - self.finish_helpers() + def finish(self, backendopt=True): + self.finish_helpers(backendopt=backendopt) self.finish_tables() def transform_generic_set(self, hop): From arigo at codespeak.net Sat Dec 22 09:19:31 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Dec 2007 09:19:31 +0100 (CET) Subject: [pypy-svn] r49997 - pypy/dist/pypy/translator/c Message-ID: <20071222081931.A5B561684E1@codespeak.net> Author: arigo Date: Sat Dec 22 09:19:31 2007 New Revision: 49997 Modified: pypy/dist/pypy/translator/c/genc.py Log: Fix the generated setup.py. Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Sat Dec 22 09:19:31 2007 @@ -764,7 +764,7 @@ ext_modules = [Extension(name = "%(modulename)s", sources = ["%(modulename)s.c"], extra_compile_args = extra_compile_args, - include_dirs = [PYPY_INCLUDE_DIR] + %(include_dirs)r, + include_dirs = (PYPY_INCLUDE_DIR,) + %(include_dirs)r, library_dirs = %(library_dirs)r, libraries = %(libraries)r)]) ''' From arigo at codespeak.net Sat Dec 22 10:17:13 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Dec 2007 10:17:13 +0100 (CET) Subject: [pypy-svn] r49998 - in pypy/dist/pypy/translator: backendopt backendopt/test c/test Message-ID: <20071222091713.B46CA168501@codespeak.net> Author: arigo Date: Sat Dec 22 10:17:12 2007 New Revision: 49998 Modified: pypy/dist/pypy/translator/backendopt/inline.py pypy/dist/pypy/translator/backendopt/test/test_all.py pypy/dist/pypy/translator/backendopt/test/test_constfold.py pypy/dist/pypy/translator/c/test/test_boehm.py Log: When doing secondary backendopts, don't inline a graph that is not in the 'graphs' list. The issue, shown by test_boehm, is that the graphs outside the list may already be exception-transformed. The inliner cannot do something sane with them. Modified: pypy/dist/pypy/translator/backendopt/inline.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/inline.py (original) +++ pypy/dist/pypy/translator/backendopt/inline.py Sat Dec 22 10:17:12 2007 @@ -625,6 +625,7 @@ def inlinable_static_callers(graphs): + ok_to_call = dict.fromkeys(graphs) result = [] for parentgraph in graphs: for block in parentgraph.iterblocks(): @@ -632,7 +633,7 @@ if op.opname == "direct_call": funcobj = get_funcobj(op.args[0].value) graph = getattr(funcobj, 'graph', None) - if graph is not None: + if graph is not None and graph in ok_to_call: if getattr(getattr(funcobj, '_callable', None), '_dont_inline_', False): continue @@ -640,7 +641,7 @@ if op.opname == "oosend": meth = get_meth_from_oosend(op) graph = getattr(meth, 'graph', None) - if graph is not None: + if graph is not None and graph in ok_to_call: result.append((parentgraph, graph)) return result Modified: pypy/dist/pypy/translator/backendopt/test/test_all.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_all.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_all.py Sat Dec 22 10:17:12 2007 @@ -230,6 +230,53 @@ res = interp.eval_graph(f_graph, [11, 22]) assert res == 33 + def test_secondary_backendopt(self): + # checks an issue with a newly added graph that calls an + # already-exception-transformed graph. This can occur e.g. + # from a late-seen destructor added by the GC transformer + # which ends up calling existing code. + def common(n): + if n > 5: + raise ValueError + def main(n): + common(n) + def later(n): + try: + common(n) + return 0 + except ValueError: + return 1 + + t = TranslationContext() + t.buildannotator().build_types(main, [int]) + t.buildrtyper(type_system='lltype').specialize() + exctransformer = t.getexceptiontransformer() + exctransformer.create_exception_handling(graphof(t, common)) + from pypy.annotation import model as annmodel + from pypy.rpython.annlowlevel import MixLevelHelperAnnotator + annhelper = MixLevelHelperAnnotator(t.rtyper) + later_graph = annhelper.getgraph(later, [annmodel.SomeInteger()], + annmodel.SomeInteger()) + annhelper.finish() + annhelper.backend_optimize() + # ^^^ as the inliner can't handle exception-transformed graphs, + # this should *not* inline common() into later(). + if conftest.option.view: + later_graph.show() + common_graph = graphof(t, common) + found = False + for block in later_graph.iterblocks(): + for op in block.operations: + if (op.opname == 'direct_call' and + op.args[0].value._obj.graph is common_graph): + found = True + assert found, "cannot find the call (buggily inlined?)" + from pypy.rpython.llinterp import LLInterpreter + llinterp = LLInterpreter(t.rtyper) + res = llinterp.eval_graph(later_graph, [10]) + assert res == 1 + + class TestOOType(BaseTester): type_system = 'ootype' check_malloc_removed = OOTypeMallocRemovalTest.check_malloc_removed Modified: pypy/dist/pypy/translator/backendopt/test/test_constfold.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/test/test_constfold.py (original) +++ pypy/dist/pypy/translator/backendopt/test/test_constfold.py Sat Dec 22 10:17:12 2007 @@ -261,7 +261,7 @@ graph, t = get_graph(fn, [int]) from pypy.translator.backendopt import removenoops, inline - inline.auto_inline_graphs(t, [graph], threshold=999) + inline.auto_inline_graphs(t, t.graphs, threshold=999) constant_fold_graph(graph) removenoops.remove_same_as(graph) if conftest.option.view: @@ -303,7 +303,7 @@ graph, t = get_graph(fn, [int]) from pypy.translator.backendopt import removenoops, inline - inline.auto_inline_graphs(t, [graph], threshold=999) + inline.auto_inline_graphs(t, t.graphs, threshold=999) removenoops.remove_same_as(graph) constant_fold_graph(graph) if conftest.option.view: Modified: pypy/dist/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_boehm.py (original) +++ pypy/dist/pypy/translator/c/test/test_boehm.py Sat Dec 22 10:17:12 2007 @@ -150,7 +150,6 @@ def test_del_raises(self): from pypy.rpython.lltypesystem.lloperation import llop - import os class A(object): def __del__(self): s.dels += 1 From arigo at codespeak.net Sat Dec 22 10:23:42 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Dec 2007 10:23:42 +0100 (CET) Subject: [pypy-svn] r49999 - pypy/dist/pypy/objspace/flow Message-ID: <20071222092342.59F9D168501@codespeak.net> Author: arigo Date: Sat Dec 22 10:23:41 2007 New Revision: 49999 Modified: pypy/dist/pypy/objspace/flow/model.py Log: Trying to check this in in order to see how many tests fail. Will revert immediately. Modified: pypy/dist/pypy/objspace/flow/model.py ============================================================================== --- pypy/dist/pypy/objspace/flow/model.py (original) +++ pypy/dist/pypy/objspace/flow/model.py Sat Dec 22 10:23:41 2007 @@ -522,6 +522,8 @@ #assert v.value != last_exc_value if op.opname == 'direct_call': assert isinstance(op.args[0], Constant) + elif op.opname == 'indirect_call': + assert isinstance(op.args[0], Variable) definevar(op.result) exc_links = {} From arigo at codespeak.net Sat Dec 22 10:26:05 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Dec 2007 10:26:05 +0100 (CET) Subject: [pypy-svn] r50000 - pypy/dist/pypy/objspace/flow Message-ID: <20071222092605.655AC168501@codespeak.net> Author: arigo Date: Sat Dec 22 10:26:04 2007 New Revision: 50000 Modified: pypy/dist/pypy/objspace/flow/model.py Log: This change is likely to break tests; I checked it in to see the results of the automated test run on wyvern. Comment it out until we know. Modified: pypy/dist/pypy/objspace/flow/model.py ============================================================================== --- pypy/dist/pypy/objspace/flow/model.py (original) +++ pypy/dist/pypy/objspace/flow/model.py Sat Dec 22 10:26:04 2007 @@ -522,8 +522,8 @@ #assert v.value != last_exc_value if op.opname == 'direct_call': assert isinstance(op.args[0], Constant) - elif op.opname == 'indirect_call': - assert isinstance(op.args[0], Variable) + #elif op.opname == 'indirect_call': + # assert isinstance(op.args[0], Variable) definevar(op.result) exc_links = {} From antocuni at codespeak.net Sat Dec 22 11:49:15 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 22 Dec 2007 11:49:15 +0100 (CET) Subject: [pypy-svn] r50003 - in pypy/dist/pypy/jit/hintannotator: . test Message-ID: <20071222104915.337AC1684F9@codespeak.net> Author: antocuni Date: Sat Dec 22 11:49:14 2007 New Revision: 50003 Modified: pypy/dist/pypy/jit/hintannotator/model.py pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Log: when ananlyzing an oosend, it doesn't matter whether the hs_self is a variable or a constant to determine the list of possible graphs; it's only the type that matters, so move oosend from SomeLLAbstractConstat to SomeLLAbstractValue Modified: pypy/dist/pypy/jit/hintannotator/model.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/model.py (original) +++ pypy/dist/pypy/jit/hintannotator/model.py Sat Dec 22 11:49:14 2007 @@ -443,47 +443,6 @@ # function return annmodel.unionof(hs_res, bookkeeper.current_op_binding()) - def oosend(hs_v1, hs_name, *args_hs): - RESTYPE = getbookkeeper().current_op_concretetype() - return variableoftype(RESTYPE) - -class __extend__(SomeLLAbstractConstant): - - def same_as(hs_c1): - # this is here to prevent setup() below from adding a different - # version of same_as() - return hs_c1 - - def hint(hs_c1, hs_flags): - if hs_flags.const.get('concrete', False): - for o in hs_c1.origins: - o.set_fixed() - hs_concrete = reorigin(hs_c1) - hs_concrete.eager_concrete = True - return hs_concrete - if hs_flags.const.get('forget', False): - assert isinstance(hs_c1, SomeLLAbstractConstant) - return reorigin(hs_c1) - return SomeLLAbstractValue.hint(hs_c1, hs_flags) - - def direct_call(hs_f1, *args_hs): - bookkeeper = getbookkeeper() - fnobj = get_funcobj(hs_f1.const) - if (bookkeeper.annotator.policy.oopspec and - hasattr(fnobj._callable, 'oopspec')): - # try to handle the call as a high-level operation - try: - return handle_highlevel_operation(bookkeeper, fnobj._callable, - *args_hs) - except NotImplementedError: - pass - - # normal call - if not hasattr(fnobj, 'graph'): - raise NotImplementedError("XXX call to externals or primitives") - - return hs_f1._call_single_graph(fnobj.graph, lltype.typeOf(fnobj).RESULT, *args_hs) - def _call_single_graph(hs_f1, graph, RESULT, *args_hs): bookkeeper = getbookkeeper() if not bookkeeper.annotator.policy.look_inside_graph(graph): @@ -529,6 +488,43 @@ # like an indirect_call return hs_c1._call_multiple_graphs(graph_list, METH.RESULT, hs_c1, *args_hs) # prepend hs_c1 to the args +class __extend__(SomeLLAbstractConstant): + + def same_as(hs_c1): + # this is here to prevent setup() below from adding a different + # version of same_as() + return hs_c1 + + def hint(hs_c1, hs_flags): + if hs_flags.const.get('concrete', False): + for o in hs_c1.origins: + o.set_fixed() + hs_concrete = reorigin(hs_c1) + hs_concrete.eager_concrete = True + return hs_concrete + if hs_flags.const.get('forget', False): + assert isinstance(hs_c1, SomeLLAbstractConstant) + return reorigin(hs_c1) + return SomeLLAbstractValue.hint(hs_c1, hs_flags) + + def direct_call(hs_f1, *args_hs): + bookkeeper = getbookkeeper() + fnobj = get_funcobj(hs_f1.const) + if (bookkeeper.annotator.policy.oopspec and + hasattr(fnobj._callable, 'oopspec')): + # try to handle the call as a high-level operation + try: + return handle_highlevel_operation(bookkeeper, fnobj._callable, + *args_hs) + except NotImplementedError: + pass + + # normal call + if not hasattr(fnobj, 'graph'): + raise NotImplementedError("XXX call to externals or primitives") + + return hs_f1._call_single_graph(fnobj.graph, lltype.typeOf(fnobj).RESULT, *args_hs) + def getfield(hs_c1, hs_fieldname): S = hs_c1.concretetype.TO FIELD_TYPE = getattr(S, hs_fieldname.const) Modified: pypy/dist/pypy/jit/hintannotator/test/test_annotator.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/test/test_annotator.py (original) +++ pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Sat Dec 22 11:49:14 2007 @@ -1113,6 +1113,3 @@ f = Foo() f.bar() hs = self.hannotate(fn, [], policy=P_OOPSPEC_NOVIRTUAL) - - def test_simple_method_call_var(self): - py.test.skip('fixme!') From antocuni at codespeak.net Sat Dec 22 11:54:14 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 22 Dec 2007 11:54:14 +0100 (CET) Subject: [pypy-svn] r50004 - pypy/dist/pypy/jit/hintannotator/test Message-ID: <20071222105414.653D21684F9@codespeak.net> Author: antocuni Date: Sat Dec 22 11:54:14 2007 New Revision: 50004 Modified: pypy/dist/pypy/jit/hintannotator/test/test_toy.py Log: hoorray, all relevant tests pass out of the box. We can consider the hintannotator to be fully ported to ootype now :-) Modified: pypy/dist/pypy/jit/hintannotator/test/test_toy.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/test/test_toy.py (original) +++ pypy/dist/pypy/jit/hintannotator/test/test_toy.py Sat Dec 22 11:54:14 2007 @@ -1,3 +1,4 @@ +import py from pypy.jit.hintannotator.test.test_annotator import AbstractAnnotatorTest from pypy.jit.hintannotator.test.test_annotator import P_OOPSPEC, P_OOPSPEC_NOVIRTUAL @@ -22,6 +23,8 @@ class TestLLType(BaseToyTest): type_system = 'lltype' -## XXX: all tests fail :-( -##class TestOOType(BaseToyTest): -## type_system = 'ootype' +class TestOOType(BaseToyTest): + type_system = 'ootype' + + def test_hannotate_tl(self): + py.test.skip('fixme? (This policy is not relevant for now)') From arigo at codespeak.net Sat Dec 22 16:03:40 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Dec 2007 16:03:40 +0100 (CET) Subject: [pypy-svn] r50007 - pypy/dist/pypy/objspace/flow Message-ID: <20071222150340.1F08C1684FE@codespeak.net> Author: arigo Date: Sat Dec 22 16:03:38 2007 New Revision: 50007 Modified: pypy/dist/pypy/objspace/flow/model.py Log: No failure - it seems we don't have an indirect_call() to a Constant in our whole test suite. Good I guess. Modified: pypy/dist/pypy/objspace/flow/model.py ============================================================================== --- pypy/dist/pypy/objspace/flow/model.py (original) +++ pypy/dist/pypy/objspace/flow/model.py Sat Dec 22 16:03:38 2007 @@ -522,8 +522,8 @@ #assert v.value != last_exc_value if op.opname == 'direct_call': assert isinstance(op.args[0], Constant) - #elif op.opname == 'indirect_call': - # assert isinstance(op.args[0], Variable) + elif op.opname == 'indirect_call': + assert isinstance(op.args[0], Variable) definevar(op.result) exc_links = {} From arigo at codespeak.net Sat Dec 22 16:16:48 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Dec 2007 16:16:48 +0100 (CET) Subject: [pypy-svn] r50009 - pypy/branch/llvmgcroot Message-ID: <20071222151648.8D70B1684E9@codespeak.net> Author: arigo Date: Sat Dec 22 16:16:48 2007 New Revision: 50009 Added: pypy/branch/llvmgcroot/ - copied from r50008, pypy/dist/ Log: A branch in which to try out the llvm.gcroot() operation. From arigo at codespeak.net Sat Dec 22 16:18:43 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Dec 2007 16:18:43 +0100 (CET) Subject: [pypy-svn] r50010 - in pypy/branch/llvmgcroot/pypy: config rpython/lltypesystem rpython/memory/gctransform translator/c Message-ID: <20071222151843.B17E11684FE@codespeak.net> Author: arigo Date: Sat Dec 22 16:18:43 2007 New Revision: 50010 Added: pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/llvmgcroot.py Modified: pypy/branch/llvmgcroot/pypy/config/translationoption.py pypy/branch/llvmgcroot/pypy/rpython/lltypesystem/lloperation.py pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/framework.py pypy/branch/llvmgcroot/pypy/translator/c/gc.py pypy/branch/llvmgcroot/pypy/translator/c/genc.py Log: In-progress: the llvmgcroot GC transformer, with its custom StackRootIterator that walks the tables produced by the llvm GC plugin. Modified: pypy/branch/llvmgcroot/pypy/config/translationoption.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/config/translationoption.py (original) +++ pypy/branch/llvmgcroot/pypy/config/translationoption.py Sat Dec 22 16:18:43 2007 @@ -62,6 +62,10 @@ requires=[("translation.gctransformer", "framework"), ("translation.stackless", True)], suggests=[("translation.gc", "marksweep")]), + BoolOption("llvmgcroot", "Use the 'llvm.gcroot' primitive to find roots", + default=False, cmdline="--llvmgcroot", + requires=[("translation.backend", "llvm"), + ("translation.gctransformer", "framework")]), BoolOption("thread", "enable use of threading primitives", default=False, cmdline="--thread", requires=[("translation.gc", "boehm")]), Modified: pypy/branch/llvmgcroot/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/llvmgcroot/pypy/rpython/lltypesystem/lloperation.py Sat Dec 22 16:18:43 2007 @@ -408,6 +408,12 @@ # stackless: 'gc_x_become': LLOp(canraise=(RuntimeError,), canunwindgc=True), + # for llvm.gcroot() support. can change at any time + 'llvm_frameaddress': LLOp(), + 'llvm_gcmap_table': LLOp(), + 'llvm_store_gcroot': LLOp(), + 'llvm_load_gcroot': LLOp(), + # NOTE NOTE NOTE! don't forget *** canunwindgc=True *** for anything that # can go through a stack unwind, in particular anything that mallocs! Modified: pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/framework.py Sat Dec 22 16:18:43 2007 @@ -732,9 +732,7 @@ def pop_alive_nopyobj(self, var, llops): pass - def push_roots(self, hop, keep_current_args=False): - if self.incr_stack_ptr is None: - return + def get_livevars_for_roots(self, hop, keep_current_args=False): if self.gcdata.gc.moving_gc and not keep_current_args: # moving GCs don't borrow, so the caller does not need to keep # the arguments alive @@ -743,6 +741,12 @@ else: livevars = hop.livevars_after_op() + hop.current_op_keeps_alive() livevars = [var for var in livevars if not var_ispyobj(var)] + return livevars + + def push_roots(self, hop, keep_current_args=False): + if self.incr_stack_ptr is None: + return + livevars = self.get_livevars_for_roots(hop, keep_current_args) self.num_pushs += len(livevars) if not livevars: return [] Added: pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/llvmgcroot.py ============================================================================== --- (empty file) +++ pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/llvmgcroot.py Sat Dec 22 16:18:43 2007 @@ -0,0 +1,159 @@ +from pypy.rpython.memory.gctransform.framework import FrameworkGCTransformer +from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rpython import rmodel +from pypy.rpython.rbuiltin import gen_cast +from pypy.rlib.debug import ll_assert + + +class LLVMGcRootFrameworkGCTransformer(FrameworkGCTransformer): + # XXX this is completely specific to the llvm backend at the moment. + + def push_roots(self, hop, keep_current_args=False): + livevars = self.get_livevars_for_roots(hop, keep_current_args) + self.num_pushs += len(livevars) + if not livevars: + return [] + for k, var in enumerate(livevars): + c_k = rmodel.inputconst(lltype.Signed, k) + v_adr = gen_cast(hop.llops, llmemory.Address, var) + hop.genop("llvm_store_gcroot", [c_k, v_adr]) + return livevars + + def pop_roots(self, hop, livevars): + if not livevars: + return + if self.gcdata.gc.moving_gc: + # for moving collectors, reload the roots into the local variables + for k, var in enumerate(livevars): + c_k = rmodel.inputconst(lltype.Signed, k) + v_newaddr = hop.genop("llvm_load_gcroot", [c_k], + resulttype=llmemory.Address) + hop.genop("gc_reload_possibly_moved", [v_newaddr, var]) + # XXX for now, the values stay in the gcroots. It might keep + # some old objects alive for a bit longer than necessary. + + def build_stack_root_iterator(self): + sizeofaddr = llmemory.sizeof(llmemory.Address) + gcdata = self.gcdata + + class StackRootIterator: + _alloc_flavor_ = 'raw' + + def setup_root_stack(): + pass + setup_root_stack = staticmethod(setup_root_stack) + + need_root_stack = False + + def append_static_root(adr): + gcdata.static_root_end.address[0] = adr + gcdata.static_root_end += sizeofaddr + append_static_root = staticmethod(append_static_root) + + def __init__(self, with_static=True): + self.stack_current = llop.llvm_frameaddress(llmemory.Address) + self.remaining_roots_in_current_frame = 0 + if with_static: + self.static_current = gcdata.static_root_end + else: + self.static_current = gcdata.static_root_nongcend + + def pop(self): + while self.static_current != gcdata.static_root_start: + self.static_current -= sizeofaddr + result = self.static_current.address[0] + if result.address[0] != llmemory.NULL: + return result + + while True: + while self.remaining_roots_in_current_frame == 0: + if not self.walk_to_parent_frame(): + return llmemory.NULL + result = self.next_gcroot_from_current_frame() + if result.address[0] != llmemory.NULL: + return result + + def walk_to_parent_frame(self): + # + # XXX assumes a 32-bit machine for simplicity. + # + # The gcmap table is a list of pointers to gcmap_t + # structures, where the shape of each gcmap_t is: + # struct { + # int32_t FrameSize; + # int32_t PointCount; + # struct { + # void *SafePointAddress; + # int32_t LiveCount; + # int32_t LiveOffsets[LiveCount]; + # } Points[PointCount]; + # } gcmap_t; + # + callee_frame = self.stack_current + # + # XXX the details are completely specific to X86!!! + # a picture of the stack may help: + # ^ ^ ^ + # | ... | to older frames + # +--------------+ + # | first word | <------ caller_frame (addr of 1st word) + # + + + # | caller frame | + # | ... | + # | frame data | <------ frame_data_base + # +--------------+ + # | ret addr | + # +--------------+ + # | first word | <------ callee_frame (addr of 1st word) + # + + + # | callee frame | + # | ... | + # | frame data | lower addresses + # +--------------+ v v v + # + retaddr = callee_frame.address[1] + # + # try to locate the caller function based on retaddr. + # XXX this is just a linear scan for now, that's + # incredibly bad. + # + gcmaptbl = llop.llvm_gcmap_table(llmemory.Address) + i = 0 + while True: + gcmap = gcmaptbl.address[i] + if not gcmap: # function not found + return False # => assume end of stack + framesize = gcmap.signed[0] + pointcount = gcmap.signed[1] + gcmap += 8 + j = 0 + while j < pointcount: + safepointaddr = gcmap.address[0] + livecount = gcmap.signed[1] + if safepointaddr == retaddr: + # + # found! Setup pointers allowing us to + # parse the caller's frame structure... + # + caller_frame = callee_frame + 4 + framesize + self.stack_current = caller_frame + self.frame_data_base = callee_frame + 8 + self.remaining_roots_in_current_frame = livecount + self.liveoffsets = gcmap + 8 + return True + + # not found + gcmap += 8 + livecount * 4 + j += 1 + i += 1 + + def next_gcroot_from_current_frame(self): + i = self.remaining_roots_in_current_frame - 1 + self.remaining_roots_in_current_frame = i + ll_assert(i >= 0, "bad call to next_gcroot_from_current_frame") + liveoffset = self.liveoffsets.signed[i] + return self.frame_data_base + liveoffset + + + return StackRootIterator Modified: pypy/branch/llvmgcroot/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/c/gc.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/c/gc.py Sat Dec 22 16:18:43 2007 @@ -5,7 +5,7 @@ typeOf, Ptr, ContainerType, RttiStruct, \ RuntimeTypeInfo, getRuntimeTypeInfo, top_container from pypy.rpython.memory.gctransform import \ - refcounting, boehm, framework, stacklessframework + refcounting, boehm, framework, stacklessframework, llvmgcroot from pypy.rpython.lltypesystem import lltype, llmemory class BasicGcPolicy(object): @@ -305,6 +305,9 @@ transformerclass = stacklessframework.StacklessFrameworkGCTransformer requires_stackless = True +class LLVMGcRootFrameworkGcPolicy(FrameworkGcPolicy): + transformerclass = llvmgcroot.LLVMGcRootFrameworkGCTransformer + name_to_gcpolicy = { 'boehm': BoehmGcPolicy, @@ -312,6 +315,7 @@ 'none': NoneGcPolicy, 'framework': FrameworkGcPolicy, 'framework+stacklessgc': StacklessFrameworkGcPolicy, + 'framework+llvmgcroot': LLVMGcRootFrameworkGcPolicy, } Modified: pypy/branch/llvmgcroot/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/c/genc.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/c/genc.py Sat Dec 22 16:18:43 2007 @@ -93,6 +93,8 @@ name = self.config.translation.gctransformer if self.config.translation.stacklessgc: name = "%s+stacklessgc" % (name,) + if self.config.translation.llvmgcroot: + name = "%s+llvmgcroot" % (name,) return gc.name_to_gcpolicy[name] return self.gcpolicy From arigo at codespeak.net Sat Dec 22 16:20:26 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Dec 2007 16:20:26 +0100 (CET) Subject: [pypy-svn] r50011 - pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform Message-ID: <20071222152026.C81A716850E@codespeak.net> Author: arigo Date: Sat Dec 22 16:20:26 2007 New Revision: 50011 Modified: pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/llvmgcroot.py Log: Add a link. Modified: pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/llvmgcroot.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/llvmgcroot.py (original) +++ pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/llvmgcroot.py Sat Dec 22 16:20:26 2007 @@ -6,6 +6,14 @@ from pypy.rlib.debug import ll_assert +# +# This implements a StackRootWalker based on the data produced by +# the llvm GC plug-in found over there: +# +# http://codespeak.net/svn/user/arigo/hack/pypy-hack/stackrootwalker +# + + class LLVMGcRootFrameworkGCTransformer(FrameworkGCTransformer): # XXX this is completely specific to the llvm backend at the moment. From arigo at codespeak.net Sat Dec 22 20:11:50 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Dec 2007 20:11:50 +0100 (CET) Subject: [pypy-svn] r50017 - in pypy/branch/llvmgcroot/pypy/translator/llvm: . module Message-ID: <20071222191150.D6FBB168480@codespeak.net> Author: arigo Date: Sat Dec 22 20:11:50 2007 New Revision: 50017 Modified: pypy/branch/llvmgcroot/pypy/translator/llvm/codewriter.py pypy/branch/llvmgcroot/pypy/translator/llvm/funcnode.py pypy/branch/llvmgcroot/pypy/translator/llvm/genllvm.py pypy/branch/llvmgcroot/pypy/translator/llvm/module/support.py pypy/branch/llvmgcroot/pypy/translator/llvm/opwriter.py Log: Intermediate check-in, before I kill some of this code. Modified: pypy/branch/llvmgcroot/pypy/translator/llvm/codewriter.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/llvm/codewriter.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/llvm/codewriter.py Sat Dec 22 20:11:50 2007 @@ -173,3 +173,14 @@ self.call(var, "i32", "@write", ['i32', 'i8*', 'i32'], ['2', arg, '%d' % node.get_length()]) + + # ____________________________________________________________ + # Special support for llvm.gcroot + + def declare_gcroots(self, gcrootscount): + assert self.db.genllvm.config.translation.llvmgcroot + for i in range(gcrootscount): + self._indent("%%gcroot%d = alloca i8*" % i) + for i in range(gcrootscount): + self._indent("call void @llvm.gcroot(i8** %%gcroot%d, i8* null)" + % i) Modified: pypy/branch/llvmgcroot/pypy/translator/llvm/funcnode.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/llvm/funcnode.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/llvm/funcnode.py Sat Dec 22 20:11:50 2007 @@ -42,6 +42,7 @@ def patch_graph(self): graph = self.graph + self.gcrootscount = 0 if self.db.gctransformer: # inline the GC helpers (malloc, write_barrier) into # a copy of the graph @@ -63,6 +64,9 @@ op.args = [v_newaddr] op.result = v_newptr rename[v_targetvar] = v_newptr + elif op.opname == 'llvm_store_gcroot': + index = op.args[0].value + self.gcrootscount = max(self.gcrootscount, index+1) if rename: block.exitswitch = rename.get(block.exitswitch, block.exitswitch) @@ -120,7 +124,7 @@ def getdecl(self): returntype, ref, args = self.getdecl_parts() - return "%s %s(%s)" % (returntype, ref, ", ".join(args)) + return '%s %s(%s) gc "gcrootsingle"' % (returntype, ref, ", ".join(args)) # ______________________________________________________________________ # helpers for block writers @@ -220,6 +224,8 @@ # actual block writers def write_startblock(self, codewriter, block): + if self.gcrootscount > 0: + codewriter.declare_gcroots(self.gcrootscount) self.write_block_operations(codewriter, block) # a start block may return also if block.exitswitch is None and len(block.exits) == 0: Modified: pypy/branch/llvmgcroot/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/llvm/genllvm.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/llvm/genllvm.py Sat Dec 22 20:11:50 2007 @@ -105,6 +105,7 @@ node.writeimpl(codewriter) self._debug(codewriter) + self.write_gcroot_table(codewriter) codewriter.comment("End of file") codewriter.close() @@ -276,3 +277,29 @@ print "Start" print self.db.dump_pbcs() print "End" + + def write_gcroot_table(self, codewriter): + # Special support for llvm.gcroot + if self.config.translation.llvmgcroot: + from pypy.translator.llvm.funcnode import FuncImplNode + entries = [] + for node in self.db.getnodes(): + if isinstance(node, FuncImplNode): + ref = node.ref + assert ref.startswith('@') + entries.append('@__gcmap_' + ref[1:]) + + codewriter.header_comment("The global gcmap table") + data = [] + for entry in entries: + codewriter._append('%s = extern_weak constant i32' % entry) + data.append('i32* %s' % entry) + codewriter._append( + '@__gcmaptable1 = internal constant [%d x i32*] [ %s ]' % + (len(data), ", ".join(data))) + codewriter._append( + '@__gcmaptable = internal constant ' + 'i8* bitcast([%d x i32*]* @__gcmaptable1 to i8*)' + % len(data)) + codewriter._append( + '@__gcmaptablelen = internal constant i32 %d' % len(data)) Modified: pypy/branch/llvmgcroot/pypy/translator/llvm/module/support.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/llvm/module/support.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/llvm/module/support.py Sat Dec 22 20:11:50 2007 @@ -34,5 +34,8 @@ %result = phi i64 [%x, %block0], [%x2, %block1] ret i64 %result } + +declare void @llvm.gcroot(i8**, i8*) nounwind +declare i8* @llvm.frameaddress(i32) nounwind """ Modified: pypy/branch/llvmgcroot/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/llvm/opwriter.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/llvm/opwriter.py Sat Dec 22 20:11:50 2007 @@ -588,3 +588,24 @@ # cheat llvm self.codewriter.cast(opr.retref, opr.rettype, 'undef', opr.rettype) + # ____________________________________________________________ + # Special support for llvm.gcroot + + def llvm_store_gcroot(self, opr): + index = opr.op.args[0].value + rootref = '%%gcroot%d' % index + var = self.db.repr_tmpvar() + self.codewriter.cast(var, opr.argtypes[1], opr.argrefs[1], 'i8*') + self.codewriter.store('i8*', var, rootref) + + def llvm_load_gcroot(self, opr): + index = opr.op.args[0].value + rootref = '%%gcroot%d' % index + self.codewriter.load(opr.retref, opr.rettype, rootref) + + def llvm_frameaddress(self, opr): + self.codewriter.call(opr.retref, opr.rettype, + "@llvm.frameaddress", ['i32'], ['0']) + + def llvm_gcmap_table(self, opr): + self.codewriter.load(opr.retref, opr.rettype, '@__gcmaptable') From arigo at codespeak.net Sat Dec 22 21:34:13 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Dec 2007 21:34:13 +0100 (CET) Subject: [pypy-svn] r50019 - in pypy/branch/llvmgcroot/pypy: rpython/lltypesystem rpython/memory/gctransform translator/llvm translator/llvm/module Message-ID: <20071222203413.4874B1684CA@codespeak.net> Author: arigo Date: Sat Dec 22 21:34:12 2007 New Revision: 50019 Modified: pypy/branch/llvmgcroot/pypy/rpython/lltypesystem/lloperation.py pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/llvmgcroot.py pypy/branch/llvmgcroot/pypy/translator/llvm/genllvm.py pypy/branch/llvmgcroot/pypy/translator/llvm/module/support.py pypy/branch/llvmgcroot/pypy/translator/llvm/opwriter.py Log: Use the "gcrootsingle" gc plugin. Modified: pypy/branch/llvmgcroot/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/llvmgcroot/pypy/rpython/lltypesystem/lloperation.py Sat Dec 22 21:34:12 2007 @@ -409,8 +409,9 @@ 'gc_x_become': LLOp(canraise=(RuntimeError,), canunwindgc=True), # for llvm.gcroot() support. can change at any time - 'llvm_frameaddress': LLOp(), - 'llvm_gcmap_table': LLOp(), + 'llvm_frameaddress': LLOp(sideeffects=False), + 'llvm_gcmapstart': LLOp(sideeffects=False), + 'llvm_gcmapend': LLOp(sideeffects=False), 'llvm_store_gcroot': LLOp(), 'llvm_load_gcroot': LLOp(), Modified: pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/llvmgcroot.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/llvmgcroot.py (original) +++ pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/llvmgcroot.py Sat Dec 22 21:34:12 2007 @@ -84,19 +84,16 @@ def walk_to_parent_frame(self): # - # XXX assumes a 32-bit machine for simplicity. - # - # The gcmap table is a list of pointers to gcmap_t - # structures, where the shape of each gcmap_t is: - # struct { - # int32_t FrameSize; - # int32_t PointCount; - # struct { - # void *SafePointAddress; - # int32_t LiveCount; - # int32_t LiveOffsets[LiveCount]; - # } Points[PointCount]; - # } gcmap_t; + # The gcmap table is a list of pairs of pointers: + # void *SafePointAddress; + # void *Shape; + # + # A "safe point" is the return address of a call. + # The "shape" of a safe point records the size of the + # frame of the function containing it, as well as a + # list of the variables that contain gc roots at that + # time. Each variable is described by its offset in + # the frame. # callee_frame = self.stack_current # @@ -126,35 +123,31 @@ # XXX this is just a linear scan for now, that's # incredibly bad. # - gcmaptbl = llop.llvm_gcmap_table(llmemory.Address) - i = 0 - while True: - gcmap = gcmaptbl.address[i] - if not gcmap: # function not found - return False # => assume end of stack - framesize = gcmap.signed[0] - pointcount = gcmap.signed[1] - gcmap += 8 - j = 0 - while j < pointcount: - safepointaddr = gcmap.address[0] - livecount = gcmap.signed[1] - if safepointaddr == retaddr: - # - # found! Setup pointers allowing us to - # parse the caller's frame structure... - # - caller_frame = callee_frame + 4 + framesize - self.stack_current = caller_frame - self.frame_data_base = callee_frame + 8 - self.remaining_roots_in_current_frame = livecount - self.liveoffsets = gcmap + 8 - return True - - # not found - gcmap += 8 + livecount * 4 - j += 1 - i += 1 + gcmapstart = llop.llvm_gcmapstart(llmemory.Address) + gcmapend = llop.llvm_gcmapend(llmemory.Address) + while gcmapstart != gcmapend: + if gcmapstart.address[0] == retaddr: + # + # found! Setup pointers allowing us to + # parse the caller's frame structure... + # + shape = gcmapstart.address[1] + # XXX assumes that .signed is 32-bit + framesize = shape.signed[0] + livecount = shape.signed[1] + caller_frame = callee_frame + 4 + framesize + self.stack_current = caller_frame + self.frame_data_base = callee_frame + 8 + self.remaining_roots_in_current_frame = livecount + self.liveoffsets = shape + 8 + return True + + gcmapstart += 2 * sizeofaddr + + # retaddr not found. Assume that this is the system function + # that called the original entry point, i.e. it's the end of + # the stack for us + return False def next_gcroot_from_current_frame(self): i = self.remaining_roots_in_current_frame - 1 Modified: pypy/branch/llvmgcroot/pypy/translator/llvm/genllvm.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/llvm/genllvm.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/llvm/genllvm.py Sat Dec 22 21:34:12 2007 @@ -105,7 +105,6 @@ node.writeimpl(codewriter) self._debug(codewriter) - self.write_gcroot_table(codewriter) codewriter.comment("End of file") codewriter.close() @@ -277,29 +276,3 @@ print "Start" print self.db.dump_pbcs() print "End" - - def write_gcroot_table(self, codewriter): - # Special support for llvm.gcroot - if self.config.translation.llvmgcroot: - from pypy.translator.llvm.funcnode import FuncImplNode - entries = [] - for node in self.db.getnodes(): - if isinstance(node, FuncImplNode): - ref = node.ref - assert ref.startswith('@') - entries.append('@__gcmap_' + ref[1:]) - - codewriter.header_comment("The global gcmap table") - data = [] - for entry in entries: - codewriter._append('%s = extern_weak constant i32' % entry) - data.append('i32* %s' % entry) - codewriter._append( - '@__gcmaptable1 = internal constant [%d x i32*] [ %s ]' % - (len(data), ", ".join(data))) - codewriter._append( - '@__gcmaptable = internal constant ' - 'i8* bitcast([%d x i32*]* @__gcmaptable1 to i8*)' - % len(data)) - codewriter._append( - '@__gcmaptablelen = internal constant i32 %d' % len(data)) Modified: pypy/branch/llvmgcroot/pypy/translator/llvm/module/support.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/llvm/module/support.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/llvm/module/support.py Sat Dec 22 21:34:12 2007 @@ -37,5 +37,8 @@ declare void @llvm.gcroot(i8**, i8*) nounwind declare i8* @llvm.frameaddress(i32) nounwind + + at __gcmapstart = external constant i8* + at __gcmapend = external constant i8* """ Modified: pypy/branch/llvmgcroot/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/llvm/opwriter.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/llvm/opwriter.py Sat Dec 22 21:34:12 2007 @@ -607,5 +607,8 @@ self.codewriter.call(opr.retref, opr.rettype, "@llvm.frameaddress", ['i32'], ['0']) - def llvm_gcmap_table(self, opr): - self.codewriter.load(opr.retref, opr.rettype, '@__gcmaptable') + def llvm_gcmapstart(self, opr): + self.codewriter.load(opr.retref, opr.rettype, '@__gcmapstart') + + def llvm_gcmapend(self, opr): + self.codewriter.load(opr.retref, opr.rettype, '@__gcmapend') From arigo at codespeak.net Sat Dec 22 22:01:47 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Dec 2007 22:01:47 +0100 (CET) Subject: [pypy-svn] r50020 - in pypy/branch/llvmgcroot/pypy/translator/llvm: . module Message-ID: <20071222210147.EEA101683F4@codespeak.net> Author: arigo Date: Sat Dec 22 22:01:47 2007 New Revision: 50020 Modified: pypy/branch/llvmgcroot/pypy/translator/llvm/module/support.py pypy/branch/llvmgcroot/pypy/translator/llvm/opwriter.py Log: Bad level of indirection. For reference, here's how I compile the generated .ll file manually: cd /tmp/usession-arigo llvm-as < new_entrypoint.ll | opt -std-compile-opts > new_entrypoint.bc ~/svn/llvm-trunk/Debug/bin/llc -load=/home/arigo/svn/arigo/hack/pypy-hack/stackrootwalker/gcrootsingle.so new_entrypoint.bc -f -o new_entrypoint.s gcc -g -pthread new_entrypoint.s module_cache/module_*.c -I ~/svn/pypy/dist2/pypy/translator/c/ Modified: pypy/branch/llvmgcroot/pypy/translator/llvm/module/support.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/llvm/module/support.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/llvm/module/support.py Sat Dec 22 22:01:47 2007 @@ -38,7 +38,7 @@ declare void @llvm.gcroot(i8**, i8*) nounwind declare i8* @llvm.frameaddress(i32) nounwind - at __gcmapstart = external constant i8* - at __gcmapend = external constant i8* + at __gcmapstart = external constant i8 + at __gcmapend = external constant i8 """ Modified: pypy/branch/llvmgcroot/pypy/translator/llvm/opwriter.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/llvm/opwriter.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/llvm/opwriter.py Sat Dec 22 22:01:47 2007 @@ -608,7 +608,7 @@ "@llvm.frameaddress", ['i32'], ['0']) def llvm_gcmapstart(self, opr): - self.codewriter.load(opr.retref, opr.rettype, '@__gcmapstart') + self.codewriter.cast(opr.retref, 'i8*', '@__gcmapstart', opr.rettype) def llvm_gcmapend(self, opr): - self.codewriter.load(opr.retref, opr.rettype, '@__gcmapend') + self.codewriter.cast(opr.retref, 'i8*', '@__gcmapend', opr.rettype) From cfbolz at codespeak.net Sat Dec 22 22:03:18 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 22 Dec 2007 22:03:18 +0100 (CET) Subject: [pypy-svn] r50021 - pypy/branch/ghop-ropes-classes Message-ID: <20071222210318.073791684CC@codespeak.net> Author: cfbolz Date: Sat Dec 22 22:03:18 2007 New Revision: 50021 Added: pypy/branch/ghop-ropes-classes/ - copied from r50020, pypy/dist/ Log: a branch for the GHOP work of Pavel Vinogradov on creating a class wrapper around our ropes-algorithms. From arigo at codespeak.net Sat Dec 22 22:06:14 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Dec 2007 22:06:14 +0100 (CET) Subject: [pypy-svn] r50022 - pypy/dist/pypy/translator/goal Message-ID: <20071222210614.4038A1684CC@codespeak.net> Author: arigo Date: Sat Dec 22 22:06:13 2007 New Revision: 50022 Modified: pypy/dist/pypy/translator/goal/gcbench.py Log: A clearer error message and no timing when a memory corruption is detected Modified: pypy/dist/pypy/translator/goal/gcbench.py ============================================================================== --- pypy/dist/pypy/translator/goal/gcbench.py (original) +++ pypy/dist/pypy/translator/goal/gcbench.py Sat Dec 22 22:06:13 2007 @@ -133,7 +133,8 @@ time_construction(d) if long_lived_tree is None or array[1000] != 1.0/1000: - println("Failed") + println("FAILED") + return t_finish = time.time() print_diagnostics() From arigo at codespeak.net Sat Dec 22 22:07:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 22 Dec 2007 22:07:09 +0100 (CET) Subject: [pypy-svn] r50023 - pypy/branch/llvmgcroot/pypy/translator/goal Message-ID: <20071222210709.ADE9B1684CC@codespeak.net> Author: arigo Date: Sat Dec 22 22:07:09 2007 New Revision: 50023 Modified: pypy/branch/llvmgcroot/pypy/translator/goal/gcbench.py Log: Port of r50022 to the branch. Modified: pypy/branch/llvmgcroot/pypy/translator/goal/gcbench.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/goal/gcbench.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/goal/gcbench.py Sat Dec 22 22:07:09 2007 @@ -133,7 +133,8 @@ time_construction(d) if long_lived_tree is None or array[1000] != 1.0/1000: - println("Failed") + println("FAILED") + return t_finish = time.time() print_diagnostics() From cfbolz at codespeak.net Sat Dec 22 22:18:20 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 22 Dec 2007 22:18:20 +0100 (CET) Subject: [pypy-svn] r50024 - in pypy/branch/ghop-ropes-classes/pypy/rlib: . test Message-ID: <20071222211820.F23451684CD@codespeak.net> Author: cfbolz Date: Sat Dec 22 22:18:20 2007 New Revision: 50024 Added: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (contents, props changed) pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py (contents, props changed) Log: a skeleton to give some hints as to how I would like things to proceed Added: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- (empty file) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Sat Dec 22 22:18:20 2007 @@ -0,0 +1,7 @@ +from pypy.rlib import rope + +class RopeString(object): + pass + +class RopeUnicode(object): + pass Added: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py ============================================================================== --- (empty file) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Sat Dec 22 22:18:20 2007 @@ -0,0 +1,45 @@ +from pypy.rlib.ropewrapper import RopeUnicode, RopeString + +class AbstractTest(object): + + def test_construction(self): + s = self.const("abc") + assert len(s) == 3 + assert s[0] == self.const("a") + assert s[1] == self.const("b") + assert s[2] == self.const("c") + + def test_add(self): + s1 = self.const("abc") + s2 = self.const("def") + s = s1 + s2 + assert s[0] == self.const("a") + assert s[1] == self.const("b") + assert s[2] == self.const("c") + assert s[3] == self.const("d") + assert s[4] == self.const("e") + assert s[5] == self.const("f") + + def test_mul(self): + t = self.const("abc") + l = [t * 5, 5 * t] + for s in l: + for i in range(5): + assert s[i * 3 + 0] == self.const("a") + assert s[i * 3 + 1] == self.const("b") + assert s[i * 3 + 2] == self.const("c") + + +class TestString(AbstractTest): + const = RopeString + +class TestPythonString(AbstractTest): + const = str + +class TestUnicode(AbstractTest): + const = RopeUnicode + +class TestPythonUnicode(AbstractTest): + const = unicode + + From vinogradov at codespeak.net Sun Dec 23 09:21:16 2007 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Sun, 23 Dec 2007 09:21:16 +0100 (CET) Subject: [pypy-svn] r50026 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071223082116.C3E831684C5@codespeak.net> Author: vinogradov Date: Sun Dec 23 09:21:15 2007 New Revision: 50026 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: Implement methods of RopeString to pass construction test Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Sun Dec 23 09:21:15 2007 @@ -1,7 +1,17 @@ from pypy.rlib import rope class RopeString(object): - pass + def __init__ (self, str): + self._node = rope.LiteralStringNode(str) + + def __len__ (self): + return self._node.length() + + def __getitem__ (self, index): + return self._node.getchar(index) + + def __eq__ (self, str): + return rope.eq (self._node, rope.LiteralStringNode(str)) class RopeUnicode(object): pass From vinogradov at codespeak.net Sun Dec 23 10:14:25 2007 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Sun, 23 Dec 2007 10:14:25 +0100 (CET) Subject: [pypy-svn] r50027 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071223091425.B0809168472@codespeak.net> Author: vinogradov Date: Sun Dec 23 10:14:23 2007 New Revision: 50027 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: Rename some arguments for RopeString methods Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Sun Dec 23 10:14:23 2007 @@ -1,8 +1,8 @@ from pypy.rlib import rope class RopeString(object): - def __init__ (self, str): - self._node = rope.LiteralStringNode(str) + def __init__ (self, s): + self._node = rope.LiteralStringNode(s) def __len__ (self): return self._node.length() @@ -10,8 +10,8 @@ def __getitem__ (self, index): return self._node.getchar(index) - def __eq__ (self, str): - return rope.eq (self._node, rope.LiteralStringNode(str)) + def __eq__ (self, other): + return rope.eq (self._node, rope.LiteralStringNode(other)) class RopeUnicode(object): pass From vinogradov at codespeak.net Sun Dec 23 10:22:46 2007 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Sun, 23 Dec 2007 10:22:46 +0100 (CET) Subject: [pypy-svn] r50028 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071223092246.25049168472@codespeak.net> Author: vinogradov Date: Sun Dec 23 10:22:46 2007 New Revision: 50028 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: Implement methods of RopeString to pass add test Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Sun Dec 23 10:22:46 2007 @@ -12,6 +12,10 @@ def __eq__ (self, other): return rope.eq (self._node, rope.LiteralStringNode(other)) + + def __add__ (self, other): + self._node = self._node + other._node + return self class RopeUnicode(object): pass From vinogradov at codespeak.net Sun Dec 23 10:59:32 2007 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Sun, 23 Dec 2007 10:59:32 +0100 (CET) Subject: [pypy-svn] r50029 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071223095932.5124A168476@codespeak.net> Author: vinogradov Date: Sun Dec 23 10:59:31 2007 New Revision: 50029 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: Remove side effect from __add__ method Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Sun Dec 23 10:59:31 2007 @@ -14,8 +14,9 @@ return rope.eq (self._node, rope.LiteralStringNode(other)) def __add__ (self, other): - self._node = self._node + other._node - return self + result = RopeString('') + result._node = self._node + other._node + return result class RopeUnicode(object): pass From vinogradov at codespeak.net Sun Dec 23 12:03:52 2007 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Sun, 23 Dec 2007 12:03:52 +0100 (CET) Subject: [pypy-svn] r50037 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071223110352.7839F1684C9@codespeak.net> Author: vinogradov Date: Sun Dec 23 12:03:51 2007 New Revision: 50037 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: Implement methods of RopeString to pass mul test Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Sun Dec 23 12:03:51 2007 @@ -17,6 +17,14 @@ result = RopeString('') result._node = self._node + other._node return result + + def __mul__ (self, n): + result = RopeString('') + result._node = rope.multiply(self._node, n) + return result + + def __rmul__ (self, n): + return self * n class RopeUnicode(object): pass From vinogradov at codespeak.net Sun Dec 23 12:36:51 2007 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Sun, 23 Dec 2007 12:36:51 +0100 (CET) Subject: [pypy-svn] r50039 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071223113651.DB98D1684D4@codespeak.net> Author: vinogradov Date: Sun Dec 23 12:36:50 2007 New Revision: 50039 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: Implement methods of RopeUnicode to pass construction test Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Sun Dec 23 12:36:50 2007 @@ -27,4 +27,16 @@ return self * n class RopeUnicode(object): - pass + def __init__ (self, s): + if isinstance (s, str): + s = unicode(s) + self._node = rope.LiteralUnicodeNode(unicode(s)) + + def __len__ (self): + return self._node.length() + + def __getitem__ (self, index): + return self._node.getunichar(index) + + def __eq__ (self, other): + return rope.eq (self._node, rope.LiteralUnicodeNode(other)) From cfbolz at codespeak.net Sun Dec 23 17:03:12 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 23 Dec 2007 17:03:12 +0100 (CET) Subject: [pypy-svn] r50042 - pypy/branch/ghop-ropes-classes/pypy/rlib/test Message-ID: <20071223160312.669711684CA@codespeak.net> Author: cfbolz Date: Sun Dec 23 17:03:11 2007 New Revision: 50042 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Log: some more failing tests Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Sun Dec 23 17:03:11 2007 @@ -29,6 +29,34 @@ assert s[i * 3 + 1] == self.const("b") assert s[i * 3 + 2] == self.const("c") + def test_slice(self): + import pdb; pdb.set_trace() + s = self.const("abcd") * 100 + self.const("efghi") * 100 + assert s[1::1] == self.const("bcd" + "abcd" * 99 + "efghi" * 100) + assert s[1:-1:1] == self.const("bcd" + "abcd" * 99 + + "efghi" * 99 + "efgh") + + def test_compare(self): + s1 = self.const("abc") + s2 = self.const("abc") + assert s1 == s2 + assert not s1 != s2 + + def test_iteration(self): + # XXX rope iteration is working but should use a custom iterator + # e.g. define an __iter__ method + s = self.const("abcdefghijkl") + i = 0 + for c in s: + assert c == s[i] + i += 1 + assert i == len(s) + + def test_hash(self): + s1 = self.const("abc") + s2 = self.const("abc") + assert hash(s1) == hash(s2) + class TestString(AbstractTest): const = RopeString From vinogradov at codespeak.net Sun Dec 23 19:51:08 2007 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Sun, 23 Dec 2007 19:51:08 +0100 (CET) Subject: [pypy-svn] r50046 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071223185108.A2CE4168460@codespeak.net> Author: vinogradov Date: Sun Dec 23 19:51:08 2007 New Revision: 50046 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: Remove spaces before () according to PEP 8 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Sun Dec 23 19:51:08 2007 @@ -1,42 +1,42 @@ from pypy.rlib import rope class RopeString(object): - def __init__ (self, s): + def __init__(self, s): self._node = rope.LiteralStringNode(s) - def __len__ (self): + def __len__(self): return self._node.length() - def __getitem__ (self, index): + def __getitem__(self, index): return self._node.getchar(index) - def __eq__ (self, other): - return rope.eq (self._node, rope.LiteralStringNode(other)) + def __eq__(self, other): + return rope.eq(self._node, rope.LiteralStringNode(other)) - def __add__ (self, other): + def __add__(self, other): result = RopeString('') result._node = self._node + other._node return result - def __mul__ (self, n): + def __mul__(self, n): result = RopeString('') result._node = rope.multiply(self._node, n) return result - def __rmul__ (self, n): + def __rmul__(self, n): return self * n class RopeUnicode(object): - def __init__ (self, s): - if isinstance (s, str): + def __init__(self, s): + if isinstance(s, str): s = unicode(s) self._node = rope.LiteralUnicodeNode(unicode(s)) - def __len__ (self): + def __len__(self): return self._node.length() - def __getitem__ (self, index): + def __getitem__(self, index): return self._node.getunichar(index) - def __eq__ (self, other): - return rope.eq (self._node, rope.LiteralUnicodeNode(other)) + def __eq__(self, other): + return rope.eq(self._node, rope.LiteralUnicodeNode(other)) From vinogradov at codespeak.net Sun Dec 23 19:57:00 2007 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Sun, 23 Dec 2007 19:57:00 +0100 (CET) Subject: [pypy-svn] r50047 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071223185700.99BD5168460@codespeak.net> Author: vinogradov Date: Sun Dec 23 19:57:00 2007 New Revision: 50047 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: RopeString constructor now accept *Rope as argument Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Sun Dec 23 19:57:00 2007 @@ -2,7 +2,10 @@ class RopeString(object): def __init__(self, s): - self._node = rope.LiteralStringNode(s) + if isinstance(s, str): + self._node = rope.LiteralStringNode(s) + if isinstance(s, rope.LiteralStringNode): + self._node = s def __len__(self): return self._node.length() @@ -14,14 +17,10 @@ return rope.eq(self._node, rope.LiteralStringNode(other)) def __add__(self, other): - result = RopeString('') - result._node = self._node + other._node - return result + return RopeString(self._node + other._node) def __mul__(self, n): - result = RopeString('') - result._node = rope.multiply(self._node, n) - return result + return RopeString(rope.multiply(self._node, n)) def __rmul__(self, n): return self * n From vinogradov at codespeak.net Sun Dec 23 19:59:52 2007 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Sun, 23 Dec 2007 19:59:52 +0100 (CET) Subject: [pypy-svn] r50048 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071223185952.598FB16845A@codespeak.net> Author: vinogradov Date: Sun Dec 23 19:59:51 2007 New Revision: 50048 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: RopeUnicode constructor now accept *Rope as argument Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Sun Dec 23 19:59:51 2007 @@ -28,8 +28,11 @@ class RopeUnicode(object): def __init__(self, s): if isinstance(s, str): - s = unicode(s) - self._node = rope.LiteralUnicodeNode(unicode(s)) + self._node = rope.LiteralUnicodeNode(unicode(s)) + if isinstance(s, unicode): + self._node = rope.LiteralUnicodeNode(s) + if isinstance(s, rope.LiteralUnicodeNode): + self._node = s def __len__(self): return self._node.length() From vinogradov at codespeak.net Sun Dec 23 20:02:12 2007 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Sun, 23 Dec 2007 20:02:12 +0100 (CET) Subject: [pypy-svn] r50049 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071223190212.3DBB216845A@codespeak.net> Author: vinogradov Date: Sun Dec 23 20:02:11 2007 New Revision: 50049 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: Implement methods of RopeUnicode to pass add test Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Sun Dec 23 20:02:11 2007 @@ -42,3 +42,7 @@ def __eq__(self, other): return rope.eq(self._node, rope.LiteralUnicodeNode(other)) + + def __add__(self, other): + return RopeUnicode(self._node + other._node) + From vinogradov at codespeak.net Sun Dec 23 20:05:50 2007 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Sun, 23 Dec 2007 20:05:50 +0100 (CET) Subject: [pypy-svn] r50050 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071223190550.2BDDA16845E@codespeak.net> Author: vinogradov Date: Sun Dec 23 20:05:45 2007 New Revision: 50050 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: Implement methods of RopeUnicode to pass mul test Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Sun Dec 23 20:05:45 2007 @@ -45,4 +45,9 @@ def __add__(self, other): return RopeUnicode(self._node + other._node) - + + def __mul__(self, n): + return RopeUnicode(rope.multiply(self._node, n)) + + def __rmul__(self, n): + return self * n From arigo at codespeak.net Sun Dec 23 21:09:44 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 23 Dec 2007 21:09:44 +0100 (CET) Subject: [pypy-svn] r50051 - pypy/dist/pypy/translator/c Message-ID: <20071223200944.03ADA168455@codespeak.net> Author: arigo Date: Sun Dec 23 21:09:42 2007 New Revision: 50051 Modified: pypy/dist/pypy/translator/c/database.py pypy/dist/pypy/translator/c/funcgen.py pypy/dist/pypy/translator/c/genc.py pypy/dist/pypy/translator/c/primitive.py pypy/dist/pypy/translator/c/support.py Log: (rxe) Some general clean-ups in genc. Modified: pypy/dist/pypy/translator/c/database.py ============================================================================== --- pypy/dist/pypy/translator/c/database.py (original) +++ pypy/dist/pypy/translator/c/database.py Sun Dec 23 21:09:42 2007 @@ -7,11 +7,10 @@ from pypy.rpython.lltypesystem.rffi import CConstant from pypy.tool.sourcetools import valid_identifier from pypy.translator.c.primitive import PrimitiveName, PrimitiveType -from pypy.translator.c.primitive import PrimitiveErrorValue from pypy.translator.c.node import StructDefNode, ArrayDefNode from pypy.translator.c.node import FixedSizeArrayDefNode, BareBoneArrayDefNode from pypy.translator.c.node import ContainerNodeFactory, ExtTypeOpaqueDefNode -from pypy.translator.c.support import cdecl, CNameManager, ErrorValue +from pypy.translator.c.support import cdecl, CNameManager from pypy.translator.c.support import log, barebonearray from pypy.translator.c.extfunc import do_the_getting from pypy import conftest @@ -170,15 +169,8 @@ return node def get(self, obj): - if isinstance(obj, ErrorValue): - T = obj.TYPE - if isinstance(T, Primitive): - return PrimitiveErrorValue[T] - elif isinstance(T, Ptr): - return 'NULL' - else: - raise Exception("don't know about %r" % (T,)) - else: + # XXX extra indent is preserve svn blame - kind of important IMHO (rxe) + if 1: if isinstance(obj, CConstant): return obj.c_name # without further checks T = typeOf(obj) Modified: pypy/dist/pypy/translator/c/funcgen.py ============================================================================== --- pypy/dist/pypy/translator/c/funcgen.py (original) +++ pypy/dist/pypy/translator/c/funcgen.py Sun Dec 23 21:09:42 2007 @@ -1,6 +1,6 @@ from __future__ import generators from pypy.translator.c.support import USESLOTS # set to False if necessary while refactoring -from pypy.translator.c.support import cdecl, ErrorValue +from pypy.translator.c.support import cdecl from pypy.translator.c.support import llvalue_from_constant, gen_assignments from pypy.translator.c.support import c_string_constant, barebonearray from pypy.objspace.flow.model import Variable, Constant, Block @@ -178,21 +178,6 @@ else: raise TypeError, "expr(%r)" % (v,) - def error_return_value(self): - returnlltype = self.lltypemap(self.graph.getreturnvar()) - return self.db.get(ErrorValue(returnlltype)) - - def return_with_error(self): - if self.exception_policy == "CPython": - assert self.lltypemap(self.graph.getreturnvar()) == PyObjPtr - v, exc_cleanup_ops = self.graph.exc_cleanup - vanishing_exc_value = self.expr(v) - yield 'RPyConvertExceptionToCPython(%s);' % vanishing_exc_value - for cleanupop in exc_cleanup_ops: - for line in self.gen_op(cleanupop): - yield line - yield 'return %s; ' % self.error_return_value() - # ____________________________________________________________ def cfunction_declarations(self): Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Sun Dec 23 21:09:42 2007 @@ -23,18 +23,13 @@ _compiled = False modulename = None - def __init__(self, translator, entrypoint, config, - eci=ExternalCompilationInfo(), gcpolicy=None): + def __init__(self, translator, entrypoint, config): self.translator = translator self.entrypoint = entrypoint self.entrypoint_name = self.entrypoint.func_name self.originalentrypoint = entrypoint - self.gcpolicy = gcpolicy - if gcpolicy is not None and gcpolicy.requires_stackless: - config.translation.stackless = True self.config = config - self.exports = {} - self.eci = eci + self.eci = ExternalCompilationInfo() def build_database(self): translator = self.translator @@ -59,11 +54,7 @@ thread_enabled=self.config.translation.thread, sandbox=self.config.translation.sandbox) self.db = db - - # we need a concrete gcpolicy to do this - self.eci = self.eci.merge(ExternalCompilationInfo( - libraries=db.gcpolicy.gc_libraries())) - + # give the gc a chance to register interest in the start-up functions it # need (we call this for its side-effects of db.get()) list(db.gcpolicy.gc_startup_code()) @@ -71,16 +62,19 @@ # build entrypoint and eventually other things to expose pf = self.getentrypointptr() pfname = db.get(pf) - self.exports[self.entrypoint_name] = pf self.c_entrypoint_name = pfname db.complete() - - self.collect_compilation_info() + + self.collect_compilation_info(db) return db have___thread = None - def collect_compilation_info(self): + def collect_compilation_info(self, db): + # we need a concrete gcpolicy to do this + self.eci = self.eci.merge(ExternalCompilationInfo( + libraries=db.gcpolicy.gc_libraries())) + all = [] for node in self.db.globalcontainers(): eci = getattr(node, 'compilation_info', None) @@ -89,12 +83,10 @@ self.eci = self.eci.merge(*all) def get_gcpolicyclass(self): - if self.gcpolicy is None: - name = self.config.translation.gctransformer - if self.config.translation.stacklessgc: - name = "%s+stacklessgc" % (name,) - return gc.name_to_gcpolicy[name] - return self.gcpolicy + name = self.config.translation.gctransformer + if self.config.translation.stacklessgc: + name = "%s+stacklessgc" % (name,) + return gc.name_to_gcpolicy[name] # use generate_source(defines=DEBUG_DEFINES) to force the #definition # of the macros that enable debugging assertions @@ -125,8 +117,7 @@ if not self.standalone: assert not self.config.translation.instrument cfile, extra = gen_source(db, modulename, targetdir, self.eci, - defines = defines, - exports = self.exports) + defines = defines) else: if self.config.translation.instrument: defines['INSTRUMENT'] = 1 @@ -687,7 +678,7 @@ return filename, sg.getextrafiles() + list(eci.separate_module_files) -def gen_source(database, modulename, targetdir, eci, defines={}, exports={}): +def gen_source(database, modulename, targetdir, eci, defines={}): assert not database.standalone if isinstance(targetdir, str): targetdir = py.path.local(targetdir) Modified: pypy/dist/pypy/translator/c/primitive.py ============================================================================== --- pypy/dist/pypy/translator/c/primitive.py (original) +++ pypy/dist/pypy/translator/c/primitive.py Sun Dec 23 21:09:42 2007 @@ -158,20 +158,6 @@ Address: 'void* @', } -PrimitiveErrorValue = { - SignedLongLong: '-1LL', - Signed: '-1', - UnsignedLongLong: '((unsigned long long) -1)', - Unsigned: '((unsigned) -1)', - Float: '-1.0', - SingleFloat: '-1.0f', - Char: '((char) -1)', - UniChar: '((unsigned) -1)', - Bool: '0 /* error */', - Void: '/* error */', - Address: 'NULL', - } - def define_c_primitive(ll_type, c_name): if ll_type in PrimitiveName: return @@ -181,7 +167,6 @@ name_str = '((%s) %%dLL)' % c_name PrimitiveName[ll_type] = lambda value, db: name_str % value PrimitiveType[ll_type] = '%s @'% c_name - PrimitiveErrorValue[ll_type] = '((%s) -1)'% c_name for ll_type, c_name in [(rffi.SIGNEDCHAR, 'signed char'), (rffi.UCHAR, 'unsigned char'), Modified: pypy/dist/pypy/translator/c/support.py ============================================================================== --- pypy/dist/pypy/translator/c/support.py (original) +++ pypy/dist/pypy/translator/c/support.py Sun Dec 23 21:09:42 2007 @@ -9,11 +9,6 @@ PyObjPtr = lltype.Ptr(lltype.PyObject) - -class ErrorValue: - def __init__(self, TYPE): - self.TYPE = TYPE - def barebonearray(ARRAY): """Check if ARRAY is a 'simple' array type, i.e. doesn't need a length nor GC headers.""" From vinogradov at codespeak.net Sun Dec 23 21:27:28 2007 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Sun, 23 Dec 2007 21:27:28 +0100 (CET) Subject: [pypy-svn] r50052 - pypy/branch/ghop-ropes-classes/pypy/rlib/test Message-ID: <20071223202728.9F0EC168469@codespeak.net> Author: vinogradov Date: Sun Dec 23 21:27:26 2007 New Revision: 50052 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Log: Remove pdb import(not needed there) Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Sun Dec 23 21:27:26 2007 @@ -30,7 +30,6 @@ assert s[i * 3 + 2] == self.const("c") def test_slice(self): - import pdb; pdb.set_trace() s = self.const("abcd") * 100 + self.const("efghi") * 100 assert s[1::1] == self.const("bcd" + "abcd" * 99 + "efghi" * 100) assert s[1:-1:1] == self.const("bcd" + "abcd" * 99 + From vinogradov at codespeak.net Sun Dec 23 21:33:55 2007 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Sun, 23 Dec 2007 21:33:55 +0100 (CET) Subject: [pypy-svn] r50053 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071223203355.7DF7D16847A@codespeak.net> Author: vinogradov Date: Sun Dec 23 21:33:54 2007 New Revision: 50053 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: Implement methods of RopeUnicode and RopeString to pass hash test Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Sun Dec 23 21:33:54 2007 @@ -25,6 +25,9 @@ def __rmul__(self, n): return self * n + def __hash__(self): + return rope.hash_rope(self._node) + class RopeUnicode(object): def __init__(self, s): if isinstance(s, str): @@ -51,3 +54,6 @@ def __rmul__(self, n): return self * n + + def __hash__(self): + return rope.hash_rope(self._node) From vinogradov at codespeak.net Sun Dec 23 21:54:53 2007 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Sun, 23 Dec 2007 21:54:53 +0100 (CET) Subject: [pypy-svn] r50054 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071223205453.920B1168461@codespeak.net> Author: vinogradov Date: Sun Dec 23 21:54:53 2007 New Revision: 50054 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: Move some functions from Rope[Unicode|String] to RopeBaseString Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Sun Dec 23 21:54:53 2007 @@ -1,34 +1,38 @@ from pypy.rlib import rope -class RopeString(object): +class RopeBaseString(object): + def __init__(self): + self._node = None + + def __len__(self): + return self._node.length() + + def __rmul__(self, n): + return self * n + + def __hash__(self): + return rope.hash_rope(self._node) + +class RopeString(RopeBaseString): def __init__(self, s): if isinstance(s, str): self._node = rope.LiteralStringNode(s) if isinstance(s, rope.LiteralStringNode): self._node = s - def __len__(self): - return self._node.length() - def __getitem__(self, index): return self._node.getchar(index) - def __eq__(self, other): - return rope.eq(self._node, rope.LiteralStringNode(other)) - def __add__(self, other): return RopeString(self._node + other._node) def __mul__(self, n): return RopeString(rope.multiply(self._node, n)) - def __rmul__(self, n): - return self * n - - def __hash__(self): - return rope.hash_rope(self._node) + def __eq__(self, other): + return rope.eq(self._node, rope.LiteralStringNode(other)) -class RopeUnicode(object): +class RopeUnicode(RopeBaseString): def __init__(self, s): if isinstance(s, str): self._node = rope.LiteralUnicodeNode(unicode(s)) @@ -37,9 +41,6 @@ if isinstance(s, rope.LiteralUnicodeNode): self._node = s - def __len__(self): - return self._node.length() - def __getitem__(self, index): return self._node.getunichar(index) @@ -51,9 +52,3 @@ def __mul__(self, n): return RopeUnicode(rope.multiply(self._node, n)) - - def __rmul__(self, n): - return self * n - - def __hash__(self): - return rope.hash_rope(self._node) From pypy-svn at codespeak.net Mon Dec 24 07:28:10 2007 From: pypy-svn at codespeak.net (VIAGRA ® Official Site) Date: Mon, 24 Dec 2007 07:28:10 +0100 (CET) Subject: [pypy-svn] December 70% OFF Message-ID: <20070611192818.4876.qmail@ppp-124.120.3.253.revip2.asianet.co.th> An HTML attachment was scrubbed... URL: From pypy-svn at codespeak.net Mon Dec 24 14:04:51 2007 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Mon, 24 Dec 2007 14:04:51 +0100 (CET) Subject: [pypy-svn] Shopper's Alert Message-ID: <20071224030451.13457.qmail@chello087206160226.chello.pl> An HTML attachment was scrubbed... URL: From arigo at codespeak.net Mon Dec 24 16:46:34 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 24 Dec 2007 16:46:34 +0100 (CET) Subject: [pypy-svn] r50061 - pypy/branch/llvmgcroot/pypy/translator/llvm Message-ID: <20071224154634.D6DD016844D@codespeak.net> Author: arigo Date: Mon Dec 24 16:46:33 2007 New Revision: 50061 Modified: pypy/branch/llvmgcroot/pypy/translator/llvm/node.py Log: Use the NameManager to build names that are valid C identifiers. Helps a lot for debugging with gdb... Modified: pypy/branch/llvmgcroot/pypy/translator/llvm/node.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/llvm/node.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/llvm/node.py Mon Dec 24 16:46:33 2007 @@ -1,29 +1,16 @@ from pypy.rpython.lltypesystem import lltype +from pypy.translator.gensupp import NameManager + +NODE_NAMES = NameManager() class Node(object): __slots__ = "name".split() prefix = '%' - nodename_count = {} - def mangle(self, name): - if name not in self.nodename_count: - self.nodename_count[name] = 1 - return name - else: - result = '%s_%d' % (name, self.nodename_count[name]) - self.nodename_count[name] += 1 - # this ensures (a) doesn exist yet, and (b) adds it to the - # dictionary just to prevent some function called xxx_42() and clashing - return self.mangle(result) - def make_name(self, name=''): " helper for creating names" - name = self.prefix + name - name = self.mangle(name) - if " " in name or "<" in name: - name = '"%s"' % name - + name = self.prefix + NODE_NAMES.uniquename(name) self.name = name def setup(self): From arigo at codespeak.net Mon Dec 24 17:42:32 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 24 Dec 2007 17:42:32 +0100 (CET) Subject: [pypy-svn] r50065 - pypy/dist/pypy/translator/c Message-ID: <20071224164232.C7364168451@codespeak.net> Author: arigo Date: Mon Dec 24 17:42:32 2007 New Revision: 50065 Modified: pypy/dist/pypy/translator/c/genc.py Log: Restore this - needed by tests. Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Mon Dec 24 17:42:32 2007 @@ -23,12 +23,15 @@ _compiled = False modulename = None - def __init__(self, translator, entrypoint, config): + def __init__(self, translator, entrypoint, config, gcpolicy=None): self.translator = translator self.entrypoint = entrypoint self.entrypoint_name = self.entrypoint.func_name self.originalentrypoint = entrypoint self.config = config + self.gcpolicy = gcpolicy # for tests only, e.g. rpython/memory/ + if gcpolicy is not None and gcpolicy.requires_stackless: + config.translation.stackless = True self.eci = ExternalCompilationInfo() def build_database(self): @@ -83,6 +86,8 @@ self.eci = self.eci.merge(*all) def get_gcpolicyclass(self): + if self.gcpolicy is not None: + return self.gcpolicy # for tests only name = self.config.translation.gctransformer if self.config.translation.stacklessgc: name = "%s+stacklessgc" % (name,) From vinogradov at codespeak.net Tue Dec 25 18:38:13 2007 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Tue, 25 Dec 2007 18:38:13 +0100 (CET) Subject: [pypy-svn] r50108 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071225173813.9AC0416845A@codespeak.net> Author: vinogradov Date: Tue Dec 25 18:38:11 2007 New Revision: 50108 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: Implement methods of RopeUnicode and RopeString to pass compare test Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Tue Dec 25 18:38:11 2007 @@ -12,6 +12,13 @@ def __hash__(self): return rope.hash_rope(self._node) + + def __eq__(self): + pass + + def __ne__(self, other): + return not self == other + class RopeString(RopeBaseString): def __init__(self, s): @@ -30,7 +37,10 @@ return RopeString(rope.multiply(self._node, n)) def __eq__(self, other): - return rope.eq(self._node, rope.LiteralStringNode(other)) + if isinstance(other, RopeBaseString): + return (rope.eq(self._node, other._node)) + else: + return rope.eq(self._node, rope.LiteralStringNode(other)) class RopeUnicode(RopeBaseString): def __init__(self, s): @@ -45,7 +55,10 @@ return self._node.getunichar(index) def __eq__(self, other): - return rope.eq(self._node, rope.LiteralUnicodeNode(other)) + if isinstance (other, RopeBaseString): + return rope.eq(self._node, other._node) + else: + return rope.eq(self._node, rope.LiteralUnicodeNode(other)) def __add__(self, other): return RopeUnicode(self._node + other._node) From vinogradov at codespeak.net Tue Dec 25 20:45:02 2007 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Tue, 25 Dec 2007 20:45:02 +0100 (CET) Subject: [pypy-svn] r50111 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071225194502.D6144168467@codespeak.net> Author: vinogradov Date: Tue Dec 25 20:45:01 2007 New Revision: 50111 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: Implement methods of RopeUnicode and RopeString to pass iteration test Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Tue Dec 25 20:45:01 2007 @@ -19,6 +19,15 @@ def __ne__(self, other): return not self == other +class RopeStringIterator(object): + def __init__(self, node): + self._iter = rope.ItemIterator(node) + + def next(self): + return self._iter.nextchar() + + def __iter__(): + return self class RopeString(RopeBaseString): def __init__(self, s): @@ -41,6 +50,19 @@ return (rope.eq(self._node, other._node)) else: return rope.eq(self._node, rope.LiteralStringNode(other)) + + def __iter__(self): + return RopeStringIterator(self._node) + +class RopeUnicodeIterator(object): + def __init__(self, node): + self._iter = rope.ItemIterator(node) + + def next(self): + return self._iter.nextunichar() + + def __iter__(): + return self class RopeUnicode(RopeBaseString): def __init__(self, s): @@ -65,3 +87,6 @@ def __mul__(self, n): return RopeUnicode(rope.multiply(self._node, n)) + + def __iter__(self): + return RopeUnicodeIterator(self._node) From cfbolz at codespeak.net Tue Dec 25 21:10:37 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 25 Dec 2007 21:10:37 +0100 (CET) Subject: [pypy-svn] r50112 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071225201037.7FFA2168440@codespeak.net> Author: cfbolz Date: Tue Dec 25 21:10:35 2007 New Revision: 50112 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: show the bug more clearly Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Tue Dec 25 21:10:35 2007 @@ -35,6 +35,7 @@ self._node = rope.LiteralStringNode(s) if isinstance(s, rope.LiteralStringNode): self._node = s + raise NotImplemtedError("don't know about %s" % (s, )) def __getitem__(self, index): return self._node.getchar(index) From cfbolz at codespeak.net Tue Dec 25 21:12:11 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 25 Dec 2007 21:12:11 +0100 (CET) Subject: [pypy-svn] r50113 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071225201211.BB4A1168440@codespeak.net> Author: cfbolz Date: Tue Dec 25 21:12:11 2007 New Revision: 50113 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: typo Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Tue Dec 25 21:12:11 2007 @@ -33,9 +33,10 @@ def __init__(self, s): if isinstance(s, str): self._node = rope.LiteralStringNode(s) - if isinstance(s, rope.LiteralStringNode): + elif isinstance(s, rope.LiteralStringNode): self._node = s - raise NotImplemtedError("don't know about %s" % (s, )) + else: + raise NotImplementedError("don't know about %s" % (s, )) def __getitem__(self, index): return self._node.getchar(index) From cfbolz at codespeak.net Tue Dec 25 21:23:17 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 25 Dec 2007 21:23:17 +0100 (CET) Subject: [pypy-svn] r50114 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071225202317.017AD168445@codespeak.net> Author: cfbolz Date: Tue Dec 25 21:23:17 2007 New Revision: 50114 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: share some more code Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Tue Dec 25 21:23:17 2007 @@ -19,6 +19,9 @@ def __ne__(self, other): return not self == other + def __mul__(self, n): + return self.__class__(rope.multiply(self._node, n)) + class RopeStringIterator(object): def __init__(self, node): self._iter = rope.ItemIterator(node) @@ -44,9 +47,6 @@ def __add__(self, other): return RopeString(self._node + other._node) - def __mul__(self, n): - return RopeString(rope.multiply(self._node, n)) - def __eq__(self, other): if isinstance(other, RopeBaseString): return (rope.eq(self._node, other._node)) @@ -87,8 +87,5 @@ def __add__(self, other): return RopeUnicode(self._node + other._node) - def __mul__(self, n): - return RopeUnicode(rope.multiply(self._node, n)) - def __iter__(self): return RopeUnicodeIterator(self._node) From cfbolz at codespeak.net Tue Dec 25 21:28:25 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 25 Dec 2007 21:28:25 +0100 (CET) Subject: [pypy-svn] r50115 - pypy/branch/ghop-ropes-classes/pypy/rlib/test Message-ID: <20071225202825.338DE1684D1@codespeak.net> Author: cfbolz Date: Tue Dec 25 21:28:23 2007 New Revision: 50115 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Log: a failing test: RopeString + RopeUnicode should give a RopeUnicode. Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Tue Dec 25 21:28:23 2007 @@ -69,4 +69,21 @@ class TestPythonUnicode(AbstractTest): const = unicode +class AbstractTestCoercion(object): + def test_add_coercion(self): + s1 = self.conststr("abc") + s2 = self.constunicode("def") + s = s1 + s2 + assert isinstance(s, self.constunicode) + assert s == self.constunicode("abcdef") + s = s2 + s1 + assert isinstance(s, self.constunicode) + assert s == self.constunicode("defabc") +class TestPythonCoercion(AbstractTestCoercion): + conststr = str + constunicode = unicode + +class TestRopesCoercion(AbstractTestCoercion): + conststr = RopeString + constunicode = RopeUnicode From vinogradov at codespeak.net Tue Dec 25 21:30:37 2007 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Tue, 25 Dec 2007 21:30:37 +0100 (CET) Subject: [pypy-svn] r50116 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071225203037.2FCE41684D1@codespeak.net> Author: vinogradov Date: Tue Dec 25 21:30:36 2007 New Revision: 50116 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: Move __add__ method to RopeBaseString class Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Tue Dec 25 21:30:36 2007 @@ -22,6 +22,9 @@ def __mul__(self, n): return self.__class__(rope.multiply(self._node, n)) + def __add__(self, other): + return self.__class__(self._node + other._node) + class RopeStringIterator(object): def __init__(self, node): self._iter = rope.ItemIterator(node) @@ -44,9 +47,6 @@ def __getitem__(self, index): return self._node.getchar(index) - def __add__(self, other): - return RopeString(self._node + other._node) - def __eq__(self, other): if isinstance(other, RopeBaseString): return (rope.eq(self._node, other._node)) @@ -83,9 +83,6 @@ return rope.eq(self._node, other._node) else: return rope.eq(self._node, rope.LiteralUnicodeNode(other)) - - def __add__(self, other): - return RopeUnicode(self._node + other._node) def __iter__(self): return RopeUnicodeIterator(self._node) From vinogradov at codespeak.net Tue Dec 25 21:40:20 2007 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Tue, 25 Dec 2007 21:40:20 +0100 (CET) Subject: [pypy-svn] r50117 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071225204020.8D8011684D1@codespeak.net> Author: vinogradov Date: Tue Dec 25 21:40:20 2007 New Revision: 50117 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: Use StringNode in rope-wrapper contructor Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Tue Dec 25 21:40:20 2007 @@ -39,10 +39,8 @@ def __init__(self, s): if isinstance(s, str): self._node = rope.LiteralStringNode(s) - elif isinstance(s, rope.LiteralStringNode): + elif isinstance(s, rope.StringNode): self._node = s - else: - raise NotImplementedError("don't know about %s" % (s, )) def __getitem__(self, index): return self._node.getchar(index) @@ -72,7 +70,7 @@ self._node = rope.LiteralUnicodeNode(unicode(s)) if isinstance(s, unicode): self._node = rope.LiteralUnicodeNode(s) - if isinstance(s, rope.LiteralUnicodeNode): + if isinstance(s, rope.StringNode): self._node = s def __getitem__(self, index): From cfbolz at codespeak.net Tue Dec 25 21:40:38 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 25 Dec 2007 21:40:38 +0100 (CET) Subject: [pypy-svn] r50118 - pypy/branch/ghop-ropes-classes/pypy/rlib/test Message-ID: <20071225204038.D45ED1684D1@codespeak.net> Author: cfbolz Date: Tue Dec 25 21:40:38 2007 New Revision: 50118 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Log: encode and decode tests Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Tue Dec 25 21:40:38 2007 @@ -1,3 +1,4 @@ +import py from pypy.rlib.ropewrapper import RopeUnicode, RopeString class AbstractTest(object): @@ -70,6 +71,28 @@ const = unicode class AbstractTestCoercion(object): + def test_encode(self): + u = self.constunicode(u"\uffff") + s = u.encode("utf-8") + assert s == self.conststr("\xef\xbf\xbf") + s = u.encode("utf-16") + assert s == self.conststr('\xff\xfe\xff\xff') + py.test.raises(UnicodeEncodeError, u.encode, "latin-1") + py.test.raises(UnicodeEncodeError, u.encode, "ascii") + + def test_decode(self): + s = self.conststr("abc") + u = s.decode("utf-8") + assert s == self.constunicode(u"abc") + u = s.decode("latin-1") + assert s == self.constunicode(u"abc") + u = s.decode("ascii") + assert s == self.constunicode(u"abc") + u = self.conststr("\xff") + s = u.decode("latin-1") + assert s == self.constunicode(u"\xff") + py.test.raises(UnicodeEncodeError, s.decode, "ascii") + def test_add_coercion(self): s1 = self.conststr("abc") s2 = self.constunicode("def") From arigo at codespeak.net Wed Dec 26 15:02:19 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 26 Dec 2007 15:02:19 +0100 (CET) Subject: [pypy-svn] r50122 - pypy/dist/pypy/module/_file/test Message-ID: <20071226140219.34B2416840C@codespeak.net> Author: arigo Date: Wed Dec 26 15:02:16 2007 New Revision: 50122 Modified: pypy/dist/pypy/module/_file/test/test_file_extra.py Log: Fix test for corner cases of the randomly-generated SAMPLE string. Modified: pypy/dist/pypy/module/_file/test/test_file_extra.py ============================================================================== --- pypy/dist/pypy/module/_file/test/test_file_extra.py (original) +++ pypy/dist/pypy/module/_file/test/test_file_extra.py Wed Dec 26 15:02:16 2007 @@ -5,11 +5,16 @@ udir = pypy.tool.udir.udir.ensure('test_file_extra', dir=1) +# XXX this file is a random test. It may only fail occasionally +# depending on the details of the random string SAMPLE. + SAMPLE = ''.join([chr(random.randrange(0, 256)) for i in range(12487)]) for extra in ['\r\r', '\r\n', '\n\r', '\n\n']: for i in range(20): j = random.randrange(0, len(SAMPLE)+1) SAMPLE = SAMPLE[:j] + extra + SAMPLE[j:] + if random.random() < 0.1: + SAMPLE += extra # occasionally, also test strings ending in an EOL def setup_module(mod): @@ -23,6 +28,10 @@ lines = self.sample.split('\n') for i in range(len(lines)-1): lines[i] += '\n' + # if self.sample ends exactly in '\n', the split() gives a + # spurious empty line at the end. Fix it: + if lines[-1] == '': + del lines[-1] return lines def test_simple_tell(self): @@ -310,7 +319,11 @@ class AppTestUniversalNewlines(AppTestFile): expected_mode = 'rU' - sample = '\n'.join(SAMPLE.splitlines(False)) + sample = '\n'.join((SAMPLE+'X').splitlines(False))[:-1] + # ^^^ if SAMPLE ends in any end-of-line character combination, read()ing + # it in 'rU' mode gives a final '\n', but splitlines(False) doesn't give + # a final empty line. Adding and removing an extra 'X' avoids this + # corner case. def test_seek(self): skip("does not apply in universal newlines mode") From arigo at codespeak.net Wed Dec 26 21:30:53 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 26 Dec 2007 21:30:53 +0100 (CET) Subject: [pypy-svn] r50129 - pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform Message-ID: <20071226203053.A1F9E1684F6@codespeak.net> Author: arigo Date: Wed Dec 26 21:30:53 2007 New Revision: 50129 Modified: pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/llvmgcroot.py Log: Sort the array when the program starts, for faster searches in StackRootIterator.pop(). Modified: pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/llvmgcroot.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/llvmgcroot.py (original) +++ pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/llvmgcroot.py Wed Dec 26 21:30:53 2007 @@ -42,14 +42,21 @@ # some old objects alive for a bit longer than necessary. def build_stack_root_iterator(self): - sizeofaddr = llmemory.sizeof(llmemory.Address) gcdata = self.gcdata class StackRootIterator: _alloc_flavor_ = 'raw' def setup_root_stack(): - pass + # The gcmap table is a list of pairs of pointers: + # void *SafePointAddress; + # void *Shape; + # Here, i.e. when the program starts, we sort it + # in-place on the SafePointAddress to allow for more + # efficient searches. + gcmapstart = llop.llvm_gcmapstart(llmemory.Address) + gcmapend = llop.llvm_gcmapend(llmemory.Address) + insertion_sort(gcmapstart, gcmapend) setup_root_stack = staticmethod(setup_root_stack) need_root_stack = False @@ -96,6 +103,10 @@ # the frame. # callee_frame = self.stack_current + if llmemory.cast_adr_to_int(callee_frame) & 1: + return False # odd bit set here when the callee_frame + # is the frame of main(), i.e. when there + # is nothing more for us in the stack. # # XXX the details are completely specific to X86!!! # a picture of the stack may help: @@ -120,33 +131,28 @@ retaddr = callee_frame.address[1] # # try to locate the caller function based on retaddr. - # XXX this is just a linear scan for now, that's - # incredibly bad. # gcmapstart = llop.llvm_gcmapstart(llmemory.Address) gcmapend = llop.llvm_gcmapend(llmemory.Address) - while gcmapstart != gcmapend: - if gcmapstart.address[0] == retaddr: - # - # found! Setup pointers allowing us to - # parse the caller's frame structure... - # - shape = gcmapstart.address[1] - # XXX assumes that .signed is 32-bit - framesize = shape.signed[0] - livecount = shape.signed[1] - caller_frame = callee_frame + 4 + framesize - self.stack_current = caller_frame - self.frame_data_base = callee_frame + 8 - self.remaining_roots_in_current_frame = livecount - self.liveoffsets = shape + 8 - return True - - gcmapstart += 2 * sizeofaddr - - # retaddr not found. Assume that this is the system function - # that called the original entry point, i.e. it's the end of - # the stack for us + item = binary_search(gcmapstart, gcmapend, retaddr) + if item.address[0] == retaddr: + # + # found! Setup pointers allowing us to + # parse the caller's frame structure... + # + shape = item.address[1] + # XXX assumes that .signed is 32-bit + framesize = shape.signed[0] # odd if it's main() + livecount = shape.signed[1] + caller_frame = callee_frame + 4 + framesize + self.stack_current = caller_frame + self.frame_data_base = callee_frame + 8 + self.remaining_roots_in_current_frame = livecount + self.liveoffsets = shape + 8 + return True + + # retaddr not found! + llop.debug_fatalerror(lltype.Void, "cannot find gc roots!") return False def next_gcroot_from_current_frame(self): @@ -158,3 +164,49 @@ return StackRootIterator + + +sizeofaddr = llmemory.sizeof(llmemory.Address) +arrayitemsize = 2 * sizeofaddr + + +def binary_search(start, end, addr1): + """Search for an element in a sorted array. + + The interval from the start address (included) to the end address + (excluded) is assumed to be a sorted arrays of pairs (addr1, addr2). + This searches for the item with a given addr1 and returns its + address. + """ + count = (end - start) // arrayitemsize + while count > 1: + middleindex = count // 2 + middle = start + middleindex * arrayitemsize + if addr1 < middle.address[0]: + count = middleindex + else: + start = middle + count -= middleindex + return start + +def insertion_sort(start, end): + """Sort an array of pairs of addresses. + + This is an insertion sort, so it's slowish unless the array is mostly + sorted already (which is what I expect, but XXX check this). + """ + next = start + while next < end: + # assuming the interval from start (included) to next (excluded) + # to be already sorted, move the next element back into the array + # until it reaches its proper place. + addr1 = next.address[0] + addr2 = next.address[1] + scan = next + while scan > start and addr1 < scan.address[-2]: + scan.address[0] = scan.address[-2] + scan.address[1] = scan.address[-1] + scan -= arrayitemsize + scan.address[0] = addr1 + scan.address[1] = addr2 + next += arrayitemsize From arigo at codespeak.net Thu Dec 27 13:12:37 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Dec 2007 13:12:37 +0100 (CET) Subject: [pypy-svn] r50135 - pypy/dist/pypy/translator/c/src Message-ID: <20071227121237.D45E21684EE@codespeak.net> Author: arigo Date: Thu Dec 27 13:12:36 2007 New Revision: 50135 Modified: pypy/dist/pypy/translator/c/src/stack.h Log: LL_stack_too_big() accounts for up to 10% of the run-time of PyPy, because it's called extremely often. Split it in a fast path and a slow path. Gives a ~3% improvement. Modified: pypy/dist/pypy/translator/c/src/stack.h ============================================================================== --- pypy/dist/pypy/translator/c/src/stack.h (original) +++ pypy/dist/pypy/translator/c/src/stack.h Thu Dec 27 13:12:36 2007 @@ -9,7 +9,26 @@ #endif void LL_stack_unwind(void); -int LL_stack_too_big(void); +int LL_stack_too_big_slowpath(void); + +extern volatile char *_LLstacktoobig_stack_base_pointer; +extern long _LLstacktoobig_stack_min; +extern long _LLstacktoobig_stack_max; + +static int LL_stack_too_big(void) +{ + /* The fast path of stack_too_big, called extremely often. + Making it static makes an *inlinable* copy of this small + function's implementation in each compilation unit. */ + char local; + long diff = &local - _LLstacktoobig_stack_base_pointer; + /* common case: we are still in the same thread as last time + we checked, and still in the allowed part of the stack */ + return ((diff < _LLstacktoobig_stack_min || + diff > _LLstacktoobig_stack_max) + /* if not, call the slow path */ + && LL_stack_too_big_slowpath()); +} #ifndef PYPY_NOT_MAIN_FILE #include @@ -33,15 +52,16 @@ return &local - parent; } -int LL_stack_too_big(void) +volatile char *_LLstacktoobig_stack_base_pointer = NULL; +long _LLstacktoobig_stack_min = 0; +long _LLstacktoobig_stack_max = 0; +RPyThreadStaticTLS _LLstacktoobig_stack_base_pointer_key; + +int LL_stack_too_big_slowpath(void) { char local; long diff; char *baseptr; - static volatile char *stack_base_pointer = NULL; - static long stack_min = 0; - static long stack_max = 0; - static RPyThreadStaticTLS stack_base_pointer_key; /* Check that the stack is less than MAX_STACK_SIZE bytes bigger than the value recorded in stack_base_pointer. The base pointer is updated to the current value if it is still NULL @@ -50,41 +70,37 @@ try to minimize its overhead by keeping a local copy in stack_pointer_pointer. */ - diff = &local - stack_base_pointer; - if (stack_min <= diff && diff <= stack_max) { - /* common case: we are still in the same thread as last time - we checked, and still in the allowed part of the stack */ - return 0; - } - - if (stack_min == stack_max /* == 0 */) { + if (_LLstacktoobig_stack_min == _LLstacktoobig_stack_max /* == 0 */) { /* not initialized */ /* XXX We assume that initialization is performed early, when there is still only one thread running. This allows us to ignore race conditions here */ - char *errmsg = RPyThreadStaticTLS_Create(&stack_base_pointer_key); + char *errmsg = RPyThreadStaticTLS_Create( + &_LLstacktoobig_stack_base_pointer_key); if (errmsg) { /* XXX should we exit the process? */ fprintf(stderr, "Internal PyPy error: %s\n", errmsg); return 1; } if (_LL_stack_growing_direction(NULL) > 0) - stack_max = MAX_STACK_SIZE; + _LLstacktoobig_stack_max = MAX_STACK_SIZE; else - stack_min = -MAX_STACK_SIZE; + _LLstacktoobig_stack_min = -MAX_STACK_SIZE; } - baseptr = (char *) RPyThreadStaticTLS_Get(stack_base_pointer_key); + baseptr = (char *) RPyThreadStaticTLS_Get( + _LLstacktoobig_stack_base_pointer_key); if (baseptr != NULL) { diff = &local - baseptr; - if (stack_min <= diff && diff <= stack_max) { + if (_LLstacktoobig_stack_min <= diff && + diff <= _LLstacktoobig_stack_max) { /* within bounds */ - stack_base_pointer = baseptr; + _LLstacktoobig_stack_base_pointer = baseptr; return 0; } - if ((stack_min == 0 && diff < 0) || - (stack_max == 0 && diff > 0)) { + if ((_LLstacktoobig_stack_min == 0 && diff < 0) || + (_LLstacktoobig_stack_max == 0 && diff > 0)) { /* we underflowed the stack, which means that the initial estimation of the stack base must be revised (see below) */ @@ -96,8 +112,8 @@ /* update the stack base pointer to the current value */ baseptr = &local; - RPyThreadStaticTLS_Set(stack_base_pointer_key, baseptr); - stack_base_pointer = baseptr; + RPyThreadStaticTLS_Set(_LLstacktoobig_stack_base_pointer_key, baseptr); + _LLstacktoobig_stack_base_pointer = baseptr; return 0; } From arigo at codespeak.net Thu Dec 27 14:57:38 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Dec 2007 14:57:38 +0100 (CET) Subject: [pypy-svn] r50136 - pypy/branch/llvmgcroot/pypy/translator/llvm Message-ID: <20071227135738.CE4621684D8@codespeak.net> Author: arigo Date: Thu Dec 27 14:57:37 2007 New Revision: 50136 Modified: pypy/branch/llvmgcroot/pypy/translator/llvm/funcnode.py Log: block.operations is not modified during this loop. Modified: pypy/branch/llvmgcroot/pypy/translator/llvm/funcnode.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/llvm/funcnode.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/llvm/funcnode.py Thu Dec 27 14:57:37 2007 @@ -52,7 +52,7 @@ # really SSA. Fix them now. for block in graph.iterblocks(): rename = {} - for op in list(block.operations): + for op in block.operations: if rename: op.args = [rename.get(v, v) for v in op.args] if op.opname == 'gc_reload_possibly_moved': From arigo at codespeak.net Thu Dec 27 16:10:27 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Dec 2007 16:10:27 +0100 (CET) Subject: [pypy-svn] r50137 - in pypy/branch/llvmgcroot/pypy: config translator/c Message-ID: <20071227151027.DD7D1168518@codespeak.net> Author: arigo Date: Thu Dec 27 16:10:26 2007 New Revision: 50137 Added: pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py (contents, props changed) Modified: pypy/branch/llvmgcroot/pypy/config/translationoption.py pypy/branch/llvmgcroot/pypy/translator/c/funcgen.py pypy/branch/llvmgcroot/pypy/translator/c/genc.py Log: Fooling around with a '.s' file parser. Modified: pypy/branch/llvmgcroot/pypy/config/translationoption.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/config/translationoption.py (original) +++ pypy/branch/llvmgcroot/pypy/config/translationoption.py Thu Dec 27 16:10:26 2007 @@ -64,8 +64,8 @@ suggests=[("translation.gc", "marksweep")]), BoolOption("llvmgcroot", "Use the 'llvm.gcroot' primitive to find roots", default=False, cmdline="--llvmgcroot", - requires=[("translation.backend", "llvm"), - ("translation.gctransformer", "framework")]), + requires=[("translation.gctransformer", "framework")], + suggests=[("translation.backend", "llvm")]), BoolOption("thread", "enable use of threading primitives", default=False, cmdline="--thread", requires=[("translation.gc", "boehm")]), Modified: pypy/branch/llvmgcroot/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/c/funcgen.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/c/funcgen.py Thu Dec 27 16:10:26 2007 @@ -218,6 +218,15 @@ def cfunction_body(self): graph = self.graph + gcrootscount = 0 + for block in graph.iterblocks(): + for op in block.operations: + if op.opname == 'llvm_store_gcroot': + index = op.args[0].value + gcrootscount = max(gcrootscount, index+1) + for i in range(gcrootscount): + yield 'void* gcroot%d;' % i + # generate the body of each block for block in graph.iterblocks(): self.currentblock = block @@ -722,5 +731,17 @@ def OP_IS_EARLY_CONSTANT(self, op): return self.expr(op.result) + ' = 0;' # Allways false - + + def OP_LLVM_STORE_GCROOT(self, op): + index = op.args[0].value + value = self.expr(op.args[1]) + return ('gcroot%d = %s; ' % (index, value) + + 'asm volatile ("/*STORE GCROOT %%0*/"::"m"(gcroot%d));' % (index,)) + + def OP_LLVM_LOAD_GCROOT(self, op): + index = op.args[0].value + result = self.expr(op.result) + return ('%s = gcroot%d; ' % (result, index) + + 'asm volatile ("/*LOAD GCROOT %%0*/"::"m"(gcroot%d));' % (index,)) + assert not USESLOTS or '__dict__' not in dir(FunctionCodeGenerator) Modified: pypy/branch/llvmgcroot/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/c/genc.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/c/genc.py Thu Dec 27 16:10:26 2007 @@ -315,7 +315,7 @@ if self.config.translation.linkerflags: compiler.link_extra.append(self.config.translation.linkerflags) cfiles = [] - ofiles = [] + sfiles = [] for fn in compiler.cfilenames: fn = py.path.local(fn) if fn.dirpath() == targetdir: @@ -324,7 +324,7 @@ assert fn.dirpath().dirpath() == udir name = '../' + fn.relto(udir) cfiles.append(name) - ofiles.append(name[:-2] + '.o') + sfiles.append(name[:-2] + '.s') if self.config.translation.cc: cc = self.config.translation.cc @@ -342,7 +342,7 @@ print >> f write_list(cfiles, 'SOURCES =') print >> f - write_list(ofiles, 'OBJECTS =') + write_list(sfiles, 'ASMFILES =') print >> f args = ['-l'+libname for libname in self.eci.libraries] print >> f, 'LIBS =', ' '.join(args) @@ -773,14 +773,14 @@ MAKEFILE = ''' -$(TARGET): $(OBJECTS) -\t$(CC) $(LDFLAGS) $(TFLAGS) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) +$(TARGET): $(ASMFILES) +\t$(CC) $(LDFLAGS) $(TFLAGS) -o $@ $(ASMFILES) $(LIBDIRS) $(LIBS) -%.o: %.c -\t$(CC) $(CFLAGS) -o $@ -c $< $(INCLUDEDIRS) +%.s: %.c +\t$(CC) $(CFLAGS) -o $@ -S $< $(INCLUDEDIRS) clean: -\trm -f $(OBJECTS) $(TARGET) +\trm -f $(ASMFILES) $(TARGET) debug: \t$(MAKE) CFLAGS="-g -DRPY_ASSERT" @@ -803,6 +803,6 @@ profopt: \t$(MAKE) CFLAGS="-fprofile-generate $(CFLAGS)" LDFLAGS="-fprofile-generate $(LDFLAGS)" \t./$(TARGET) $(PROFOPT) -\trm -f $(OBJECTS) $(TARGET) +\trm -f $(ASMFILES) $(TARGET) \t$(MAKE) CFLAGS="-fprofile-use $(CFLAGS)" LDFLAGS="-fprofile-use $(LDFLAGS)" ''' Added: pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py ============================================================================== --- (empty file) +++ pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py Thu Dec 27 16:10:26 2007 @@ -0,0 +1,247 @@ +#! /usr/bin/env python + +import re, sys + +r_functionstart = re.compile(r"\t.type\s+(\w+),\s*[@]function\s*$") +r_functionend = re.compile(r"\t.size\s+\w+,\s*[.]-\w+\s*$") +r_label = re.compile(r"([.]?\w+)[:]\s*$") +r_insn = re.compile(r"\t([a-z]\w*)\s") +r_jump = re.compile(r"\tj\w+\s+([.]?\w+)\s*$") +OPERAND = r"[\w$%-+]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)]" +r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+")\s*$") +r_binaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+"),\s*("+OPERAND+")\s*$") +r_gcroot_marker = re.compile(r"\t/[*](STORE|LOAD) GCROOT ") +r_gcroot_op = re.compile(r"\t/[*](STORE|LOAD) GCROOT (\d*)[(]%esp[)][*]/\s*$") + +# for sanity-checking, %esp should only appear as a way to access locals, +# i.e. inside parenthesis, except if explicitly recognized otherwise +r_esp_outside_paren = re.compile(r"(.+[)])?[^(]*[%]esp") + + +class GcRootTracker(object): + + def __init__(self): + self.gcmaptable = [] + + def dump(self, output): + pass + + def process(self, iterlines): + functionlines = None + for line in iterlines: + if r_functionstart.match(line): + assert functionlines is None, ( + "missed the end of the previous function") + functionlines = [] + if functionlines is not None: + functionlines.append(line) + if r_functionend.match(line): + assert functionlines is not None, ( + "missed the start of the current function") + self.process_function(functionlines) + functionlines = None + + def process_function(self, lines): + tracker = FunctionGcRootTracker(lines) + print >> sys.stderr, tracker.funcname + self.gcmaptable.extend(tracker.computegcmaptable()) + + +class FunctionGcRootTracker(object): + VISIT_OPERATION = {} + + def __init__(self, lines): + match = r_functionstart.match(lines[0]) + self.funcname = match.group(1) + self.lines = lines + + def computegcmaptable(self): + self.findlabels() + self.calls = {} + self.follow_control_flow() + print self.calls + self.table = [] + #xxx + return self.table + + def findlabels(self): + self.labels = {} + for i, line in enumerate(self.lines): + match = r_label.match(line) + if match: + label = match.group(1) + assert label not in self.labels, "duplicate label" + self.labels[label] = i + + def follow_control_flow(self): + # 'states' is a list [(framesize, gcroot0, gcroot1, gcroot2...)] + self.states = [None] * len(self.lines) + self.pending = [] + self.framesize = 0 + self.gcroots = {} + self.propagate_state_to(1) + while self.pending: + lin = self.pending.pop() + self.follow_basic_block(lin) + + def getstate(self): + gcroots = self.gcroots.keys() + gcroots.sort() + return (self.framesize,) + tuple(gcroots) + + def propagate_state_to(self, lin): + state = self.getstate() + if self.states[lin] is None: + self.states[lin] = state + self.pending.append(lin) + else: + assert self.states[lin] == state + + def follow_basic_block(self, lin): + state = self.states[lin] + self.framesize = state[0] + self.gcroots = dict.fromkeys(state[1:]) + line = '?' + try: + while 1: + self.currentlinenum = lin + line = self.lines[lin] + match = r_insn.match(line) + if match: + insn = match.group(1) + try: + meth = self.VISIT_OPERATION[insn] + except KeyError: + meth = self.find_visitor(insn) + meth(self, line) + elif r_label.match(line): + self.propagate_state_to(lin+1) + break + elif r_gcroot_marker.match(line): + self.handle_gcroot_marker(line) + lin += 1 + except LeaveBasicBlock: + pass + except Exception, e: + print >> sys.stderr, '*'*60 + print >> sys.stderr, "%s while processing line:" % ( + e.__class__.__name__,) + print >> sys.stderr, line + raise + + def handle_gcroot_marker(self, line): + match = r_gcroot_op.match(line) + op = match.group(1) + position = int(match.group(2) or '0') + assert position % 4 == 0 + if op == 'STORE': + assert position not in self.gcroots + self.gcroots[position] = None + elif op == 'LOAD': + assert position in self.gcroots + del self.gcroots[position] + else: + raise UnrecognizedOperation(line) + + def find_visitor(self, insn): + opname = insn + while 1: + try: + meth = getattr(self.__class__, 'visit_' + opname) + break + except AttributeError: + assert opname + opname = opname[:-1] + self.VISIT_OPERATION[insn] = meth + return meth + + def visit_(self, line): + # fallback for all operations. By default, ignore the operation, + # unless it appears to do something with %esp + if r_esp_outside_paren.match(line): + raise UnrecognizedOperation(line) + + def visit_push(self, line): + raise UnrecognizedOperation(line) + + def visit_pushl(self, line): + self.framesize += 4 + + def visit_pop(self, line): + raise UnrecognizedOperation(line) + + def visit_popl(self, line): + self.framesize -= 4 + assert self.framesize >= 0, "stack underflow" + + def visit_subl(self, line): + match = r_binaryinsn.match(line) + if match.group(2) == '%esp': + count = match.group(1) + assert count.startswith('$') + count = int(count[1:]) + assert count % 4 == 0 + self.framesize += count + + def visit_addl(self, line): + match = r_binaryinsn.match(line) + if match.group(2) == '%esp': + count = match.group(1) + assert count.startswith('$') + count = int(count[1:]) + assert count % 4 == 0 + self.framesize -= count + assert self.framesize >= 0, "stack underflow" + + def visit_ret(self, line): + raise LeaveBasicBlock + + def visit_j(self, line): + raise UnrecognizedOperation(line) + + def visit_jmp(self, line): + try: + self.conditional_jump(line) + except KeyError: + pass # label not found: assume a tail-call turned into a jump + raise LeaveBasicBlock + + def conditional_jump(self, line): + match = r_jump.match(line) + label = match.group(1) + targetlin = self.labels[label] + self.propagate_state_to(targetlin) + + visit_je = conditional_jump + visit_jne = conditional_jump + visit_jg = conditional_jump + visit_jge = conditional_jump + visit_jl = conditional_jump + visit_jle = conditional_jump + visit_ja = conditional_jump + visit_jae = conditional_jump + visit_jb = conditional_jump + visit_jbe = conditional_jump + + def visit_call(self, line): + match = r_unaryinsn.match(line) + target = match.group(1) + if target == "abort": + raise LeaveBasicBlock + self.calls[self.currentlinenum] = self.getstate() + + +class LeaveBasicBlock(Exception): + pass + +class UnrecognizedOperation(Exception): + pass + + +if __name__ == '__main__': + tracker = GcRootTracker() + for fn in sys.argv[1:]: + f = open(fn, 'r') + tracker.process(f) + f.close() + tracker.dump(sys.stdout) From arigo at codespeak.net Thu Dec 27 16:24:49 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Dec 2007 16:24:49 +0100 (CET) Subject: [pypy-svn] r50138 - pypy/branch/llvmgcroot/pypy/translator/c Message-ID: <20071227152449.86BBF168518@codespeak.net> Author: arigo Date: Thu Dec 27 16:24:49 2007 New Revision: 50138 Modified: pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py Log: Add labels after each 'call' in the .s file. Modified: pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py Thu Dec 27 16:24:49 2007 @@ -1,6 +1,6 @@ #! /usr/bin/env python -import re, sys +import re, sys, os r_functionstart = re.compile(r"\t.type\s+(\w+),\s*[@]function\s*$") r_functionend = re.compile(r"\t.size\s+\w+,\s*[.]-\w+\s*$") @@ -26,7 +26,7 @@ def dump(self, output): pass - def process(self, iterlines): + def process(self, iterlines, newfile): functionlines = None for line in iterlines: if r_functionstart.match(line): @@ -35,16 +35,19 @@ functionlines = [] if functionlines is not None: functionlines.append(line) + else: + newfile.write(line) if r_functionend.match(line): assert functionlines is not None, ( "missed the start of the current function") - self.process_function(functionlines) + self.process_function(functionlines, newfile) functionlines = None - def process_function(self, lines): + def process_function(self, lines, newfile): tracker = FunctionGcRootTracker(lines) print >> sys.stderr, tracker.funcname self.gcmaptable.extend(tracker.computegcmaptable()) + newfile.writelines(tracker.lines) class FunctionGcRootTracker(object): @@ -57,11 +60,14 @@ def computegcmaptable(self): self.findlabels() - self.calls = {} + self.calls = {} # {label_after_call: state} + self.missing_labels_after_call = [] self.follow_control_flow() - print self.calls + for label, state in self.calls.values(): + print >> sys.stderr, label, '\t', state self.table = [] #xxx + self.extend_calls_with_labels() return self.table def findlabels(self): @@ -73,6 +79,12 @@ assert label not in self.labels, "duplicate label" self.labels[label] = i + def extend_calls_with_labels(self): + self.missing_labels_after_call.sort() + self.missing_labels_after_call.reverse() + for linenum, label in self.missing_labels_after_call: + self.lines.insert(linenum, '%s:\n' % (label,)) + def follow_control_flow(self): # 'states' is a list [(framesize, gcroot0, gcroot1, gcroot2...)] self.states = [None] * len(self.lines) @@ -203,7 +215,10 @@ try: self.conditional_jump(line) except KeyError: - pass # label not found: assume a tail-call turned into a jump + # label not found: check if it's a tail-call turned into a jump + match = r_unaryinsn.match(line) + target = match.group(1) + assert not target.startswith('.') raise LeaveBasicBlock def conditional_jump(self, line): @@ -228,7 +243,21 @@ target = match.group(1) if target == "abort": raise LeaveBasicBlock - self.calls[self.currentlinenum] = self.getstate() + nextline = self.lines[self.currentlinenum+1] + match = r_label.match(nextline) + if match and not match.group(1).startswith('.'): + label = match.group(1) + else: + k = 0 + while 1: + label = '__gcmap_IN_%s_%d' % (self.funcname, k) + if label not in self.labels: + break + k += 1 + self.labels[label] = self.currentlinenum+1 + self.missing_labels_after_call.append( + (self.currentlinenum+1, label)) + self.calls[self.currentlinenum] = label, self.getstate() class LeaveBasicBlock(Exception): @@ -241,7 +270,12 @@ if __name__ == '__main__': tracker = GcRootTracker() for fn in sys.argv[1:]: + tmpfn = fn + '.TMP' f = open(fn, 'r') - tracker.process(f) + g = open(tmpfn, 'w') + tracker.process(f, g) f.close() + g.close() + os.unlink(fn) + os.rename(tmpfn, fn) tracker.dump(sys.stdout) From arigo at codespeak.net Thu Dec 27 16:32:00 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Dec 2007 16:32:00 +0100 (CET) Subject: [pypy-svn] r50139 - pypy/branch/llvmgcroot/pypy/translator/c Message-ID: <20071227153200.6B80F168513@codespeak.net> Author: arigo Date: Thu Dec 27 16:31:58 2007 New Revision: 50139 Modified: pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py Log: More sanity checks. Modified: pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py Thu Dec 27 16:31:58 2007 @@ -61,14 +61,20 @@ def computegcmaptable(self): self.findlabels() self.calls = {} # {label_after_call: state} + self.ignore_calls = {} self.missing_labels_after_call = [] self.follow_control_flow() - for label, state in self.calls.values(): - print >> sys.stderr, label, '\t', state - self.table = [] - #xxx + table = self.gettable() self.extend_calls_with_labels() - return self.table + return table + + def gettable(self): + table = self.calls.items() + table.sort() # by line number + table = [value for key, value in table] + for label, state in table: + print >> sys.stderr, label, '\t', state + return table def findlabels(self): self.labels = {} @@ -95,6 +101,7 @@ while self.pending: lin = self.pending.pop() self.follow_basic_block(lin) + self.check_all_calls_seen() def getstate(self): gcroots = self.gcroots.keys() @@ -141,6 +148,15 @@ print >> sys.stderr, line raise + def check_all_calls_seen(self): + for i, line in enumerate(self.lines): + match = r_insn.match(line) + if match: + insn = match.group(1) + if insn == 'call': + assert i in self.calls or i in self.ignore_calls, ( + "unreachable call!" + line) + def handle_gcroot_marker(self, line): match = r_gcroot_op.match(line) op = match.group(1) @@ -242,7 +258,11 @@ match = r_unaryinsn.match(line) target = match.group(1) if target == "abort": + self.ignore_calls[self.currentlinenum] = None raise LeaveBasicBlock + # we need a label just after the call. Reuse one if it is already + # there (e.g. from a previous run of this script); otherwise + # invent a name and schedule the line insertion. nextline = self.lines[self.currentlinenum+1] match = r_label.match(nextline) if match and not match.group(1).startswith('.'): From arigo at codespeak.net Thu Dec 27 16:42:49 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Dec 2007 16:42:49 +0100 (CET) Subject: [pypy-svn] r50140 - pypy/branch/llvmgcroot/pypy/translator/c Message-ID: <20071227154249.4E25D168513@codespeak.net> Author: arigo Date: Thu Dec 27 16:42:48 2007 New Revision: 50140 Modified: pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py Log: Generate the gc map table collected from all the input .s files. Modified: pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py Thu Dec 27 16:42:48 2007 @@ -20,11 +20,33 @@ class GcRootTracker(object): - def __init__(self): + def __init__(self, verbose=False): self.gcmaptable = [] + self.verbose = verbose def dump(self, output): - pass + shapes = {} + print >> output, '\t.data' + print >> output, '\t.align\t4' + print >> output, '__gcmapstart:' + for label, state in self.gcmaptable: + if state not in shapes: + lst = ['__gcmap_shape'] + lst.extend(map(str, state)) + shapes[state] = '_'.join(lst) + print >> output, '\t.long\t%s' % (label,) + print >> output, '\t.long\t%s' % (shapes[state],) + print >> output, '__gcmapend:' + print >> output, '\t.section\trodata' + print >> output, '\t.align\t4' + keys = shapes.keys() + keys.sort() + for state in keys: + print >> output, '%s:' % (shapes[state],) + print >> output, '\t.long\t%d' % (state[0],) # frame size + print >> output, '\t.long\t%d' % (len(state)-1,) # gcroot count + for p in state[1:]: + print >> output, '\t.long\t%d' % (p,) # gcroots def process(self, iterlines, newfile): functionlines = None @@ -45,8 +67,13 @@ def process_function(self, lines, newfile): tracker = FunctionGcRootTracker(lines) - print >> sys.stderr, tracker.funcname - self.gcmaptable.extend(tracker.computegcmaptable()) + if self.verbose: + print >> sys.stderr, tracker.funcname + table = tracker.computegcmaptable() + if self.verbose: + for label, state in table: + print >> sys.stderr, label, '\t', state + self.gcmaptable.extend(table) newfile.writelines(tracker.lines) @@ -69,11 +96,10 @@ return table def gettable(self): + "Returns a list [(label_after_call, (framesize, gcroot0, gcroot1,..))]" table = self.calls.items() table.sort() # by line number table = [value for key, value in table] - for label, state in table: - print >> sys.stderr, label, '\t', state return table def findlabels(self): @@ -288,7 +314,7 @@ if __name__ == '__main__': - tracker = GcRootTracker() + tracker = GcRootTracker(verbose=True) for fn in sys.argv[1:]: tmpfn = fn + '.TMP' f = open(fn, 'r') From arigo at codespeak.net Thu Dec 27 18:12:07 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Dec 2007 18:12:07 +0100 (CET) Subject: [pypy-svn] r50141 - in pypy/branch/llvmgcroot/pypy: rpython/memory/gctransform translator/c translator/c/src Message-ID: <20071227171207.E27961684F6@codespeak.net> Author: arigo Date: Thu Dec 27 18:12:05 2007 New Revision: 50141 Modified: pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/llvmgcroot.py pypy/branch/llvmgcroot/pypy/translator/c/genc.py pypy/branch/llvmgcroot/pypy/translator/c/src/mem.h pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py Log: targetgcbench works now via genc using the --llvmgcroot logic. Only for 386-compatible Linux boxes. To try it out, use translate.py --llvmgcroot --backend=c --gc=xxx --text targetxxx and use the generated Makefile to process the result. Modified: pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/llvmgcroot.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/llvmgcroot.py (original) +++ pypy/branch/llvmgcroot/pypy/rpython/memory/gctransform/llvmgcroot.py Thu Dec 27 18:12:05 2007 @@ -69,6 +69,15 @@ def __init__(self, with_static=True): self.stack_current = llop.llvm_frameaddress(llmemory.Address) self.remaining_roots_in_current_frame = 0 + # We must walk at least a couple of frames up the stack + # *now*, i.e. before we leave __init__, otherwise + # self.stack_current ends up pointing to a dead frame. + # We can walk until we find a real GC root; then we're + # definitely out of the GC code itself. + while self.remaining_roots_in_current_frame == 0: + if not self.walk_to_parent_frame(): + break # not a single GC root? unlikely but not + # impossible I guess if with_static: self.static_current = gcdata.static_root_end else: Modified: pypy/branch/llvmgcroot/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/c/genc.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/c/genc.py Thu Dec 27 18:12:05 2007 @@ -1,6 +1,6 @@ import autopath import py -import sys +import sys, os from pypy.translator.c.node import PyObjectNode, FuncNode from pypy.translator.c.database import LowLevelDatabase from pypy.translator.c.extfunc import pre_include_code_lines @@ -275,6 +275,10 @@ self.eci, compiler_exe = cc, profbased = profbased) def compile(self): + if self.config.translation.llvmgcroot: + raise Exception("Dunno how to compile with --llvmgcroot. " + "Just go to the %s directory and type 'make'." + % (self.targetdir,)) assert self.c_source_filename assert not self._compiled eci = self.eci.merge(ExternalCompilationInfo(includes= @@ -344,6 +348,9 @@ print >> f write_list(sfiles, 'ASMFILES =') print >> f + print >> f, 'TRACKGCROOT = "%s"' % (os.path.join(autopath.this_dir, + 'trackgcroot.py'),) + print >> f args = ['-l'+libname for libname in self.eci.libraries] print >> f, 'LIBS =', ' '.join(args) args = ['-L'+path for path in self.eci.library_dirs] @@ -773,14 +780,17 @@ MAKEFILE = ''' -$(TARGET): $(ASMFILES) -\t$(CC) $(LDFLAGS) $(TFLAGS) -o $@ $(ASMFILES) $(LIBDIRS) $(LIBS) +$(TARGET): $(ASMFILES) gcmaptable.s +\t$(CC) $(LDFLAGS) $(TFLAGS) -o $@ $(ASMFILES) gcmaptable.s $(LIBDIRS) $(LIBS) %.s: %.c \t$(CC) $(CFLAGS) -o $@ -S $< $(INCLUDEDIRS) +gcmaptable.s: $(ASMFILES) +\t$(TRACKGCROOT) $(ASMFILES) > $@ || (rm -f $@ && exit 1) + clean: -\trm -f $(ASMFILES) $(TARGET) +\trm -f $(ASMFILES) gcmaptable.s $(TARGET) debug: \t$(MAKE) CFLAGS="-g -DRPY_ASSERT" @@ -803,6 +813,6 @@ profopt: \t$(MAKE) CFLAGS="-fprofile-generate $(CFLAGS)" LDFLAGS="-fprofile-generate $(LDFLAGS)" \t./$(TARGET) $(PROFOPT) -\trm -f $(ASMFILES) $(TARGET) +\trm -f $(ASMFILES) gcmaptable.s $(TARGET) \t$(MAKE) CFLAGS="-fprofile-use $(CFLAGS)" LDFLAGS="-fprofile-use $(LDFLAGS)" ''' Modified: pypy/branch/llvmgcroot/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/c/src/mem.h (original) +++ pypy/branch/llvmgcroot/pypy/translator/c/src/mem.h Thu Dec 27 18:12:05 2007 @@ -8,6 +8,17 @@ #define ROUND_UP_FOR_ALLOCATION(x) \ (((x) + (MEMORY_ALIGNMENT-1)) & ~(MEMORY_ALIGNMENT-1)) +extern char __gcmapstart; +extern char __gcmapend; +extern char* __gcmap_frame_address(void); + +#define OP_LLVM_GCMAPSTART(r) r = &__gcmapstart +#define OP_LLVM_GCMAPEND(r) r = &__gcmapend +#define OP_LLVM_FRAMEADDRESS(r) asm ("pypygetframeaddress %0" : "=r" (r)) +/* NB. we cannot use __builtin_frame_address(0) - apparently, gcc thinks + it can return %ebp even if -fomit-frame-pointer is specified, which + doesn't work. So we need a bit of help from trackgcroot.py... */ + #define RAW_MALLOC_ZERO_FILLED 0 Modified: pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py Thu Dec 27 18:12:05 2007 @@ -5,10 +5,12 @@ r_functionstart = re.compile(r"\t.type\s+(\w+),\s*[@]function\s*$") r_functionend = re.compile(r"\t.size\s+\w+,\s*[.]-\w+\s*$") r_label = re.compile(r"([.]?\w+)[:]\s*$") +r_globl = re.compile(r"\t[.]globl\t(\w+)\s*$") r_insn = re.compile(r"\t([a-z]\w*)\s") r_jump = re.compile(r"\tj\w+\s+([.]?\w+)\s*$") -OPERAND = r"[\w$%-+]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)]" +OPERAND = r"[-\w$%+.:@]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)]" r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+")\s*$") +r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+([*]"+OPERAND+")\s*$") r_binaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+"),\s*("+OPERAND+")\s*$") r_gcroot_marker = re.compile(r"\t/[*](STORE|LOAD) GCROOT ") r_gcroot_op = re.compile(r"\t/[*](STORE|LOAD) GCROOT (\d*)[(]%esp[)][*]/\s*$") @@ -28,17 +30,22 @@ shapes = {} print >> output, '\t.data' print >> output, '\t.align\t4' + print >> output, '\t.globl\t__gcmapstart' print >> output, '__gcmapstart:' for label, state in self.gcmaptable: if state not in shapes: lst = ['__gcmap_shape'] - lst.extend(map(str, state)) + for n in state: + if n < 0: + n = 'minus%d' % (-n,) + lst.append(str(n)) shapes[state] = '_'.join(lst) print >> output, '\t.long\t%s' % (label,) print >> output, '\t.long\t%s' % (shapes[state],) + print >> output, '\t.globl\t__gcmapend' print >> output, '__gcmapend:' - print >> output, '\t.section\trodata' - print >> output, '\t.align\t4' + #print >> output, '\t.section\trodata' + #print >> output, '\t.align\t4' keys = shapes.keys() keys.sort() for state in keys: @@ -48,7 +55,7 @@ for p in state[1:]: print >> output, '\t.long\t%d' % (p,) # gcroots - def process(self, iterlines, newfile): + def process(self, iterlines, newfile, entrypoint='main', filename='?'): functionlines = None for line in iterlines: if r_functionstart.match(line): @@ -62,17 +69,29 @@ if r_functionend.match(line): assert functionlines is not None, ( "missed the start of the current function") - self.process_function(functionlines, newfile) + self.process_function(functionlines, newfile, entrypoint, + filename) functionlines = None - def process_function(self, lines, newfile): + def process_function(self, lines, newfile, entrypoint, filename): tracker = FunctionGcRootTracker(lines) if self.verbose: - print >> sys.stderr, tracker.funcname + print >> sys.stderr, '[trackgcroot:%s] %s' % (filename, + tracker.funcname) table = tracker.computegcmaptable() - if self.verbose: + #if self.verbose: + # for label, state in table: + # print >> sys.stderr, label, '\t', state + if tracker.funcname == entrypoint: for label, state in table: - print >> sys.stderr, label, '\t', state + assert len(state) == 1, ( + "XXX for now the entry point should not have any gc roots") + table = [(label, (-1,)) for label, _ in table] + # ^^^ we set the framesize of the entry point to -1 as a marker + else: + assert not tracker.uses_frame_pointer, ( + "Function %r uses the frame pointer %ebp. FIXME: implement it" + % (tracker.funcname,)) self.gcmaptable.extend(table) newfile.writelines(tracker.lines) @@ -87,6 +106,7 @@ def computegcmaptable(self): self.findlabels() + self.uses_frame_pointer = False self.calls = {} # {label_after_call: state} self.ignore_calls = {} self.missing_labels_after_call = [] @@ -116,6 +136,7 @@ self.missing_labels_after_call.reverse() for linenum, label in self.missing_labels_after_call: self.lines.insert(linenum, '%s:\n' % (label,)) + self.lines.insert(linenum, '\t.globl\t%s\n' % (label,)) def follow_control_flow(self): # 'states' is a list [(framesize, gcroot0, gcroot1, gcroot2...)] @@ -212,8 +233,9 @@ def visit_(self, line): # fallback for all operations. By default, ignore the operation, # unless it appears to do something with %esp - if r_esp_outside_paren.match(line): - raise UnrecognizedOperation(line) + if not self.uses_frame_pointer: + if r_esp_outside_paren.match(line): + raise UnrecognizedOperation(line) def visit_push(self, line): raise UnrecognizedOperation(line) @@ -247,6 +269,17 @@ self.framesize -= count assert self.framesize >= 0, "stack underflow" + def visit_movl(self, line): + match = r_binaryinsn.match(line) + if match.group(1) == '%esp': + # only for movl %esp, %ebp + if match.group(2) != '%ebp': + raise UnrecognizedOperation(line) + assert self.framesize == 4 # only %ebp should have been pushed + self.uses_frame_pointer = True + elif match.group(2) == '%esp': + raise UnrecognizedOperation(line) + def visit_ret(self, line): raise LeaveBasicBlock @@ -279,21 +312,36 @@ visit_jae = conditional_jump visit_jb = conditional_jump visit_jbe = conditional_jump + visit_jp = conditional_jump + visit_jnp = conditional_jump + visit_js = conditional_jump + visit_jns = conditional_jump + visit_jo = conditional_jump + visit_jno = conditional_jump def visit_call(self, line): match = r_unaryinsn.match(line) - target = match.group(1) - if target == "abort": - self.ignore_calls[self.currentlinenum] = None - raise LeaveBasicBlock - # we need a label just after the call. Reuse one if it is already - # there (e.g. from a previous run of this script); otherwise - # invent a name and schedule the line insertion. - nextline = self.lines[self.currentlinenum+1] - match = r_label.match(nextline) - if match and not match.group(1).startswith('.'): - label = match.group(1) + if match is None: + assert r_unaryinsn_star.match(line) # indirect call else: + target = match.group(1) + if target in FUNCTIONS_NOT_RETURNING: + self.ignore_calls[self.currentlinenum] = None + raise LeaveBasicBlock + # we need a globally-declared label just after the call. + # Reuse one if it is already there (e.g. from a previous run of this + # script); otherwise invent a name and schedule the line insertion. + label = None + # this checks for a ".globl NAME" followed by "NAME:" + match = r_globl.match(self.lines[self.currentlinenum+1]) + if match: + label1 = match.group(1) + match = r_label.match(self.lines[self.currentlinenum+2]) + if match: + label2 = match.group(1) + if label1 == label2: + label = label2 + if label is None: k = 0 while 1: label = '__gcmap_IN_%s_%d' % (self.funcname, k) @@ -305,6 +353,18 @@ (self.currentlinenum+1, label)) self.calls[self.currentlinenum] = label, self.getstate() + def visit_pypygetframeaddress(self, line): + # this is a pseudo-instruction that is emitted to find the first + # frame address on the stack. We cannot just use + # __builtin_frame_address(0) - apparently, gcc thinks it can + # return %ebp even if -fomit-frame-pointer is specified, which + # doesn't work. + match = r_unaryinsn.match(line) + reg = match.group(1) + newline = '\tleal\t%d(%%esp), %s\t/* pypygetframeaddress */\n' % ( + self.framesize-4, reg) + self.lines[self.currentlinenum] = newline + class LeaveBasicBlock(Exception): pass @@ -312,6 +372,11 @@ class UnrecognizedOperation(Exception): pass +FUNCTIONS_NOT_RETURNING = { + 'abort': None, + '__assert_fail': None, + } + if __name__ == '__main__': tracker = GcRootTracker(verbose=True) @@ -319,7 +384,7 @@ tmpfn = fn + '.TMP' f = open(fn, 'r') g = open(tmpfn, 'w') - tracker.process(f, g) + tracker.process(f, g, filename=fn) f.close() g.close() os.unlink(fn) From arigo at codespeak.net Thu Dec 27 18:30:49 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Dec 2007 18:30:49 +0100 (CET) Subject: [pypy-svn] r50142 - pypy/branch/llvmgcroot/pypy/translator/c Message-ID: <20071227173049.C0C6D16844A@codespeak.net> Author: arigo Date: Thu Dec 27 18:30:48 2007 New Revision: 50142 Modified: pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py Log: Fixes to support targetrpystonedalone. Modified: pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py Thu Dec 27 18:30:48 2007 @@ -11,6 +11,9 @@ OPERAND = r"[-\w$%+.:@]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)]" r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+")\s*$") r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+([*]"+OPERAND+")\s*$") +r_jmp_switch = re.compile(r"\tjmp\t[*]([.]?\w+)[(]") +r_jmptable_item = re.compile(r"\t.long\t([.]?\w+)\s*$") +r_jmptable_end = re.compile(r"\t.text\s*$") r_binaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+"),\s*("+OPERAND+")\s*$") r_gcroot_marker = re.compile(r"\t/[*](STORE|LOAD) GCROOT ") r_gcroot_op = re.compile(r"\t/[*](STORE|LOAD) GCROOT (\d*)[(]%esp[)][*]/\s*$") @@ -44,8 +47,8 @@ print >> output, '\t.long\t%s' % (shapes[state],) print >> output, '\t.globl\t__gcmapend' print >> output, '__gcmapend:' - #print >> output, '\t.section\trodata' - #print >> output, '\t.align\t4' + print >> output, '\t.section\t.rodata' + print >> output, '\t.align\t4' keys = shapes.keys() keys.sort() for state in keys: @@ -168,8 +171,9 @@ self.framesize = state[0] self.gcroots = dict.fromkeys(state[1:]) line = '?' - try: - while 1: + self.in_APP = False + while 1: + try: self.currentlinenum = lin line = self.lines[lin] match = r_insn.match(line) @@ -181,19 +185,32 @@ meth = self.find_visitor(insn) meth(self, line) elif r_label.match(line): - self.propagate_state_to(lin+1) + if not self.in_APP: # ignore labels inside #APP/#NO_APP + self.propagate_state_to(lin+1) break elif r_gcroot_marker.match(line): self.handle_gcroot_marker(line) + elif line == '#APP\n': + self.in_APP = True + elif line == '#NO_APP\n': + self.in_APP = False lin += 1 - except LeaveBasicBlock: - pass - except Exception, e: - print >> sys.stderr, '*'*60 - print >> sys.stderr, "%s while processing line:" % ( - e.__class__.__name__,) - print >> sys.stderr, line - raise + except LeaveBasicBlock: + if self.in_APP: # ignore control flow inside #APP/#NO_APP + lin += 1 + continue + break + except UnrecognizedOperation: + if self.in_APP: # ignore strange ops inside #APP/#NO_APP + lin += 1 + continue + raise + except Exception, e: + print >> sys.stderr, '*'*60 + print >> sys.stderr, "%s while processing line:" % ( + e.__class__.__name__,) + print >> sys.stderr, line + raise def check_all_calls_seen(self): for i, line in enumerate(self.lines): @@ -287,6 +304,23 @@ raise UnrecognizedOperation(line) def visit_jmp(self, line): + if self.in_APP: + return # ignore jumps inside a #APP/#NO_APP block + match = r_jmp_switch.match(line) + if match: + # this is a jmp *Label(%index), used for table-based switches. + # Assume that the table is just a list of lines looking like + # .long LABEL or .long 0, ending in a .text. + tablelabel = match.group(1) + tablelin = self.labels[tablelabel] + 1 + while not r_jmptable_end.match(self.lines[tablelin]): + match = r_jmptable_item.match(self.lines[tablelin]) + label = match.group(1) + if label != '0': + targetlin = self.labels[label] + self.propagate_state_to(targetlin) + tablelin += 1 + raise LeaveBasicBlock try: self.conditional_jump(line) except KeyError: @@ -297,6 +331,8 @@ raise LeaveBasicBlock def conditional_jump(self, line): + if self.in_APP: + return # ignore jumps inside a #APP/#NO_APP block match = r_jump.match(line) label = match.group(1) targetlin = self.labels[label] @@ -320,6 +356,9 @@ visit_jno = conditional_jump def visit_call(self, line): + if self.in_APP: + self.ignore_calls[self.currentlinenum] = None + return # ignore calls inside a #APP/#NO_APP block match = r_unaryinsn.match(line) if match is None: assert r_unaryinsn_star.match(line) # indirect call From arigo at codespeak.net Thu Dec 27 18:48:37 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Dec 2007 18:48:37 +0100 (CET) Subject: [pypy-svn] r50143 - pypy/branch/llvmgcroot/pypy/translator/c Message-ID: <20071227174837.4950D16843D@codespeak.net> Author: arigo Date: Thu Dec 27 18:48:36 2007 New Revision: 50143 Modified: pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py Log: More hacks until targetprologstandalone works. Modified: pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py Thu Dec 27 18:48:36 2007 @@ -106,14 +106,18 @@ match = r_functionstart.match(lines[0]) self.funcname = match.group(1) self.lines = lines + self.inconsistent_state = {} def computegcmaptable(self): self.findlabels() - self.uses_frame_pointer = False - self.calls = {} # {label_after_call: state} - self.ignore_calls = {} - self.missing_labels_after_call = [] - self.follow_control_flow() + try: + self.uses_frame_pointer = False + self.calls = {} # {label_after_call: state} + self.ignore_calls = {} + self.missing_labels_after_call = [] + self.follow_control_flow() + except ReflowCompletely: + return self.computegcmaptable() table = self.gettable() self.extend_calls_with_labels() return table @@ -154,8 +158,11 @@ self.check_all_calls_seen() def getstate(self): - gcroots = self.gcroots.keys() - gcroots.sort() + if self.gcroots is Bogus: + gcroots = () + else: + gcroots = self.gcroots.keys() + gcroots.sort() return (self.framesize,) + tuple(gcroots) def propagate_state_to(self, lin): @@ -163,13 +170,19 @@ if self.states[lin] is None: self.states[lin] = state self.pending.append(lin) - else: - assert self.states[lin] == state + elif self.states[lin] != state: + if lin not in self.inconsistent_state: + self.inconsistent_state[lin] = (self.states[lin], state) + raise ReflowCompletely def follow_basic_block(self, lin): state = self.states[lin] self.framesize = state[0] self.gcroots = dict.fromkeys(state[1:]) + if lin in self.inconsistent_state: # in case of inconsistent gcroots, + self.framesize = Bogus # assume that we're about to leave + if self.framesize is Bogus: # the function or fail an assert + self.gcroots = Bogus line = '?' self.in_APP = False while 1: @@ -205,6 +218,8 @@ lin += 1 continue raise + except ReflowCompletely: + raise except Exception, e: print >> sys.stderr, '*'*60 print >> sys.stderr, "%s while processing line:" % ( @@ -321,6 +336,9 @@ self.propagate_state_to(targetlin) tablelin += 1 raise LeaveBasicBlock + if r_unaryinsn_star.match(line): + # that looks like an indirect tail-call. + raise LeaveBasicBlock try: self.conditional_jump(line) except KeyError: @@ -411,6 +429,13 @@ class UnrecognizedOperation(Exception): pass +class ReflowCompletely(Exception): + pass + +class BogusObject(object): + pass +Bogus = BogusObject() + FUNCTIONS_NOT_RETURNING = { 'abort': None, '__assert_fail': None, From arigo at codespeak.net Thu Dec 27 18:54:53 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Dec 2007 18:54:53 +0100 (CET) Subject: [pypy-svn] r50144 - pypy/branch/llvmgcroot/pypy/translator/c Message-ID: <20071227175453.DB81E168443@codespeak.net> Author: arigo Date: Thu Dec 27 18:54:53 2007 New Revision: 50144 Modified: pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py Log: Fix for wyvern's version of gcc. Modified: pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py Thu Dec 27 18:54:53 2007 @@ -78,6 +78,8 @@ def process_function(self, lines, newfile, entrypoint, filename): tracker = FunctionGcRootTracker(lines) + if tracker.funcname == entrypoint: + tracker.can_use_frame_pointer = True if self.verbose: print >> sys.stderr, '[trackgcroot:%s] %s' % (filename, tracker.funcname) @@ -85,16 +87,15 @@ #if self.verbose: # for label, state in table: # print >> sys.stderr, label, '\t', state - if tracker.funcname == entrypoint: + if tracker.can_use_frame_pointer: + # XXX for now we have no logic to track the gc roots of + # functions using %ebp for label, state in table: assert len(state) == 1, ( "XXX for now the entry point should not have any gc roots") + if tracker.funcname == entrypoint: table = [(label, (-1,)) for label, _ in table] # ^^^ we set the framesize of the entry point to -1 as a marker - else: - assert not tracker.uses_frame_pointer, ( - "Function %r uses the frame pointer %ebp. FIXME: implement it" - % (tracker.funcname,)) self.gcmaptable.extend(table) newfile.writelines(tracker.lines) @@ -107,11 +108,11 @@ self.funcname = match.group(1) self.lines = lines self.inconsistent_state = {} + self.can_use_frame_pointer = False # unless changed by caller def computegcmaptable(self): self.findlabels() try: - self.uses_frame_pointer = False self.calls = {} # {label_after_call: state} self.ignore_calls = {} self.missing_labels_after_call = [] @@ -265,7 +266,7 @@ def visit_(self, line): # fallback for all operations. By default, ignore the operation, # unless it appears to do something with %esp - if not self.uses_frame_pointer: + if not self.can_use_frame_pointer: if r_esp_outside_paren.match(line): raise UnrecognizedOperation(line) @@ -307,8 +308,8 @@ # only for movl %esp, %ebp if match.group(2) != '%ebp': raise UnrecognizedOperation(line) + assert self.can_use_frame_pointer # only if we can have a frame ptr assert self.framesize == 4 # only %ebp should have been pushed - self.uses_frame_pointer = True elif match.group(2) == '%esp': raise UnrecognizedOperation(line) From jacob at codespeak.net Thu Dec 27 19:35:28 2007 From: jacob at codespeak.net (jacob at codespeak.net) Date: Thu, 27 Dec 2007 19:35:28 +0100 (CET) Subject: [pypy-svn] r50145 - pypy/extradoc/planning/roadmap Message-ID: <20071227183528.2EE0D168451@codespeak.net> Author: jacob Date: Thu Dec 27 19:35:25 2007 New Revision: 50145 Modified: pypy/extradoc/planning/roadmap/task_catch_up_with_2_x.txt pypy/extradoc/planning/roadmap/task_cpython_api.txt pypy/extradoc/planning/roadmap/task_ctypes.txt pypy/extradoc/planning/roadmap/task_gui_support.txt pypy/extradoc/planning/roadmap/task_manual_optimizations.txt pypy/extradoc/planning/roadmap/task_modules_3rdparty.txt pypy/extradoc/planning/roadmap/task_modules_rffi.txt pypy/extradoc/planning/roadmap/task_multi_platform.txt pypy/extradoc/planning/roadmap/task_optimize_jit.txt pypy/extradoc/planning/roadmap/task_parser_cleanup.txt pypy/extradoc/planning/roadmap/task_refactor_jit.txt pypy/extradoc/planning/roadmap/task_separate_compilation.txt pypy/extradoc/planning/roadmap/task_wrapper_generator.txt Log: Modified all CPython targets to the common format. Added text in some places. Modified: pypy/extradoc/planning/roadmap/task_catch_up_with_2_x.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_catch_up_with_2_x.txt (original) +++ pypy/extradoc/planning/roadmap/task_catch_up_with_2_x.txt Thu Dec 27 19:35:25 2007 @@ -2,8 +2,9 @@ ============================= status: - - PyPy already implements a few features of 2.5 (PEP 309, PEP 338, PEP 341, PEP 357, - partition, any, all) + + - PyPy already implements a few features of 2.5 (PEP 309, PEP 338, + PEP 341, PEP 357, partition, any, all) todo: - Implement these features that were new in CPython 2.5: @@ -24,10 +25,12 @@ - class C(): pass - Merge Stdlib of Python 2.5/2.6 - - Implement Python 2.6 features (not so important if 2.5 is the goal for this task) + - Implement Python 2.6 features (not so important if 2.5 is the goal + for this task) expected outcome: - - PyPy supports Python 2.5 semantics and makes people with apps happy that use these + - PyPy supports Python 2.5 semantics and makes people with apps that use + these happy. dependencies: - None Modified: pypy/extradoc/planning/roadmap/task_cpython_api.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_cpython_api.txt (original) +++ pypy/extradoc/planning/roadmap/task_cpython_api.txt Thu Dec 27 19:35:25 2007 @@ -1,9 +1,20 @@ Providing the CPython C-API =========================== -Separate compilation would be interesting ground work for a try to compile -cpython modules with a pypy-supplied Python.h file and link them to pypy. This -would require writing a lot of new functions (for the CPython API), though. -Also it is not clear how large the expected speed impact would be (because of -e.g. pinning). -We need to think more about this before we can be more precise. +status: + + - Not started + +todo: + + - Separate compilation would be interesting ground work for a try to + compile cpython modules with a pypy-supplied Python.h file and link + them to pypy. This would require writing a lot of new functions + (for the CPython API), though. Also it is not clear how large the + expected speed impact would be (because of e.g. pinning). + + - We need to think more about this before we can be more precise. + +expected outcome: + +dependencies: \ No newline at end of file Modified: pypy/extradoc/planning/roadmap/task_ctypes.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_ctypes.txt (original) +++ pypy/extradoc/planning/roadmap/task_ctypes.txt Thu Dec 27 19:35:25 2007 @@ -1,10 +1,17 @@ Support app-level ctypes ======================== -As a first approach for providing C-level bindings, we should implement -the ctypes module on top of already existing _ffi wrapper. This would -allow to access quite a bit of libraries which use ctypes already -(like SDL wrapper). +status: -This is probably a medium-sized task. The details of the ctypes semantics -are messy and can only be found by poking around the implementation. +todo: + - As a first approach for providing C-level bindings, we should implement + the ctypes module on top of already existing _ffi wrapper. This would + allow to access quite a bit of libraries which use ctypes already + (like SDL wrapper). + + - This is probably a medium-sized task. The details of the ctypes semantics + are messy and can only be found by poking around the implementation. + +expected outcome: + +dependencies: \ No newline at end of file Modified: pypy/extradoc/planning/roadmap/task_gui_support.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_gui_support.txt (original) +++ pypy/extradoc/planning/roadmap/task_gui_support.txt Thu Dec 27 19:35:25 2007 @@ -1,11 +1,19 @@ Support a GUI ============= -One of the C extension modules that we need to have in order to get -traction is support for one of the multi-platform GUI toolkits. The -choice is probably between gtk (in use by olpc and others), Qt (written in c++, -using SIP to get a cpython package) and wxWindows, with Tkinter as a -possible fourth. Someone with more understanding of the Python -community than us should make the pick. Which one would bring more -users to PyPy? Would any one of them prompt proponents of other -GUI toolkits to make their own ports? +status: + +todo: + + - One of the C extension modules that we need to have in order to get + traction is support for one of the multi-platform GUI toolkits. The + choice is probably between gtk (in use by olpc and others), Qt + (written in c++, using SIP to get a cpython package) and wxWindows, + with Tkinter as a possible fourth. Someone with more understanding + of the Python community than us should make the pick. Which one + would bring more users to PyPy? Would any one of them prompt + proponents of other GUI toolkits to make their own ports? + +expected outcome: + +dependencies: Modified: pypy/extradoc/planning/roadmap/task_manual_optimizations.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_manual_optimizations.txt (original) +++ pypy/extradoc/planning/roadmap/task_manual_optimizations.txt Thu Dec 27 19:35:25 2007 @@ -1,10 +1,22 @@ Hand optimizations ================== -There are still some hand optimizations of the Python interpreter that -are well worth doing, since they can be expected to yield significant -performance improvements. For instance, some of our mircobenchmarks -show particularly poor performance in some areas. +status: -The amount of effort required to get what kind of speed-up for -what kind of application remains unknown at this point in time. + - Improvements to the interpreter is an ongoing task. Some microbenchmarks + show particularly bad performance in specific areas. + +todo: + + - There are still some hand optimizations of the Python interpreter + that are well worth doing, since they can be expected to yield + significant performance improvements. + +expected outcome: + +- The amount of effort required to get what kind of speed-up for + what kind of application remains unknown at this point in time. + +dependencies: + +- None \ No newline at end of file Modified: pypy/extradoc/planning/roadmap/task_modules_3rdparty.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_modules_3rdparty.txt (original) +++ pypy/extradoc/planning/roadmap/task_modules_3rdparty.txt Thu Dec 27 19:35:25 2007 @@ -1,9 +1,18 @@ Third party module support ========================== -There are some third party modules that would bring early adopters to -PyPy. They would also serve as code coverage cases. Our time estimate -is a guess on how much time we would need to spend to get enough -traction in the community. Exactly what modules to support needs to -be determined after some of the infrastructure in other tasks is -in place. +status: + +todo: + + - There are some third party modules that would bring early adopters + to PyPy. They would also serve as code coverage cases. Our time + estimate is a guess on how much time we would need to spend to get + enough traction in the community. + +expected outcome: + +dependencies: + + - We should have our infrastucture for extension modules in place before + deciding what modules make most sense to port. \ No newline at end of file Modified: pypy/extradoc/planning/roadmap/task_modules_rffi.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_modules_rffi.txt (original) +++ pypy/extradoc/planning/roadmap/task_modules_rffi.txt Thu Dec 27 19:35:25 2007 @@ -1,28 +1,61 @@ Port more modules to rffi style =============================== -Some standard modules written in C have already been ported to the -rffi style, but more remain. -XXX Flesh out with some more details. Examples for example. +status: +todo: + + - Some standard modules written in C have already been ported to the + rffi style, but more remain. + + - XXX Flesh out with some more details. Examples for example. + +expected outcome: + +dependencies: Move modules from app to interpreter level ========================================== -Some modules are implemented in Python at application level. For -preformance reasons, many of them should eventually be implemented -at interpreter level. - -These modules are: array, binascii, cPickle, cStringIO, cmath, -collections, datetime, functional, functools(incomplete), imp, -itertools. +status: + +todo: + + - Some modules are implemented in Python at application level. For + preformance reasons, many of them should eventually be implemented + at interpreter level. + + - These modules are: + + - array + - binascii + - cPickle + - cStringIO + - cmath + - collections + - datetime + - functional + - functools(incomplete) + - imp + - itertools + +expected outcome: + +dependencies: + + - None Write new modules ================= -There are still some modules in the standard library that need writing -before PyPy is a reasonable replacement for CPython. Exactly which -ones needs some investigation. -XXX Flesh out with some more details. Examples for example. +status: + +todo: + + - There are still some modules in the standard library that need writing + before PyPy is a reasonable replacement for CPython. Exactly which + ones needs some investigation. + + - XXX Flesh out with some more details. Examples for example. -XXX these tasks should be merged, I think +XXX The three previous tasks should be merged, I think Modified: pypy/extradoc/planning/roadmap/task_multi_platform.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_multi_platform.txt (original) +++ pypy/extradoc/planning/roadmap/task_multi_platform.txt Thu Dec 27 19:35:25 2007 @@ -1,12 +1,39 @@ Multi-platform support ====================== -In order to be a viable alternative to CPython, PyPy needs to be -tested and run cleanly on more platforms than it currently does -(Linux/IA32, OS/X, and Windows). Amd64 with Linux is probably -the most urgent one, but there are probably others that have -some problems. +status: + + - We currently have PyPy with Posix support running on the following + platforms: + - Linux ia32 + - Mac OS/X ia32 + - Mac OS/X powerpc + - Windows ia32 + +todo: + + - In order to be a viable alternative to CPython, PyPy needs to be + tested and run cleanly on more platforms than it currently does. + Amd64 with Linux is probably the most urgent one, but there are + probably others that have some problems. It probably makes sense to put this item off until we are closer to having a production ready system for the i386. We also have a possible volunteer effort under way. It remains to be seen if it takes off. + +expected outcome: + + - PyPy passes the suite of tests to the same level on Linux/amd64 + as on Linux/ia32. + +dependencies: + + - None + + - However, it probably makes sense to put this item off until we are + closer to having a production ready system for the ia32. + + - Somebody showed up o the PyPy IRC channel with an interest in + putting together a group of people doing the port to amd64. + It remains to be seen if this group takes off. If so, we may have + a port earlier than expected. \ No newline at end of file Modified: pypy/extradoc/planning/roadmap/task_optimize_jit.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_optimize_jit.txt (original) +++ pypy/extradoc/planning/roadmap/task_optimize_jit.txt Thu Dec 27 19:35:25 2007 @@ -1,6 +1,29 @@ Add optimizations to the JIT ============================ -A base set of optimizations include common operations on floats and -strings as well as general call optimization. Fixing bugs in the -current JIT implementation is also needed. +status: + + - Optimizations for integer operations have been implemented. Benchmarks + show speeds at the level of unoptimized C code, translating to a + factor 60 faster than CPython. + +todo: + + - A base set of optimizations include common operations on floats and + strings as well as general call optimization. Fixing bugs in the + current JIT implementation is also needed. + +expected outcome: + + - Call optimizations should improve the excecution speed of general + Python programs to be faster than under CPython. + + - String optimizations should speed up string intensive programs + like web frameworks by at least an order of magnitude over CPython. + + - Float operations should run at a speed comparable to unoptimized C + code; about 2 orders of magnitude faster than CPython. + +dependencies: + + - The JIT refactoring should be done before in order not to do extra work. \ No newline at end of file Modified: pypy/extradoc/planning/roadmap/task_parser_cleanup.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_parser_cleanup.txt (original) +++ pypy/extradoc/planning/roadmap/task_parser_cleanup.txt Thu Dec 27 19:35:25 2007 @@ -1,10 +1,23 @@ Parser cleanup ============== -The parser is in a rather sorry state. Making it support various -versions of Python is currently a rather nasty job. It needs replacing -with a more robust design. The estimates for this are probably too low, -since they were based on the idea of improving the existing parser. -Subsequent attempts to improve the parser have prompted the idea of -a rewrite. In the parser work, there is also time needed to upgrade -the interpreter to handle all of Python 2.5. +status: + + - The parser is in a rather sorry state. Making it support various + versions of Python is currently a rather nasty job. It needs + replacing with a more robust design. The estimates for this are + probably too low, since they were based on the idea of improving + the existing parser. Subsequent attempts to improve the parser + have prompted the idea of a rewrite. + +todo: + + - Select a suitable model/toolkit for the implementation. + + - Build the parser. + +expected outcome: + +dependencies: + + - None Modified: pypy/extradoc/planning/roadmap/task_refactor_jit.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_refactor_jit.txt (original) +++ pypy/extradoc/planning/roadmap/task_refactor_jit.txt Thu Dec 27 19:35:25 2007 @@ -1,8 +1,18 @@ Refactor the JIT ================ -The JIT is the result of programming under severe time pressure. This -means that the existing JIT framework needs a fair bit of cleanup and -restructuring. Armin also has some ideas for a refactoring based on an -interpreter, rather than annotated graphs. -XXX maybe phrase this a bit more clearly. +status: + + - The JIT is the result of programming under severe time pressure. This + means that the existing JIT framework needs a fair bit of cleanup and + restructuring. Armin also has some ideas for a refactoring based on an + interpreter, rather than annotated graphs. + + - XXX maybe phrase this a bit more clearly. + +todo: + +expected outcome: + +dependencies: + - None \ No newline at end of file Modified: pypy/extradoc/planning/roadmap/task_separate_compilation.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_separate_compilation.txt (original) +++ pypy/extradoc/planning/roadmap/task_separate_compilation.txt Thu Dec 27 19:35:25 2007 @@ -1,7 +1,20 @@ Separate compilation ==================== -To make porting third party modules reasonable, it should not be necessary -to recompile all of PyPy every time you want to integrate a new extension -module. This requires supporting separate compilation in our translation -tool chain. +status: + + - Currently PyPy has to be compiled as a monolithic system with all + extension modules included at compile time. + +todo: + + - To make porting third party modules reasonable, it should not be + necessary to recompile all of PyPy every time you want to integrate + a new extension module. This requires supporting separate + compilation in our translation tool chain. + +expected outcome: + +dependencies: + + - None \ No newline at end of file Modified: pypy/extradoc/planning/roadmap/task_wrapper_generator.txt ============================================================================== --- pypy/extradoc/planning/roadmap/task_wrapper_generator.txt (original) +++ pypy/extradoc/planning/roadmap/task_wrapper_generator.txt Thu Dec 27 19:35:25 2007 @@ -1,6 +1,16 @@ Support a tool like Swig or Boost ================================= -To make it simpler to integrate C libraries with PyPy, we should support -at least one of the wrapper generators that are already in use in the -Python community. +status: + + - Not started + +todo: + + - To make it simpler to integrate C libraries with PyPy, we should + support at least one of the wrapper generators that are already in + use in the Python community. + +expected outcome: + +dependencies: \ No newline at end of file From arigo at codespeak.net Thu Dec 27 20:12:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 27 Dec 2007 20:12:09 +0100 (CET) Subject: [pypy-svn] r50146 - pypy/branch/llvmgcroot/pypy/translator/c Message-ID: <20071227191209.3E40C168472@codespeak.net> Author: arigo Date: Thu Dec 27 20:12:08 2007 New Revision: 50146 Modified: pypy/branch/llvmgcroot/pypy/translator/c/genc.py Log: Tweak the Makefile generation so that it still works in the non-llvmgcroot case. Modified: pypy/branch/llvmgcroot/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/c/genc.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/c/genc.py Thu Dec 27 20:12:08 2007 @@ -319,7 +319,7 @@ if self.config.translation.linkerflags: compiler.link_extra.append(self.config.translation.linkerflags) cfiles = [] - sfiles = [] + ofiles = [] for fn in compiler.cfilenames: fn = py.path.local(fn) if fn.dirpath() == targetdir: @@ -328,7 +328,10 @@ assert fn.dirpath().dirpath() == udir name = '../' + fn.relto(udir) cfiles.append(name) - sfiles.append(name[:-2] + '.s') + if self.config.translation.llvmgcroot: + ofiles.append(name[:-2] + '.s') + else: + ofiles.append(name[:-2] + '.o') if self.config.translation.cc: cc = self.config.translation.cc @@ -346,10 +349,15 @@ print >> f write_list(cfiles, 'SOURCES =') print >> f - write_list(sfiles, 'ASMFILES =') + if self.config.translation.llvmgcroot: + write_list(ofiles, 'ASMFILES =') + print >> f, 'OBJECTS = $(ASMFILES) gcmaptable.s' + else: + write_list(ofiles, 'OBJECTS =') print >> f - print >> f, 'TRACKGCROOT = "%s"' % (os.path.join(autopath.this_dir, - 'trackgcroot.py'),) + if self.config.translation.llvmgcroot: + print >> f, 'TRACKGCROOT="%s"' % (os.path.join(autopath.this_dir, + 'trackgcroot.py'),) print >> f args = ['-l'+libname for libname in self.eci.libraries] print >> f, 'LIBS =', ' '.join(args) @@ -780,8 +788,11 @@ MAKEFILE = ''' -$(TARGET): $(ASMFILES) gcmaptable.s -\t$(CC) $(LDFLAGS) $(TFLAGS) -o $@ $(ASMFILES) gcmaptable.s $(LIBDIRS) $(LIBS) +$(TARGET): $(OBJECTS) +\t$(CC) $(LDFLAGS) $(TFLAGS) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) + +%.o: %.c +\t$(CC) $(CFLAGS) -o $@ -c $< $(INCLUDEDIRS) %.s: %.c \t$(CC) $(CFLAGS) -o $@ -S $< $(INCLUDEDIRS) @@ -790,7 +801,7 @@ \t$(TRACKGCROOT) $(ASMFILES) > $@ || (rm -f $@ && exit 1) clean: -\trm -f $(ASMFILES) gcmaptable.s $(TARGET) +\trm -f $(OBJECTS) $(TARGET) debug: \t$(MAKE) CFLAGS="-g -DRPY_ASSERT" @@ -813,6 +824,6 @@ profopt: \t$(MAKE) CFLAGS="-fprofile-generate $(CFLAGS)" LDFLAGS="-fprofile-generate $(LDFLAGS)" \t./$(TARGET) $(PROFOPT) -\trm -f $(ASMFILES) gcmaptable.s $(TARGET) +\trm -f $(OBJECTS) $(TARGET) \t$(MAKE) CFLAGS="-fprofile-use $(CFLAGS)" LDFLAGS="-fprofile-use $(LDFLAGS)" ''' From arigo at codespeak.net Fri Dec 28 13:47:01 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 Dec 2007 13:47:01 +0100 (CET) Subject: [pypy-svn] r50161 - pypy/branch/llvmgcroot/pypy/translator/c Message-ID: <20071228124701.B76261684CE@codespeak.net> Author: arigo Date: Fri Dec 28 13:46:59 2007 New Revision: 50161 Modified: pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py Log: Check the name at the end of the function. Modified: pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py ============================================================================== --- pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py (original) +++ pypy/branch/llvmgcroot/pypy/translator/c/trackgcroot.py Fri Dec 28 13:46:59 2007 @@ -3,7 +3,7 @@ import re, sys, os r_functionstart = re.compile(r"\t.type\s+(\w+),\s*[@]function\s*$") -r_functionend = re.compile(r"\t.size\s+\w+,\s*[.]-\w+\s*$") +r_functionend = re.compile(r"\t.size\s+(\w+),\s*[.]-(\w+)\s*$") r_label = re.compile(r"([.]?\w+)[:]\s*$") r_globl = re.compile(r"\t[.]globl\t(\w+)\s*$") r_insn = re.compile(r"\t([a-z]\w*)\s") @@ -106,6 +106,9 @@ def __init__(self, lines): match = r_functionstart.match(lines[0]) self.funcname = match.group(1) + match = r_functionend.match(lines[-1]) + assert self.funcname == match.group(1) + assert self.funcname == match.group(2) self.lines = lines self.inconsistent_state = {} self.can_use_frame_pointer = False # unless changed by caller From arigo at codespeak.net Fri Dec 28 19:33:13 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 Dec 2007 19:33:13 +0100 (CET) Subject: [pypy-svn] r50163 - pypy/dist/pypy/rlib Message-ID: <20071228183313.2B6551684C5@codespeak.net> Author: arigo Date: Fri Dec 28 19:33:11 2007 New Revision: 50163 Modified: pypy/dist/pypy/rlib/rstack.py Log: Tweak - avoids a wrapper level, less work for GCC's inliner probably. Modified: pypy/dist/pypy/rlib/rstack.py ============================================================================== --- pypy/dist/pypy/rlib/rstack.py (original) +++ pypy/dist/pypy/rlib/rstack.py Fri Dec 28 19:33:11 2007 @@ -37,6 +37,7 @@ stack_too_big = rffi.llexternal('LL_stack_too_big', [], rffi.INT, compilation_info=compilation_info, + _nowrapper=True, _callable=lambda: 0) def stack_check(): From arigo at codespeak.net Fri Dec 28 19:35:44 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 28 Dec 2007 19:35:44 +0100 (CET) Subject: [pypy-svn] r50164 - in pypy/dist/pypy: rlib rpython Message-ID: <20071228183544.71BA41684C5@codespeak.net> Author: arigo Date: Fri Dec 28 19:35:44 2007 New Revision: 50164 Modified: pypy/dist/pypy/rlib/rarithmetic.py pypy/dist/pypy/rpython/rfloat.py Log: Ordering issue: the lazy registering hacks for SomeSingleFloat() didn't quite work, because SomeSingleFloat() can also be found via the annotation_to_llmap table in annotation/model.py and then the rtyper_makerepr() method is missing. Modified: pypy/dist/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/dist/pypy/rlib/rarithmetic.py (original) +++ pypy/dist/pypy/rlib/rarithmetic.py Fri Dec 28 19:35:44 2007 @@ -478,13 +478,15 @@ _type_ = r_singlefloat def compute_annotation(self): - return _somesinglefloat() + from pypy.annotation import model as annmodel + return annmodel.SomeSingleFloat() class For_r_singlefloat_type_Entry(extregistry.ExtRegistryEntry): _about_ = r_singlefloat def compute_result_annotation(self, *args_s, **kwds_s): - return _somesinglefloat() + from pypy.annotation import model as annmodel + return annmodel.SomeSingleFloat() def specialize_call(self, hop): from pypy.rpython.lltypesystem import lltype @@ -493,31 +495,3 @@ # we use cast_primitive to go between Float and SingleFloat. return hop.genop('cast_primitive', [v], resulttype = lltype.SingleFloat) - -def _somesinglefloat(): - """Returns SomeSingleFloat(), but also lazily register the rtyping support - for SomeSingleFloat. - """ - from pypy.annotation import model as annmodel - - if 'rtyper_makerepr' not in annmodel.SomeSingleFloat.__dict__: - from pypy.rpython.lltypesystem import lltype - from pypy.rpython.rmodel import Repr - - class SingleFloatRepr(Repr): - lowleveltype = lltype.SingleFloat - - def rtype_float(self, hop): - v, = hop.inputargs(lltype.SingleFloat) - hop.exception_cannot_occur() - # we use cast_primitive to go between Float and SingleFloat. - return hop.genop('cast_primitive', [v], - resulttype = lltype.Float) - - class __extend__(annmodel.SomeSingleFloat): - def rtyper_makerepr(self, rtyper): - return SingleFloatRepr() - def rtyper_makekey(self): - return self.__class__, - - return annmodel.SomeSingleFloat() Modified: pypy/dist/pypy/rpython/rfloat.py ============================================================================== --- pypy/dist/pypy/rpython/rfloat.py (original) +++ pypy/dist/pypy/rpython/rfloat.py Fri Dec 28 19:35:44 2007 @@ -216,3 +216,25 @@ resulttype=pyobj_repr, _callable=lambda x: pyobjectptr(x)) return NotImplemented + +# ____________________________________________________________ +# Support for r_singlefloat from pypy.rlib.rarithmetic + +from pypy.rpython.lltypesystem import lltype +from pypy.rpython.rmodel import Repr + +class __extend__(annmodel.SomeSingleFloat): + def rtyper_makerepr(self, rtyper): + return SingleFloatRepr() + def rtyper_makekey(self): + return self.__class__, + +class SingleFloatRepr(Repr): + lowleveltype = lltype.SingleFloat + + def rtype_float(self, hop): + v, = hop.inputargs(lltype.SingleFloat) + hop.exception_cannot_occur() + # we use cast_primitive to go between Float and SingleFloat. + return hop.genop('cast_primitive', [v], + resulttype = lltype.Float) From arigo at codespeak.net Sat Dec 29 18:34:08 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 29 Dec 2007 18:34:08 +0100 (CET) Subject: [pypy-svn] r50175 - in pypy/dist/pypy: interpreter/test module/__builtin__ Message-ID: <20071229173408.93FAA168538@codespeak.net> Author: arigo Date: Sat Dec 29 18:34:07 2007 New Revision: 50175 Modified: pypy/dist/pypy/interpreter/test/test_compiler.py pypy/dist/pypy/module/__builtin__/importing.py Log: issue315 testing Test and fix: the import logic should not inherit the flags from the caller while compile()ing the imported module. Modified: pypy/dist/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/dist/pypy/interpreter/test/test_compiler.py (original) +++ pypy/dist/pypy/interpreter/test/test_compiler.py Sat Dec 29 18:34:07 2007 @@ -548,6 +548,40 @@ w_res = space.getitem(w_d, space.wrap('res')) assert space.eq_w(w_res, space.wrap("var")) + def test_dont_inherit_flag(self): + space = self.space + s1 = str(py.code.Source(""" + from __future__ import division + exec compile('x = 1/2', '?', 'exec', 0, 1) + """)) + w_result = space.appexec([space.wrap(s1)], """(s1): + exec s1 + return x + """) + assert space.float_w(w_result) == 0 + + def test_dont_inherit_across_import(self): + from pypy.tool.udir import udir + udir.join('test_dont_inherit_across_import.py').write('x = 1/2\n') + space = self.space + s1 = str(py.code.Source(""" + from __future__ import division + from test_dont_inherit_across_import import x + """)) + w_result = space.appexec([space.wrap(str(udir)), space.wrap(s1)], + """(udir, s1): + import sys + copy = sys.path[:] + sys.path.insert(0, udir) + try: + exec s1 + finally: + sys.path[:] = copy + return x + """) + assert space.float_w(w_result) == 0 + + class TestECCompiler(BaseTestCompiler): def setup_method(self, method): self.compiler = self.space.getexecutioncontext().compiler Modified: pypy/dist/pypy/module/__builtin__/importing.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/importing.py (original) +++ pypy/dist/pypy/module/__builtin__/importing.py Sat Dec 29 18:34:07 2007 @@ -431,12 +431,8 @@ def parse_source_module(space, pathname, source): """ Parse a source file and return the corresponding code object """ - w = space.wrap - w_source = w(source) - w_mode = w("exec") - w_pathname = w(pathname) - w_code = space.builtin.call('compile', w_source, w_pathname, w_mode) - pycode = space.interp_w(Code, w_code) + ec = space.getexecutioncontext() + pycode = ec.compiler.compile(source, pathname, 'exec', 0) return pycode def load_source_module(space, w_modulename, w_mod, pathname, source, From arigo at codespeak.net Sat Dec 29 20:04:53 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 29 Dec 2007 20:04:53 +0100 (CET) Subject: [pypy-svn] r50177 - in pypy/dist/pypy/interpreter/pyparser/test: . output Message-ID: <20071229190453.AB8D116854E@codespeak.net> Author: arigo Date: Sat Dec 29 20:04:48 2007 New Revision: 50177 Added: pypy/dist/pypy/interpreter/pyparser/test/output/ pypy/dist/pypy/interpreter/pyparser/test/output/__init__.py (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/regen.py (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_1.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_2.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_3.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_4.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_classes.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_comment.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_decorators.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_docstring.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_encoding_declaration.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_encoding_declaration2.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_encoding_declaration3.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_exceptions.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_function_calls.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_generator.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_import_statements.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_list_comps.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_listlinenos.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_multiline.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_numbers.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_only_one_comment.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_redirected_prints.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_samples.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_several_statements.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_simple_assignment.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_simple_class.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_simple_for_loop.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_simple_function.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_simple_in_expr.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_slice.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_while.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_whilelineno.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/output/snippet_whitespaces.txt (contents, props changed) pypy/dist/pypy/interpreter/pyparser/test/test_snippet_out.py (contents, props changed) Modified: pypy/dist/pypy/interpreter/pyparser/test/test_astcompiler.py Log: A new style for some of the ast snippet tests: compare the compiler's output with a previously-saved output, instead of with the "stable" (ha!) compiler output. This should allow for easier experimentation: although it's not ideal, we can at least call regen.py after changes in the compiler and see with 'svn diff' if the produced bytecode still looks reasonable... Added: pypy/dist/pypy/interpreter/pyparser/test/output/__init__.py ============================================================================== Added: pypy/dist/pypy/interpreter/pyparser/test/output/regen.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/regen.py Sat Dec 29 20:04:48 2007 @@ -0,0 +1,20 @@ +""" +Run this script to regenerate all the output/snippet_*.txt. +Then use 'svn diff' to review all the changes! +""" + +from pypy.interpreter.pyparser.test import test_snippet_out + +def test_main(): + import pypy.conftest + space = pypy.conftest.gettestobjspace('std') + for snippet_name in test_snippet_out.SNIPPETS: + print snippet_name + output = test_snippet_out.generate_output(snippet_name, space) + path = test_snippet_out.get_output_path(snippet_name) + f = open(path, 'w') + f.write(output) + f.close() + +if __name__ == '__main__': + test_main() Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_1.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_1.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,17 @@ + +x = y + 1 + + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 2 0 LOAD_NAME 0 (y) + 3 LOAD_CONST 1 (1) + 6 BINARY_ADD + 7 STORE_NAME 1 (x) + 10 LOAD_CONST 0 (None) + 13 RETURN_VALUE +co_consts: +[0] None +[1] 1 Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_2.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_2.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,88 @@ + + +L = [] +print L[0:10] + +def f(): + print 1 + # commentaire foireux +x = 1 +s = "asd" + +class A: + def f(): + pass + + + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 3 0 BUILD_LIST 0 + 3 STORE_NAME 0 (L) + + 4 6 LOAD_NAME 0 (L) + 9 LOAD_CONST 1 (0) + 12 LOAD_CONST 2 (10) + 15 SLICE+3 + 16 PRINT_ITEM + 17 PRINT_NEWLINE + + 6 18 LOAD_CONST 3 () + 21 MAKE_FUNCTION 0 + 24 STORE_NAME 1 (f) + + 9 27 LOAD_CONST 4 (1) + 30 STORE_NAME 2 (x) + + 10 33 LOAD_CONST 5 ('asd') + 36 STORE_NAME 3 (s) + + 12 39 LOAD_CONST 6 ('A') + 42 BUILD_TUPLE 0 + 45 LOAD_CONST 7 () + 48 MAKE_FUNCTION 0 + 51 CALL_FUNCTION 0 + 54 BUILD_CLASS + 55 STORE_NAME 4 (A) + 58 LOAD_CONST 0 (None) + 61 RETURN_VALUE +co_consts: +[0] None +[1] 0 +[2] 10 +[3] co_filename = '' + co_varnames = () + co_flags = 67 + 7 0 LOAD_CONST 1 (1) + 3 PRINT_ITEM + 4 PRINT_NEWLINE + 5 LOAD_CONST 0 (None) + 8 RETURN_VALUE + co_consts: + [0] None + [1] 1 +[4] 1 +[5] 'asd' +[6] 'A' +[7] co_filename = '' + co_varnames = () + co_flags = 66 + 12 0 LOAD_GLOBAL 0 (__name__) + 3 STORE_NAME 1 (__module__) + + 13 6 LOAD_CONST 1 () + 9 MAKE_FUNCTION 0 + 12 STORE_NAME 2 (f) + 15 LOAD_LOCALS + 16 RETURN_VALUE + co_consts: + [0] None + [1] co_filename = '' + co_varnames = () + co_flags = 67 + 13 0 LOAD_CONST 0 (None) + 3 RETURN_VALUE + co_consts: + [0] None Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_3.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_3.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,15 @@ +a[1:] + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 LOAD_NAME 0 (a) + 3 LOAD_CONST 1 (1) + 6 SLICE+1 + 7 POP_TOP + 8 LOAD_CONST 0 (None) + 11 RETURN_VALUE +co_consts: +[0] None +[1] 1 Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_4.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_4.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,14 @@ +a is not None + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 LOAD_NAME 0 (a) + 3 LOAD_CONST 0 (None) + 6 COMPARE_OP 9 (is not) + 9 POP_TOP + 10 LOAD_CONST 0 (None) + 13 RETURN_VALUE +co_consts: +[0] None Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_classes.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_classes.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,100 @@ +class A: + def with_white_spaces_before(self): + pass + + def another_method(self, foo): + bar = foo + + +class B(object, A): + def foo(self, bar): + a = 2 + return "spam" + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 LOAD_CONST 1 ('A') + 3 BUILD_TUPLE 0 + 6 LOAD_CONST 2 () + 9 MAKE_FUNCTION 0 + 12 CALL_FUNCTION 0 + 15 BUILD_CLASS + 16 STORE_NAME 0 (A) + + 9 19 LOAD_CONST 3 ('B') + 22 LOAD_NAME 1 (object) + 25 LOAD_NAME 0 (A) + 28 BUILD_TUPLE 2 + 31 LOAD_CONST 4 () + 34 MAKE_FUNCTION 0 + 37 CALL_FUNCTION 0 + 40 BUILD_CLASS + 41 STORE_NAME 2 (B) + 44 LOAD_CONST 0 (None) + 47 RETURN_VALUE +co_consts: +[0] None +[1] 'A' +[2] co_filename = '' + co_varnames = () + co_flags = 66 + 1 0 LOAD_GLOBAL 0 (__name__) + 3 STORE_NAME 1 (__module__) + + 2 6 LOAD_CONST 1 () + 9 MAKE_FUNCTION 0 + 12 STORE_NAME 2 (with_white_spaces_before) + + 5 15 LOAD_CONST 2 () + 18 MAKE_FUNCTION 0 + 21 STORE_NAME 3 (another_method) + 24 LOAD_LOCALS + 25 RETURN_VALUE + co_consts: + [0] None + [1] co_filename = '' + co_varnames = ('self',) + co_flags = 67 + 2 0 LOAD_CONST 0 (None) + 3 RETURN_VALUE + co_consts: + [0] None + [2] co_filename = '' + co_varnames = ('self', 'foo', 'bar') + co_flags = 67 + 6 0 LOAD_FAST 1 (foo) + 3 STORE_FAST 2 (bar) + 6 LOAD_CONST 0 (None) + 9 RETURN_VALUE + co_consts: + [0] None +[3] 'B' +[4] co_filename = '' + co_varnames = () + co_flags = 66 + 9 0 LOAD_GLOBAL 0 (__name__) + 3 STORE_NAME 1 (__module__) + + 10 6 LOAD_CONST 1 () + 9 MAKE_FUNCTION 0 + 12 STORE_NAME 2 (foo) + 15 LOAD_LOCALS + 16 RETURN_VALUE + co_consts: + [0] None + [1] co_filename = '' + co_varnames = ('self', 'bar', 'a') + co_flags = 67 + 11 0 LOAD_CONST 1 (2) + 3 STORE_FAST 2 (a) + + 12 6 LOAD_CONST 2 ('spam') + 9 RETURN_VALUE + 10 LOAD_CONST 0 (None) + 13 RETURN_VALUE + co_consts: + [0] None + [1] 2 + [2] 'spam' Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_comment.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_comment.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,26 @@ +x = 0x1L # comment +a = 1 # yo + # hello +# world +a = 2 +# end + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 LOAD_CONST 1 (1L) + 3 STORE_NAME 0 (x) + + 2 6 LOAD_CONST 2 (1) + 9 STORE_NAME 1 (a) + + 5 12 LOAD_CONST 3 (2) + 15 STORE_NAME 1 (a) + 18 LOAD_CONST 0 (None) + 21 RETURN_VALUE +co_consts: +[0] None +[1] 1L +[2] 1 +[3] 2 Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_decorators.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_decorators.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,104 @@ +# Function(Decorators([Name('foo')]), 'f', ['a', 'b'], [], 0, None, Stmt([Pass()])) + at foo +def f(a, b): + pass + + at accepts(int, (int,float)) + at returns((int,float)) +def func0(arg1, arg2): + return arg1 * arg2 + + +## Stmt([Function(Decorators([CallFunc(Getattr(Getattr(Name('mod1'), 'mod2'), 'accepts'), [Name('int'), Tuple([Name('int'), Name('float')])], None, None), +## CallFunc(Getattr(Getattr(Name('mod1'), 'mod2'), 'returns'), [Tuple([Name('int'), Name('float')])], None, None)]), +## 'func', ['arg1', 'arg2'], [], 0, None, Stmt([Return(Mul((Name('arg1'), Name('arg2'))))]))]) + at mod1.mod2.accepts(int, (int,float)) + at mod1.mod2.returns((int,float)) +def func(arg1, arg2): + return arg1 * arg2 + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 2 0 LOAD_NAME 0 (foo) + + 3 3 LOAD_CONST 1 () + 6 MAKE_FUNCTION 0 + 9 CALL_FUNCTION 1 + 12 STORE_NAME 1 (f) + + 6 15 LOAD_NAME 2 (accepts) + 18 LOAD_NAME 3 (int) + 21 LOAD_NAME 3 (int) + 24 LOAD_NAME 4 (float) + 27 BUILD_TUPLE 2 + 30 CALL_FUNCTION 2 + + 7 33 LOAD_NAME 5 (returns) + 36 LOAD_NAME 3 (int) + 39 LOAD_NAME 4 (float) + 42 BUILD_TUPLE 2 + 45 CALL_FUNCTION 1 + + 8 48 LOAD_CONST 2 () + 51 MAKE_FUNCTION 0 + 54 CALL_FUNCTION 1 + 57 CALL_FUNCTION 1 + 60 STORE_NAME 6 (func0) + + 15 63 LOAD_NAME 7 (mod1) + 66 LOAD_ATTR 8 (mod2) + 69 LOAD_ATTR 2 (accepts) + 72 LOAD_NAME 3 (int) + 75 LOAD_NAME 3 (int) + 78 LOAD_NAME 4 (float) + 81 BUILD_TUPLE 2 + 84 CALL_FUNCTION 2 + + 16 87 LOAD_NAME 7 (mod1) + 90 LOAD_ATTR 8 (mod2) + 93 LOAD_ATTR 5 (returns) + 96 LOAD_NAME 3 (int) + 99 LOAD_NAME 4 (float) + 102 BUILD_TUPLE 2 + 105 CALL_FUNCTION 1 + + 17 108 LOAD_CONST 3 () + 111 MAKE_FUNCTION 0 + 114 CALL_FUNCTION 1 + 117 CALL_FUNCTION 1 + 120 STORE_NAME 9 (func) + 123 LOAD_CONST 0 (None) + 126 RETURN_VALUE +co_consts: +[0] None +[1] co_filename = '' + co_varnames = ('a', 'b') + co_flags = 67 + 3 0 LOAD_CONST 0 (None) + 3 RETURN_VALUE + co_consts: + [0] None +[2] co_filename = '' + co_varnames = ('arg1', 'arg2') + co_flags = 67 + 9 0 LOAD_FAST 0 (arg1) + 3 LOAD_FAST 1 (arg2) + 6 BINARY_MULTIPLY + 7 RETURN_VALUE + 8 LOAD_CONST 0 (None) + 11 RETURN_VALUE + co_consts: + [0] None +[3] co_filename = '' + co_varnames = ('arg1', 'arg2') + co_flags = 67 + 18 0 LOAD_FAST 0 (arg1) + 3 LOAD_FAST 1 (arg2) + 6 BINARY_MULTIPLY + 7 RETURN_VALUE + 8 LOAD_CONST 0 (None) + 11 RETURN_VALUE + co_consts: + [0] None Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_docstring.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_docstring.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,81 @@ +"""module docstring""" + + +"""hello + +""" + +class A: + """class doctring + + on several lines + """ + +def foo(self): + """function docstring""" + +def boo(x): + """Docstring""";print 1 + +"""world""" + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 LOAD_CONST 1 ('module docstring') + 3 STORE_NAME 0 (__doc__) + + 8 6 LOAD_CONST 2 ('A') + 9 BUILD_TUPLE 0 + 12 LOAD_CONST 3 () + 15 MAKE_FUNCTION 0 + 18 CALL_FUNCTION 0 + 21 BUILD_CLASS + 22 STORE_NAME 1 (A) + + 14 25 LOAD_CONST 4 () + 28 MAKE_FUNCTION 0 + 31 STORE_NAME 2 (foo) + + 17 34 LOAD_CONST 5 () + 37 MAKE_FUNCTION 0 + 40 STORE_NAME 3 (boo) + 43 LOAD_CONST 6 (None) + 46 RETURN_VALUE +co_consts: +[0] 'Docstring' +[1] 'module docstring' +[2] 'A' +[3] co_filename = '' + co_varnames = () + co_flags = 66 + 8 0 LOAD_GLOBAL 0 (__name__) + 3 STORE_NAME 1 (__module__) + 6 LOAD_CONST 0 ('class doctring\n\n on several lines\n ') + 9 STORE_NAME 2 (__doc__) + 12 LOAD_LOCALS + 13 RETURN_VALUE + co_consts: + [0] 'class doctring\n\n on several lines\n ' +[4] co_filename = '' + co_varnames = ('self',) + co_flags = 67 + 14 0 LOAD_CONST 1 (None) + 3 RETURN_VALUE + co_consts: + [0] 'function docstring' + [1] None +[5] co_filename = '' + co_varnames = ('x',) + co_flags = 67 + 18 0 LOAD_CONST 1 (1) + 3 PRINT_ITEM + 4 PRINT_NEWLINE + 5 LOAD_CONST 2 (None) + 8 RETURN_VALUE + co_consts: + [0] 'Docstring' + [1] 1 + [2] None +[6] None Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_encoding_declaration.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_encoding_declaration.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,14 @@ +# -*- coding: ISO-8859-1 -*- +a = 1 # keep this statement for now (see test_only_one_comment.py) + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 2 0 LOAD_CONST 1 (1) + 3 STORE_NAME 0 (a) + 6 LOAD_CONST 0 (None) + 9 RETURN_VALUE +co_consts: +[0] None +[1] 1 Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_encoding_declaration2.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_encoding_declaration2.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# coding: ISO_LATIN_1 +a = 1 + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 3 0 LOAD_CONST 1 (1) + 3 STORE_NAME 0 (a) + 6 LOAD_CONST 0 (None) + 9 RETURN_VALUE +co_consts: +[0] None +[1] 1 Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_encoding_declaration3.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_encoding_declaration3.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,17 @@ + + +# coding: ISO-8859-1 +# encoding on the third line <=> no encoding +a = 1 + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 5 0 LOAD_CONST 1 (1) + 3 STORE_NAME 0 (a) + 6 LOAD_CONST 0 (None) + 9 RETURN_VALUE +co_consts: +[0] None +[1] 1 Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_exceptions.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_exceptions.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,249 @@ +try: + a + b +except: + pass + +try: + a + b +except NameError: + pass + +try: + a + b +except NameError, err: + pass + +try: + a + b +except (NameError, ValueError): + pass + + +try: + a + b +except (NameError, ValueError), err: + pass + +try: + a +except NameError, err: + pass +except ValueError, err: + pass + +try: + a +except NameError, err: + pass +except ValueError, err: + pass +else: + pass + +try: + a +finally: + b + + + + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 SETUP_EXCEPT 12 (to 15) + + 2 3 LOAD_NAME 0 (a) + 6 POP_TOP + + 3 7 LOAD_NAME 1 (b) + 10 POP_TOP + 11 POP_BLOCK + 12 JUMP_FORWARD 7 (to 22) + >> 15 POP_TOP + 16 POP_TOP + 17 POP_TOP + + 5 18 JUMP_FORWARD 1 (to 22) + 21 END_FINALLY + + 7 >> 22 SETUP_EXCEPT 12 (to 37) + + 8 25 LOAD_NAME 0 (a) + 28 POP_TOP + + 9 29 LOAD_NAME 1 (b) + 32 POP_TOP + 33 POP_BLOCK + 34 JUMP_FORWARD 19 (to 56) + + 10 >> 37 DUP_TOP + 38 LOAD_NAME 2 (NameError) + 41 COMPARE_OP 10 (exception match) + 44 JUMP_IF_FALSE 7 (to 54) + 47 POP_TOP + 48 POP_TOP + 49 POP_TOP + 50 POP_TOP + + 11 51 JUMP_FORWARD 2 (to 56) + >> 54 POP_TOP + 55 END_FINALLY + + 13 >> 56 SETUP_EXCEPT 12 (to 71) + + 14 59 LOAD_NAME 0 (a) + 62 POP_TOP + + 15 63 LOAD_NAME 1 (b) + 66 POP_TOP + 67 POP_BLOCK + 68 JUMP_FORWARD 21 (to 92) + + 16 >> 71 DUP_TOP + 72 LOAD_NAME 2 (NameError) + 75 COMPARE_OP 10 (exception match) + 78 JUMP_IF_FALSE 9 (to 90) + 81 POP_TOP + 82 POP_TOP + 83 STORE_NAME 3 (err) + 86 POP_TOP + + 17 87 JUMP_FORWARD 2 (to 92) + >> 90 POP_TOP + 91 END_FINALLY + + 19 >> 92 SETUP_EXCEPT 12 (to 107) + + 20 95 LOAD_NAME 0 (a) + 98 POP_TOP + + 21 99 LOAD_NAME 1 (b) + 102 POP_TOP + 103 POP_BLOCK + 104 JUMP_FORWARD 25 (to 132) + + 22 >> 107 DUP_TOP + 108 LOAD_NAME 2 (NameError) + 111 LOAD_NAME 4 (ValueError) + 114 BUILD_TUPLE 2 + 117 COMPARE_OP 10 (exception match) + 120 JUMP_IF_FALSE 7 (to 130) + 123 POP_TOP + 124 POP_TOP + 125 POP_TOP + 126 POP_TOP + + 23 127 JUMP_FORWARD 2 (to 132) + >> 130 POP_TOP + 131 END_FINALLY + + 26 >> 132 SETUP_EXCEPT 12 (to 147) + + 27 135 LOAD_NAME 0 (a) + 138 POP_TOP + + 28 139 LOAD_NAME 1 (b) + 142 POP_TOP + 143 POP_BLOCK + 144 JUMP_FORWARD 27 (to 174) + + 29 >> 147 DUP_TOP + 148 LOAD_NAME 2 (NameError) + 151 LOAD_NAME 4 (ValueError) + 154 BUILD_TUPLE 2 + 157 COMPARE_OP 10 (exception match) + 160 JUMP_IF_FALSE 9 (to 172) + 163 POP_TOP + 164 POP_TOP + 165 STORE_NAME 3 (err) + 168 POP_TOP + + 30 169 JUMP_FORWARD 2 (to 174) + >> 172 POP_TOP + 173 END_FINALLY + + 32 >> 174 SETUP_EXCEPT 8 (to 185) + + 33 177 LOAD_NAME 0 (a) + 180 POP_TOP + 181 POP_BLOCK + 182 JUMP_FORWARD 41 (to 226) + + 34 >> 185 DUP_TOP + 186 LOAD_NAME 2 (NameError) + 189 COMPARE_OP 10 (exception match) + 192 JUMP_IF_FALSE 9 (to 204) + 195 POP_TOP + 196 POP_TOP + 197 STORE_NAME 3 (err) + 200 POP_TOP + + 35 201 JUMP_FORWARD 22 (to 226) + >> 204 POP_TOP + + 36 205 DUP_TOP + 206 LOAD_NAME 4 (ValueError) + 209 COMPARE_OP 10 (exception match) + 212 JUMP_IF_FALSE 9 (to 224) + 215 POP_TOP + 216 POP_TOP + 217 STORE_NAME 3 (err) + 220 POP_TOP + + 37 221 JUMP_FORWARD 2 (to 226) + >> 224 POP_TOP + 225 END_FINALLY + + 39 >> 226 SETUP_EXCEPT 8 (to 237) + + 40 229 LOAD_NAME 0 (a) + 232 POP_TOP + 233 POP_BLOCK + 234 JUMP_FORWARD 41 (to 278) + + 41 >> 237 DUP_TOP + 238 LOAD_NAME 2 (NameError) + 241 COMPARE_OP 10 (exception match) + 244 JUMP_IF_FALSE 9 (to 256) + 247 POP_TOP + 248 POP_TOP + 249 STORE_NAME 3 (err) + 252 POP_TOP + + 42 253 JUMP_FORWARD 22 (to 278) + >> 256 POP_TOP + + 43 257 DUP_TOP + 258 LOAD_NAME 4 (ValueError) + 261 COMPARE_OP 10 (exception match) + 264 JUMP_IF_FALSE 9 (to 276) + 267 POP_TOP + 268 POP_TOP + 269 STORE_NAME 3 (err) + 272 POP_TOP + + 44 273 JUMP_FORWARD 2 (to 278) + >> 276 POP_TOP + 277 END_FINALLY + + 48 >> 278 SETUP_FINALLY 8 (to 289) + + 49 281 LOAD_NAME 0 (a) + 284 POP_TOP + 285 POP_BLOCK + 286 LOAD_CONST 0 (None) + + 51 >> 289 LOAD_NAME 1 (b) + 292 POP_TOP + 293 END_FINALLY + 294 LOAD_CONST 0 (None) + 297 RETURN_VALUE +co_consts: +[0] None Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_function_calls.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_function_calls.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,80 @@ +f() +f(a) +f(a,) +f(a,b) +f(a, b,) +f(*args) +f(**kwargs) +f(*args, **kwargs) +f(a, *args, **kwargs) +f(a, b, *args, **kwargs) +a = 1 + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 LOAD_NAME 0 (f) + 3 CALL_FUNCTION 0 + 6 POP_TOP + + 2 7 LOAD_NAME 0 (f) + 10 LOAD_NAME 1 (a) + 13 CALL_FUNCTION 1 + 16 POP_TOP + + 3 17 LOAD_NAME 0 (f) + 20 LOAD_NAME 1 (a) + 23 CALL_FUNCTION 1 + 26 POP_TOP + + 4 27 LOAD_NAME 0 (f) + 30 LOAD_NAME 1 (a) + 33 LOAD_NAME 2 (b) + 36 CALL_FUNCTION 2 + 39 POP_TOP + + 5 40 LOAD_NAME 0 (f) + 43 LOAD_NAME 1 (a) + 46 LOAD_NAME 2 (b) + 49 CALL_FUNCTION 2 + 52 POP_TOP + + 6 53 LOAD_NAME 0 (f) + 56 LOAD_NAME 3 (args) + 59 CALL_FUNCTION_VAR 0 + 62 POP_TOP + + 7 63 LOAD_NAME 0 (f) + 66 LOAD_NAME 4 (kwargs) + 69 CALL_FUNCTION_KW 0 + 72 POP_TOP + + 8 73 LOAD_NAME 0 (f) + 76 LOAD_NAME 3 (args) + 79 LOAD_NAME 4 (kwargs) + 82 CALL_FUNCTION_VAR_KW 0 + 85 POP_TOP + + 9 86 LOAD_NAME 0 (f) + 89 LOAD_NAME 1 (a) + 92 LOAD_NAME 3 (args) + 95 LOAD_NAME 4 (kwargs) + 98 CALL_FUNCTION_VAR_KW 1 + 101 POP_TOP + + 10 102 LOAD_NAME 0 (f) + 105 LOAD_NAME 1 (a) + 108 LOAD_NAME 2 (b) + 111 LOAD_NAME 3 (args) + 114 LOAD_NAME 4 (kwargs) + 117 CALL_FUNCTION_VAR_KW 2 + 120 POP_TOP + + 11 121 LOAD_CONST 1 (1) + 124 STORE_NAME 1 (a) + 127 LOAD_CONST 0 (None) + 130 RETURN_VALUE +co_consts: +[0] None +[1] 1 Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_generator.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_generator.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,34 @@ +def f(n): + for i in range(n): + yield n + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 LOAD_CONST 1 () + 3 MAKE_FUNCTION 0 + 6 STORE_NAME 0 (f) + 9 LOAD_CONST 0 (None) + 12 RETURN_VALUE +co_consts: +[0] None +[1] co_filename = '' + co_varnames = ('n', 'i') + co_flags = 99 + 2 0 SETUP_LOOP 24 (to 27) + 3 LOAD_GLOBAL 0 (range) + 6 LOAD_FAST 0 (n) + 9 CALL_FUNCTION 1 + 12 GET_ITER + >> 13 FOR_ITER 10 (to 26) + 16 STORE_FAST 1 (i) + + 3 19 LOAD_FAST 0 (n) + 22 YIELD_VALUE + 23 JUMP_ABSOLUTE 13 + >> 26 POP_BLOCK + >> 27 LOAD_CONST 0 (None) + 30 RETURN_VALUE + co_consts: + [0] None Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_import_statements.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_import_statements.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,31 @@ +import os +import os.path as osp +from sets import Set, ImmutableSet + + + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 LOAD_CONST 0 (None) + 3 IMPORT_NAME 0 (os) + 6 STORE_NAME 0 (os) + + 2 9 LOAD_CONST 0 (None) + 12 IMPORT_NAME 1 (os.path) + 15 LOAD_ATTR 2 (path) + 18 STORE_NAME 3 (osp) + + 3 21 LOAD_CONST 1 (('Set', 'ImmutableSet')) + 24 IMPORT_NAME 4 (sets) + 27 IMPORT_FROM 5 (Set) + 30 STORE_NAME 5 (Set) + 33 IMPORT_FROM 6 (ImmutableSet) + 36 STORE_NAME 6 (ImmutableSet) + 39 POP_TOP + 40 LOAD_CONST 0 (None) + 43 RETURN_VALUE +co_consts: +[0] None +[1] ('Set', 'ImmutableSet') Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_list_comps.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_list_comps.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,120 @@ +[i for i in range(10) if i%2 == 0] +# same list on several lines +[i for i in range(10) + if i%2 == 0] +[i for i in 1,2] +[(i,j) for i in 1,2 for j in 3,4] + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 BUILD_LIST 0 + 3 DUP_TOP + 4 LOAD_ATTR 0 (append) + 7 STORE_NAME 1 ($append0) + 10 LOAD_NAME 2 (range) + 13 LOAD_CONST 1 (10) + 16 CALL_FUNCTION 1 + 19 GET_ITER + >> 20 FOR_ITER 37 (to 60) + 23 STORE_NAME 3 (i) + 26 LOAD_NAME 3 (i) + 29 LOAD_CONST 2 (2) + 32 BINARY_MODULO + 33 LOAD_CONST 3 (0) + 36 COMPARE_OP 2 (==) + 39 JUMP_IF_FALSE 14 (to 56) + 42 POP_TOP + 43 LOAD_NAME 1 ($append0) + 46 LOAD_NAME 3 (i) + 49 CALL_FUNCTION 1 + 52 POP_TOP + 53 JUMP_FORWARD 1 (to 57) + >> 56 POP_TOP + >> 57 JUMP_ABSOLUTE 20 + >> 60 DELETE_NAME 1 ($append0) + 63 POP_TOP + + 3 64 BUILD_LIST 0 + 67 DUP_TOP + 68 LOAD_ATTR 0 (append) + 71 STORE_NAME 1 ($append0) + 74 LOAD_NAME 2 (range) + 77 LOAD_CONST 1 (10) + 80 CALL_FUNCTION 1 + 83 GET_ITER + >> 84 FOR_ITER 37 (to 124) + 87 STORE_NAME 3 (i) + + 4 90 LOAD_NAME 3 (i) + 93 LOAD_CONST 2 (2) + 96 BINARY_MODULO + 97 LOAD_CONST 3 (0) + 100 COMPARE_OP 2 (==) + 103 JUMP_IF_FALSE 14 (to 120) + 106 POP_TOP + 107 LOAD_NAME 1 ($append0) + 110 LOAD_NAME 3 (i) + 113 CALL_FUNCTION 1 + 116 POP_TOP + 117 JUMP_FORWARD 1 (to 121) + >> 120 POP_TOP + >> 121 JUMP_ABSOLUTE 84 + >> 124 DELETE_NAME 1 ($append0) + 127 POP_TOP + + 5 128 BUILD_LIST 0 + 131 DUP_TOP + 132 LOAD_ATTR 0 (append) + 135 STORE_NAME 1 ($append0) + 138 LOAD_CONST 4 (1) + 141 LOAD_CONST 2 (2) + 144 BUILD_TUPLE 2 + 147 GET_ITER + >> 148 FOR_ITER 16 (to 167) + 151 STORE_NAME 3 (i) + 154 LOAD_NAME 1 ($append0) + 157 LOAD_NAME 3 (i) + 160 CALL_FUNCTION 1 + 163 POP_TOP + 164 JUMP_ABSOLUTE 148 + >> 167 DELETE_NAME 1 ($append0) + 170 POP_TOP + + 6 171 BUILD_LIST 0 + 174 DUP_TOP + 175 LOAD_ATTR 0 (append) + 178 STORE_NAME 1 ($append0) + 181 LOAD_CONST 4 (1) + 184 LOAD_CONST 2 (2) + 187 BUILD_TUPLE 2 + 190 GET_ITER + >> 191 FOR_ITER 41 (to 235) + 194 STORE_NAME 3 (i) + 197 LOAD_CONST 5 (3) + 200 LOAD_CONST 6 (4) + 203 BUILD_TUPLE 2 + 206 GET_ITER + >> 207 FOR_ITER 22 (to 232) + 210 STORE_NAME 4 (j) + 213 LOAD_NAME 1 ($append0) + 216 LOAD_NAME 3 (i) + 219 LOAD_NAME 4 (j) + 222 BUILD_TUPLE 2 + 225 CALL_FUNCTION 1 + 228 POP_TOP + 229 JUMP_ABSOLUTE 207 + >> 232 JUMP_ABSOLUTE 191 + >> 235 DELETE_NAME 1 ($append0) + 238 POP_TOP + 239 LOAD_CONST 0 (None) + 242 RETURN_VALUE +co_consts: +[0] None +[1] 10 +[2] 2 +[3] 0 +[4] 1 +[5] 3 +[6] 4 Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_listlinenos.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_listlinenos.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,41 @@ +l = [ "foo", "bar", + "baz"] + +l = [ + "foo", + "bar", + "baz", + ] + +l = [] +l = [ + ] + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 LOAD_CONST 1 ('foo') + 3 LOAD_CONST 2 ('bar') + 6 LOAD_CONST 3 ('baz') + 9 BUILD_LIST 3 + 12 STORE_NAME 0 (l) + + 4 15 LOAD_CONST 1 ('foo') + 18 LOAD_CONST 2 ('bar') + 21 LOAD_CONST 3 ('baz') + 24 BUILD_LIST 3 + 27 STORE_NAME 0 (l) + + 10 30 BUILD_LIST 0 + 33 STORE_NAME 0 (l) + + 11 36 BUILD_LIST 0 + 39 STORE_NAME 0 (l) + 42 LOAD_CONST 0 (None) + 45 RETURN_VALUE +co_consts: +[0] None +[1] 'foo' +[2] 'bar' +[3] 'baz' Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_multiline.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_multiline.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,43 @@ +from foo import bar, \ + baz + +if True and \ + False \ + and True: + print "excellent !" + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 LOAD_CONST 1 (('bar', 'baz')) + 3 IMPORT_NAME 0 (foo) + 6 IMPORT_FROM 1 (bar) + 9 STORE_NAME 1 (bar) + 12 IMPORT_FROM 2 (baz) + 15 STORE_NAME 2 (baz) + 18 POP_TOP + + 4 19 LOAD_NAME 3 (True) + 22 JUMP_IF_FALSE 11 (to 36) + 25 POP_TOP + + 5 26 LOAD_NAME 4 (False) + 29 JUMP_IF_FALSE 4 (to 36) + 32 POP_TOP + + 6 33 LOAD_NAME 3 (True) + >> 36 JUMP_IF_FALSE 9 (to 48) + 39 POP_TOP + + 7 40 LOAD_CONST 2 ('excellent !') + 43 PRINT_ITEM + 44 PRINT_NEWLINE + 45 JUMP_FORWARD 1 (to 49) + >> 48 POP_TOP + >> 49 LOAD_CONST 0 (None) + 52 RETURN_VALUE +co_consts: +[0] None +[1] ('bar', 'baz') +[2] 'excellent !' Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_numbers.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_numbers.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,49 @@ +a = 1 +a = -1 +a = 1. +a = .2 +a = 1.2 +a = 1e3 +a = 1.3e4 +a = -1.3 + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 LOAD_CONST 1 (1) + 3 STORE_NAME 0 (a) + + 2 6 LOAD_CONST 1 (1) + 9 UNARY_NEGATIVE + 10 STORE_NAME 0 (a) + + 3 13 LOAD_CONST 2 (1.0) + 16 STORE_NAME 0 (a) + + 4 19 LOAD_CONST 3 (0.20000000000000001) + 22 STORE_NAME 0 (a) + + 5 25 LOAD_CONST 4 (1.2) + 28 STORE_NAME 0 (a) + + 6 31 LOAD_CONST 5 (1000.0) + 34 STORE_NAME 0 (a) + + 7 37 LOAD_CONST 6 (13000.0) + 40 STORE_NAME 0 (a) + + 8 43 LOAD_CONST 7 (1.3) + 46 UNARY_NEGATIVE + 47 STORE_NAME 0 (a) + 50 LOAD_CONST 0 (None) + 53 RETURN_VALUE +co_consts: +[0] None +[1] 1 +[2] 1.0 +[3] 0.20000000000000001 +[4] 1.2 +[5] 1000.0 +[6] 13000.0 +[7] 1.3 Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_only_one_comment.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_only_one_comment.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,10 @@ +# only one comment + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 0 0 LOAD_CONST 0 (None) + 3 RETURN_VALUE +co_consts: +[0] None Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_redirected_prints.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_redirected_prints.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,12 @@ +print >> f + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 LOAD_NAME 0 (f) + 3 PRINT_NEWLINE_TO + 4 LOAD_CONST 0 (None) + 7 RETURN_VALUE +co_consts: +[0] None Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_samples.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_samples.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,702 @@ + + + +import os, os.path as osp +import sys +from ebnf import parse_grammar +from python import python_parse, pypy_parse, set_debug +from pprint import pprint +import grammar +grammar.DEBUG = False +from symbol import sym_name + + +def name(elt): + return "%s[%d]"% (sym_name.get(elt,elt),elt) + +def read_samples_dir(): + return [osp.join('samples', fname) for fname in os.listdir('samples') + if fname.endswith('.py')] + + +def print_sym_tuple( tup ): + print "\n(", + for elt in tup: + if type(elt)==int: + print name(elt), + elif type(elt)==str: + print repr(elt), + else: + print_sym_tuple(elt) + print ")", + +def assert_tuples_equal(tup1, tup2, curpos = (), disp=""): + if disp: + print "\n"+disp+"(", + for index, (elt1, elt2) in enumerate(zip(tup1, tup2)): + if disp and elt1==elt2 and type(elt1)==int: + print name(elt1), + if elt1 != elt2: + if type(elt1) is tuple and type(elt2) is tuple: + if disp: + disp=disp+" " + assert_tuples_equal(elt1, elt2, curpos + (index,), disp) + print + print "TUP1" + print_sym_tuple(tup1) + print + print "TUP2" + print_sym_tuple(tup2) + + raise AssertionError('Found difference at %s : %s != %s' % + (curpos, name(elt1), name(elt2) ), curpos) + if disp: + print ")", + +def test_samples( samples ): + for sample in samples: + pypy_tuples = pypy_parse(sample) + python_tuples = python_parse(sample) + print "="*20 + print file(sample).read() + print "-"*10 + pprint(pypy_tuples) + print "-"*10 + pprint(python_tuples) + try: + assert_tuples_equal( python_tuples, pypy_tuples, disp=" " ) + assert python_tuples == pypy_tuples + except AssertionError,e: + print + print "python_tuples" + show( python_tuples, e.args[-1] ) + print + print "pypy_tuples" + show( pypy_tuples, e.args[-1] ) + raise + + +def show( tup, idxs ): + for level, i in enumerate(idxs): + print " "*level , tup + tup=tup[i] + print tup + +if __name__=="__main__": + import getopt + opts, args = getopt.getopt( sys.argv[1:], "d:", [] ) + for opt, val in opts: + if opt=="-d": + set_debug(int(val)) + if args: + samples = args + else: + samples = read_samples_dir() + + test_samples( samples ) + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 4 0 LOAD_CONST 0 (None) + 3 IMPORT_NAME 0 (os) + 6 STORE_NAME 0 (os) + 9 LOAD_CONST 0 (None) + 12 IMPORT_NAME 1 (os.path) + 15 LOAD_ATTR 2 (path) + 18 STORE_NAME 3 (osp) + + 5 21 LOAD_CONST 0 (None) + 24 IMPORT_NAME 4 (sys) + 27 STORE_NAME 4 (sys) + + 6 30 LOAD_CONST 1 (('parse_grammar',)) + 33 IMPORT_NAME 5 (ebnf) + 36 IMPORT_FROM 6 (parse_grammar) + 39 STORE_NAME 6 (parse_grammar) + 42 POP_TOP + + 7 43 LOAD_CONST 2 (('python_parse', 'pypy_parse', 'set_debug')) + 46 IMPORT_NAME 7 (python) + 49 IMPORT_FROM 8 (python_parse) + 52 STORE_NAME 8 (python_parse) + 55 IMPORT_FROM 9 (pypy_parse) + 58 STORE_NAME 9 (pypy_parse) + 61 IMPORT_FROM 10 (set_debug) + 64 STORE_NAME 10 (set_debug) + 67 POP_TOP + + 8 68 LOAD_CONST 3 (('pprint',)) + 71 IMPORT_NAME 11 (pprint) + 74 IMPORT_FROM 11 (pprint) + 77 STORE_NAME 11 (pprint) + 80 POP_TOP + + 9 81 LOAD_CONST 0 (None) + 84 IMPORT_NAME 12 (grammar) + 87 STORE_NAME 12 (grammar) + + 10 90 LOAD_NAME 13 (False) + 93 LOAD_NAME 12 (grammar) + 96 STORE_ATTR 14 (DEBUG) + + 11 99 LOAD_CONST 4 (('sym_name',)) + 102 IMPORT_NAME 15 (symbol) + 105 IMPORT_FROM 16 (sym_name) + 108 STORE_NAME 16 (sym_name) + 111 POP_TOP + + 14 112 LOAD_CONST 5 () + 115 MAKE_FUNCTION 0 + 118 STORE_NAME 17 (name) + + 17 121 LOAD_CONST 6 () + 124 MAKE_FUNCTION 0 + 127 STORE_NAME 18 (read_samples_dir) + + 22 130 LOAD_CONST 7 () + 133 MAKE_FUNCTION 0 + 136 STORE_NAME 19 (print_sym_tuple) + + 33 139 BUILD_TUPLE 0 + 142 LOAD_CONST 8 ('') + 145 LOAD_CONST 9 () + 148 MAKE_FUNCTION 2 + 151 STORE_NAME 20 (assert_tuples_equal) + + 56 154 LOAD_CONST 10 () + 157 MAKE_FUNCTION 0 + 160 STORE_NAME 21 (test_samples) + + 79 163 LOAD_CONST 11 () + 166 MAKE_FUNCTION 0 + 169 STORE_NAME 22 (show) + + 85 172 LOAD_NAME 23 (__name__) + 175 LOAD_CONST 12 ('__main__') + 178 COMPARE_OP 2 (==) + 181 JUMP_IF_FALSE 139 (to 323) + 184 POP_TOP + + 86 185 LOAD_CONST 0 (None) + 188 IMPORT_NAME 24 (getopt) + 191 STORE_NAME 24 (getopt) + + 87 194 LOAD_NAME 24 (getopt) + 197 LOAD_ATTR 24 (getopt) + 200 LOAD_NAME 4 (sys) + 203 LOAD_ATTR 25 (argv) + 206 LOAD_CONST 13 (1) + 209 SLICE+1 + 210 LOAD_CONST 14 ('d:') + 213 BUILD_LIST 0 + 216 CALL_FUNCTION 3 + 219 UNPACK_SEQUENCE 2 + 222 STORE_NAME 26 (opts) + 225 STORE_NAME 27 (args) + + 88 228 SETUP_LOOP 53 (to 284) + 231 LOAD_NAME 26 (opts) + 234 GET_ITER + >> 235 FOR_ITER 45 (to 283) + 238 UNPACK_SEQUENCE 2 + 241 STORE_NAME 28 (opt) + 244 STORE_NAME 29 (val) + + 89 247 LOAD_NAME 28 (opt) + 250 LOAD_CONST 15 ('-d') + 253 COMPARE_OP 2 (==) + 256 JUMP_IF_FALSE 20 (to 279) + 259 POP_TOP + + 90 260 LOAD_NAME 10 (set_debug) + 263 LOAD_NAME 30 (int) + 266 LOAD_NAME 29 (val) + 269 CALL_FUNCTION 1 + 272 CALL_FUNCTION 1 + 275 POP_TOP + 276 JUMP_FORWARD 1 (to 280) + >> 279 POP_TOP + >> 280 JUMP_ABSOLUTE 235 + >> 283 POP_BLOCK + + 91 >> 284 LOAD_NAME 27 (args) + 287 JUMP_IF_FALSE 10 (to 300) + 290 POP_TOP + + 92 291 LOAD_NAME 27 (args) + 294 STORE_NAME 31 (samples) + 297 JUMP_FORWARD 10 (to 310) + >> 300 POP_TOP + + 94 301 LOAD_NAME 18 (read_samples_dir) + 304 CALL_FUNCTION 0 + 307 STORE_NAME 31 (samples) + + 96 >> 310 LOAD_NAME 21 (test_samples) + 313 LOAD_NAME 31 (samples) + 316 CALL_FUNCTION 1 + 319 POP_TOP + 320 JUMP_FORWARD 1 (to 324) + >> 323 POP_TOP + >> 324 LOAD_CONST 0 (None) + 327 RETURN_VALUE +co_consts: +[0] None +[1] ('parse_grammar',) +[2] ('python_parse', 'pypy_parse', 'set_debug') +[3] ('pprint',) +[4] ('sym_name',) +[5] co_filename = '' + co_varnames = ('elt',) + co_flags = 67 + 15 0 LOAD_CONST 1 ('%s[%d]') + 3 LOAD_GLOBAL 0 (sym_name) + 6 LOAD_ATTR 1 (get) + 9 LOAD_FAST 0 (elt) + 12 LOAD_FAST 0 (elt) + 15 CALL_FUNCTION 2 + 18 LOAD_FAST 0 (elt) + 21 BUILD_TUPLE 2 + 24 BINARY_MODULO + 25 RETURN_VALUE + 26 LOAD_CONST 0 (None) + 29 RETURN_VALUE + co_consts: + [0] None + [1] '%s[%d]' +[6] co_filename = '' + co_varnames = ('$append0', 'fname') + co_flags = 67 + 18 0 BUILD_LIST 0 + 3 DUP_TOP + 4 LOAD_ATTR 0 (append) + 7 STORE_FAST 0 ($append0) + 10 LOAD_GLOBAL 2 (os) + 13 LOAD_ATTR 3 (listdir) + 16 LOAD_CONST 1 ('samples') + 19 CALL_FUNCTION 1 + 22 GET_ITER + >> 23 FOR_ITER 48 (to 74) + 26 STORE_FAST 1 (fname) + + 19 29 LOAD_FAST 1 (fname) + 32 LOAD_ATTR 5 (endswith) + 35 LOAD_CONST 2 ('.py') + 38 CALL_FUNCTION 1 + 41 JUMP_IF_FALSE 26 (to 70) + 44 POP_TOP + 45 LOAD_FAST 0 ($append0) + 48 LOAD_GLOBAL 6 (osp) + 51 LOAD_ATTR 7 (join) + 54 LOAD_CONST 1 ('samples') + 57 LOAD_FAST 1 (fname) + 60 CALL_FUNCTION 2 + 63 CALL_FUNCTION 1 + 66 POP_TOP + 67 JUMP_FORWARD 1 (to 71) + >> 70 POP_TOP + >> 71 JUMP_ABSOLUTE 23 + >> 74 DELETE_FAST 0 ($append0) + 77 RETURN_VALUE + 78 LOAD_CONST 0 (None) + 81 RETURN_VALUE + co_consts: + [0] None + [1] 'samples' + [2] '.py' +[7] co_filename = '' + co_varnames = ('tup', 'elt') + co_flags = 67 + 23 0 LOAD_CONST 1 ('\n(') + 3 PRINT_ITEM + + 24 4 SETUP_LOOP 90 (to 97) + 7 LOAD_FAST 0 (tup) + 10 GET_ITER + >> 11 FOR_ITER 82 (to 96) + 14 STORE_FAST 1 (elt) + + 25 17 LOAD_GLOBAL 2 (type) + 20 LOAD_FAST 1 (elt) + 23 CALL_FUNCTION 1 + 26 LOAD_GLOBAL 3 (int) + 29 COMPARE_OP 2 (==) + 32 JUMP_IF_FALSE 14 (to 49) + 35 POP_TOP + + 26 36 LOAD_GLOBAL 4 (name) + 39 LOAD_FAST 1 (elt) + 42 CALL_FUNCTION 1 + 45 PRINT_ITEM + 46 JUMP_FORWARD 44 (to 93) + >> 49 POP_TOP + + 27 50 LOAD_GLOBAL 2 (type) + 53 LOAD_FAST 1 (elt) + 56 CALL_FUNCTION 1 + 59 LOAD_GLOBAL 5 (str) + 62 COMPARE_OP 2 (==) + 65 JUMP_IF_FALSE 14 (to 82) + 68 POP_TOP + + 28 69 LOAD_GLOBAL 6 (repr) + 72 LOAD_FAST 1 (elt) + 75 CALL_FUNCTION 1 + 78 PRINT_ITEM + 79 JUMP_FORWARD 11 (to 93) + >> 82 POP_TOP + + 30 83 LOAD_GLOBAL 7 (print_sym_tuple) + 86 LOAD_FAST 1 (elt) + 89 CALL_FUNCTION 1 + 92 POP_TOP + >> 93 JUMP_ABSOLUTE 11 + >> 96 POP_BLOCK + + 31 >> 97 LOAD_CONST 2 (')') + 100 PRINT_ITEM + 101 LOAD_CONST 0 (None) + 104 RETURN_VALUE + co_consts: + [0] None + [1] '\n(' + [2] ')' +[8] '' +[9] co_filename = '' + co_varnames = ('tup1', 'tup2', 'curpos', 'disp', 'index', 'elt1', 'elt2') + co_flags = 67 + 34 0 LOAD_FAST 3 (disp) + 3 JUMP_IF_FALSE 16 (to 22) + 6 POP_TOP + + 35 7 LOAD_CONST 1 ('\n') + 10 LOAD_FAST 3 (disp) + 13 BINARY_ADD + 14 LOAD_CONST 2 ('(') + 17 BINARY_ADD + 18 PRINT_ITEM + 19 JUMP_FORWARD 1 (to 23) + >> 22 POP_TOP + + 36 >> 23 SETUP_LOOP 272 (to 298) + 26 LOAD_GLOBAL 1 (enumerate) + 29 LOAD_GLOBAL 2 (zip) + 32 LOAD_FAST 0 (tup1) + 35 LOAD_FAST 1 (tup2) + 38 CALL_FUNCTION 2 + 41 CALL_FUNCTION 1 + 44 GET_ITER + >> 45 FOR_ITER 249 (to 297) + 48 UNPACK_SEQUENCE 2 + 51 STORE_FAST 4 (index) + 54 UNPACK_SEQUENCE 2 + 57 STORE_FAST 5 (elt1) + 60 STORE_FAST 6 (elt2) + + 37 63 LOAD_FAST 3 (disp) + 66 JUMP_IF_FALSE 29 (to 98) + 69 POP_TOP + 70 LOAD_FAST 5 (elt1) + 73 LOAD_FAST 6 (elt2) + 76 COMPARE_OP 2 (==) + 79 JUMP_IF_FALSE 16 (to 98) + 82 POP_TOP + 83 LOAD_GLOBAL 8 (type) + 86 LOAD_FAST 5 (elt1) + 89 CALL_FUNCTION 1 + 92 LOAD_GLOBAL 9 (int) + 95 COMPARE_OP 2 (==) + >> 98 JUMP_IF_FALSE 14 (to 115) + 101 POP_TOP + + 38 102 LOAD_GLOBAL 10 (name) + 105 LOAD_FAST 5 (elt1) + 108 CALL_FUNCTION 1 + 111 PRINT_ITEM + 112 JUMP_FORWARD 1 (to 116) + >> 115 POP_TOP + + 39 >> 116 LOAD_FAST 5 (elt1) + 119 LOAD_FAST 6 (elt2) + 122 COMPARE_OP 3 (!=) + 125 JUMP_IF_FALSE 165 (to 293) + 128 POP_TOP + + 40 129 LOAD_GLOBAL 8 (type) + 132 LOAD_FAST 5 (elt1) + 135 CALL_FUNCTION 1 + 138 LOAD_GLOBAL 11 (tuple) + 141 COMPARE_OP 8 (is) + 144 JUMP_IF_FALSE 16 (to 163) + 147 POP_TOP + 148 LOAD_GLOBAL 8 (type) + 151 LOAD_FAST 6 (elt2) + 154 CALL_FUNCTION 1 + 157 LOAD_GLOBAL 11 (tuple) + 160 COMPARE_OP 8 (is) + >> 163 JUMP_IF_FALSE 51 (to 217) + 166 POP_TOP + + 41 167 LOAD_FAST 3 (disp) + 170 JUMP_IF_FALSE 14 (to 187) + 173 POP_TOP + + 42 174 LOAD_FAST 3 (disp) + 177 LOAD_CONST 3 (' ') + 180 BINARY_ADD + 181 STORE_FAST 3 (disp) + 184 JUMP_FORWARD 1 (to 188) + >> 187 POP_TOP + + 43 >> 188 LOAD_GLOBAL 12 (assert_tuples_equal) + 191 LOAD_FAST 5 (elt1) + 194 LOAD_FAST 6 (elt2) + 197 LOAD_FAST 2 (curpos) + 200 LOAD_FAST 4 (index) + 203 BUILD_TUPLE 1 + 206 BINARY_ADD + 207 LOAD_FAST 3 (disp) + 210 CALL_FUNCTION 4 + 213 POP_TOP + 214 JUMP_FORWARD 1 (to 218) + >> 217 POP_TOP + + 44 >> 218 PRINT_NEWLINE + + 45 219 LOAD_CONST 4 ('TUP1') + 222 PRINT_ITEM + 223 PRINT_NEWLINE + + 46 224 LOAD_GLOBAL 14 (print_sym_tuple) + 227 LOAD_FAST 0 (tup1) + 230 CALL_FUNCTION 1 + 233 POP_TOP + + 47 234 PRINT_NEWLINE + + 48 235 LOAD_CONST 5 ('TUP2') + 238 PRINT_ITEM + 239 PRINT_NEWLINE + + 49 240 LOAD_GLOBAL 14 (print_sym_tuple) + 243 LOAD_FAST 1 (tup2) + 246 CALL_FUNCTION 1 + 249 POP_TOP + + 51 250 LOAD_GLOBAL 15 (AssertionError) + 253 LOAD_CONST 6 ('Found difference at %s : %s != %s') + + 52 256 LOAD_FAST 2 (curpos) + 259 LOAD_GLOBAL 10 (name) + 262 LOAD_FAST 5 (elt1) + 265 CALL_FUNCTION 1 + 268 LOAD_GLOBAL 10 (name) + 271 LOAD_FAST 6 (elt2) + 274 CALL_FUNCTION 1 + 277 BUILD_TUPLE 3 + 280 BINARY_MODULO + 281 LOAD_FAST 2 (curpos) + 284 CALL_FUNCTION 2 + 287 RAISE_VARARGS 1 + 290 JUMP_FORWARD 1 (to 294) + >> 293 POP_TOP + >> 294 JUMP_ABSOLUTE 45 + >> 297 POP_BLOCK + + 53 >> 298 LOAD_FAST 3 (disp) + 301 JUMP_IF_FALSE 8 (to 312) + 304 POP_TOP + + 54 305 LOAD_CONST 7 (')') + 308 PRINT_ITEM + 309 JUMP_FORWARD 1 (to 313) + >> 312 POP_TOP + >> 313 LOAD_CONST 0 (None) + 316 RETURN_VALUE + co_consts: + [0] None + [1] '\n' + [2] '(' + [3] ' ' + [4] 'TUP1' + [5] 'TUP2' + [6] 'Found difference at %s : %s != %s' + [7] ')' +[10] co_filename = '' + co_varnames = ('samples', 'sample', 'pypy_tuples', 'python_tuples', 'e') + co_flags = 67 + 57 0 SETUP_LOOP 226 (to 229) + 3 LOAD_FAST 0 (samples) + 6 GET_ITER + >> 7 FOR_ITER 218 (to 228) + 10 STORE_FAST 1 (sample) + + 58 13 LOAD_GLOBAL 2 (pypy_parse) + 16 LOAD_FAST 1 (sample) + 19 CALL_FUNCTION 1 + 22 STORE_FAST 2 (pypy_tuples) + + 59 25 LOAD_GLOBAL 4 (python_parse) + 28 LOAD_FAST 1 (sample) + 31 CALL_FUNCTION 1 + 34 STORE_FAST 3 (python_tuples) + + 60 37 LOAD_CONST 1 ('=') + 40 LOAD_CONST 2 (20) + 43 BINARY_MULTIPLY + 44 PRINT_ITEM + 45 PRINT_NEWLINE + + 61 46 LOAD_GLOBAL 6 (file) + 49 LOAD_FAST 1 (sample) + 52 CALL_FUNCTION 1 + 55 LOAD_ATTR 7 (read) + 58 CALL_FUNCTION 0 + 61 PRINT_ITEM + 62 PRINT_NEWLINE + + 62 63 LOAD_CONST 3 ('-') + 66 LOAD_CONST 4 (10) + 69 BINARY_MULTIPLY + 70 PRINT_ITEM + 71 PRINT_NEWLINE + + 63 72 LOAD_GLOBAL 8 (pprint) + 75 LOAD_FAST 2 (pypy_tuples) + 78 CALL_FUNCTION 1 + 81 POP_TOP + + 64 82 LOAD_CONST 3 ('-') + 85 LOAD_CONST 4 (10) + 88 BINARY_MULTIPLY + 89 PRINT_ITEM + 90 PRINT_NEWLINE + + 65 91 LOAD_GLOBAL 8 (pprint) + 94 LOAD_FAST 3 (python_tuples) + 97 CALL_FUNCTION 1 + 100 POP_TOP + + 66 101 SETUP_EXCEPT 43 (to 147) + + 67 104 LOAD_GLOBAL 9 (assert_tuples_equal) + 107 LOAD_FAST 3 (python_tuples) + 110 LOAD_FAST 2 (pypy_tuples) + 113 LOAD_CONST 5 ('disp') + 116 LOAD_CONST 6 (' ') + 119 CALL_FUNCTION 258 + 122 POP_TOP + + 68 123 LOAD_FAST 3 (python_tuples) + 126 LOAD_FAST 2 (pypy_tuples) + 129 COMPARE_OP 2 (==) + 132 JUMP_IF_TRUE 7 (to 142) + 135 POP_TOP + 136 LOAD_GLOBAL 10 (AssertionError) + 139 RAISE_VARARGS 1 + >> 142 POP_TOP + 143 POP_BLOCK + 144 JUMP_FORWARD 78 (to 225) + + 69 >> 147 DUP_TOP + 148 LOAD_GLOBAL 10 (AssertionError) + 151 COMPARE_OP 10 (exception match) + 154 JUMP_IF_FALSE 66 (to 223) + 157 POP_TOP + 158 POP_TOP + 159 STORE_FAST 4 (e) + 162 POP_TOP + + 70 163 PRINT_NEWLINE + + 71 164 LOAD_CONST 7 ('python_tuples') + 167 PRINT_ITEM + 168 PRINT_NEWLINE + + 72 169 LOAD_GLOBAL 12 (show) + 172 LOAD_FAST 3 (python_tuples) + 175 LOAD_FAST 4 (e) + 178 LOAD_ATTR 13 (args) + 181 LOAD_CONST 8 (1) + 184 UNARY_NEGATIVE + 185 BINARY_SUBSCR + 186 CALL_FUNCTION 2 + 189 POP_TOP + + 73 190 PRINT_NEWLINE + + 74 191 LOAD_CONST 9 ('pypy_tuples') + 194 PRINT_ITEM + 195 PRINT_NEWLINE + + 75 196 LOAD_GLOBAL 12 (show) + 199 LOAD_FAST 2 (pypy_tuples) + 202 LOAD_FAST 4 (e) + 205 LOAD_ATTR 13 (args) + 208 LOAD_CONST 8 (1) + 211 UNARY_NEGATIVE + 212 BINARY_SUBSCR + 213 CALL_FUNCTION 2 + 216 POP_TOP + + 76 217 RAISE_VARARGS 0 + 220 JUMP_FORWARD 2 (to 225) + >> 223 POP_TOP + 224 END_FINALLY + >> 225 JUMP_ABSOLUTE 7 + >> 228 POP_BLOCK + >> 229 LOAD_CONST 0 (None) + 232 RETURN_VALUE + co_consts: + [0] None + [1] '=' + [2] 20 + [3] '-' + [4] 10 + [5] 'disp' + [6] ' ' + [7] 'python_tuples' + [8] 1 + [9] 'pypy_tuples' +[11] co_filename = '' + co_varnames = ('tup', 'idxs', 'level', 'i') + co_flags = 67 + 80 0 SETUP_LOOP 49 (to 52) + 3 LOAD_GLOBAL 0 (enumerate) + 6 LOAD_FAST 1 (idxs) + 9 CALL_FUNCTION 1 + 12 GET_ITER + >> 13 FOR_ITER 35 (to 51) + 16 UNPACK_SEQUENCE 2 + 19 STORE_FAST 2 (level) + 22 STORE_FAST 3 (i) + + 81 25 LOAD_CONST 1 (' ') + 28 LOAD_FAST 2 (level) + 31 BINARY_MULTIPLY + 32 PRINT_ITEM + 33 LOAD_FAST 0 (tup) + 36 PRINT_ITEM + 37 PRINT_NEWLINE + + 82 38 LOAD_FAST 0 (tup) + 41 LOAD_FAST 3 (i) + 44 BINARY_SUBSCR + 45 STORE_FAST 0 (tup) + 48 JUMP_ABSOLUTE 13 + >> 51 POP_BLOCK + + 83 >> 52 LOAD_FAST 0 (tup) + 55 PRINT_ITEM + 56 PRINT_NEWLINE + 57 LOAD_CONST 0 (None) + 60 RETURN_VALUE + co_consts: + [0] None + [1] ' ' +[12] '__main__' +[13] 1 +[14] 'd:' +[15] '-d' Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_several_statements.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_several_statements.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,24 @@ +a = 1 +b = 2 +c = a + b + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 LOAD_CONST 1 (1) + 3 STORE_NAME 0 (a) + + 2 6 LOAD_CONST 2 (2) + 9 STORE_NAME 1 (b) + + 3 12 LOAD_NAME 0 (a) + 15 LOAD_NAME 1 (b) + 18 BINARY_ADD + 19 STORE_NAME 2 (c) + 22 LOAD_CONST 0 (None) + 25 RETURN_VALUE +co_consts: +[0] None +[1] 1 +[2] 2 Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_simple_assignment.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_simple_assignment.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,13 @@ +x = 1 + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 LOAD_CONST 1 (1) + 3 STORE_NAME 0 (x) + 6 LOAD_CONST 0 (None) + 9 RETURN_VALUE +co_consts: +[0] None +[1] 1 Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_simple_class.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_simple_class.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,64 @@ +class A: + + def with_white_spaces_before(self): + pass + + + def another_method(self, foo): + """with a docstring + on several lines + # with a sharpsign + """ + self.bar = foo + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 LOAD_CONST 1 ('A') + 3 BUILD_TUPLE 0 + 6 LOAD_CONST 2 () + 9 MAKE_FUNCTION 0 + 12 CALL_FUNCTION 0 + 15 BUILD_CLASS + 16 STORE_NAME 0 (A) + 19 LOAD_CONST 0 (None) + 22 RETURN_VALUE +co_consts: +[0] None +[1] 'A' +[2] co_filename = '' + co_varnames = () + co_flags = 66 + 1 0 LOAD_GLOBAL 0 (__name__) + 3 STORE_NAME 1 (__module__) + + 3 6 LOAD_CONST 1 () + 9 MAKE_FUNCTION 0 + 12 STORE_NAME 2 (with_white_spaces_before) + + 7 15 LOAD_CONST 2 () + 18 MAKE_FUNCTION 0 + 21 STORE_NAME 3 (another_method) + 24 LOAD_LOCALS + 25 RETURN_VALUE + co_consts: + [0] 'with a docstring\n on several lines\n # with a sharpsign\n ' + [1] co_filename = '' + co_varnames = ('self',) + co_flags = 67 + 3 0 LOAD_CONST 0 (None) + 3 RETURN_VALUE + co_consts: + [0] None + [2] co_filename = '' + co_varnames = ('self', 'foo') + co_flags = 67 + 12 0 LOAD_FAST 1 (foo) + 3 LOAD_FAST 0 (self) + 6 STORE_ATTR 2 (bar) + 9 LOAD_CONST 1 (None) + 12 RETURN_VALUE + co_consts: + [0] 'with a docstring\n on several lines\n # with a sharpsign\n ' + [1] None Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_simple_for_loop.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_simple_for_loop.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,90 @@ +for x in range(10): + pass + +for x in range(5): + a += x + b += 2 + if False: + break + else: + continue +else: + c = 3 + +for index, val in enumerate(range(5)): + val *= 2 + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 SETUP_LOOP 20 (to 23) + 3 LOAD_NAME 0 (range) + 6 LOAD_CONST 1 (10) + 9 CALL_FUNCTION 1 + 12 GET_ITER + >> 13 FOR_ITER 6 (to 22) + 16 STORE_NAME 1 (x) + + 2 19 JUMP_ABSOLUTE 13 + >> 22 POP_BLOCK + + 4 >> 23 SETUP_LOOP 61 (to 87) + 26 LOAD_NAME 0 (range) + 29 LOAD_CONST 2 (5) + 32 CALL_FUNCTION 1 + 35 GET_ITER + >> 36 FOR_ITER 41 (to 80) + 39 STORE_NAME 1 (x) + + 5 42 LOAD_NAME 2 (a) + 45 LOAD_NAME 1 (x) + 48 INPLACE_ADD + 49 STORE_NAME 2 (a) + + 6 52 LOAD_NAME 3 (b) + 55 LOAD_CONST 3 (2) + 58 INPLACE_ADD + 59 STORE_NAME 3 (b) + + 7 62 LOAD_NAME 4 (False) + 65 JUMP_IF_FALSE 5 (to 73) + 68 POP_TOP + + 8 69 BREAK_LOOP + 70 JUMP_FORWARD 4 (to 77) + >> 73 POP_TOP + + 10 74 JUMP_ABSOLUTE 36 + >> 77 JUMP_ABSOLUTE 36 + >> 80 POP_BLOCK + + 12 81 LOAD_CONST 4 (3) + 84 STORE_NAME 5 (c) + + 14 >> 87 SETUP_LOOP 42 (to 132) + 90 LOAD_NAME 6 (enumerate) + 93 LOAD_NAME 0 (range) + 96 LOAD_CONST 2 (5) + 99 CALL_FUNCTION 1 + 102 CALL_FUNCTION 1 + 105 GET_ITER + >> 106 FOR_ITER 22 (to 131) + 109 UNPACK_SEQUENCE 2 + 112 STORE_NAME 7 (index) + 115 STORE_NAME 8 (val) + + 15 118 LOAD_NAME 8 (val) + 121 LOAD_CONST 3 (2) + 124 INPLACE_MULTIPLY + 125 STORE_NAME 8 (val) + 128 JUMP_ABSOLUTE 106 + >> 131 POP_BLOCK + >> 132 LOAD_CONST 0 (None) + 135 RETURN_VALUE +co_consts: +[0] None +[1] 10 +[2] 5 +[3] 2 +[4] 3 Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_simple_function.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_simple_function.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,57 @@ +def f(a, b=1, *args, **kwargs): + if args: + a += len(args) + if kwargs: + a += len(kwargs) + return a*b + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 LOAD_CONST 1 (1) + 3 LOAD_CONST 2 () + 6 MAKE_FUNCTION 1 + 9 STORE_NAME 0 (f) + 12 LOAD_CONST 0 (None) + 15 RETURN_VALUE +co_consts: +[0] None +[1] 1 +[2] co_filename = '' + co_varnames = ('a', 'b', 'args', 'kwargs') + co_flags = 79 + 2 0 LOAD_FAST 2 (args) + 3 JUMP_IF_FALSE 20 (to 26) + 6 POP_TOP + + 3 7 LOAD_FAST 0 (a) + 10 LOAD_GLOBAL 2 (len) + 13 LOAD_FAST 2 (args) + 16 CALL_FUNCTION 1 + 19 INPLACE_ADD + 20 STORE_FAST 0 (a) + 23 JUMP_FORWARD 1 (to 27) + >> 26 POP_TOP + + 4 >> 27 LOAD_FAST 3 (kwargs) + 30 JUMP_IF_FALSE 20 (to 53) + 33 POP_TOP + + 5 34 LOAD_FAST 0 (a) + 37 LOAD_GLOBAL 2 (len) + 40 LOAD_FAST 3 (kwargs) + 43 CALL_FUNCTION 1 + 46 INPLACE_ADD + 47 STORE_FAST 0 (a) + 50 JUMP_FORWARD 1 (to 54) + >> 53 POP_TOP + + 6 >> 54 LOAD_FAST 0 (a) + 57 LOAD_FAST 1 (b) + 60 BINARY_MULTIPLY + 61 RETURN_VALUE + 62 LOAD_CONST 0 (None) + 65 RETURN_VALUE + co_consts: + [0] None Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_simple_in_expr.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_simple_in_expr.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,17 @@ +x in range(10) + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 LOAD_NAME 0 (x) + 3 LOAD_NAME 1 (range) + 6 LOAD_CONST 1 (10) + 9 CALL_FUNCTION 1 + 12 COMPARE_OP 6 (in) + 15 POP_TOP + 16 LOAD_CONST 0 (None) + 19 RETURN_VALUE +co_consts: +[0] None +[1] 10 Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_slice.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_slice.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,15 @@ +a[1:] + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 LOAD_NAME 0 (a) + 3 LOAD_CONST 1 (1) + 6 SLICE+1 + 7 POP_TOP + 8 LOAD_CONST 0 (None) + 11 RETURN_VALUE +co_consts: +[0] None +[1] 1 Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_while.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_while.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,54 @@ +index = 0 +while index < 10: + index += 1 + foo = 10 - index + if False: + break +else: + foo = 12 + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 LOAD_CONST 1 (0) + 3 STORE_NAME 0 (index) + + 2 6 SETUP_LOOP 56 (to 65) + >> 9 LOAD_NAME 0 (index) + 12 LOAD_CONST 2 (10) + 15 COMPARE_OP 0 (<) + 18 JUMP_IF_FALSE 36 (to 57) + 21 POP_TOP + + 3 22 LOAD_NAME 0 (index) + 25 LOAD_CONST 3 (1) + 28 INPLACE_ADD + 29 STORE_NAME 0 (index) + + 4 32 LOAD_CONST 2 (10) + 35 LOAD_NAME 0 (index) + 38 BINARY_SUBTRACT + 39 STORE_NAME 1 (foo) + + 5 42 LOAD_NAME 2 (False) + 45 JUMP_IF_FALSE 5 (to 53) + 48 POP_TOP + + 6 49 BREAK_LOOP + 50 JUMP_FORWARD 1 (to 54) + >> 53 POP_TOP + >> 54 JUMP_ABSOLUTE 9 + >> 57 POP_TOP + 58 POP_BLOCK + + 8 59 LOAD_CONST 4 (12) + 62 STORE_NAME 1 (foo) + >> 65 LOAD_CONST 0 (None) + 68 RETURN_VALUE +co_consts: +[0] None +[1] 0 +[2] 10 +[3] 1 +[4] 12 Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_whilelineno.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_whilelineno.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,117 @@ +while (a < b and c < d + and e < f): + pass + +while (a < b and + c < d + and e < f): + pass + +while (a < b + and c < d + and e < f): + pass + +while (a < b + and c < d and + e < f): + pass + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 SETUP_LOOP 44 (to 47) + >> 3 LOAD_NAME 0 (a) + 6 LOAD_NAME 1 (b) + 9 COMPARE_OP 0 (<) + 12 JUMP_IF_FALSE 23 (to 38) + 15 POP_TOP + 16 LOAD_NAME 2 (c) + 19 LOAD_NAME 3 (d) + 22 COMPARE_OP 0 (<) + 25 JUMP_IF_FALSE 10 (to 38) + 28 POP_TOP + + 2 29 LOAD_NAME 4 (e) + 32 LOAD_NAME 5 (f) + 35 COMPARE_OP 0 (<) + >> 38 JUMP_IF_FALSE 4 (to 45) + 41 POP_TOP + + 3 42 JUMP_ABSOLUTE 3 + >> 45 POP_TOP + 46 POP_BLOCK + + 5 >> 47 SETUP_LOOP 44 (to 94) + >> 50 LOAD_NAME 0 (a) + 53 LOAD_NAME 1 (b) + 56 COMPARE_OP 0 (<) + 59 JUMP_IF_FALSE 23 (to 85) + 62 POP_TOP + + 6 63 LOAD_NAME 2 (c) + 66 LOAD_NAME 3 (d) + 69 COMPARE_OP 0 (<) + 72 JUMP_IF_FALSE 10 (to 85) + 75 POP_TOP + + 7 76 LOAD_NAME 4 (e) + 79 LOAD_NAME 5 (f) + 82 COMPARE_OP 0 (<) + >> 85 JUMP_IF_FALSE 4 (to 92) + 88 POP_TOP + + 8 89 JUMP_ABSOLUTE 50 + >> 92 POP_TOP + 93 POP_BLOCK + + 10 >> 94 SETUP_LOOP 44 (to 141) + >> 97 LOAD_NAME 0 (a) + 100 LOAD_NAME 1 (b) + 103 COMPARE_OP 0 (<) + 106 JUMP_IF_FALSE 23 (to 132) + 109 POP_TOP + + 11 110 LOAD_NAME 2 (c) + 113 LOAD_NAME 3 (d) + 116 COMPARE_OP 0 (<) + 119 JUMP_IF_FALSE 10 (to 132) + 122 POP_TOP + + 12 123 LOAD_NAME 4 (e) + 126 LOAD_NAME 5 (f) + 129 COMPARE_OP 0 (<) + >> 132 JUMP_IF_FALSE 4 (to 139) + 135 POP_TOP + + 13 136 JUMP_ABSOLUTE 97 + >> 139 POP_TOP + 140 POP_BLOCK + + 15 >> 141 SETUP_LOOP 44 (to 188) + >> 144 LOAD_NAME 0 (a) + 147 LOAD_NAME 1 (b) + 150 COMPARE_OP 0 (<) + 153 JUMP_IF_FALSE 23 (to 179) + 156 POP_TOP + + 16 157 LOAD_NAME 2 (c) + 160 LOAD_NAME 3 (d) + 163 COMPARE_OP 0 (<) + 166 JUMP_IF_FALSE 10 (to 179) + 169 POP_TOP + + 17 170 LOAD_NAME 4 (e) + 173 LOAD_NAME 5 (f) + 176 COMPARE_OP 0 (<) + >> 179 JUMP_IF_FALSE 4 (to 186) + 182 POP_TOP + + 18 183 JUMP_ABSOLUTE 144 + >> 186 POP_TOP + 187 POP_BLOCK + >> 188 LOAD_CONST 0 (None) + 191 RETURN_VALUE +co_consts: +[0] None Added: pypy/dist/pypy/interpreter/pyparser/test/output/snippet_whitespaces.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/output/snippet_whitespaces.txt Sat Dec 29 20:04:48 2007 @@ -0,0 +1,20 @@ +l = [] +l . append ( 12 ) + +=============================================================================== +co_filename = '' +co_varnames = () +co_flags = 64 + 1 0 BUILD_LIST 0 + 3 STORE_NAME 0 (l) + + 2 6 LOAD_NAME 0 (l) + 9 LOAD_ATTR 1 (append) + 12 LOAD_CONST 1 (12) + 15 CALL_FUNCTION 1 + 18 POP_TOP + 19 LOAD_CONST 0 (None) + 22 RETURN_VALUE +co_consts: +[0] None +[1] 12 Modified: pypy/dist/pypy/interpreter/pyparser/test/test_astcompiler.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/test/test_astcompiler.py (original) +++ pypy/dist/pypy/interpreter/pyparser/test/test_astcompiler.py Sat Dec 29 20:04:48 2007 @@ -146,12 +146,6 @@ for expr in family: yield check_compile, expr -def test_snippets(): - for snippet_name in SNIPPETS: - filepath = os.path.join(os.path.dirname(__file__), 'samples', snippet_name) - source = file(filepath).read() - yield check_compile, source, 'exec' - STDLIB_PATH = os.path.dirname(os.__file__) def test_on_stdlib(): py.test.skip('too ambitious for now (and time consuming)') Added: pypy/dist/pypy/interpreter/pyparser/test/test_snippet_out.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/interpreter/pyparser/test/test_snippet_out.py Sat Dec 29 20:04:48 2007 @@ -0,0 +1,74 @@ +import os, sys, re +import dis, StringIO +from pypy.interpreter.pyparser.test import test_astbuilder +from pypy.interpreter.pyparser.test import test_astcompiler + +SNIPPETS = test_astbuilder.SNIPPETS + +# ____________________________________________________________ + +def generate_output(snippet_name, space): + filepath = os.path.join(os.path.dirname(__file__), + 'samples', snippet_name) + source = file(filepath).read() + ac_code = test_astcompiler.compile_with_astcompiler(source, + mode='exec', + space=space) + text = dump_code(ac_code, space) + return '%s\n%s\n%s' % (source, + '=' * 79, + text) + +def get_output_path(snippet_name): + outputpath = os.path.join(os.path.dirname(__file__), + 'output', snippet_name) + assert outputpath.endswith('.py') + outputpath = outputpath[:-3] + '.txt' + return outputpath + +# ____________________________________________________________ + +r_addr = re.compile(r" object at -?0x[0-9a-fA-F]+") + +def dump_code(co, space=None): + if not hasattr(co, 'co_consts'): + co = test_astcompiler.to_code(co, space) + saved = sys.stdout + try: + sys.stdout = f = StringIO.StringIO() + print 'co_filename = %r' % (co.co_filename,) + print 'co_varnames = %r' % (co.co_varnames,) + print 'co_flags = %r' % (co.co_flags,) + dis.dis(co) + print 'co_consts:' + for i, x in enumerate(co.co_consts): + if hasattr(x, 'co_code'): + sub = dump_code(x, space) + x = sub.replace('\n', '\n ').rstrip() + else: + x = repr(x) + print '[%d]' % i, x + finally: + sys.stdout = saved + text = f.getvalue() + text = r_addr.sub(" object at xxx", text) + return text + +# ____________________________________________________________ + +def setup_module(mod): + import pypy.conftest + mod.std_space = pypy.conftest.gettestobjspace('std') + +for snippet_name in SNIPPETS: + def _test_snippet(snippet_name=snippet_name): + result = generate_output(snippet_name, std_space) + print result + '===RESULT=END===' + f = open(get_output_path(snippet_name), 'r') + expected = f.read() + f.close() + if result != expected: + print expected + '===EXPECTED=END===' + assert False, "expected a different result!" + + globals()['test_' + snippet_name] = _test_snippet From arigo at codespeak.net Sat Dec 29 20:05:45 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 29 Dec 2007 20:05:45 +0100 (CET) Subject: [pypy-svn] r50178 - pypy/dist/pypy/interpreter/astcompiler Message-ID: <20071229190545.E974E16854E@codespeak.net> Author: arigo Date: Sat Dec 29 20:05:45 2007 New Revision: 50178 Modified: pypy/dist/pypy/interpreter/astcompiler/pycodegen.py Log: issue336 in-progress LOAD_GLOBAL None => LOAD_CONST None Modified: pypy/dist/pypy/interpreter/astcompiler/pycodegen.py ============================================================================== --- pypy/dist/pypy/interpreter/astcompiler/pycodegen.py (original) +++ pypy/dist/pypy/interpreter/astcompiler/pycodegen.py Sat Dec 29 20:05:45 2007 @@ -229,6 +229,9 @@ self._nameOp('DELETE', name) def _nameOp(self, prefix, name): + if name == 'None': # always use LOAD_CONST to load None + self.emitop_obj('LOAD_CONST', self.space.w_None) + return name = self.mangle(name) scope = self.scope.check_name(name) if scope == SC_LOCAL: From arigo at codespeak.net Sun Dec 30 11:52:47 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 Dec 2007 11:52:47 +0100 (CET) Subject: [pypy-svn] r50186 - pypy/branch/astcompilertests Message-ID: <20071230105247.6A1831684CB@codespeak.net> Author: arigo Date: Sun Dec 30 11:52:46 2007 New Revision: 50186 Added: pypy/branch/astcompilertests/ - copied from r50185, pypy/dist/ Log: A branch to write direct tests for the ast compiler without relying on getting exactly the same output as the stable (ha!) compiler from the stdlib. From arigo at codespeak.net Sun Dec 30 11:54:02 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 Dec 2007 11:54:02 +0100 (CET) Subject: [pypy-svn] r50187 - in pypy/branch/astcompilertests/pypy/interpreter/pyparser/test: . output Message-ID: <20071230105402.6A9081684CB@codespeak.net> Author: arigo Date: Sun Dec 30 11:54:01 2007 New Revision: 50187 Removed: pypy/branch/astcompilertests/pypy/interpreter/pyparser/test/output/ pypy/branch/astcompilertests/pypy/interpreter/pyparser/test/test_astcompiler.py Log: Kill kill. From arigo at codespeak.net Sun Dec 30 12:10:53 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 Dec 2007 12:10:53 +0100 (CET) Subject: [pypy-svn] r50189 - pypy/branch/astcompilertests/pypy/interpreter/astcompiler/test Message-ID: <20071230111053.7B7B0168525@codespeak.net> Author: arigo Date: Sun Dec 30 12:10:52 2007 New Revision: 50189 Added: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/test/test_compiler.py (contents, props changed) Log: Tests by running the compiled code objects with our interpreter. Added: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- (empty file) +++ pypy/branch/astcompilertests/pypy/interpreter/astcompiler/test/test_compiler.py Sun Dec 30 12:10:52 2007 @@ -0,0 +1,64 @@ +import py +from pypy.interpreter.astcompiler import misc, pycodegen +from pypy.interpreter.pyparser.test.test_astbuilder import source2ast +from pypy.interpreter.pycode import PyCode + +def compile_with_astcompiler(expr, mode, space): + ast = source2ast(expr, mode, space) + misc.set_filename('', ast) + if mode == 'exec': + Generator = pycodegen.ModuleCodeGenerator + elif mode == 'single': + Generator = pycodegen.InteractiveCodeGenerator + elif mode == 'eval': + Generator = pycodegen.ExpressionCodeGenerator + codegen = Generator(space, ast) + rcode = codegen.getCode() + assert isinstance(rcode, PyCode) + assert rcode.co_filename == '' + return rcode + + +class TestCompiler: + """These tests compile snippets of code and check them by + running them with our own interpreter. These are thus not + completely *unit* tests, but given that our interpreter is + pretty stable now it is the best way I could find to check + the compiler. + """ + + def run(self, source): + source = str(py.code.Source(source)) + space = self.space + code = compile_with_astcompiler(source, 'exec', space) + w_dict = space.newdict() + code.exec_code(space, w_dict, w_dict) + return w_dict + + def check(self, w_dict, evalexpr, expected): + # for now, we compile evalexpr with CPython's compiler but run + # it with our own interpreter to extract the data from w_dict + co_expr = compile(evalexpr, '', 'eval') + space = self.space + pyco_expr = PyCode._from_code(space, co_expr) + w_res = pyco_expr.exec_code(space, w_dict, w_dict) + res = space.str_w(space.repr(w_res)) + assert res == repr(expected) + + def test_argtuple_1(self): + w_g = self.run("""def f( x, (y,z) ): + return x,y,z + """) + self.check(w_g, "f((1,2),(3,4))", ((1,2),3,4)) + + def test_argtuple_2(self): + w_g = self.run("""def f( x, (y,(z,t)) ): + return x,y,z,t + """) + self.check(w_g, "f(1,(2,(3,4)))", (1,2,3,4)) + + def test_argtuple_3(self): + w_g = self.run("""def f( ((((x,),y),z),t), u ): + return x,y,z,t,u + """) + self.check(w_g, "f(((((1,),2),3),4),5)", (1,2,3,4,5)) From arigo at codespeak.net Sun Dec 30 12:44:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 Dec 2007 12:44:24 +0100 (CET) Subject: [pypy-svn] r50191 - pypy/branch/astcompilertests/pypy/interpreter/astcompiler/test Message-ID: <20071230114424.B4E281684D3@codespeak.net> Author: arigo Date: Sun Dec 30 12:44:23 2007 New Revision: 50191 Modified: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/test/test_compiler.py Log: More tests. Modified: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/branch/astcompilertests/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/branch/astcompilertests/pypy/interpreter/astcompiler/test/test_compiler.py Sun Dec 30 12:44:23 2007 @@ -1,6 +1,7 @@ import py from pypy.interpreter.astcompiler import misc, pycodegen from pypy.interpreter.pyparser.test.test_astbuilder import source2ast +from pypy.interpreter.pyparser.test import expressions from pypy.interpreter.pycode import PyCode def compile_with_astcompiler(expr, mode, space): @@ -45,20 +46,117 @@ res = space.str_w(space.repr(w_res)) assert res == repr(expected) - def test_argtuple_1(self): - w_g = self.run("""def f( x, (y,z) ): - return x,y,z - """) - self.check(w_g, "f((1,2),(3,4))", ((1,2),3,4)) - - def test_argtuple_2(self): - w_g = self.run("""def f( x, (y,(z,t)) ): - return x,y,z,t - """) - self.check(w_g, "f(1,(2,(3,4)))", (1,2,3,4)) - - def test_argtuple_3(self): - w_g = self.run("""def f( ((((x,),y),z),t), u ): - return x,y,z,t,u - """) - self.check(w_g, "f(((((1,),2),3),4),5)", (1,2,3,4,5)) + def simple_test(self, source, evalexpr, expected): + w_g = self.run(source) + self.check(w_g, evalexpr, expected) + + st = simple_test + + def test_argtuple(self): + yield (self.simple_test, "def f( x, (y,z) ): return x,y,z", + "f((1,2),(3,4))", ((1,2),3,4)) + yield (self.simple_test, "def f( x, (y,(z,t)) ): return x,y,z,t", + "f(1,(2,(3,4)))", (1,2,3,4)) + yield (self.simple_test, "def f(((((x,),y),z),t),u): return x,y,z,t,u", + "f(((((1,),2),3),4),5)", (1,2,3,4,5)) + + def test_constants(self): + for c in expressions.constants: + yield (self.simple_test, "x="+c, "x", eval(c)) + + def test_tuple_assign(self): + yield self.simple_test, "x,= 1,", "x", 1 + yield self.simple_test, "x,y = 1,2", "x,y", (1, 2) + yield self.simple_test, "x,y,z = 1,2,3", "x,y,z", (1, 2, 3) + yield self.simple_test, "x,y,z,t = 1,2,3,4", "x,y,z,t", (1, 2, 3, 4) + yield self.simple_test, "x,y,x,t = 1,2,3,4", "x,y,t", (3, 2, 4) + yield self.simple_test, "[x]= 1,", "x", 1 + yield self.simple_test, "[x,y] = [1,2]", "x,y", (1, 2) + yield self.simple_test, "[x,y,z] = 1,2,3", "x,y,z", (1, 2, 3) + yield self.simple_test, "[x,y,z,t] = [1,2,3,4]", "x,y,z,t", (1, 2, 3,4) + yield self.simple_test, "[x,y,x,t] = 1,2,3,4", "x,y,t", (3, 2, 4) + + def test_binary_operator(self): + for operator in ['+', '-', '*', '**', '/', '&', '|', '^', '//', + '<<', '>>', 'and', 'or']: + expected = eval("17 %s 5" % operator) + yield self.simple_test, "x = 17 %s 5" % operator, "x", expected + expected = eval("0 %s 11" % operator) + yield self.simple_test, "x = 0 %s 11" % operator, "x", expected + + def test_augmented_assignment(self): + for operator in ['+', '-', '*', '**', '/', '&', '|', '^', '//', + '<<', '>>']: + expected = eval("17 %s 5" % operator) + yield self.simple_test, "x = 17; x %s= 5" % operator, "x", expected + + def test_subscript(self): + yield self.simple_test, "d={2:3}; x=d[2]", "x", 3 + yield self.simple_test, "d={(2,):3}; x=d[2,]", "x", 3 + yield self.simple_test, "d={}; d[1]=len(d); x=d[len(d)]", "x", 0 + yield self.simple_test, "d={}; d[1]=3; del d[1]", "len(d)", 0 + + def test_attribute(self): + yield self.simple_test, """ + class A: + pass + a1 = A() + a2 = A() + a1.bc = A() + a1.bc.de = a2 + a2.see = 4 + a1.bc.de.see += 3 + x = a1.bc.de.see + """, 'x', 7 + + def test_slice(self): + decl = py.code.Source(""" + class A(object): + def __getitem__(self, x): + global got + got = x.start, x.stop, x.step + def __setitem__(self, x, y): + global set + set = x.start, x.stop, x.step + def __delitem__(self, x): + global deleted + deleted = x.start, x.stop, x.step + a = A() + """) + decl = str(decl) + '\n' + yield self.st, decl + "a[:]", "got", (None, None, None) + yield self.st, decl + "a[2:]", "got", (2, None, None) + yield self.st, decl + "a[:2]", "got", (None, 2, None) + yield self.st, decl + "a[4:7]", "got", (4, 7, None) + yield self.st, decl + "a[::]", "got", (None, None, None) + yield self.st, decl + "a[2::]", "got", (2, None, None) + yield self.st, decl + "a[:2:]", "got", (None, 2, None) + yield self.st, decl + "a[4:7:]", "got", (4, 7, None) + yield self.st, decl + "a[::3]", "got", (None, None, 3) + yield self.st, decl + "a[2::3]", "got", (2, None, 3) + yield self.st, decl + "a[:2:3]", "got", (None, 2, 3) + yield self.st, decl + "a[4:7:3]", "got", (4, 7, 3) + + def test_funccalls(self): + decl = py.code.Source(""" + def f(*args, **kwds): + kwds = kwds.items() + kwds.sort() + return list(args) + kwds + """) + decl = str(decl) + '\n' + yield self.st, decl + "x=f()", "x", [] + yield self.st, decl + "x=f(5)", "x", [5] + yield self.st, decl + "x=f(5, 6, 7, 8)", "x", [5, 6, 7, 8] + yield self.st, decl + "x=f(a=2, b=5)", "x", [('a',2), ('b',5)] + yield self.st, decl + "x=f(5, b=2, *[6,7])", "x", [5, 6, 7, ('b', 2)] + yield self.st, decl + "x=f(5, b=2, **{'a': 8})", "x", [5, ('a', 8), + ('b', 2)] + + def test_listmakers(self): + yield (self.st, + "l = [(j, i) for j in range(10) for i in range(j)" + + " if (i+j)%2 == 0 and i%3 == 0]", + "l", + [(2, 0), (4, 0), (5, 3), (6, 0), + (7, 3), (8, 0), (8, 6), (9, 3)]) From arigo at codespeak.net Sun Dec 30 13:09:37 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 Dec 2007 13:09:37 +0100 (CET) Subject: [pypy-svn] r50192 - pypy/branch/astcompilertests/pypy/interpreter/astcompiler/test Message-ID: <20071230120937.C7B271684C1@codespeak.net> Author: arigo Date: Sun Dec 30 13:09:36 2007 New Revision: 50192 Modified: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/test/test_compiler.py Log: More tests. Modified: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/branch/astcompilertests/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/branch/astcompilertests/pypy/interpreter/astcompiler/test/test_compiler.py Sun Dec 30 13:09:36 2007 @@ -78,7 +78,8 @@ def test_binary_operator(self): for operator in ['+', '-', '*', '**', '/', '&', '|', '^', '//', - '<<', '>>', 'and', 'or']: + '<<', '>>', 'and', 'or', '<', '>', '<=', '>=', + 'is', 'is not']: expected = eval("17 %s 5" % operator) yield self.simple_test, "x = 17 %s 5" % operator, "x", expected expected = eval("0 %s 11" % operator) @@ -114,28 +115,38 @@ class A(object): def __getitem__(self, x): global got - got = x.start, x.stop, x.step + got = x def __setitem__(self, x, y): global set - set = x.start, x.stop, x.step + set = x def __delitem__(self, x): global deleted - deleted = x.start, x.stop, x.step + deleted = x a = A() """) decl = str(decl) + '\n' - yield self.st, decl + "a[:]", "got", (None, None, None) - yield self.st, decl + "a[2:]", "got", (2, None, None) - yield self.st, decl + "a[:2]", "got", (None, 2, None) - yield self.st, decl + "a[4:7]", "got", (4, 7, None) - yield self.st, decl + "a[::]", "got", (None, None, None) - yield self.st, decl + "a[2::]", "got", (2, None, None) - yield self.st, decl + "a[:2:]", "got", (None, 2, None) - yield self.st, decl + "a[4:7:]", "got", (4, 7, None) - yield self.st, decl + "a[::3]", "got", (None, None, 3) - yield self.st, decl + "a[2::3]", "got", (2, None, 3) - yield self.st, decl + "a[:2:3]", "got", (None, 2, 3) - yield self.st, decl + "a[4:7:3]", "got", (4, 7, 3) + testcases = ['[:]', '[:,9]', '[8,:]', + '[2:]', '[2:,9]', '[8,2:]', + '[:2]', '[:2,9]', '[8,:2]', + '[4:7]', '[4:7,9]', '[8,4:7]', + '[::]', '[::,9]', '[8,::]', + '[2::]', '[2::,9]', '[8,2::]', + '[:2:]', '[:2:,9]', '[8,:2:]', + '[4:7:]', '[4:7:,9]', '[8,4:7:]', + '[::3]', '[::3,9]', '[8,::3]', + '[2::3]', '[2::3,9]', '[8,2::3]', + '[:2:3]', '[:2:3,9]', '[8,:2:3]', + '[4:7:3]','[4:7:3,9]','[8,4:7:3]', + ] + class Checker(object): + def __getitem__(self, x): + self.got = x + checker = Checker() + for testcase in testcases: + exec "checker" + testcase + yield self.st, decl + "a" + testcase, "got", checker.got + yield self.st, decl + "a" + testcase + ' = 5', "set", checker.got + yield self.st, decl + "del a" + testcase, "deleted", checker.got def test_funccalls(self): decl = py.code.Source(""" @@ -160,3 +171,87 @@ "l", [(2, 0), (4, 0), (5, 3), (6, 0), (7, 3), (8, 0), (8, 6), (9, 3)]) + + def test_genexprs(self): + yield (self.st, + "l = list((j, i) for j in range(10) for i in range(j)" + + " if (i+j)%2 == 0 and i%3 == 0)", + "l", + [(2, 0), (4, 0), (5, 3), (6, 0), + (7, 3), (8, 0), (8, 6), (9, 3)]) + + def test_comparisons(self): + yield self.st, "x = 3 in {3: 5}", "x", True + yield self.st, "x = 3 not in {3: 5}", "x", False + yield self.st, "t = True; x = t is True", "x", True + yield self.st, "t = True; x = t is False", "x", False + yield self.st, "t = True; x = t is None", "x", False + yield self.st, "n = None; x = n is True", "x", False + yield self.st, "n = None; x = n is False", "x", False + yield self.st, "n = None; x = n is None", "x", True + + def test_multiexpr(self): + yield self.st, "z = 2+3; x = y = z", "x,y,z", (5,5,5) + + def test_imports(self): + import os + yield self.st, "import sys", "sys.__name__", "sys" + yield self.st, "import sys as y", "y.__name__", "sys" + yield (self.st, "import sys, os", + "sys.__name__, os.__name__", ("sys", "os")) + yield (self.st, "import sys as x, os.path as y", + "x.__name__, y.__name__", ("sys", os.path.__name__)) + yield self.st, 'import os.path', "os.path.__name__", os.path.__name__ + yield (self.st, 'import os.path, sys', + "os.path.__name__, sys.__name__", (os.path.__name__, "sys")) + yield (self.st, 'import sys, os.path as osp', + "osp.__name__, sys.__name__", (os.path.__name__, "sys")) + yield (self.st, 'import os.path as osp', + "osp.__name__", os.path.__name__) + yield (self.st, 'from os import path', + "path.__name__", os.path.__name__) + yield (self.st, 'from os import path, sep', + "path.__name__, sep", (os.path.__name__, os.sep)) + yield (self.st, 'from os import path as p', + "p.__name__", os.path.__name__) + yield (self.st, 'from os import *', + "path.__name__, sep", (os.path.__name__, os.sep)) + + def test_if_stmts(self): + yield self.st, "a = 42\nif a > 10: a += 2", "a", 44 + yield self.st, "a=5\nif 0: a=7", "a", 5 + yield self.st, "a=5\nif 1: a=7", "a", 7 + yield self.st, "a=5\nif a and not not (a<10): a=7", "a", 7 + yield self.st, """ + lst = [] + for a in range(10): + if a < 3: + a += 20 + elif a > 3 and a < 8: + a += 30 + else: + a += 40 + lst.append(a) + """, "lst", [20, 21, 22, 43, 34, 35, 36, 37, 48, 49] + + def test_docstrings(self): + for source, expected in [ + ('''def foo(): return 1''', None), + ('''class foo: pass''', None), + ('''class foo: "foo"''', "foo"), + ('''def foo(): + """foo docstring""" + return 1 + ''', "foo docstring"), + ('''def foo(): + """foo docstring""" + a = 1 + """bar""" + return a + ''', "foo docstring"), + ('''def foo(): + """doc"""; print 1 + a=1 + ''', "doc"), + ]: + yield self.simple_test, source, "foo.__doc__", expected From arigo at codespeak.net Sun Dec 30 18:24:50 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 Dec 2007 18:24:50 +0100 (CET) Subject: [pypy-svn] r50209 - in pypy/branch/astcompilertests/pypy: interpreter interpreter/astcompiler interpreter/astcompiler/test objspace/std Message-ID: <20071230172450.E9F351684C0@codespeak.net> Author: arigo Date: Sun Dec 30 18:24:49 2007 New Revision: 50209 Added: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/opt.py (contents, props changed) Modified: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/pycodegen.py pypy/branch/astcompilertests/pypy/interpreter/astcompiler/test/test_compiler.py pypy/branch/astcompilertests/pypy/interpreter/pycode.py pypy/branch/astcompilertests/pypy/objspace/std/complexobject.py Log: Generate better bytecode for code like 'if x and y:'. Added: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/opt.py ============================================================================== --- (empty file) +++ pypy/branch/astcompilertests/pypy/interpreter/astcompiler/opt.py Sun Dec 30 18:24:49 2007 @@ -0,0 +1,73 @@ +from pypy.tool.pairtype import extendabletype +from pypy.interpreter.astcompiler import ast + +# Extra bytecode optimizations. The visitor pattern is a bit of a mess +# for these, so we simply stick new methods on the nodes. + +OPTIMIZE = True + +def is_constant_false(space, node): + return isinstance(node, ast.Const) and not space.is_true(node.value) + +def is_constant_true(space, node): + return isinstance(node, ast.Const) and space.is_true(node.value) + + +class __extend__(ast.Node): + __metaclass__ = extendabletype + + def opt_accept_jump_if(self, codegen, condition, target): + """Generate code equivalent to: + self.accept() + JUMP_IF_condition target + except that the value left on the stack afterwards (both if the + branch is taken or not) can be garbage. + """ + self.accept(codegen) + if condition: + codegen.emitop_block('JUMP_IF_TRUE', target) + else: + codegen.emitop_block('JUMP_IF_FALSE', target) + + +if OPTIMIZE: + + class __extend__(ast.Not): + __metaclass__ = extendabletype + + def opt_accept_jump_if(self, codegen, condition, target): + self.expr.opt_accept_jump_if(codegen, not condition, target) + + + class __extend__(ast.AbstractTest): + __metaclass__ = extendabletype + + def _accept_jump_if_any_is(self, codegen, condition, target): + # generate a "jump if any of the nodes' truth value is 'condition'" + garbage_on_stack = False + for node in self.nodes: + if garbage_on_stack: + codegen.emit('POP_TOP') + node.opt_accept_jump_if(codegen, condition, target) + garbage_on_stack = True + assert garbage_on_stack + + def opt_accept_jump_if(self, codegen, condition, target): + if condition == self.is_and: + # jump only if all the nodes' truth values are equal to + # 'condition' + end = codegen.newBlock() + self._accept_jump_if_any_is(codegen, not condition, end) + codegen.emitop_block('JUMP_FORWARD', target) + codegen.nextBlock(end) + else: + self._accept_jump_if_any_is(codegen, condition, target) + + + class __extend__(ast.And): + __metaclass__ = extendabletype + is_and = True + + class __extend__(ast.Or): + __metaclass__ = extendabletype + is_and = False Modified: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/pycodegen.py ============================================================================== --- pypy/branch/astcompilertests/pypy/interpreter/astcompiler/pycodegen.py (original) +++ pypy/branch/astcompilertests/pypy/interpreter/astcompiler/pycodegen.py Sun Dec 30 18:24:49 2007 @@ -13,6 +13,8 @@ CO_NEWLOCALS, CO_NESTED, CO_GENERATOR, CO_GENERATOR_ALLOWED, \ CO_FUTURE_DIVISION, CO_FUTURE_WITH_STATEMENT from pypy.interpreter.pyparser.error import SyntaxError +from pypy.interpreter.astcompiler.opt import is_constant_false +from pypy.interpreter.astcompiler.opt import is_constant_true # drop VERSION dependency since it the ast transformer for 2.4 doesn't work with 2.3 anyway VERSION = 2 @@ -123,18 +125,6 @@ mtime = struct.pack('" % (w_self.realval, w_self.imagval) From cfbolz at codespeak.net Sun Dec 30 18:30:38 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 30 Dec 2007 18:30:38 +0100 (CET) Subject: [pypy-svn] r50211 - pypy/extradoc/sprintinfo/leysin-winter-2008 Message-ID: <20071230173038.AC119168473@codespeak.net> Author: cfbolz Date: Sun Dec 30 18:30:37 2007 New Revision: 50211 Added: pypy/extradoc/sprintinfo/leysin-winter-2008/ pypy/extradoc/sprintinfo/leysin-winter-2008/people.txt (contents, props changed) Log: start a people file for the leysin sprint Added: pypy/extradoc/sprintinfo/leysin-winter-2008/people.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/leysin-winter-2008/people.txt Sun Dec 30 18:30:37 2007 @@ -0,0 +1,52 @@ + +People coming to the Leysin sprint Winter 2008 +================================================== + +People who have a ``?`` in their arrive/depart or accomodation +column are known to be coming but there are no details +available yet from them. + + +==================== ============== ===================== + Name Arrive/Depart Accomodation +==================== ============== ===================== +Armin Rigo -- private +Carl Friedrich Bolz 12th-19th? Ermina +==================== ============== ===================== + +People on the following list were present at previous sprints: + +==================== ============== ===================== + Name Arrive/Depart Accomodation +==================== ============== ===================== +Christian Tismer ? ? +Michael Hudson ? ? +Anders Lehmann ? ? +Niklaus Haldimann ? ? +Lene Wagner ? ? +Amaury Forgeot d'Arc ? ? +Valentino Volonghi ? ? +Boris Feigin ? ? +Andrew Thompson ? ? +Bert Freudenberg ? ? +Beatrice Duering ? ? +Richard Emslie ? ? +Johan Hahn ? ? +Stephan Diehl ? ? +Niko Matsakis ? ? +Alexander Schremmer ? ? +Anders Chrigstroem ? ? +Samuele Pedroni ? ? +Michael Hudson ? ? +Niko Matsakis ? ? +Antonio Cuni ? ? +Maciej Fijalkowski ? ? +Eric van Riet Paap ? ? +Jacob Hallen ? ? +Laura Creighton ? ? +Holger Krekel ? ? +Guido Wesdorp ? ? +Leonardo Santagada ? ? +Alexandre Fayolle ? ? +Sylvain Th?nault ? ? +==================== ============== ===================== From cfbolz at codespeak.net Sun Dec 30 18:40:57 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 30 Dec 2007 18:40:57 +0100 (CET) Subject: [pypy-svn] r50212 - pypy/extradoc/sprintinfo/leysin-winter-2008 Message-ID: <20071230174057.1208C168473@codespeak.net> Author: cfbolz Date: Sun Dec 30 18:40:56 2007 New Revision: 50212 Added: pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt (contents, props changed) Log: first stab at the announcement Added: pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt Sun Dec 30 18:40:56 2007 @@ -0,0 +1,63 @@ +===================================================================== + PyPy Leysin Winter Sprint (12-19th January 2008) +===================================================================== +.. image:: http://www.ermina.ch/002.JPG + +XXX nice introduction + +------------------------------ +Goals and topics of the sprint +------------------------------ + +* Like previous winter, the main side goal is to have fun in winter + sports :-) We can take a couple of days off for ski; at this time of + year, ski days end before 4pm, which still leaves plenty of time + to recover (er, I mean hack). + +* the overall idea of the sprint is to continue working on making PyPy ready + for general use. + +XXX more tasks + +----------------------- +Location & Accomodation +----------------------- + +Leysin, Switzerland, "same place as before". Let me refresh your +memory: both the sprint venue and the lodging will be in a very spacious +pair of chalets built specifically for bed & breakfast: +http://www.ermina.ch/. The place has a baseline ADSL Internet connexion +(600Kb) with wireless installed. You can of course arrange your own +lodging anywhere (so long as you are in Leysin, you cannot be more than a +15 minute walk away from the sprint venue), but I definitely recommend +lodging there too -- you won't find a better view anywhere else (though you +probably won't get much worse ones easily, either :-) + +I made pre-reservations in the Chalet, so please *confirm* quickly that +you are coming so that we can adjust the reservations as appropriate. +The rate so far has been 60 CHF a night all included in 2-person rooms, +with breakfast. There are larger rooms too (less expensive, possibly +more available too) and the possibility to get a single room if you +really want to. + +Please register by svn: + + http://codespeak.net/svn/pypy/extradoc/sprintinfo/leysin-winter-2008/people.txt + +or on the pypy-sprint mailing list if you do not yet have check-in rights: + + http://codespeak.net/mailman/listinfo/pypy-sprint + +You need a Swiss-to-(insert country here) power adapter. There will be +some Swiss-to-EU adapters around - bring a EU-format power strip if you +have one. + +----------- +Exact times +----------- + +XXX is this correct? +Officially, 12th-19th January 2008. Both dates are flexible, you can +arrive or leave earlier or later. We will give introductions and +tutorials on the 13th, in the morning if everybody is there, or in the +afternoon otherwise. From arigo at codespeak.net Sun Dec 30 18:43:16 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 Dec 2007 18:43:16 +0100 (CET) Subject: [pypy-svn] r50215 - pypy/branch/astcompilertests/pypy/interpreter/astcompiler Message-ID: <20071230174316.5C945168471@codespeak.net> Author: arigo Date: Sun Dec 30 18:43:16 2007 New Revision: 50215 Removed: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/syntax.py Modified: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/pycodegen.py Log: Kill old file. Modified: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/pycodegen.py ============================================================================== --- pypy/branch/astcompilertests/pypy/interpreter/astcompiler/pycodegen.py (original) +++ pypy/branch/astcompilertests/pypy/interpreter/astcompiler/pycodegen.py Sun Dec 30 18:43:16 2007 @@ -4,7 +4,6 @@ import struct import sys -#from pypy.interpreter.astcompiler import ast, parse, walk, syntax from pypy.interpreter.astcompiler import ast from pypy.interpreter.astcompiler import pyassem, misc, future, symbols from pypy.interpreter.astcompiler.consts import SC_LOCAL, SC_GLOBAL, \ @@ -74,7 +73,7 @@ def _get_tree(self): tree = parse(self.source, self.mode) misc.set_filename(self.filename, tree) - syntax.check(tree) + #syntax.check(tree) return tree def compile(self): From arigo at codespeak.net Sun Dec 30 18:48:18 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 Dec 2007 18:48:18 +0100 (CET) Subject: [pypy-svn] r50216 - pypy/branch/astcompilertests/pypy/interpreter/pyparser/test Message-ID: <20071230174818.9F07F168473@codespeak.net> Author: arigo Date: Sun Dec 30 18:48:18 2007 New Revision: 50216 Removed: pypy/branch/astcompilertests/pypy/interpreter/pyparser/test/test_snippet_out.py Modified: pypy/branch/astcompilertests/pypy/interpreter/pyparser/test/stdlib_testall.py Log: Leftovers. Modified: pypy/branch/astcompilertests/pypy/interpreter/pyparser/test/stdlib_testall.py ============================================================================== --- pypy/branch/astcompilertests/pypy/interpreter/pyparser/test/stdlib_testall.py (original) +++ pypy/branch/astcompilertests/pypy/interpreter/pyparser/test/stdlib_testall.py Sun Dec 30 18:48:18 2007 @@ -1,3 +1,4 @@ +# XXX this code needs to be updated import autopath import py from test_astcompiler import compile_with_astcompiler From arigo at codespeak.net Sun Dec 30 18:52:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 Dec 2007 18:52:09 +0100 (CET) Subject: [pypy-svn] r50218 - pypy/branch/astcompilertests/pypy/interpreter/astcompiler Message-ID: <20071230175209.55F7D168473@codespeak.net> Author: arigo Date: Sun Dec 30 18:52:08 2007 New Revision: 50218 Modified: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/visitor.py Log: Let's keep the old 'def walk()' uncommented just before the new 'def walk()' just to see how the API changed. Good idea. Modified: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/visitor.py ============================================================================== --- pypy/branch/astcompilertests/pypy/interpreter/astcompiler/visitor.py (original) +++ pypy/branch/astcompilertests/pypy/interpreter/astcompiler/visitor.py Sun Dec 30 18:52:08 2007 @@ -97,14 +97,14 @@ # XXX this is an API change -_walker = ASTVisitor -def walk(tree, visitor, walker=None, verbose=None): - if walker is None: - walker = _walker() - if verbose is not None: - walker.VERBOSE = verbose - walker.preorder(tree, visitor) - return walker.visitor +##_walker = ASTVisitor +##def walk(tree, visitor, walker=None, verbose=None): +## if walker is None: +## walker = _walker() +## if verbose is not None: +## walker.VERBOSE = verbose +## walker.preorder(tree, visitor) +## return walker.visitor def walk(tree, visitor, verbose=-1): tree.accept(visitor) From arigo at codespeak.net Sun Dec 30 19:53:56 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 Dec 2007 19:53:56 +0100 (CET) Subject: [pypy-svn] r50219 - pypy/branch/astcompilertests/pypy/interpreter/astcompiler Message-ID: <20071230185356.6403B1684DC@codespeak.net> Author: arigo Date: Sun Dec 30 19:53:55 2007 New Revision: 50219 Modified: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/astgen.py Log: ast.py is generated. A manual edit gets lost after regeneration. Modified: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/astgen.py ============================================================================== --- pypy/branch/astcompilertests/pypy/interpreter/astcompiler/astgen.py (original) +++ pypy/branch/astcompilertests/pypy/interpreter/astcompiler/astgen.py Sun Dec 30 19:53:55 2007 @@ -723,6 +723,9 @@ lst = self.getChildNodes() return space.newlist( [ space.wrap( it ) for it in lst ] ) + def get_value(self): + pass + def descr_node_accept( space, w_self, w_visitor ): return space.call_method( w_visitor, 'visitNode', w_self ) From arigo at codespeak.net Sun Dec 30 19:54:29 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 Dec 2007 19:54:29 +0100 (CET) Subject: [pypy-svn] r50220 - pypy/branch/astcompilertests/pypy/interpreter/astcompiler Message-ID: <20071230185429.6CB3A1684D2@codespeak.net> Author: arigo Date: Sun Dec 30 19:54:28 2007 New Revision: 50220 Modified: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/ast.py pypy/branch/astcompilertests/pypy/interpreter/astcompiler/ast.txt Log: Bug. Modified: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/ast.py ============================================================================== --- pypy/branch/astcompilertests/pypy/interpreter/astcompiler/ast.py (original) +++ pypy/branch/astcompilertests/pypy/interpreter/astcompiler/ast.py Sun Dec 30 19:54:28 2007 @@ -5429,15 +5429,17 @@ if expr1 is not None: - newhandlers.append(expr1.mutate(visitor)) + expr1 = expr1.mutate(visitor) if expr2 is not None: - newhandlers.append(expr2.mutate(visitor)) + expr2 = expr2.mutate(visitor) if body is not None: - newhandlers.append(body.mutate(visitor)) + body = body.mutate(visitor) + + newhandlers.append((expr1, expr2, body)) self.handlers[:] = newhandlers Modified: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/ast.txt ============================================================================== --- pypy/branch/astcompilertests/pypy/interpreter/astcompiler/ast.txt (original) +++ pypy/branch/astcompilertests/pypy/interpreter/astcompiler/ast.txt Sun Dec 30 19:54:28 2007 @@ -143,11 +143,12 @@ newhandlers = [] for expr1, expr2, body in self.handlers: if expr1 is not None: - newhandlers.append(expr1.mutate(visitor)) + expr1 = expr1.mutate(visitor) if expr2 is not None: - newhandlers.append(expr2.mutate(visitor)) + expr2 = expr2.mutate(visitor) if body is not None: - newhandlers.append(body.mutate(visitor)) + body = body.mutate(visitor) + newhandlers.append((expr1, expr2, body)) self.handlers[:] = newhandlers flatten_nodes(Dict.items): From arigo at codespeak.net Sun Dec 30 19:57:43 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 30 Dec 2007 19:57:43 +0100 (CET) Subject: [pypy-svn] r50221 - in pypy/branch/astcompilertests/pypy/interpreter: . astcompiler astcompiler/test Message-ID: <20071230185743.935991684D2@codespeak.net> Author: arigo Date: Sun Dec 30 19:57:42 2007 New Revision: 50221 Modified: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/opt.py pypy/branch/astcompilertests/pypy/interpreter/astcompiler/test/test_compiler.py pypy/branch/astcompilertests/pypy/interpreter/pycompiler.py Log: More optimizations, mostly constant-folding. Modified: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/opt.py ============================================================================== --- pypy/branch/astcompilertests/pypy/interpreter/astcompiler/opt.py (original) +++ pypy/branch/astcompilertests/pypy/interpreter/astcompiler/opt.py Sun Dec 30 19:57:42 2007 @@ -1,8 +1,11 @@ from pypy.tool.pairtype import extendabletype from pypy.interpreter.astcompiler import ast +from pypy.interpreter.error import OperationError -# Extra bytecode optimizations. The visitor pattern is a bit of a mess -# for these, so we simply stick new methods on the nodes. +# Extra bytecode optimizations. Two styles: a visitor (a mutator really) +# that modifies the AST tree in-place, and hooks for pycodegen to produce +# better bytecode. The visitor pattern is a bit of a mess for the latter, +# so we simply stick new methods on the nodes. OPTIMIZE = True @@ -30,7 +33,11 @@ codegen.emitop_block('JUMP_IF_FALSE', target) -if OPTIMIZE: +if not OPTIMIZE: + def optimize_ast_tree(space, tree): + return tree + +else: class __extend__(ast.Not): __metaclass__ = extendabletype @@ -71,3 +78,169 @@ class __extend__(ast.Or): __metaclass__ = extendabletype is_and = False + + + class OptimizerMutator(ast.ASTVisitor): + def __init__(self, space): + self.space = space + + def default(self, node): + for child in node.getChildNodes(): + child.accept(self) + return node + + def _visitUnaryOp(self, node, constant_fold): + expr = node.expr + if isinstance(expr, ast.Const): + try: + w_newvalue = constant_fold(self.space, expr.value) + except OperationError: + pass + else: + return ast.Const(w_newvalue) + return node + + def visitBackquote(self, node): + return self._visitUnaryOp(node, _spacewrapper1('repr')) + def visitInvert(self, node): + return self._visitUnaryOp(node, _spacewrapper1('invert')) + def visitNot(self, node): + return self._visitUnaryOp(node, _spacewrapper1('not_')) + def visitUnaryAdd(self, node): + return self._visitUnaryOp(node, _spacewrapper1('pos')) + def visitUnarySub(self, node): + return self._visitUnaryOp(node, _spacewrapper1('neg')) + + def _visitBinaryOp(self, node, constant_fold): + left = node.left + right = node.right + if isinstance(left, ast.Const) and isinstance(right, ast.Const): + try: + w_newvalue = constant_fold(self.space, left.value, + right.value) + except OperationError: + pass + else: + # to avoid creating too large .pyc files, we don't + # constant-fold operations that create long sequences, + # like '(5,) * 500'. This is the same as CPython. + try: + size = self.space.int_w(self.space.len(w_newvalue)) + except OperationError: + size = -1 + if size <= 20: + return ast.Const(w_newvalue) + return node + + def visitAdd(self, node): + return self._visitBinaryOp(node, _spacewrapper2('add')) + #def visitDiv(self, node): + # cannot constant-fold because the result depends on + # whether future division is enabled or not + def visitFloorDiv(self, node): + return self._visitBinaryOp(node, _spacewrapper2('floordiv')) + def visitLeftShift(self, node): + return self._visitBinaryOp(node, _spacewrapper2('lshift')) + def visitMod(self, node): + return self._visitBinaryOp(node, _spacewrapper2('mod')) + def visitMul(self, node): + return self._visitBinaryOp(node, _spacewrapper2('mul')) + def visitPower(self, node): + return self._visitBinaryOp(node, constant_fold_pow) + def visitRightShift(self, node): + return self._visitBinaryOp(node, _spacewrapper2('rshift')) + def visitSub(self, node): + return self._visitBinaryOp(node, _spacewrapper2('sub')) + + #def visitSubscript(self, node): XXX + + def _visitBitOp(self, node, constant_fold): + while len(node.nodes) >= 2: + left = node.nodes[0] + if not isinstance(left, ast.Const): + return node # done + right = node.nodes[1] + if not isinstance(right, ast.Const): + return node # done + try: + w_newvalue = constant_fold(self.space, left.value, + right.value) + except OperationError: + return node # done + del node.nodes[1] + node.nodes[0] = ast.Const(w_newvalue) + else: + # if reduced to a single node, just returns it + return node.nodes[0] + + def visitBitand(self, node): + return self._visitBitOp(node, _spacewrapper2('and_')) + def visitBitor(self, node): + return self._visitBitOp(node, _spacewrapper2('or_')) + def visitBitxor(self, node): + return self._visitBitOp(node, _spacewrapper2('xor')) + + #def visitCompare(self, node): XXX + + def _visitAbstractTest(self, node, is_and): + # Logic for And nodes: + # 1. if any of the nodes is True, it can be removed + # 2. if any of the nodes is False, all nodes after it can be killed + # For Or nodes, the conditions are reversed. + i = 0 + nodes = node.nodes + while i < len(nodes) - 1: + subnode = nodes[i] + if isinstance(subnode, ast.Const): + if (not self.space.is_true(subnode.value)) == is_and: + del nodes[i+1:] # case 2. + break + else: + del nodes[i] + continue # case 1. + i += 1 + if len(nodes) > 1: + return node + else: + return nodes[0] # a single item left + + def visitAnd(self, node): + return self._visitAbstractTest(node, True) + def visitOr(self, node): + return self._visitAbstractTest(node, False) + + def visitTuple(self, node): + consts_w = [] + for subnode in node.nodes: + if not isinstance(subnode, ast.Const): + return node # not all constants + consts_w.append(subnode.value) + return ast.Const(self.space.newtuple(consts_w)) + + + def _spacewrapper1(name): + """Make a wrapper around the method: space.(w_x) + to avoid taking bound method objects, which creates issues + depending on the details of the real space method, e.g. default args. + """ + def constant_fold(space, w_x): + return getattr(space, name)(w_x) + return constant_fold + _spacewrapper1._annspecialcase_ = 'specialize:memo' + + def _spacewrapper2(name): + """Make a wrapper around the method: space.(w_x, w_y) + to avoid taking bound method objects, which creates issues + depending on the details of the real space method, e.g. default args. + """ + def constant_fold(space, w_x, w_y): + return getattr(space, name)(w_x, w_y) + return constant_fold + _spacewrapper2._annspecialcase_ = 'specialize:memo' + + def constant_fold_pow(space, w_x, w_y): + return space.pow(w_x, w_y, space.w_None) + + + def optimize_ast_tree(space, tree): + return tree.mutate(OptimizerMutator(space)) Modified: pypy/branch/astcompilertests/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/branch/astcompilertests/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/branch/astcompilertests/pypy/interpreter/astcompiler/test/test_compiler.py Sun Dec 30 19:57:42 2007 @@ -1,5 +1,5 @@ import py -from pypy.interpreter.astcompiler import misc, pycodegen +from pypy.interpreter.astcompiler import misc, pycodegen, opt from pypy.interpreter.pyparser.test.test_astbuilder import source2ast from pypy.interpreter.pyparser.test import expressions from pypy.interpreter.pycode import PyCode @@ -7,6 +7,7 @@ def compile_with_astcompiler(expr, mode, space): ast = source2ast(expr, mode, space) misc.set_filename('', ast) + ast = opt.optimize_ast_tree(space, ast) if mode == 'exec': Generator = pycodegen.ModuleCodeGenerator elif mode == 'single': Modified: pypy/branch/astcompilertests/pypy/interpreter/pycompiler.py ============================================================================== --- pypy/branch/astcompilertests/pypy/interpreter/pycompiler.py (original) +++ pypy/branch/astcompilertests/pypy/interpreter/pycompiler.py Sun Dec 30 19:57:42 2007 @@ -233,6 +233,7 @@ from pypy.interpreter.astcompiler.pycodegen import InteractiveCodeGenerator from pypy.interpreter.astcompiler.pycodegen import ExpressionCodeGenerator from pypy.interpreter.astcompiler.ast import Node + from pypy.interpreter.astcompiler import opt from pyparser.astbuilder import AstBuilder from pypy.interpreter.pycode import PyCode from pypy.interpreter.function import Function @@ -254,6 +255,8 @@ raise OperationError(space.w_SyntaxError, e.wrap_info(space, filename)) + ast_tree = opt.optimize_ast_tree(space, ast_tree) + if not space.is_w(self.w_compile_hook, space.w_None): try: w_ast_tree = space.call_function(self.w_compile_hook, From fijal at codespeak.net Mon Dec 31 14:42:37 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 31 Dec 2007 14:42:37 +0100 (CET) Subject: [pypy-svn] r50223 - pypy/dist/pypy/module/zipimport Message-ID: <20071231134237.05458168453@codespeak.net> Author: fijal Date: Mon Dec 31 14:42:36 2007 New Revision: 50223 Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py Log: Fix zipimport failures. Modified: pypy/dist/pypy/module/zipimport/interp_zipimport.py ============================================================================== --- pypy/dist/pypy/module/zipimport/interp_zipimport.py (original) +++ pypy/dist/pypy/module/zipimport/interp_zipimport.py Mon Dec 31 14:42:36 2007 @@ -42,11 +42,33 @@ space.setattr(w_mod, w('__loader__'), space.wrap(self)) return result + def check_newer_pyfile(self, space, filename, timestamp): + w = space.wrap + try: + w_info = space.call_function(space.getattr(self.w_dir, + w('getinfo')), w(filename)) + w_all = space.getattr(w_info, w('date_time')) + except OperationError, e: + # in either case, this is a fallback + return False + else: + w_mktime = space.getattr(space.getbuiltinmodule('time'), + w('mktime')) + # XXX this is incredible fishing around module limitations + # in order to compare timestamps of .py and .pyc files + all = space.unpackiterable(w_all) + all += [w(0), w(1), w(-1)] + mtime = int(space.float_w(space.call_function(w_mktime, space.newtuple(all)))) + return mtime > timestamp + def import_pyc_file(self, space, modname, filename, w_buf, pkgpath): w = space.wrap buf = space.str_w(w_buf) magic = importing._get_long(buf[:4]) timestamp = importing._get_long(buf[4:8]) + if self.check_newer_pyfile(space, filename[:-1], timestamp): + return self.import_py_file(space, modname, filename[:-1], w_buf, + pkgpath) buf = buf[8:] # XXX ugly copy, should use sequential read instead w_mod = w(Module(space, w(modname))) real_name = self.name + os.path.sep + filename @@ -100,16 +122,16 @@ pkgpath = None if compiled: return self.import_pyc_file(space, fullname, fname, - w_buf, pkgpath) + w_buf, pkgpath) else: return self.import_py_file(space, fullname, fname, w_buf, pkgpath) except OperationError, e: last_exc = e w_mods = space.sys.get('modules') - space.call_method(w_mods,'pop', w(fullname), space.w_None) + space.call_method(w_mods, 'pop', w(fullname), space.w_None) if last_exc: - raise OperationError(w_ZipImportError, last_exc.w_value) + raise OperationError(space.w_ImportError, last_exc.w_value) # should never happen I think return space.w_None load_module.unwrap_spec = ['self', ObjSpace, str] @@ -119,8 +141,8 @@ filename = filename.replace(os.path.sep, ZIPSEP) w = space.wrap try: - return space.call(space.getattr(self.w_dir, w('read')), - space.newlist([w(filename)])) + return space.call_function(space.getattr(self.w_dir, w('read')), + w(filename)) except OperationError, e: raise OperationError(space.w_IOError, e.w_value) get_data.unwrap_spec = ['self', ObjSpace, str] From fijal at codespeak.net Mon Dec 31 14:43:46 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 31 Dec 2007 14:43:46 +0100 (CET) Subject: [pypy-svn] r50224 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20071231134346.1B1E9168453@codespeak.net> Author: fijal Date: Mon Dec 31 14:43:45 2007 New Revision: 50224 Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Log: Rough implementation of rffi callbacks. Does not include: * exception handling * GIL handling * stackless explosion etc... Modified: pypy/dist/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rffi.py Mon Dec 31 14:43:45 2007 @@ -11,8 +11,12 @@ from pypy.tool.sourcetools import func_with_new_name from pypy.rpython.tool.rfficache import platform from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.translator.backendopt.canraise import RaiseAnalyzer import os +class UnhandledRPythonException(Exception): + pass + class CConstant(Symbolic): """ A C-level constant, maybe #define, rendered directly. """ @@ -26,6 +30,13 @@ def lltype(self): return self.TP +def isfunctype(TP): + """ Evil hack to get rid of flow objspace inability + to accept .TO when TP is not a pointer + """ + return isinstance(TP, lltype.Ptr) and isinstance(TP.TO, lltype.FuncType) +isfunctype._annspecialcase_ = 'specialize:memo' + def llexternal(name, args, result, _callable=None, compilation_info=ExternalCompilationInfo(), sandboxsafe=False, threadsafe='auto', @@ -92,6 +103,8 @@ # XXX leaks if a str2charp() fails with MemoryError # and was not the first in this function freeme = arg + elif isfunctype(TARGET): + arg = _callback(arg, TARGET) else: SOURCE = lltype.typeOf(arg) if SOURCE != TARGET: @@ -121,6 +134,7 @@ wrapper._always_inline_ = True # for debugging, stick ll func ptr to that wrapper._ptr = funcptr + return func_with_new_name(wrapper, name) AroundFnPtr = lltype.Ptr(lltype.FuncType([], lltype.Void)) @@ -132,6 +146,54 @@ aroundstate = AroundState() aroundstate._freeze_() +def _callback(arg, TP): + return lltype.functionptr(TP.TO, arg.func_name, _callable=arg) +_callback._annspecialcase_ = 'specialize:arg(1)' + +class Entry(ExtRegistryEntry): + _about_ = _callback + _CACHE = {} + + def compute_result_annotation(self, s_pbc, s_TP): + assert s_TP.is_constant() + assert s_pbc.is_constant() + # XXX in general this can be non-constant, but get_unique_llfn + # will not work in this case + TP = s_TP.const + bk = self.bookkeeper + args_s = [annmodel.lltype_to_annotation(ll_arg) for ll_arg in TP.TO.ARGS] + res = bk.emulate_pbc_call(s_pbc, s_pbc, args_s) + return annmodel.SomePtr(TP) + + # this is some wrapper creation for handling exceptions. + # I think this is ugly and better way is needed for that, + # but we definitely need a way to express exception raising inside + # the callback function + + #def _get_or_create_wrapper_pbc(self, bk, func): + # try: + # return self._CACHE[func] + # except: + # def wrapper(*args): + # try: + # return func(*args) + # except Exception, e: + # os.write(2, "Unhandled Fatal RPython exception %s in callback" % str(e)) + # return 0 + # # we ignore exception here, we can exit the program as well + # # not sure what is the best way + # s_pbc = annmodel.SomePBC([bk.getdesc(wrapper)]) + # self._CACHE[func] = s_pbc + # return s_pbc + + def specialize_call(self, hop): + #hop.exception_cannot_occur() + ## XXX fish a bit to have a wrapper here, not sure if annmixlevel + ## is not waaaay better here + #repr = hop.rtyper.getrepr(self._CACHE[hop.args_s[0].const]) + repr = hop.args_r[0] + return repr.get_unique_llfn() + # ____________________________________________________________ TYPES = [] @@ -219,6 +281,10 @@ return lltype.Ptr(CArray(tp)) CArray._annspecialcase_ = 'specialize:memo' +def CCallback(args, res): + return lltype.Ptr(lltype.FuncType(args, res)) +CCallback._annspecialcase_ = 'specialize:memo' + def COpaque(name, hints=None, compilation_info=None): if compilation_info is None: compilation_info = ExternalCompilationInfo() @@ -297,6 +363,9 @@ # void * - for now, represented as char * VOIDP = lltype.Ptr(lltype.Array(lltype.Char, hints={'nolength': True})) +# void ** +VOIDPP = CArrayPtr(VOIDP) + # char * CCHARP = lltype.Ptr(lltype.Array(lltype.Char, hints={'nolength': True})) Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Mon Dec 31 14:43:45 2007 @@ -709,3 +709,48 @@ res = f() assert res == 16 assert g() == "c" + + def test_c_callback(self): + c_source = py.code.Source(""" + int eating_callback(int arg, int(*call)(int)) + { + return call(arg); + } + """) + + eci = ExternalCompilationInfo(separate_module_sources=[c_source]) + + args = [rffi.INT, rffi.CCallback([rffi.INT], rffi.INT)] + eating_callback = rffi.llexternal('eating_callback', args, rffi.INT, + compilation_info=eci) + + def g(i): + return i + 3 + + def f(): + return eating_callback(3, g) + + assert f() == 6 + + def test_qsort(self): + TP = rffi.CArrayPtr(rffi.INT) + a = lltype.malloc(TP.TO, 5, flavor='raw') + a[0] = 5 + a[1] = 3 + a[2] = 2 + a[3] = 1 + a[4] = 4 + + def compare(a, b): + if a[0] > b[0]: + return 1 + else: + return -1 + + CALLBACK = rffi.CCallback([rffi.VOIDP, rffi.VOIDP], rffi.INT) + qsort = rffi.llexternal('qsort', [rffi.VOIDP, rffi.INT, + rffi.INT, CALLBACK], lltype.Void) + + qsort(rffi.cast(rffi.VOIDP, a), 5, rffi.sizeof(rffi.INT), compare) + for i in range(5): + assert a[i] == i + 1 Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_rffi.py Mon Dec 31 14:43:45 2007 @@ -319,6 +319,48 @@ assert fn() == 16 gn = self.compile(g, []) assert gn() == "c" + + def test_c_callback(self): + h_source = py.code.Source(""" + int eating_callback(int arg, int(*call)(int)) + { + return call(arg); + } + """) + + h_include = udir.join('callback.h') + h_include.write(h_source) + + eci = ExternalCompilationInfo(includes=['callback.h'], + include_dirs=[str(udir)]) + + args = [INT, CCallback([INT], INT)] + eating_callback = llexternal('eating_callback', args, INT, + compilation_info=eci) + + def g(i): + return i + 3 + + def z(i): + if i: + raise ValueError() + else: + return 0 + + def f(): + return eating_callback(3, g) + + fn = self.compile(f, []) + assert fn() == 6 + + def z2(): + return eating_callback(3, z) + + # this should complain if there are unhandled exceptions inside + # callbacks, or complain if really raising exception + #fn = self.compile(z2, []) + #fn() + #raises(UnhandledRPythonException, "self.compile(z2, [])") class TestRffiInternals: def test_struct_create(self): From fijal at codespeak.net Mon Dec 31 14:46:48 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 31 Dec 2007 14:46:48 +0100 (CET) Subject: [pypy-svn] r50225 - pypy/dist/pypy/objspace/std Message-ID: <20071231134648.25054168453@codespeak.net> Author: fijal Date: Mon Dec 31 14:46:47 2007 New Revision: 50225 Modified: pypy/dist/pypy/objspace/std/objspace.py Log: A comment about some encountered problems Modified: pypy/dist/pypy/objspace/std/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/objspace.py (original) +++ pypy/dist/pypy/objspace/std/objspace.py Mon Dec 31 14:46:47 2007 @@ -370,6 +370,9 @@ def createframe(self, code, w_globals, closure=None): from pypy.objspace.std.fake import CPythonFakeCode, CPythonFakeFrame + # XXX this function is not inlined. I don't get why, but + # either I'm wrong or it shows some problems with our inlining + # [fijal] if not we_are_translated() and isinstance(code, CPythonFakeCode): return CPythonFakeFrame(self, code, w_globals) else: From fijal at codespeak.net Mon Dec 31 15:32:35 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 31 Dec 2007 15:32:35 +0100 (CET) Subject: [pypy-svn] r50226 - pypy/dist/pypy/objspace/std Message-ID: <20071231143235.8376716845B@codespeak.net> Author: fijal Date: Mon Dec 31 15:32:34 2007 New Revision: 50226 Modified: pypy/dist/pypy/objspace/std/objspace.py Log: I understand now. Modified: pypy/dist/pypy/objspace/std/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/objspace.py (original) +++ pypy/dist/pypy/objspace/std/objspace.py Mon Dec 31 15:32:34 2007 @@ -370,9 +370,6 @@ def createframe(self, code, w_globals, closure=None): from pypy.objspace.std.fake import CPythonFakeCode, CPythonFakeFrame - # XXX this function is not inlined. I don't get why, but - # either I'm wrong or it shows some problems with our inlining - # [fijal] if not we_are_translated() and isinstance(code, CPythonFakeCode): return CPythonFakeFrame(self, code, w_globals) else: From cfbolz at codespeak.net Mon Dec 31 15:40:49 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 31 Dec 2007 15:40:49 +0100 (CET) Subject: [pypy-svn] r50227 - pypy/branch/applevel-ctypes/pypy/lib/_ctypes Message-ID: <20071231144049.C5B0A168461@codespeak.net> Author: cfbolz Date: Mon Dec 31 15:40:49 2007 New Revision: 50227 Modified: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/primitive.py Log: use a _ffi struct for simple values. I think this is conceptually the right thing to do, but it is terribly inefficient. Modified: pypy/branch/applevel-ctypes/pypy/lib/_ctypes/primitive.py ============================================================================== --- pypy/branch/applevel-ctypes/pypy/lib/_ctypes/primitive.py (original) +++ pypy/branch/applevel-ctypes/pypy/lib/_ctypes/primitive.py Mon Dec 31 15:40:49 2007 @@ -1,49 +1,17 @@ +import _ffi + SIMPLE_TYPE_CHARS = "cbBhHiIlLdfuzZqQPXOv" -# type converters -def convert_intlike(val): - if not isinstance(val, (int, long)): - raise TypeError("int expected, got %s" % (type(val), )) - return val - -def convert_floatlike(val): - if not isinstance(val, (float, int, long)): - raise TypeError("float expected, got %s" % (type(val), )) - return float(val) - -def convert_char(val): - if not isinstance(val, str) or not len(val) == 1: - raise TypeError("one character string expected") - return val - -def convert_nothing(val): - return val - - -TP_TO_CONVERTER = { - 'c': convert_char, - 'b': convert_intlike, - 'B': convert_intlike, - 'h': convert_intlike, - 'H': convert_intlike, - 'i': convert_intlike, - 'I': convert_intlike, - 'l': convert_intlike, - 'L': convert_intlike, - 'q': convert_intlike, - 'Q': convert_intlike, - 'f': convert_floatlike, - 'd': convert_floatlike, - 'P': convert_nothing, #XXX - # not part of struct - 'O': convert_nothing, - 'z': convert_nothing, #XXX -} - class NULL(object): pass NULL = NULL() +TP_TO_FFITP = { + 'O': 'P', + 'z': 's', +} + + TP_TO_DEFAULT = { 'c': '\x00', 'b': 0, @@ -74,14 +42,15 @@ tp not in SIMPLE_TYPE_CHARS): raise ValueError('%s is not a type character' % (tp)) default = TP_TO_DEFAULT[tp] - converter = TP_TO_CONVERTER[tp] + ffitp = TP_TO_FFITP.get(tp, tp) + ffistruct = _ffi.Structure([("value", ffitp)]) def __init__(self, value=DEFAULT_VALUE): - self._value = default + self._struct = ffistruct() if value is not DEFAULT_VALUE: - self.value = value + self._struct.value = value dct['__init__'] = __init__ - dct['_converter'] = staticmethod(TP_TO_CONVERTER[tp]) result = type.__new__(self, name, bases, dct) + result._ffistruct = ffistruct return result def __mul__(self, other): @@ -90,16 +59,17 @@ class _SimpleCData(object): __metaclass__ = SimpleType _type_ = 'i' - def from_param(self, *args, **kwargs): + def from_param(cls, *args, **kwargs): pass + from_param = classmethod(from_param) def _getvalue(self): - return self._value + return self._struct.value def _setvalue(self, val): - self._value = self._converter(val) + self._struct.value = value + value = property(_getvalue, _setvalue) def __repr__(self): return "%s(%s)" % (type(self).__name__, self.value) - value = property(_getvalue, _setvalue) From arigo at codespeak.net Mon Dec 31 16:21:11 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 Dec 2007 16:21:11 +0100 (CET) Subject: [pypy-svn] r50228 - pypy/extradoc/sprintinfo/leysin-winter-2008 Message-ID: <20071231152111.55B3A168471@codespeak.net> Author: arigo Date: Mon Dec 31 16:21:10 2007 New Revision: 50228 Modified: pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt Log: Starting on some tasks. Modified: pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt ============================================================================== --- pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt (original) +++ pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt Mon Dec 31 16:21:10 2007 @@ -9,15 +9,25 @@ Goals and topics of the sprint ------------------------------ -* Like previous winter, the main side goal is to have fun in winter +* Like previous winters, the main side goal is to have fun in winter sports :-) We can take a couple of days off for ski; at this time of year, ski days end before 4pm, which still leaves plenty of time to recover (er, I mean hack). * the overall idea of the sprint is to continue working on making PyPy ready - for general use. + for general use. A few more specific tasks: -XXX more tasks + - app-level ctypes: getting to a basically usable point + would be really nice. + + - JIT: there is a long-standing timeshifter refactoring, + towards making the JIT be more interpreter-like. Starting + it during the sprint might be a good way to share some of + the knowledge of how the JIT really works. Alternatively, + we can work on supporting ootype in the timeshifter. + + - Testing: e.g. we run various nightly test runs but the + results are not summarized in a single page yet. ----------------------- Location & Accomodation @@ -26,19 +36,18 @@ Leysin, Switzerland, "same place as before". Let me refresh your memory: both the sprint venue and the lodging will be in a very spacious pair of chalets built specifically for bed & breakfast: -http://www.ermina.ch/. The place has a baseline ADSL Internet connexion -(600Kb) with wireless installed. You can of course arrange your own +http://www.ermina.ch/. The place has a good ADSL Internet connexion +with wireless installed. You can of course arrange your own lodging anywhere (so long as you are in Leysin, you cannot be more than a 15 minute walk away from the sprint venue), but I definitely recommend lodging there too -- you won't find a better view anywhere else (though you probably won't get much worse ones easily, either :-) I made pre-reservations in the Chalet, so please *confirm* quickly that -you are coming so that we can adjust the reservations as appropriate. -The rate so far has been 60 CHF a night all included in 2-person rooms, -with breakfast. There are larger rooms too (less expensive, possibly -more available too) and the possibility to get a single room if you -really want to. +you are coming so that we can adjust the reservations as appropriate. The +rate so far has been around 60 CHF a night all included in 2-person rooms, +with breakfast. There are larger rooms too (less expensive) and maybe the +possibility to get a single room if you really want to. Please register by svn: @@ -56,7 +65,6 @@ Exact times ----------- -XXX is this correct? Officially, 12th-19th January 2008. Both dates are flexible, you can arrive or leave earlier or later. We will give introductions and tutorials on the 13th, in the morning if everybody is there, or in the From cfbolz at codespeak.net Mon Dec 31 16:29:53 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 31 Dec 2007 16:29:53 +0100 (CET) Subject: [pypy-svn] r50229 - pypy/extradoc/sprintinfo/leysin-winter-2008 Message-ID: <20071231152953.15C1E168436@codespeak.net> Author: cfbolz Date: Mon Dec 31 16:29:50 2007 New Revision: 50229 Modified: pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt Log: stress that other tasks are possible too Modified: pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt ============================================================================== --- pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt (original) +++ pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt Mon Dec 31 16:29:50 2007 @@ -29,6 +29,9 @@ - Testing: e.g. we run various nightly test runs but the results are not summarized in a single page yet. +* We are open to all sorts of other tasks during the sprint, just + propose something. + ----------------------- Location & Accomodation ----------------------- From arigo at codespeak.net Mon Dec 31 16:55:19 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 Dec 2007 16:55:19 +0100 (CET) Subject: [pypy-svn] r50230 - pypy/extradoc/sprintinfo/leysin-winter-2008 Message-ID: <20071231155519.D38AA1684D3@codespeak.net> Author: arigo Date: Mon Dec 31 16:55:19 2007 New Revision: 50230 Modified: pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt Log: Short introduction. Modified: pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt ============================================================================== --- pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt (original) +++ pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt Mon Dec 31 16:55:19 2007 @@ -3,7 +3,9 @@ ===================================================================== .. image:: http://www.ermina.ch/002.JPG -XXX nice introduction +The next PyPy sprint will be in Leysin, Switzerland, for the fifth +time. This is a fully public sprint. Newcomers and other topics +than those suggested below are welcome. ------------------------------ Goals and topics of the sprint From arigo at codespeak.net Mon Dec 31 16:58:11 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 Dec 2007 16:58:11 +0100 (CET) Subject: [pypy-svn] r50231 - pypy/extradoc/sprintinfo/leysin-winter-2008 Message-ID: <20071231155811.111D41684D9@codespeak.net> Author: arigo Date: Mon Dec 31 16:58:11 2007 New Revision: 50231 Modified: pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt Log: Minor rephrasing. Modified: pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt ============================================================================== --- pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt (original) +++ pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt Mon Dec 31 16:58:11 2007 @@ -3,9 +3,9 @@ ===================================================================== .. image:: http://www.ermina.ch/002.JPG -The next PyPy sprint will be in Leysin, Switzerland, for the fifth -time. This is a fully public sprint. Newcomers and other topics -than those suggested below are welcome. +The next PyPy sprint will be in Leysin, Switzerland, for the +fifth time. This is a fully public sprint: newcomers and +topics other than those proposed below are welcome. ------------------------------ Goals and topics of the sprint From arigo at codespeak.net Mon Dec 31 17:02:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 Dec 2007 17:02:09 +0100 (CET) Subject: [pypy-svn] r50232 - pypy/extradoc/sprintinfo/leysin-winter-2008 Message-ID: <20071231160209.762521684D9@codespeak.net> Author: arigo Date: Mon Dec 31 17:02:08 2007 New Revision: 50232 Modified: pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt Log: Be more flexible about tutorials. Modified: pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt ============================================================================== --- pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt (original) +++ pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt Mon Dec 31 17:02:08 2007 @@ -72,5 +72,4 @@ Officially, 12th-19th January 2008. Both dates are flexible, you can arrive or leave earlier or later. We will give introductions and -tutorials on the 13th, in the morning if everybody is there, or in the -afternoon otherwise. +tutorials depending on who needs them, either on the 13th or the 14th. From arigo at codespeak.net Mon Dec 31 17:15:21 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 Dec 2007 17:15:21 +0100 (CET) Subject: [pypy-svn] r50233 - pypy/extradoc/sprintinfo/leysin-winter-2008 Message-ID: <20071231161521.E2AB91684E0@codespeak.net> Author: arigo Date: Mon Dec 31 17:15:20 2007 New Revision: 50233 Modified: pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt Log: Mention LLVM. Modified: pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt ============================================================================== --- pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt (original) +++ pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.txt Mon Dec 31 17:15:20 2007 @@ -31,6 +31,11 @@ - Testing: e.g. we run various nightly test runs but the results are not summarized in a single page yet. + - LLVM: llvm 2 is now at version 2.1 and nicely stable + again. Our llvm backend has improved in the last few + months, but refactoring it together with the genc backend + to share code more directly would be a nice task. + * We are open to all sorts of other tasks during the sprint, just propose something. From arigo at codespeak.net Mon Dec 31 17:27:18 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 Dec 2007 17:27:18 +0100 (CET) Subject: [pypy-svn] r50234 - pypy/dist/pypy/doc Message-ID: <20071231162718.B090D1684FC@codespeak.net> Author: arigo Date: Mon Dec 31 17:27:16 2007 New Revision: 50234 Modified: pypy/dist/pypy/doc/news.txt Log: News entry about the Leysin sprint. Modified: pypy/dist/pypy/doc/news.txt ============================================================================== --- pypy/dist/pypy/doc/news.txt (original) +++ pypy/dist/pypy/doc/news.txt Mon Dec 31 17:27:16 2007 @@ -7,6 +7,27 @@ .. _Python: http://www.python.org/doc/current/ref/ref.html .. _`more...`: architecture.html#mission-statement + +Leysin Winter Sports Sprint, 12th - 19th January 2008 +================================================================== + +.. raw:: html + +
+ +The next PyPy sprint will be in Leysin, Switzerland, for the fifth time. +Proposed topics: ctypes, JIT, testing, LLVM. This is a fully public +sprint: newcomers and topics other than those proposed are welcome. See +the `sprint announcement`__ for details. + +.. raw:: html + +   +
+ +.. __: http://codespeak.net/pypy/extradoc/sprintinfo/leysin-winter-2008/announcement.html + + PyPy blog started ================= From arigo at codespeak.net Mon Dec 31 17:54:29 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 31 Dec 2007 17:54:29 +0100 (CET) Subject: [pypy-svn] r50235 - pypy/dist/pypy/doc Message-ID: <20071231165429.3C4D31684EA@codespeak.net> Author: arigo Date: Mon Dec 31 17:54:28 2007 New Revision: 50235 Modified: pypy/dist/pypy/doc/news.txt Log: Tweaks to the news entry. Modified: pypy/dist/pypy/doc/news.txt ============================================================================== --- pypy/dist/pypy/doc/news.txt (original) +++ pypy/dist/pypy/doc/news.txt Mon Dec 31 17:54:28 2007 @@ -13,12 +13,21 @@ .. raw:: html -
+
+ +The next PyPy sprint will be held in Leysin, Switzerland, for +the fifth time. The overall idea of the sprint is to continue +working on making PyPy ready for general use. + +.. raw:: html + +
-The next PyPy sprint will be in Leysin, Switzerland, for the fifth time. -Proposed topics: ctypes, JIT, testing, LLVM. This is a fully public -sprint: newcomers and topics other than those proposed are welcome. See -the `sprint announcement`__ for details. +The proposed topics are: ctypes, JIT, testing, LLVM. This is +a fully public sprint, so newcomers and other topics are +welcome. And like previous winters, the main side goal is to +have fun in winter sports :-) See the `sprint announcement`__ +for details. .. raw:: html From vinogradov at codespeak.net Mon Dec 31 18:49:40 2007 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Mon, 31 Dec 2007 18:49:40 +0100 (CET) Subject: [pypy-svn] r50236 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071231174940.42EED16841D@codespeak.net> Author: vinogradov Date: Mon Dec 31 18:49:39 2007 New Revision: 50236 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: Update RopeString and RopeUnicode class to pass slice, add coerction test. Some work on encode/decode support Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Mon Dec 31 18:49:39 2007 @@ -23,8 +23,28 @@ return self.__class__(rope.multiply(self._node, n)) def __add__(self, other): - return self.__class__(self._node + other._node) + if isinstance(self, RopeUnicode) or isinstance(other, RopeUnicode): + return RopeUnicode(self._node + other._node) + else: + return self.__class__(self._node + other._node) + def __getitem__(self, index): + if isinstance(index, int): + return self.getchar(index) + if isinstance(index, slice): + start, stop, step = index.start, index.stop, index.step + start = start or 0 + stop = (stop and (stop < 0 and len(self) + stop or stop)) or len(self) + step = step or 1 + return self.__class__(rope.getslice(self._node, start, stop, step)) + + def getchar(self,index): + if isinstance(self, RopeString): + return self._node.getchar(index) + if isinstance(self, RopeUnicode): + return self._node.getunichar(index) + raise NotImplementedError("Index type not known.") + class RopeStringIterator(object): def __init__(self, node): self._iter = rope.ItemIterator(node) @@ -42,9 +62,6 @@ elif isinstance(s, rope.StringNode): self._node = s - def __getitem__(self, index): - return self._node.getchar(index) - def __eq__(self, other): if isinstance(other, RopeBaseString): return (rope.eq(self._node, other._node)) @@ -53,6 +70,16 @@ def __iter__(self): return RopeStringIterator(self._node) + + def decode(self, codepage): + if codepage == "utf-8": + return RopeUnicode(rope.str_decode_utf8(self._node)) + if codepage == "latin1": + #Need rewrite + return RopeUnicode(rope.str_decode_latin1(self._node)) + if codepage == "ascii": + #Need rewrite + return RopeUnicode(rope.str_decode_ascii(self._node)) class RopeUnicodeIterator(object): def __init__(self, node): @@ -73,9 +100,6 @@ if isinstance(s, rope.StringNode): self._node = s - def __getitem__(self, index): - return self._node.getunichar(index) - def __eq__(self, other): if isinstance (other, RopeBaseString): return rope.eq(self._node, other._node) @@ -84,3 +108,21 @@ def __iter__(self): return RopeUnicodeIterator(self._node) + + def encode(self, codepage): + if codepage == "utf-8": + return RopeString(rope.unicode_encode_utf8(self._node)) + if codepage == "utf-16": + raise NotImplemented("How i can encode utf-16 string?") + if codepage == "latin-1": + result = rope.unicode_encode_latin1(self._node) + if result: + return RopeString(result) + else: + raise NotImplementedError("Do i need implement such latin-1 encoding?") + if codepage == "ascii": + result = rope.unicode_encode_ascii(self._node) + if result: + return RopeString(result) + else: + raise NotImplementedError("Do i need implement such ascii encoding?") From cfbolz at codespeak.net Mon Dec 31 19:05:38 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 31 Dec 2007 19:05:38 +0100 (CET) Subject: [pypy-svn] r50237 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20071231180538.E1B21168419@codespeak.net> Author: cfbolz Date: Mon Dec 31 19:05:37 2007 New Revision: 50237 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: be a bit less nonsensical about slices Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Mon Dec 31 19:05:37 2007 @@ -32,18 +32,10 @@ if isinstance(index, int): return self.getchar(index) if isinstance(index, slice): - start, stop, step = index.start, index.stop, index.step - start = start or 0 - stop = (stop and (stop < 0 and len(self) + stop or stop)) or len(self) - step = step or 1 + start, stop, step = index.indices(self._node.length()) return self.__class__(rope.getslice(self._node, start, stop, step)) - - def getchar(self,index): - if isinstance(self, RopeString): - return self._node.getchar(index) - if isinstance(self, RopeUnicode): - return self._node.getunichar(index) raise NotImplementedError("Index type not known.") + class RopeStringIterator(object): def __init__(self, node): @@ -81,6 +73,9 @@ #Need rewrite return RopeUnicode(rope.str_decode_ascii(self._node)) + def getchar(self,index): + return self._node.getchar(index) + class RopeUnicodeIterator(object): def __init__(self, node): self._iter = rope.ItemIterator(node) @@ -126,3 +121,6 @@ return RopeString(result) else: raise NotImplementedError("Do i need implement such ascii encoding?") + + def getchar(self,index): + return self._node.getunichar(index) From cfbolz at codespeak.net Mon Dec 31 19:36:20 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 31 Dec 2007 19:36:20 +0100 (CET) Subject: [pypy-svn] r50238 - pypy/branch/ghop-ropes-classes/pypy/rlib/test Message-ID: <20071231183620.4EF85168459@codespeak.net> Author: cfbolz Date: Mon Dec 31 19:36:18 2007 New Revision: 50238 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Log: check that type coercion raises a proper error Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Mon Dec 31 19:36:18 2007 @@ -103,6 +103,11 @@ assert isinstance(s, self.constunicode) assert s == self.constunicode("defabc") + def test_add_coercion_decodes(self): + s1 = self.conststr("\xff") + s2 = self.constunicode("a") + py.test.raises(UnicodeDecodeError, "s1 + s2") + class TestPythonCoercion(AbstractTestCoercion): conststr = str constunicode = unicode From cfbolz at codespeak.net Mon Dec 31 19:45:59 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 31 Dec 2007 19:45:59 +0100 (CET) Subject: [pypy-svn] r50239 - pypy/branch/ghop-ropes-classes/pypy/rlib/test Message-ID: <20071231184559.618E01684DB@codespeak.net> Author: cfbolz Date: Mon Dec 31 19:45:58 2007 New Revision: 50239 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Log: more tests for encoding and decoding Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Mon Dec 31 19:45:58 2007 @@ -80,18 +80,38 @@ py.test.raises(UnicodeEncodeError, u.encode, "latin-1") py.test.raises(UnicodeEncodeError, u.encode, "ascii") + def test_encode_errors(self): + u = self.constunicode(u"b\uffffa\xff") + s = u.encode("latin-1", "replace") + assert s == self.conststr('b?a\xff') + s = u.encode("latin-1", "ignore") + assert s == self.conststr('ba\xff') + s = u.encode("ascii", "replace") + assert s == self.conststr('b?a?') + s = u.encode("ascii", "ignore") + assert s == self.conststr('ba') + def test_decode(self): - s = self.conststr("abc") - u = s.decode("utf-8") - assert s == self.constunicode(u"abc") - u = s.decode("latin-1") - assert s == self.constunicode(u"abc") - u = s.decode("ascii") - assert s == self.constunicode(u"abc") - u = self.conststr("\xff") + u = self.conststr("abc") + s = u.decode("utf-8") + assert u == self.constunicode(u"abc") s = u.decode("latin-1") - assert s == self.constunicode(u"\xff") - py.test.raises(UnicodeEncodeError, s.decode, "ascii") + assert u == self.constunicode(u"abc") + s = u.decode("ascii") + assert u == self.constunicode(u"abc") + s = self.conststr("\xff") + u = s.decode("latin-1") + assert u == self.constunicode(u"\xff") + py.test.raises(UnicodeDecodeError, u.decode, "ascii") + + def test_decode_errors(self): + s = self.conststr("a\xffb") + u = s.decode("ascii", "replace") + assert u == self.constunicode(u"a\ufffdb") + u = s.decode("ascii", "ignore") + assert u == self.constunicode(u"ab") + + def test_add_coercion(self): s1 = self.conststr("abc")