From cfbolz at codespeak.net Fri Feb 1 13:47:42 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 1 Feb 2008 13:47:42 +0100 (CET) Subject: [pypy-svn] r51174 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter Message-ID: <20080201124742.44FB8168441@codespeak.net> Author: cfbolz Date: Fri Feb 1 13:47:40 2008 New Revision: 51174 Added: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (contents, props changed) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Log: the first test where the rainbow interpreter actually runs passes. required a very messy test runner and some whacking. Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Fri Feb 1 13:47:40 2008 @@ -1,3 +1,4 @@ +from pypy.rlib.rarithmetic import intmask from pypy.objspace.flow import model as flowmodel from pypy.rpython.lltypesystem import lltype from pypy.jit.hintannotator.model import originalconcretetype @@ -24,15 +25,40 @@ def _freeze_(self): return True +SIGN_EXTEND2 = 1 << 15 + +STOP = object() + class JitInterpreter(object): def __init__(self): self.opcode_implementations = [] self.opcode_descs = [] self.opname_to_index = {} self.jitstate = None + self.queue = None self.bytecode = None + self.pc = -1 self._add_implemented_opcodes() + def run(self, jitstate, bytecode, greenargs, redargs): + self.jitstate = jitstate + self.queue = rtimeshift.ensure_queue(jitstate, + rtimeshift.BaseDispatchQueue) + rtimeshift.enter_frame(self.jitstate, self.queue) + self.frame = self.jitstate.frame + self.frame.pc = 0 + self.frame.bytecode = bytecode + self.frame.local_boxes = redargs + self.frame.local_green = greenargs + self.bytecode_loop() + + def bytecode_loop(self): + while 1: + bytecode = self.load_2byte() + result = self.opcode_implementations[bytecode](self) + if result is STOP: + return + # construction-time interface def _add_implemented_opcodes(self): @@ -44,7 +70,6 @@ self.opcode_implementations.append(getattr(self, name).im_func) self.opcode_descs.append(None) - def find_opcode(self, name): return self.opname_to_index.get(name, -1) @@ -54,7 +79,9 @@ def implementation(self): args = () for i in numargs: - args.append(self.get_greenarg()) + genconst = self.get_greenarg() + arg = self.jitstate.curbuilder.revealconst(opdesc.ARGS[i]) + args += (arg, ) result = opdesc.llop(*args) self.green_result(result) elif color == "red": @@ -65,12 +92,9 @@ else: XXX def implementation(self): - XXX - # the following is nonsense: the green arguments are - # GenConsts, so there are revealconsts missing - args = (self.jitstate, ) + args = (opdesc, self.jitstate, ) for i in numargs: - args.append(self.get_redarg()) + args += (self.get_redarg(), ) result = impl(*args) self.red_result(result) else: @@ -83,12 +107,46 @@ return index - # operation implemetations + # operation helper functions + + def load_2byte(self): + pc = self.frame.pc + result = ((ord(self.frame.bytecode.code[pc]) << 8) | + ord(self.frame.bytecode.code[pc + 1])) + self.frame.pc = pc + 2 + return intmask((result ^ SIGN_EXTEND2) - SIGN_EXTEND2) + + def load_4byte(self): + pc = self.frame.pc + result = ((ord(self.frame.bytecode.code[pc + 0]) << 24) | + (ord(self.frame.bytecode.code[pc + 1]) << 16) | + (ord(self.frame.bytecode.code[pc + 2]) << 8) | + (ord(self.frame.bytecode.code[pc + 3]) << 0)) + self.frame.pc = pc + 4 + return intmask(result) + + def get_greenarg(self): + i = self.load_2byte() + if i < 0: + return self.frame.bytecode.constants[~i] + return self.frame.local_green[i] + + def get_redarg(self): + return self.frame.local_boxes[self.load_2byte()] + + def red_result(self, box): + self.frame.local_boxes.append(box) + + def green_result(self, gv): + self.frame.local_green.append(gv) + + # operation implementations def opimpl_make_redbox(self): XXX def opimpl_goto(self): - XXX + target = self.load_4byte() + self.frame.pc = target def opimpl_green_goto_iftrue(self): XXX @@ -97,20 +155,35 @@ XXX def opimpl_red_return(self): - XXX + rtimeshift.save_return(self.jitstate) + # XXX for now + newstate = rtimeshift.leave_graph_red(self.queue, is_portal=True) + self.jitstate = newstate + return STOP def opimpl_green_return(self): XXX + return STOP # XXX wrong, of course def opimpl_make_new_redvars(self): # an opcode with a variable number of args # num_args arg_old_1 arg_new_1 ... - XXX + num = self.load_2byte() + newlocalboxes = [] + for i in range(num): + newlocalboxes.append(self.get_redarg()) + self.frame.local_boxes = newlocalboxes def opimpl_make_new_greenvars(self): # an opcode with a variable number of args # num_args arg_old_1 arg_new_1 ... - XXX + num = self.load_2byte() + newgreens = [] + for i in range(num): + newgreens.append(self.get_greenarg()) + self.frame.local_green = newgreens + + class BytecodeWriter(object): @@ -268,7 +341,6 @@ return color def register_redvar(self, arg): - print "register_redvar", arg assert arg not in self.redvar_positions self.redvar_positions[arg] = result = self.free_red[self.current_block] self.free_red[self.current_block] += 1 @@ -288,7 +360,7 @@ def green_position(self, arg): if isinstance(arg, flowmodel.Variable): return self.greenvar_positions[arg] - return -self.const_position(arg) - 1 + return ~self.const_position(arg) def const_position(self, const): if const in self.const_positions: @@ -374,15 +446,15 @@ -# XXX too lazy to fix the interface of make_opdesc +# XXX too lazy to fix the interface of make_opdesc, ExceptionDesc class PseudoHOP(object): def __init__(self, op, args_s, s_result, RGenOp): self.spaceop = op self.args_s = args_s self.s_result = s_result - self.rtyper = PseudoHRTyper(RGenOp) + self.rtyper = PseudoHRTyper(RGenOp=RGenOp) class PseudoHRTyper(object): - def __init__(self, RGenOp): - self.RGenOp = RGenOp + def __init__(self, **args): + self.__dict__.update(**args) Added: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- (empty file) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Fri Feb 1 13:47:40 2008 @@ -0,0 +1,140 @@ +from pypy.translator.translator import TranslationContext, graphof +from pypy.jit.hintannotator.annotator import HintAnnotator +from pypy.jit.hintannotator.policy import StopAtXPolicy, HintAnnotatorPolicy +from pypy.jit.hintannotator.model import SomeLLAbstractConstant, OriginFlags +from pypy.jit.hintannotator.model import originalconcretetype +from pypy.jit.rainbow.bytecode import BytecodeWriter, label, tlabel, assemble +from pypy.jit.codegen.llgraph.rgenop import RGenOp +from pypy.jit.rainbow.test.test_serializegraph import AbstractSerializationTest +from pypy.jit.rainbow import bytecode +from pypy.jit.timeshifter import rtimeshift, exception, rvalue +from pypy.rpython.lltypesystem import lltype, rstr +from pypy.rpython.llinterp import LLInterpreter +from pypy.annotation import model as annmodel +from pypy import conftest + +def getargtypes(annotator, values): + return [annotation(annotator, x) for x in values] + +def annotation(a, x): + T = lltype.typeOf(x) + if T == lltype.Ptr(rstr.STR): + t = str + else: + t = annmodel.lltype_to_annotation(T) + return a.typeannotation(t) + +P_OOPSPEC_NOVIRTUAL = HintAnnotatorPolicy(oopspec=True, + novirtualcontainer=True, + entrypoint_returns_red=False) + +class AbstractInterpretationTest(object): + + def serialize(self, func, values, backendoptimize=False): + # build the normal ll graphs for ll_function + t = TranslationContext() + a = t.buildannotator() + argtypes = getargtypes(a, values) + a.build_types(func, argtypes) + rtyper = t.buildrtyper(type_system = self.type_system) + rtyper.specialize() + self.rtyper = rtyper + if backendoptimize: + from pypy.translator.backendopt.all import backend_optimizations + backend_optimizations(t) + graph1 = graphof(t, func) + + # build hint annotator types + hannotator = HintAnnotator(base_translator=t, policy=P_OOPSPEC_NOVIRTUAL) + hs = hannotator.build_types(graph1, [SomeLLAbstractConstant(v.concretetype, + {OriginFlags(): True}) + for v in graph1.getargs()]) + hannotator.simplify() + t = hannotator.translator + self.hannotator = hannotator + if conftest.option.view: + t.view() + graph2 = graphof(t, func) + self.graph = graph2 + writer = BytecodeWriter(t, hannotator, RGenOp) + jitcode = writer.make_bytecode(graph2) + argcolors = [] + for i, ll_val in enumerate(values): + color = writer.varcolor(graph2.startblock.inputargs[i]) + argcolors.append(color) + return writer, jitcode, argcolors + + def interpret(self, ll_function, values, opt_consts=[], *args, **kwds): + # XXX clean this mess up + writer, jitcode, argcolors = self.serialize(ll_function, values) + hrtyper = bytecode.PseudoHRTyper(RGenOp=writer.RGenOp, + annotator=writer.translator.annotator, + rtyper=writer.translator.annotator.base_translator.rtyper) + edesc = exception.ExceptionDesc(hrtyper, False) + rgenop = writer.RGenOp() + # make residual functype + FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) + ha = self.hannotator + RESTYPE = originalconcretetype(self.hannotator.binding(self.graph.getreturnvar())) + ARGS = [] + for var in self.graph.getargs(): + # XXX ignoring virtualizables for now + ARGS.append(originalconcretetype(self.hannotator.binding(var))) + FUNC = lltype.FuncType(ARGS, RESTYPE) + sigtoken = rgenop.sigToken(FUNC) + builder, gv_generated, inputargs_gv = rgenop.newgraph(sigtoken, "generated") + print builder, builder.rgenop, rgenop + builder.start_writing() + jitstate = rtimeshift.JITState(builder, None, + edesc.null_exc_type_box, + edesc.null_exc_value_box) + def ll_finish_jitstate(jitstate, exceptiondesc, graphsigtoken): + returnbox = rtimeshift.getreturnbox(jitstate) + gv_ret = returnbox.getgenvar(jitstate) + builder = jitstate.curbuilder + for virtualizable_box in jitstate.virtualizables: + assert isinstance(virtualizable_box, rvalue.PtrRedBox) + content = virtualizable_box.content + assert isinstance(content, rcontainer.VirtualizableStruct) + content.store_back(jitstate) + exceptiondesc.store_global_excdata(jitstate) + jitstate.curbuilder.finish_and_return(graphsigtoken, gv_ret) + # build arguments + greenargs = [] + redargs = [] + residualargs = [] + i = 0 + for color, ll_val in zip(argcolors, values): + if color == "green": + greenargs.append(writer.RGenOp.constPrebuiltGlobal(const.value)) + else: + TYPE = lltype.typeOf(ll_val) + kind = rgenop.kindToken(TYPE) + boxcls = rvalue.ll_redboxcls(TYPE) + redargs.append(boxcls(kind, inputargs_gv[i])) + residualargs.append(ll_val) + i += 1 + writer.interpreter.run(jitstate, jitcode, greenargs, redargs) + ll_finish_jitstate(writer.interpreter.jitstate, edesc, sigtoken) + builder.end() + generated = gv_generated.revealconst(lltype.Ptr(FUNC)) + graph = generated._obj.graph + if conftest.option.view: + graph.show() + llinterp = LLInterpreter(self.rtyper) + res = llinterp.eval_graph(graph, residualargs) + return res + + def Xtest_return_green(self): + def f(): + return 1 + self.interpret(f, []) + + def test_very_simple(self): + def f(x, y): + return x + y + res = self.interpret(f, [1, 2]) + assert res == 3 + +class TestLLType(AbstractInterpretationTest): + type_system = "lltype" Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Fri Feb 1 13:47:40 2008 @@ -78,7 +78,7 @@ def ll_gen1(opdesc, jitstate, argbox): ARG0 = opdesc.ARG0 RESULT = opdesc.RESULT - opname = opdesc.name + opname = opdesc.opname if opdesc.tryfold and argbox.is_constant(): arg = rvalue.ll_getvalue(argbox, ARG0) if not opdesc.canraise: @@ -106,7 +106,7 @@ ARG0 = opdesc.ARG0 ARG1 = opdesc.ARG1 RESULT = opdesc.RESULT - opname = opdesc.name + opname = opdesc.opname if opdesc.tryfold and argbox0.is_constant() and argbox1.is_constant(): # const propagate arg0 = rvalue.ll_getvalue(argbox0, ARG0) @@ -991,6 +991,7 @@ self.backframe = backframe self.dispatchqueue = dispatchqueue #self.local_boxes = ... set by callers + #self.local_green = ... set by callers def enter_block(self, incoming, memo): for box in self.local_boxes: From cfbolz at codespeak.net Fri Feb 1 13:50:35 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 1 Feb 2008 13:50:35 +0100 (CET) Subject: [pypy-svn] r51175 - pypy/branch/jit-refactoring/pypy/jit/rainbow Message-ID: <20080201125035.3B261168471@codespeak.net> Author: cfbolz Date: Fri Feb 1 13:50:34 2008 New Revision: 51175 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Log: reshuffle some metho Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Fri Feb 1 13:50:34 2008 @@ -59,53 +59,6 @@ if result is STOP: return - # construction-time interface - - def _add_implemented_opcodes(self): - for name in dir(self): - if not name.startswith("opimpl_"): - continue - opname = name[len("opimpl_"):] - self.opname_to_index[opname] = len(self.opcode_implementations) - self.opcode_implementations.append(getattr(self, name).im_func) - self.opcode_descs.append(None) - - def find_opcode(self, name): - return self.opname_to_index.get(name, -1) - - def make_opcode_implementation(self, color, opdesc): - numargs = unrolling_iterable(range(opdesc.nb_args)) - if color == "green": - def implementation(self): - args = () - for i in numargs: - genconst = self.get_greenarg() - arg = self.jitstate.curbuilder.revealconst(opdesc.ARGS[i]) - args += (arg, ) - result = opdesc.llop(*args) - self.green_result(result) - elif color == "red": - if opdesc.nb_args == 1: - impl = rtimeshift.ll_gen1 - elif opdesc.nb_args == 2: - impl = rtimeshift.ll_gen2 - else: - XXX - def implementation(self): - args = (opdesc, self.jitstate, ) - for i in numargs: - args += (self.get_redarg(), ) - result = impl(*args) - self.red_result(result) - else: - assert 0, "unknown color" - implementation.func_name = "opimpl_%s_%s" % (color, opdesc.opname) - opname = "%s_%s" % (color, opdesc.opname) - index = self.opname_to_index[opname] = len(self.opcode_implementations) - self.opcode_implementations.append(implementation) - self.opcode_descs.append(opdesc) - return index - # operation helper functions @@ -183,7 +136,53 @@ newgreens.append(self.get_greenarg()) self.frame.local_green = newgreens + # construction-time interface + + def _add_implemented_opcodes(self): + for name in dir(self): + if not name.startswith("opimpl_"): + continue + opname = name[len("opimpl_"):] + self.opname_to_index[opname] = len(self.opcode_implementations) + self.opcode_implementations.append(getattr(self, name).im_func) + self.opcode_descs.append(None) + def find_opcode(self, name): + return self.opname_to_index.get(name, -1) + + def make_opcode_implementation(self, color, opdesc): + numargs = unrolling_iterable(range(opdesc.nb_args)) + if color == "green": + def implementation(self): + args = () + for i in numargs: + genconst = self.get_greenarg() + arg = self.jitstate.curbuilder.revealconst(opdesc.ARGS[i]) + args += (arg, ) + result = opdesc.llop(*args) + self.green_result(result) + elif color == "red": + if opdesc.nb_args == 1: + impl = rtimeshift.ll_gen1 + elif opdesc.nb_args == 2: + impl = rtimeshift.ll_gen2 + else: + XXX + def implementation(self): + args = (opdesc, self.jitstate, ) + for i in numargs: + args += (self.get_redarg(), ) + result = impl(*args) + self.red_result(result) + else: + assert 0, "unknown color" + implementation.func_name = "opimpl_%s_%s" % (color, opdesc.opname) + opname = "%s_%s" % (color, opdesc.opname) + index = self.opname_to_index[opname] = len(self.opcode_implementations) + self.opcode_implementations.append(implementation) + self.opcode_descs.append(opdesc) + return index + class BytecodeWriter(object): From cfbolz at codespeak.net Fri Feb 1 14:11:25 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 1 Feb 2008 14:11:25 +0100 (CET) Subject: [pypy-svn] r51177 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080201131125.35A03168477@codespeak.net> Author: cfbolz Date: Fri Feb 1 14:11:24 2008 New Revision: 51177 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: first test with a green switch passes Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Fri Feb 1 14:11:24 2008 @@ -102,7 +102,11 @@ self.frame.pc = target def opimpl_green_goto_iftrue(self): - XXX + genconst = self.get_greenarg() + target = self.load_4byte() + arg = genconst.revealconst(lltype.Bool) + if arg: + self.frame.pc = target def opimpl_red_goto_iftrue(self): XXX @@ -115,7 +119,10 @@ return STOP def opimpl_green_return(self): - XXX + rtimeshift.save_return(self.jitstate) + newstate = rtimeshift.leave_graph_yellow(self.queue) + self.jitstate = newstate + return STOP return STOP # XXX wrong, of course def opimpl_make_new_redvars(self): @@ -154,12 +161,13 @@ numargs = unrolling_iterable(range(opdesc.nb_args)) if color == "green": def implementation(self): - args = () + args = (opdesc.RESULT, ) for i in numargs: genconst = self.get_greenarg() - arg = self.jitstate.curbuilder.revealconst(opdesc.ARGS[i]) + arg = genconst.revealconst(opdesc.ARGS[i]) args += (arg, ) - result = opdesc.llop(*args) + rgenop = self.jitstate.curbuilder.rgenop + result = rgenop.genconst(opdesc.llop(*args)) self.green_result(result) elif color == "red": if opdesc.nb_args == 1: Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Fri Feb 1 14:11:24 2008 @@ -11,6 +11,7 @@ from pypy.rpython.lltypesystem import lltype, rstr from pypy.rpython.llinterp import LLInterpreter from pypy.annotation import model as annmodel +from pypy.rlib.jit import hint from pypy import conftest def getargtypes(annotator, values): @@ -79,7 +80,9 @@ ARGS = [] for var in self.graph.getargs(): # XXX ignoring virtualizables for now - ARGS.append(originalconcretetype(self.hannotator.binding(var))) + binding = self.hannotator.binding(var) + if not binding.is_green(): + ARGS.append(originalconcretetype(binding)) FUNC = lltype.FuncType(ARGS, RESTYPE) sigtoken = rgenop.sigToken(FUNC) builder, gv_generated, inputargs_gv = rgenop.newgraph(sigtoken, "generated") @@ -106,7 +109,7 @@ i = 0 for color, ll_val in zip(argcolors, values): if color == "green": - greenargs.append(writer.RGenOp.constPrebuiltGlobal(const.value)) + greenargs.append(writer.RGenOp.constPrebuiltGlobal(ll_val)) else: TYPE = lltype.typeOf(ll_val) kind = rgenop.kindToken(TYPE) @@ -136,5 +139,16 @@ res = self.interpret(f, [1, 2]) assert res == 3 + def test_green_switch(self): + def f(green, x, y): + green = hint(green, concrete=True) + if green: + return x + y + return x - y + res = self.interpret(f, [1, 1, 2]) + assert res == 3 + res = self.interpret(f, [0, 1, 2]) + assert res == -1 + class TestLLType(AbstractInterpretationTest): type_system = "lltype" From antocuni at codespeak.net Fri Feb 1 14:31:26 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 1 Feb 2008 14:31:26 +0100 (CET) Subject: [pypy-svn] r51178 - pypy/dist/pypy/jit/codegen/dump/test Message-ID: <20080201133126.65C8D168471@codespeak.net> Author: antocuni Date: Fri Feb 1 14:31:24 2008 New Revision: 51178 Modified: pypy/dist/pypy/jit/codegen/dump/test/test_rgenop.py Log: fix test Modified: pypy/dist/pypy/jit/codegen/dump/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/dump/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/dump/test/test_rgenop.py Fri Feb 1 14:31:24 2008 @@ -2,8 +2,7 @@ from pypy.jit.codegen.dump.rgenop import RDumpGenOp from pypy.jit.timeshifter.test.test_timeshift import Whatever from pypy.rpython.lltypesystem import lltype -from pypy.jit.codegen.test.rgenop_tests import AbstractRGenOpTests, FUNC, FUNC2 -from ctypes import cast, c_int, c_void_p, CFUNCTYPE +from pypy.jit.codegen.test.rgenop_tests import AbstractRGenOpTests class TestRDumpGenop(AbstractRGenOpTests): RGenOp = RDumpGenOp From antocuni at codespeak.net Fri Feb 1 14:41:58 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 1 Feb 2008 14:41:58 +0100 (CET) Subject: [pypy-svn] r51179 - in pypy/dist/pypy/translator/cli: . src test Message-ID: <20080201134158.93CF4168052@codespeak.net> Author: antocuni Date: Fri Feb 1 14:41:58 2008 New Revision: 51179 Modified: pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/query.py pypy/dist/pypy/translator/cli/src/pypylib.cs pypy/dist/pypy/translator/cli/src/query.cs pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: fix typeof of types defined in external assemblies Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Fri Feb 1 14:41:58 2008 @@ -495,8 +495,7 @@ def typeof(cliClass): TYPE = cliClass._INSTANCE - name = '%s.%s' % (TYPE._namespace, TYPE._classname) - return PythonNet.System.Type.GetType(name) + return PythonNet.System.Type.GetType(TYPE._assembly_qualified_name) class Entry(ExtRegistryEntry): _about_ = typeof Modified: pypy/dist/pypy/translator/cli/query.py ============================================================================== --- pypy/dist/pypy/translator/cli/query.py (original) +++ pypy/dist/pypy/translator/cli/query.py Fri Feb 1 14:41:58 2008 @@ -33,7 +33,8 @@ except ImportError: pass else: - Assembly.LoadFrom(dll) + ass = Assembly.LoadFrom(dll) + assert ass is not None clr.AddReference(pypylib) load_assembly(pypylib) @@ -96,6 +97,7 @@ desc = ClassDesc() desc.Assembly = mscorlib desc.FullName = name + desc.AssemblyQualifiedName = name # XXX desc.BaseType = 'System.Object' desc.IsArray = True desc.ElementType = 'System.Object' # not really true, but we need something @@ -105,6 +107,7 @@ desc = ClassDesc() desc.Assembly = mscorlib desc.FullName = name + desc.AssemblyQualifiedName = name # XXX desc.BaseType = 'System.Array' desc.ElementType = itemdesc.FullName desc.IsArray = True @@ -154,6 +157,7 @@ # no superclass for now, will add it later TYPE = NativeInstance(assembly, namespace, name, None, {}, {}) TYPE._is_value_type = self.IsValueType + TYPE._assembly_qualified_name = self.AssemblyQualifiedName Class = CliClass(TYPE, {}, {}) self._cliclass = Class # we need to check also for System.Array to prevent a circular recursion Modified: pypy/dist/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/dist/pypy/translator/cli/src/pypylib.cs Fri Feb 1 14:41:58 2008 @@ -53,6 +53,7 @@ namespace pypy.runtime { + public delegate int DelegateType_int__int(int a); public delegate int DelegateType_int__int_int(int a, int b); public class Utils Modified: pypy/dist/pypy/translator/cli/src/query.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/query.cs (original) +++ pypy/dist/pypy/translator/cli/src/query.cs Fri Feb 1 14:41:58 2008 @@ -58,6 +58,7 @@ outfile.WriteLine("desc = ClassDesc()"); outfile.WriteLine("desc.Assembly = '{0}'", t.Assembly.FullName); outfile.WriteLine("desc.FullName = '{0}'", t.FullName); + outfile.WriteLine("desc.AssemblyQualifiedName = '{0}'", t.AssemblyQualifiedName); outfile.WriteLine("desc.BaseType = '{0}'", GetBaseType(t)); outfile.WriteLine("desc.IsArray = {0}", t.IsArray); outfile.WriteLine("desc.IsValueType = {0}", t.IsValueType); Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Fri Feb 1 14:41:58 2008 @@ -355,6 +355,13 @@ res = self.interpret(fn, []) assert res is True + def test_typeof_pypylib(self): + DelegateType = CLR.pypy.runtime.DelegateType_int__int_int + def fn(): + return typeof(DelegateType) is not None + res = self.interpret(fn, []) + assert res is True + def test_mix_None_and_instance(self): def g(x): return x From antocuni at codespeak.net Fri Feb 1 15:15:54 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 1 Feb 2008 15:15:54 +0100 (CET) Subject: [pypy-svn] r51182 - in pypy/dist/pypy/translator/cli: . test Message-ID: <20080201141554.D6499168440@codespeak.net> Author: antocuni Date: Fri Feb 1 15:15:52 2008 New Revision: 51182 Modified: pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: add an helper to allow downcasts of cli objects Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Fri Feb 1 15:15:52 2008 @@ -528,6 +528,26 @@ c_methodname = hop.inputconst(ootype.Void, methodname) return hop.genop('cli_eventhandler', [v_obj, c_methodname], hop.r_result.lowleveltype) + +def clidowncast(cliClass, obj): + return obj + +class Entry(ExtRegistryEntry): + _about_ = clidowncast + + def compute_result_annotation(self, s_type, s_value): + assert s_type.is_constant() + cliClass = s_type.const + TYPE = cliClass._INSTANCE + assert ootype.isSubclass(TYPE, s_value.ootype) + return SomeOOInstance(TYPE) + + def specialize_call(self, hop): + assert isinstance(hop.args_s[0].const, CliClass) + assert isinstance(hop.args_s[1], annmodel.SomeOOInstance) + v_inst = hop.inputarg(hop.args_r[1], arg=1) + return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype) + from pypy.translator.cli.query import CliNamespace CLR = CliNamespace(None) CLR._buildtree() Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Fri Feb 1 15:15:52 2008 @@ -7,7 +7,7 @@ from pypy.translator.cli.test.runtest import CliTest from pypy.translator.cli.dotnet import SomeCliClass, SomeCliStaticMethod,\ NativeInstance, CLR, box, unbox, OverloadingResolver, NativeException,\ - native_exc, new_array, init_array, typeof, eventhandler + native_exc, new_array, init_array, typeof, eventhandler, clidowncast System = CLR.System ArrayList = CLR.System.Collections.ArrayList @@ -362,6 +362,18 @@ res = self.interpret(fn, []) assert res is True + def test_clidowncast(self): + def fn(): + a = ArrayList() + b = ArrayList() + a.Add(b) + c = a.get_Item(0) # type of c is Object + c = clidowncast(ArrayList, c) + c.Add(None) + return c.get_Item(0) + res = self.interpret(fn, []) + assert res is None + def test_mix_None_and_instance(self): def g(x): return x From antocuni at codespeak.net Fri Feb 1 15:17:05 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 1 Feb 2008 15:17:05 +0100 (CET) Subject: [pypy-svn] r51183 - pypy/dist/pypy/translator/cli/test Message-ID: <20080201141705.1C44916845F@codespeak.net> Author: antocuni Date: Fri Feb 1 15:17:05 2008 New Revision: 51183 Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: fix test to work also when interpreted Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Fri Feb 1 15:17:05 2008 @@ -469,9 +469,7 @@ def test_dynamic_method(self): from pypy.rpython.ootypesystem import ootype - self._skip_pythonnet("does not work") DelegateType = CLR.pypy.runtime.DelegateType_int__int_int - DELEGATETYPE = DelegateType._INSTANCE Utils = CLR.pypy.runtime.Utils def fn(): tInt = typeof(System.Int32) @@ -483,7 +481,7 @@ il.Emit(OpCodes.Add) il.Emit(OpCodes.Ret) myfunc = meth.CreateDelegate(typeof(DelegateType)) - myfunc = ootype.oodowncast(DELEGATETYPE, myfunc) # XXX messy + myfunc = clidowncast(DelegateType, myfunc) return myfunc.Invoke(30, 12) res = self.interpret(fn, []) assert res == 42 From cfbolz at codespeak.net Fri Feb 1 15:20:44 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 1 Feb 2008 15:20:44 +0100 (CET) Subject: [pypy-svn] r51184 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080201142044.299FE168464@codespeak.net> Author: cfbolz Date: Fri Feb 1 15:20:43 2008 New Revision: 51184 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Log: converting green things into red things works Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Fri Feb 1 15:20:43 2008 @@ -18,16 +18,20 @@ green consts are negative indexes """ - def __init__(self, code, constants): + def __init__(self, code, constants, typekinds, redboxclasses): self.code = code self.constants = constants + self.typekinds = typekinds + self.redboxclasses = redboxclasses def _freeze_(self): return True SIGN_EXTEND2 = 1 << 15 -STOP = object() +class STOP(object): + pass +STOP = STOP() class JitInterpreter(object): def __init__(self): @@ -58,6 +62,8 @@ result = self.opcode_implementations[bytecode](self) if result is STOP: return + else: + assert result is None # operation helper functions @@ -95,7 +101,11 @@ # operation implementations def opimpl_make_redbox(self): - XXX + genconst = self.get_greenarg() + typeindex = self.load_2byte() + kind = self.frame.bytecode.typekinds[typeindex] + redboxcls = self.frame.bytecode.redboxclasses[typeindex] + self.red_result(redboxcls(kind, genconst)) def opimpl_goto(self): target = self.load_4byte() @@ -123,7 +133,6 @@ newstate = rtimeshift.leave_graph_yellow(self.queue) self.jitstate = newstate return STOP - return STOP # XXX wrong, of course def opimpl_make_new_redvars(self): # an opcode with a variable number of args @@ -206,18 +215,29 @@ self.seen_blocks = {} self.assembler = [] self.constants = [] + self.typekinds = [] + self.redboxclasses = [] + # mapping constant -> index in constants self.const_positions = {} + # mapping blocks to True self.seen_blocks = {} - self.additional_positions = {} self.redvar_positions = {} + # mapping block to the free red index self.free_red = {} self.greenvar_positions = {} + # mapping block to the free green index self.free_green = {} + # mapping TYPE to index + self.type_positions = {} + self.graph = graph self.entrymap = flowmodel.mkentrymap(graph) self.make_bytecode_block(graph.startblock) assert self.current_block is None - return JitCode(assemble(self.interpreter, *self.assembler), self.constants) + return JitCode(assemble(self.interpreter, *self.assembler), + self.constants, + self.typekinds, + self.redboxclasses) def make_bytecode_block(self, block, insert_goto=False): if block in self.seen_blocks: @@ -330,6 +350,7 @@ resultindex = self.register_redvar((arg, block)) argindex = self.green_position(arg) self.emit(argindex) + self.emit(self.type_position(arg.concretetype)) return resultindex def opcolor(self, op): @@ -377,6 +398,15 @@ self.constants.append(const) self.const_positions[const] = result return result + + def type_position(self, TYPE): + if TYPE in self.type_positions: + return self.type_positions[TYPE] + self.typekinds.append(self.RGenOp.kindToken(TYPE)) + self.redboxclasses.append(rvalue.ll_redboxcls(TYPE)) + result = len(self.type_positions) + self.type_positions[TYPE] = result + return result def emit(self, stuff): assert stuff is not None Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Fri Feb 1 15:20:43 2008 @@ -139,6 +139,12 @@ res = self.interpret(f, [1, 2]) assert res == 3 + def test_convert_const_to_red(self): + def f(x): + return x + 1 + res = self.interpret(f, [2]) + assert res == 3 + def test_green_switch(self): def f(green, x, y): green = hint(green, concrete=True) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Fri Feb 1 15:20:43 2008 @@ -63,18 +63,21 @@ "make_new_greenvars", 0, "red_return", 0) assert len(jitcode.constants) == 0 + assert len(jitcode.typekinds) == 0 def test_constant(self): def f(x): return x + 1 writer, jitcode = self.serialize(f, [int]) assert jitcode.code == assemble(writer.interpreter, - "make_redbox", -1, + "make_redbox", -1, 0, "red_int_add", 0, 1, "make_new_redvars", 1, 2, "make_new_greenvars", 0, "red_return", 0) assert len(jitcode.constants) == 1 + assert len(jitcode.typekinds) == 1 + assert len(jitcode.redboxclasses) == 1 def test_green_switch(self): def f(x, y, z): @@ -98,6 +101,7 @@ ) assert jitcode.code == expected assert len(jitcode.constants) == 0 + assert len(jitcode.typekinds) == 0 def test_green_switch2(self): def f(x, y, z): @@ -129,7 +133,7 @@ ) assert jitcode.code == expected assert len(jitcode.constants) == 0 - + assert len(jitcode.typekinds) == 0 class TestLLType(AbstractSerializationTest): From cfbolz at codespeak.net Fri Feb 1 15:27:02 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 1 Feb 2008 15:27:02 +0100 (CET) Subject: [pypy-svn] r51185 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080201142702.85514168465@codespeak.net> Author: cfbolz Date: Fri Feb 1 15:27:01 2008 New Revision: 51185 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: a bit bigger test that contains mostly green vars Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Fri Feb 1 15:27:01 2008 @@ -156,5 +156,23 @@ res = self.interpret(f, [0, 1, 2]) assert res == -1 + def test_arith_plus_minus(self): + def ll_plus_minus(encoded_insn, nb_insn, x, y): + acc = x + pc = 0 + hint(nb_insn, concrete=True) + while pc < nb_insn: + op = (encoded_insn >> (pc*4)) & 0xF + op = hint(op, concrete=True) + if op == 0xA: + acc += y + elif op == 0x5: + acc -= y + pc += 1 + return acc + assert ll_plus_minus(0xA5A, 3, 32, 10) == 42 + res = self.interpret(ll_plus_minus, [0xA5A, 3, 32, 10]) + assert res == 42 + class TestLLType(AbstractInterpretationTest): type_system = "lltype" From exarkun at codespeak.net Fri Feb 1 16:02:11 2008 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Fri, 1 Feb 2008 16:02:11 +0100 (CET) Subject: [pypy-svn] r51187 - pypy/build/buildbot Message-ID: <20080201150211.A44A7168465@codespeak.net> Author: exarkun Date: Fri Feb 1 16:02:10 2008 New Revision: 51187 Modified: pypy/build/buildbot/master.cfg Log: re-enable pypy tests on 64bit slave Modified: pypy/build/buildbot/master.cfg ============================================================================== --- pypy/build/buildbot/master.cfg (original) +++ pypy/build/buildbot/master.cfg Fri Feb 1 16:02:10 2008 @@ -55,7 +55,7 @@ {"name": "pypy-c-allworkingmodules-faassen-64", "slavenames": ["linux-dvs0"], "builddir": "pypy-c-allworkingmodules-faassen-64", - "factory": PyPyBuildFactory(None, + "factory": PyPyBuildFactory(["--boxed"], [], ["--allworkingmodules", "--faassen"])}, From cfbolz at codespeak.net Sat Feb 2 02:44:12 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 02:44:12 +0100 (CET) Subject: [pypy-svn] r51196 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080202014412.83C671683EE@codespeak.net> Author: cfbolz Date: Sat Feb 2 02:44:11 2008 New Revision: 51196 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: implement red switches Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Sat Feb 2 02:44:11 2008 @@ -55,6 +55,7 @@ self.frame.local_boxes = redargs self.frame.local_green = greenargs self.bytecode_loop() + return self.jitstate def bytecode_loop(self): while 1: @@ -70,6 +71,7 @@ def load_2byte(self): pc = self.frame.pc + assert pc >= 0 result = ((ord(self.frame.bytecode.code[pc]) << 8) | ord(self.frame.bytecode.code[pc + 1])) self.frame.pc = pc + 2 @@ -77,6 +79,7 @@ def load_4byte(self): pc = self.frame.pc + assert pc >= 0 result = ((ord(self.frame.bytecode.code[pc + 0]) << 24) | (ord(self.frame.bytecode.code[pc + 1]) << 16) | (ord(self.frame.bytecode.code[pc + 2]) << 8) | @@ -99,6 +102,13 @@ def green_result(self, gv): self.frame.local_green.append(gv) + def newjitstate(self, newjitstate): + self.jitstate = newjitstate + if newjitstate is not None: + self.frame = newjitstate.frame + else: + self.frame = None + # operation implementations def opimpl_make_redbox(self): genconst = self.get_greenarg() @@ -119,14 +129,24 @@ self.frame.pc = target def opimpl_red_goto_iftrue(self): - XXX + switchbox = self.get_redarg() + target = self.load_4byte() + # XXX not sure about passing no green vars + descision = rtimeshift.split(self.jitstate, switchbox, self.frame.pc) + if descision: + self.frame.pc = target def opimpl_red_return(self): rtimeshift.save_return(self.jitstate) - # XXX for now - newstate = rtimeshift.leave_graph_red(self.queue, is_portal=True) - self.jitstate = newstate - return STOP + newjitstate = rtimeshift.dispatch_next(self.queue) + resumepoint = rtimeshift.getresumepoint(newjitstate) + if resumepoint == -1: + # XXX for now + newjitstate = rtimeshift.leave_graph_red(self.queue, is_portal=True) + self.newjitstate(newjitstate) + return STOP + self.newjitstate(newjitstate) + self.frame.pc = resumepoint def opimpl_green_return(self): rtimeshift.save_return(self.jitstate) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Feb 2 02:44:11 2008 @@ -117,8 +117,9 @@ redargs.append(boxcls(kind, inputargs_gv[i])) residualargs.append(ll_val) i += 1 - writer.interpreter.run(jitstate, jitcode, greenargs, redargs) - ll_finish_jitstate(writer.interpreter.jitstate, edesc, sigtoken) + jitstate = writer.interpreter.run(jitstate, jitcode, greenargs, redargs) + if jitstate is not None: + ll_finish_jitstate(jitstate, edesc, sigtoken) builder.end() generated = gv_generated.revealconst(lltype.Ptr(FUNC)) graph = generated._obj.graph @@ -174,5 +175,13 @@ res = self.interpret(ll_plus_minus, [0xA5A, 3, 32, 10]) assert res == 42 + def test_red_switch(self): + def f(x, y): + if x: + return x + return y + res = self.interpret(f, [1, 2]) + + class TestLLType(AbstractInterpretationTest): type_system = "lltype" From xoraxax at codespeak.net Sat Feb 2 12:57:38 2008 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 2 Feb 2008 12:57:38 +0100 (CET) Subject: [pypy-svn] r51201 - pypy/dist/pypy/tool Message-ID: <20080202115738.964D9168411@codespeak.net> Author: xoraxax Date: Sat Feb 2 12:57:36 2008 New Revision: 51201 Modified: pypy/dist/pypy/tool/cache.py Log: Add a repr for the Cache class. Modified: pypy/dist/pypy/tool/cache.py ============================================================================== --- pypy/dist/pypy/tool/cache.py (original) +++ pypy/dist/pypy/tool/cache.py Sat Feb 2 12:57:36 2008 @@ -58,6 +58,9 @@ if lock: lock.release() getorbuild._annspecialcase_ = "specialize:memo" + def __repr__(self): + return "" % (self.__class__.__name__, len(self.content)) + def _ready(self, result): pass From xoraxax at codespeak.net Sat Feb 2 12:58:44 2008 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 2 Feb 2008 12:58:44 +0100 (CET) Subject: [pypy-svn] r51202 - pypy/dist/pypy/rpython Message-ID: <20080202115844.A3D17168411@codespeak.net> Author: xoraxax Date: Sat Feb 2 12:58:44 2008 New Revision: 51202 Modified: pypy/dist/pypy/rpython/rpbc.py Log: Remove confusing getattr test (which does not really make any sense IMHO) in the rpbc code that generates the Desc warning. Modified: pypy/dist/pypy/rpython/rpbc.py ============================================================================== --- pypy/dist/pypy/rpython/rpbc.py (original) +++ pypy/dist/pypy/rpython/rpbc.py Sat Feb 2 12:58:44 2008 @@ -14,6 +14,7 @@ from pypy.rpython import callparse + def small_cand(rtyper, s_pbc): if 1 < len(s_pbc.descriptions) < rtyper.getconfig().translation.withsmallfuncsets and \ hasattr(rtyper.type_system.rpbc, 'SmallFunctionSetPBCRepr'): @@ -482,10 +483,7 @@ try: thisattrvalue = frozendesc.attrcache[attr] except KeyError: - # don't issue warning if this attribute can be read, but - # is not used - if not hasattr(frozendesc.pyobj, attr): - warning("Desc %r has no attribute %r" % (frozendesc, attr)) + warning("Desc %r has no attribute %r" % (frozendesc, attr)) continue llvalue = r_value.convert_const(thisattrvalue) setattr(result, mangled_name, llvalue) From cfbolz at codespeak.net Sat Feb 2 13:06:15 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 13:06:15 +0100 (CET) Subject: [pypy-svn] r51203 - pypy/branch/jit-refactoring/pypy/jit/timeshifter Message-ID: <20080202120615.97747168437@codespeak.net> Author: cfbolz Date: Sat Feb 2 13:06:14 2008 New Revision: 51203 Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Log: oops, forgot to check this in yesterday Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Sat Feb 2 13:06:14 2008 @@ -992,6 +992,8 @@ self.dispatchqueue = dispatchqueue #self.local_boxes = ... set by callers #self.local_green = ... set by callers + #self.pc = ... set by callers + #self.bytecode = ... set by callers def enter_block(self, incoming, memo): for box in self.local_boxes: @@ -1014,6 +1016,9 @@ newbackframe = self.backframe.copy(memo) result = VirtualFrame(newbackframe, self.dispatchqueue) result.local_boxes = [box.copy(memo) for box in self.local_boxes] + result.pc = self.pc + result.bytecode = self.bytecode + result.local_green = self.local_green[:] return result def replace(self, memo): From cfbolz at codespeak.net Sat Feb 2 13:11:27 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 13:11:27 +0100 (CET) Subject: [pypy-svn] r51204 - in pypy/branch/jit-refactoring/pypy/jit/timeshifter: . test Message-ID: <20080202121127.DBDE8168436@codespeak.net> Author: cfbolz Date: Sat Feb 2 13:11:27 2008 New Revision: 51204 Added: pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py (contents, props changed) pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py (contents, props changed) Log: add a key wrapper class to be able to use lists of genconsts in the states_dicts as keys Added: pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py ============================================================================== --- (empty file) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py Sat Feb 2 13:11:27 2008 @@ -0,0 +1,72 @@ +from pypy.rpython.annlowlevel import cachedtype +from pypy.rlib.rarithmetic import intmask +from pypy.rlib.unroll import unrolling_iterable +from pypy.rpython.lltypesystem import lltype + +class KeyDesc(object): + __metaclass__ = cachedtype + + def __init__(self, RGenOp, *TYPES): + self.RGenOp = RGenOp + self.TYPES = TYPES + TARGETTYPES = [] + for TYPE in TYPES: + # XXX more cases? + TARGET = lltype.Signed + if TYPE == lltype.Float: + TARGET = TYPE + TARGETTYPES.append(TARGET) + + iterator = unrolling_iterable(enumerate(TARGETTYPES)) + length = len(TYPES) + def greenhash(self, rgenop): + retval = 0x345678 + mult = 1000003 + for i, TARGET in iterator: + genconst = self.values[i] + item = genconst.revealconst(TARGET) + retval = intmask((retval ^ hash(item)) * intmask(mult)) + mult = mult + 82520 + 2*length + return retval + self.hash = greenhash + def greencompare(self, other, rgenop): + for i, TARGET in iterator: + genconst = self.values[i] + item_self = genconst.revealconst(TARGET) + genconst = other.values[i] + item_other = genconst.revealconst(TARGET) + if item_self != item_other: + return False + return True + self.compare = greencompare + + def _freeze_(self): + return True + + +class GreenKey(object): + def __init__(self, values, desc, rgenop): + self.desc = desc + self.values = values + self.rgenop = rgenop + + def __eq__(self, other): + raise TypeError("don't store GreenKeys in a normal dict") + + def __ne__(self, other): + raise TypeError("don't store GreenKeys in a normal dict") + + def __hash__(self): + raise TypeError("not hashable") + +def greenkey_eq(self, other): + assert self.rgenop is other.rgenop + if self is other: + return True + if self.desc is not other.desc: + return False + return self.desc.compare(self, other, self.rgenop) + +def greenkey_hash(self): + return self.desc.hash(self, self.rgenop) + Added: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py ============================================================================== --- (empty file) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py Sat Feb 2 13:11:27 2008 @@ -0,0 +1,30 @@ +from pypy.jit.timeshifter.greenkey import GreenKey, KeyDesc +from pypy.jit.timeshifter.greenkey import greenkey_hash, greenkey_eq +from pypy.jit.codegen.llgraph.rgenop import RGenOp +from pypy.rlib.objectmodel import r_dict +from pypy.rpython.lltypesystem import lltype + +rgenop = RGenOp() + +class TestGreenKeys(object): + def newdict(self): + return r_dict(greenkey_eq, greenkey_hash) + + def newkey(self, *values): + desc = KeyDesc(RGenOp, *[lltype.typeOf(val) for val in values]) + return GreenKey([rgenop.genconst(val) for val in values], desc, rgenop) + + def test_simple(self): + d = self.newdict() + d[self.newkey(1, 2)] = 1 + assert d[self.newkey(1, 2)] == 1 + + def test_check_types(self): + d = self.newdict() + d[self.newkey(1, 2)] = 1 + assert self.newkey(True, 2) not in d + + def test_check_lengths(self): + d = self.newdict() + d[self.newkey(1, 2)] = 1 + assert self.newkey(1, 2, 0) not in d From xoraxax at codespeak.net Sat Feb 2 13:16:33 2008 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 2 Feb 2008 13:16:33 +0100 (CET) Subject: [pypy-svn] r51205 - in pypy/dist/pypy: annotation tool/algo tool/algo/test Message-ID: <20080202121633.650771683EE@codespeak.net> Author: xoraxax Date: Sat Feb 2 13:16:31 2008 New Revision: 51205 Added: pypy/dist/pypy/tool/algo/test/test_unionfind.py Modified: pypy/dist/pypy/annotation/specialize.py pypy/dist/pypy/tool/algo/unionfind.py Log: Fix the mysterious RTyper warnings. Before, the memo code was registering a callback on the finish method of every ever created MemoTable instance. But the union find code was merging these tables together. Now the callback is ignored if it is called for a dominated MemoTable, i.e. one that was merged into another one. The accompanied unittest tests the new behaviour of the unionfind code, I do not know how to test the real issue reproducibly. The result of this checkin is that translation runs show only a single RTyper warning of this kind: [rtyper:WARNING] Desc > has no attribute '$memofield_getorbuild_0' Modified: pypy/dist/pypy/annotation/specialize.py ============================================================================== --- pypy/dist/pypy/annotation/specialize.py (original) +++ pypy/dist/pypy/annotation/specialize.py Sat Feb 2 13:16:31 2008 @@ -96,11 +96,19 @@ self.funcdesc = funcdesc self.table = {args: value} self.graph = None + self.do_not_process = False + + def register_finish(self): + bookkeeper = self.funcdesc.bookkeeper + bookkeeper.pending_specializations.append(self.finish) def update(self, other): self.table.update(other.table) self.graph = None # just in case + def cleanup(self): + self.do_not_process = True + fieldnamecounter = 0 def getuniquefieldname(self): @@ -110,6 +118,8 @@ return fieldname def finish(self): + if self.do_not_process: + return from pypy.annotation.model import unionof assert self.graph is None, "MemoTable already finished" # list of which argument positions can take more than one value @@ -273,7 +283,7 @@ def compute_one_result(args): value = func(*args) memotable = MemoTable(funcdesc, args, value) - bookkeeper.pending_specializations.append(memotable.finish) + memotable.register_finish() return memotable memotables = UnionFind(compute_one_result) Added: pypy/dist/pypy/tool/algo/test/test_unionfind.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/tool/algo/test/test_unionfind.py Sat Feb 2 13:16:31 2008 @@ -0,0 +1,26 @@ +from pypy.tool.algo.unionfind import UnionFind + + +def test_cleanup(): + state = [] + class ReferencedByExternalState(object): + def __init__(self, obj): + state.append(self) + self.obj = obj + + def update(self, other): + pass + + def cleanup(self): + state.remove(self) + + uf = UnionFind(ReferencedByExternalState) + uf.find(1) + for i in xrange(1, 10, 2): + uf.union(i, 1) + uf.find(2) + for i in xrange(2, 20, 2): + uf.union(i, 2) + assert len(state) < 3 + + Modified: pypy/dist/pypy/tool/algo/unionfind.py ============================================================================== --- pypy/dist/pypy/tool/algo/unionfind.py (original) +++ pypy/dist/pypy/tool/algo/unionfind.py Sat Feb 2 13:16:31 2008 @@ -87,6 +87,8 @@ self.link_to_parent[rep2] = rep1 del self.weight[rep2] + if hasattr(info2, "cleanup"): + info2.cleanup() del self.root_info[rep2] self.weight[rep1] = w From cfbolz at codespeak.net Sat Feb 2 13:27:18 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 13:27:18 +0100 (CET) Subject: [pypy-svn] r51206 - in pypy/branch/jit-refactoring/pypy/jit/timeshifter: . test Message-ID: <20080202122718.1D432168439@codespeak.net> Author: cfbolz Date: Sat Feb 2 13:27:17 2008 New Revision: 51206 Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py Log: put a newgreendict function in the main file Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py Sat Feb 2 13:27:17 2008 @@ -1,6 +1,7 @@ from pypy.rpython.annlowlevel import cachedtype from pypy.rlib.rarithmetic import intmask from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.objectmodel import r_dict from pypy.rpython.lltypesystem import lltype class KeyDesc(object): @@ -70,3 +71,7 @@ def greenkey_hash(self): return self.desc.hash(self, self.rgenop) +def newgreendict(): + return r_dict(greenkey_eq, greenkey_hash) + + Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py Sat Feb 2 13:27:17 2008 @@ -1,30 +1,26 @@ -from pypy.jit.timeshifter.greenkey import GreenKey, KeyDesc +from pypy.jit.timeshifter.greenkey import GreenKey, KeyDesc, newgreendict from pypy.jit.timeshifter.greenkey import greenkey_hash, greenkey_eq from pypy.jit.codegen.llgraph.rgenop import RGenOp -from pypy.rlib.objectmodel import r_dict from pypy.rpython.lltypesystem import lltype rgenop = RGenOp() class TestGreenKeys(object): - def newdict(self): - return r_dict(greenkey_eq, greenkey_hash) - def newkey(self, *values): desc = KeyDesc(RGenOp, *[lltype.typeOf(val) for val in values]) return GreenKey([rgenop.genconst(val) for val in values], desc, rgenop) def test_simple(self): - d = self.newdict() + d = newgreendict() d[self.newkey(1, 2)] = 1 assert d[self.newkey(1, 2)] == 1 def test_check_types(self): - d = self.newdict() + d = newgreendict() d[self.newkey(1, 2)] = 1 assert self.newkey(True, 2) not in d def test_check_lengths(self): - d = self.newdict() + d = newgreendict() d[self.newkey(1, 2)] = 1 assert self.newkey(1, 2, 0) not in d From cfbolz at codespeak.net Sat Feb 2 13:34:40 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 13:34:40 +0100 (CET) Subject: [pypy-svn] r51207 - in pypy/branch/jit-refactoring/pypy/jit/timeshifter: . test Message-ID: <20080202123440.0E1AA1683FB@codespeak.net> Author: cfbolz Date: Sat Feb 2 13:34:40 2008 New Revision: 51207 Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py Log: some cleanups, introduce a special empty key Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py Sat Feb 2 13:34:40 2008 @@ -7,10 +7,16 @@ class KeyDesc(object): __metaclass__ = cachedtype - def __init__(self, RGenOp, *TYPES): + def __init__(self, RGenOp=None, *TYPES): self.RGenOp = RGenOp self.TYPES = TYPES TARGETTYPES = [] + + if RGenOp is None: + assert len(TYPES) == 0 + self.hash = lambda self: 0 + self.compare = lambda self, other: True + for TYPE in TYPES: # XXX more cases? TARGET = lltype.Signed @@ -20,7 +26,7 @@ iterator = unrolling_iterable(enumerate(TARGETTYPES)) length = len(TYPES) - def greenhash(self, rgenop): + def greenhash(self): retval = 0x345678 mult = 1000003 for i, TARGET in iterator: @@ -30,7 +36,7 @@ mult = mult + 82520 + 2*length return retval self.hash = greenhash - def greencompare(self, other, rgenop): + def greencompare(self, other): for i, TARGET in iterator: genconst = self.values[i] item_self = genconst.revealconst(TARGET) @@ -46,10 +52,9 @@ class GreenKey(object): - def __init__(self, values, desc, rgenop): + def __init__(self, values, desc): self.desc = desc self.values = values - self.rgenop = rgenop def __eq__(self, other): raise TypeError("don't store GreenKeys in a normal dict") @@ -61,17 +66,16 @@ raise TypeError("not hashable") def greenkey_eq(self, other): - assert self.rgenop is other.rgenop if self is other: return True if self.desc is not other.desc: return False - return self.desc.compare(self, other, self.rgenop) + return self.desc.compare(self, other) def greenkey_hash(self): - return self.desc.hash(self, self.rgenop) + return self.desc.hash(self) def newgreendict(): return r_dict(greenkey_eq, greenkey_hash) - +empty_key = GreenKey([], KeyDesc()) Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_greenkey.py Sat Feb 2 13:34:40 2008 @@ -1,5 +1,5 @@ from pypy.jit.timeshifter.greenkey import GreenKey, KeyDesc, newgreendict -from pypy.jit.timeshifter.greenkey import greenkey_hash, greenkey_eq +from pypy.jit.timeshifter.greenkey import greenkey_hash, greenkey_eq, empty_key from pypy.jit.codegen.llgraph.rgenop import RGenOp from pypy.rpython.lltypesystem import lltype @@ -8,7 +8,7 @@ class TestGreenKeys(object): def newkey(self, *values): desc = KeyDesc(RGenOp, *[lltype.typeOf(val) for val in values]) - return GreenKey([rgenop.genconst(val) for val in values], desc, rgenop) + return GreenKey([rgenop.genconst(val) for val in values], desc) def test_simple(self): d = newgreendict() @@ -24,3 +24,10 @@ d = newgreendict() d[self.newkey(1, 2)] = 1 assert self.newkey(1, 2, 0) not in d + + def test_empty_key(self): + d = newgreendict() + assert empty_key not in d + d[empty_key] = 1 + assert d[empty_key] == 1 + assert self.newkey(1, 2, 0) not in d From xoraxax at codespeak.net Sat Feb 2 13:37:35 2008 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sat, 2 Feb 2008 13:37:35 +0100 (CET) Subject: [pypy-svn] r51208 - pypy/dist/pypy/tool/algo/test Message-ID: <20080202123735.0FFE61683FB@codespeak.net> Author: xoraxax Date: Sat Feb 2 13:37:34 2008 New Revision: 51208 Modified: pypy/dist/pypy/tool/algo/test/test_unionfind.py Log: Make the assert in the union find test stricter. Modified: pypy/dist/pypy/tool/algo/test/test_unionfind.py ============================================================================== --- pypy/dist/pypy/tool/algo/test/test_unionfind.py (original) +++ pypy/dist/pypy/tool/algo/test/test_unionfind.py Sat Feb 2 13:37:34 2008 @@ -21,6 +21,6 @@ uf.find(2) for i in xrange(2, 20, 2): uf.union(i, 2) - assert len(state) < 3 + assert len(state) == 2 # we have exactly 2 partitions From cfbolz at codespeak.net Sat Feb 2 16:29:42 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 16:29:42 +0100 (CET) Subject: [pypy-svn] r51211 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter Message-ID: <20080202152942.E1CB116845B@codespeak.net> Author: cfbolz Date: Sat Feb 2 16:29:41 2008 New Revision: 51211 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/hrtyper.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Log: Implement (local) merge points in the rainbow interpreter. This breaks many (most?) existing jit tests because some details in rtimeshift.py changed. Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Sat Feb 2 16:29:41 2008 @@ -1,9 +1,10 @@ from pypy.rlib.rarithmetic import intmask +from pypy.rlib.unroll import unrolling_iterable from pypy.objspace.flow import model as flowmodel from pypy.rpython.lltypesystem import lltype from pypy.jit.hintannotator.model import originalconcretetype from pypy.jit.timeshifter import rtimeshift, rvalue -from pypy.rlib.unroll import unrolling_iterable +from pypy.jit.timeshifter.greenkey import KeyDesc, empty_key, GreenKey class JitCode(object): """ @@ -18,11 +19,14 @@ green consts are negative indexes """ - def __init__(self, code, constants, typekinds, redboxclasses): + def __init__(self, code, constants, typekinds, redboxclasses, keydescs, + num_mergepoints): self.code = code self.constants = constants self.typekinds = typekinds self.redboxclasses = redboxclasses + self.keydescs = keydescs + self.num_mergepoints = num_mergepoints def _freeze_(self): return True @@ -46,8 +50,7 @@ def run(self, jitstate, bytecode, greenargs, redargs): self.jitstate = jitstate - self.queue = rtimeshift.ensure_queue(jitstate, - rtimeshift.BaseDispatchQueue) + self.queue = rtimeshift.DispatchQueue(bytecode.num_mergepoints) rtimeshift.enter_frame(self.jitstate, self.queue) self.frame = self.jitstate.frame self.frame.pc = 0 @@ -66,6 +69,17 @@ else: assert result is None + def dispatch(self): + newjitstate = rtimeshift.dispatch_next(self.queue) + resumepoint = rtimeshift.getresumepoint(newjitstate) + self.newjitstate(newjitstate) + if resumepoint == -1: + # XXX what about green returns? + newjitstate = rtimeshift.leave_graph_red(self.queue, is_portal=True) + self.newjitstate(newjitstate) + return STOP + else: + self.frame.pc = resumepoint # operation helper functions @@ -138,21 +152,10 @@ def opimpl_red_return(self): rtimeshift.save_return(self.jitstate) - newjitstate = rtimeshift.dispatch_next(self.queue) - resumepoint = rtimeshift.getresumepoint(newjitstate) - if resumepoint == -1: - # XXX for now - newjitstate = rtimeshift.leave_graph_red(self.queue, is_portal=True) - self.newjitstate(newjitstate) - return STOP - self.newjitstate(newjitstate) - self.frame.pc = resumepoint + return self.dispatch() def opimpl_green_return(self): - rtimeshift.save_return(self.jitstate) - newstate = rtimeshift.leave_graph_yellow(self.queue) - self.jitstate = newstate - return STOP + XXX def opimpl_make_new_redvars(self): # an opcode with a variable number of args @@ -167,11 +170,28 @@ # an opcode with a variable number of args # num_args arg_old_1 arg_new_1 ... num = self.load_2byte() + if num == 0 and len(self.frame.local_green) == 0: + # fast (very common) case + return newgreens = [] for i in range(num): newgreens.append(self.get_greenarg()) self.frame.local_green = newgreens + def opimpl_merge(self): + mergepointnum = self.load_2byte() + keydescnum = self.load_2byte() + if keydescnum == -1: + key = empty_key + else: + keydesc = self.frame.bytecode.keydescs[keydescnum] + key = GreenKey(self.frame.local_green[:keydesc.nb_vals], keydesc) + states_dic = self.queue.local_caches[mergepointnum] + done = rtimeshift.retrieve_jitstate_for_merge(states_dic, self.jitstate, + key, None) + if done: + return self.dispatch() + # construction-time interface def _add_implemented_opcodes(self): @@ -219,7 +239,8 @@ self.opcode_implementations.append(implementation) self.opcode_descs.append(opdesc) return index - + + class BytecodeWriter(object): @@ -237,6 +258,7 @@ self.constants = [] self.typekinds = [] self.redboxclasses = [] + self.keydescs = [] # mapping constant -> index in constants self.const_positions = {} # mapping blocks to True @@ -249,6 +271,10 @@ self.free_green = {} # mapping TYPE to index self.type_positions = {} + # mapping tuple of green TYPES to index + self.keydesc_positions = {} + + self.num_mergepoints = 0 self.graph = graph self.entrymap = flowmodel.mkentrymap(graph) @@ -257,7 +283,9 @@ return JitCode(assemble(self.interpreter, *self.assembler), self.constants, self.typekinds, - self.redboxclasses) + self.redboxclasses, + self.keydescs, + self.num_mergepoints) def make_bytecode_block(self, block, insert_goto=False): if block in self.seen_blocks: @@ -271,16 +299,16 @@ self.free_green[block] = 0 self.free_red[block] = 0 self.current_block = block + self.emit(label(block)) reds, greens = self.sort_by_color(block.inputargs) for arg in reds: self.register_redvar(arg) for arg in greens: self.register_greenvar(arg) - #self.insert_merges(block) + self.insert_merges(block) for op in block.operations: self.serialize_op(op) - #self.insert_splits(block) self.insert_exits(block) self.current_block = oldblock @@ -288,6 +316,7 @@ if block.exits == (): returnvar, = block.inputargs color = self.varcolor(returnvar) + assert color == "red" # XXX green return values not supported yet index = self.serialize_oparg(color, returnvar) self.emit("%s_return" % color) self.emit(index) @@ -312,6 +341,30 @@ else: XXX + def insert_merges(self, block): + if block is self.graph.returnblock: + return + if len(self.entrymap[block]) <= 1: + return + num = self.num_mergepoints + self.num_mergepoints += 1 + # make keydesc + key = () + for arg in self.sort_by_color(block.inputargs)[1]: + TYPE = arg.concretetype + key += (TYPE, ) + if not key: + keyindex = -1 # use prebuilt empty_key + elif key not in self.keydesc_positions: + keyindex = len(self.keydesc_positions) + self.keydesc_positions[key] = keyindex + self.keydescs.append(KeyDesc(self.RGenOp, *key)) + else: + keyindex = self.keydesc_positions[key] + self.emit("merge") + self.emit(num) + self.emit(keyindex) + def insert_renaming(self, args): reds, greens = self.sort_by_color(args) for color, args in [("red", reds), ("green", greens)]: Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Feb 2 16:29:41 2008 @@ -181,7 +181,17 @@ return x return y res = self.interpret(f, [1, 2]) + assert res == 1 + def test_merge(self): + def f(x, y, z): + if x: + a = y - z + else: + a = y + z + return 1 + a + res = self.interpret(f, [1, 2, 3]) + assert res == 0 class TestLLType(AbstractInterpretationTest): type_system = "lltype" Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Sat Feb 2 16:29:41 2008 @@ -135,6 +135,40 @@ assert len(jitcode.constants) == 0 assert len(jitcode.typekinds) == 0 + def test_merge(self): + def f(x, y, z): + if x: + a = y - z + else: + a = y + z + return 1 + a + writer, jitcode = self.serialize(f, [int, int, int]) + expected = assemble(writer.interpreter, + "red_int_is_true", 0, + "red_goto_iftrue", 3, tlabel("add"), + "make_new_redvars", 2, 1, 2, + "make_new_greenvars", 0, + "red_int_add", 0, 1, + "make_new_redvars", 1, 2, + "make_new_greenvars", 0, + label("after"), + "merge", 0, -1, + "make_redbox", -1, 0, + "red_int_add", 1, 0, + "make_new_redvars", 1, 2, + "make_new_greenvars", 0, + "red_return", 0, + label("add"), + "make_new_redvars", 2, 1, 2, + "make_new_greenvars", 0, + "red_int_sub", 0, 1, + "make_new_redvars", 1, 2, + "make_new_greenvars", 0, + "goto", tlabel("after"), + ) + assert jitcode.code == expected + assert len(jitcode.constants) == 1 + assert len(jitcode.typekinds) == 1 class TestLLType(AbstractSerializationTest): type_system = "lltype" Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py Sat Feb 2 16:29:41 2008 @@ -10,13 +10,16 @@ def __init__(self, RGenOp=None, *TYPES): self.RGenOp = RGenOp self.TYPES = TYPES - TARGETTYPES = [] + self.nb_vals = len(TYPES) + if not TYPES: + assert RGenOp is None if RGenOp is None: assert len(TYPES) == 0 self.hash = lambda self: 0 self.compare = lambda self, other: True + TARGETTYPES = [] for TYPE in TYPES: # XXX more cases? TARGET = lltype.Signed @@ -53,6 +56,7 @@ class GreenKey(object): def __init__(self, values, desc): + assert len(values) == desc.nb_vals self.desc = desc self.values = values Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/hrtyper.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/hrtyper.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/hrtyper.py Sat Feb 2 16:29:41 2008 @@ -1269,6 +1269,7 @@ attrname = hop.args_v[1].value DispatchQueueSubclass = self.get_dispatch_subclass(mpfamily) + py.test.skip("broken due to different key handling") if global_resumer is not None: states_dic = {} def merge_point(jitstate, *key): Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Sat Feb 2 16:29:41 2008 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem import lltype, lloperation, llmemory from pypy.jit.hintannotator.model import originalconcretetype from pypy.jit.timeshifter import rvalue, rcontainer, rvirtualizable +from pypy.jit.timeshifter.greenkey import newgreendict, empty_key from pypy.rlib.unroll import unrolling_iterable from pypy.rpython.annlowlevel import cachedtype, base_ptr_lltype from pypy.rpython.annlowlevel import cast_instance_to_base_ptr @@ -878,31 +879,26 @@ # ____________________________________________________________ -class BaseDispatchQueue(object): +class DispatchQueue(object): resuming = None - def __init__(self): + def __init__(self, num_local_caches=0): self.split_chain = None self.global_merge_chain = None self.return_chain = None + self.num_local_caches = num_local_caches self.clearlocalcaches() def clearlocalcaches(self): self.mergecounter = 0 + self.local_caches = [newgreendict() + for i in range(self.num_local_caches)] def clear(self): - self.__init__() + self.__init__(self.num_local_caches) def build_dispatch_subclass(attrnames): - if len(attrnames) == 0: - return BaseDispatchQueue - attrnames = unrolling_iterable(attrnames) - class DispatchQueue(BaseDispatchQueue): - def clearlocalcaches(self): - BaseDispatchQueue.clearlocalcaches(self) - for name in attrnames: - setattr(self, name, {}) # the new dicts have various types! - return DispatchQueue + py.test.skip("no longer exists") class FrozenVirtualFrame(object): @@ -1252,10 +1248,6 @@ return jitstate -def ensure_queue(jitstate, DispatchQueueClass): - return DispatchQueueClass() -ensure_queue._annspecialcase_ = 'specialize:arg(1)' - def replayable_ensure_queue(jitstate, DispatchQueueClass): if jitstate.frame is None: # common case return DispatchQueueClass() @@ -1291,14 +1283,14 @@ def merge_returning_jitstates(dispatchqueue, force_merge=False): return_chain = dispatchqueue.return_chain - return_cache = {} + return_cache = newgreendict() still_pending = None opened = None while return_chain is not None: jitstate = return_chain return_chain = return_chain.next opened = start_writing(jitstate, opened) - res = retrieve_jitstate_for_merge(return_cache, jitstate, (), + res = retrieve_jitstate_for_merge(return_cache, jitstate, empty_key, return_marker, force_merge=force_merge) if res is False: # not finished @@ -1311,13 +1303,13 @@ # more general one. return_chain = still_pending if return_chain is not None: - return_cache = {} + return_cache = newgreendict() still_pending = None while return_chain is not None: jitstate = return_chain return_chain = return_chain.next opened = start_writing(jitstate, opened) - res = retrieve_jitstate_for_merge(return_cache, jitstate, (), + res = retrieve_jitstate_for_merge(return_cache, jitstate, empty_key, return_marker, force_merge=force_merge) if res is False: # not finished From cfbolz at codespeak.net Sat Feb 2 17:01:36 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 17:01:36 +0100 (CET) Subject: [pypy-svn] r51214 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080202160136.893EF168477@codespeak.net> Author: cfbolz Date: Sat Feb 2 17:01:35 2008 New Revision: 51214 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Log: two tests about loops and a small bug fix Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Sat Feb 2 17:01:35 2008 @@ -322,7 +322,7 @@ self.emit(index) elif len(block.exits) == 1: link, = block.exits - self.insert_renaming(link.args) + self.insert_renaming(link) self.make_bytecode_block(link.target, insert_goto=True) elif len(block.exits) == 2: linkfalse, linktrue = block.exits @@ -333,10 +333,10 @@ self.emit("%s_goto_iftrue" % color) self.emit(index) self.emit(tlabel(linktrue)) - self.insert_renaming(linkfalse.args) + self.insert_renaming(linkfalse) self.make_bytecode_block(linkfalse.target, insert_goto=True) self.emit(label(linktrue)) - self.insert_renaming(linktrue.args) + self.insert_renaming(linktrue) self.make_bytecode_block(linktrue.target, insert_goto=True) else: XXX @@ -365,8 +365,8 @@ self.emit(num) self.emit(keyindex) - def insert_renaming(self, args): - reds, greens = self.sort_by_color(args) + def insert_renaming(self, link): + reds, greens = self.sort_by_color(link.args, link.target.inputargs) for color, args in [("red", reds), ("green", greens)]: result = [] for v in args: Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Feb 2 17:01:35 2008 @@ -193,5 +193,16 @@ res = self.interpret(f, [1, 2, 3]) assert res == 0 + def test_loop_merging(self): + def ll_function(x, y): + tot = 0 + while x: + tot += y + x -= 1 + return tot + res = self.interpret(ll_function, [7, 2]) + assert res == 14 + + class TestLLType(AbstractInterpretationTest): type_system = "lltype" Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Sat Feb 2 17:01:35 2008 @@ -170,5 +170,34 @@ assert len(jitcode.constants) == 1 assert len(jitcode.typekinds) == 1 + def test_loop(self): + def f(x): + r = 0 + while x: + r += x + x -= 1 + return r + writer, jitcode = self.serialize(f, [int]) + expected = assemble(writer.interpreter, + "make_redbox", -1, 0, + "make_new_redvars", 2, 0, 1, + "make_new_greenvars", 0, + label("while"), + "merge", 0, -1, + "red_int_is_true", 0, + "red_goto_iftrue", 2, tlabel("body"), + "make_new_redvars", 1, 0, + "make_new_greenvars", 0, + "red_return", 0, + label("body"), + "make_new_redvars", 2, 0, 1, + "make_new_greenvars", 0, + "red_int_add", 1, 0, + "make_redbox", -2, 0, + "red_int_sub", 0, 3, + "make_new_redvars", 2, 2, 4, + "goto", tlabel("while")) + + class TestLLType(AbstractSerializationTest): type_system = "lltype" From cfbolz at codespeak.net Sat Feb 2 17:23:36 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 17:23:36 +0100 (CET) Subject: [pypy-svn] r51215 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080202162336.2689C1684D3@codespeak.net> Author: cfbolz Date: Sat Feb 2 17:23:36 2008 New Revision: 51215 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: a slightly more complex test Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Feb 2 17:23:36 2008 @@ -203,6 +203,18 @@ res = self.interpret(ll_function, [7, 2]) assert res == 14 + def test_loop_merging2(self): + def ll_function(x, y): + tot = 0 + while x: + if tot < 3: + tot *= y + else: + tot += y + x -= 1 + return tot + res = self.interpret(ll_function, [7, 2]) + assert res == 0 class TestLLType(AbstractInterpretationTest): type_system = "lltype" From cfbolz at codespeak.net Sat Feb 2 17:27:01 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 17:27:01 +0100 (CET) Subject: [pypy-svn] r51216 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080202162701.A48E91684D3@codespeak.net> Author: cfbolz Date: Sat Feb 2 17:27:01 2008 New Revision: 51216 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: add ability to check for the correct amount of generated instructions Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Feb 2 17:27:01 2008 @@ -11,6 +11,7 @@ from pypy.rpython.lltypesystem import lltype, rstr from pypy.rpython.llinterp import LLInterpreter from pypy.annotation import model as annmodel +from pypy.objspace.flow.model import summary from pypy.rlib.jit import hint from pypy import conftest @@ -123,12 +124,20 @@ builder.end() generated = gv_generated.revealconst(lltype.Ptr(FUNC)) graph = generated._obj.graph + self.residual_graph = graph if conftest.option.view: graph.show() llinterp = LLInterpreter(self.rtyper) res = llinterp.eval_graph(graph, residualargs) return res + def check_insns(self, expected=None, **counts): + self.insns = summary(self.residual_graph) + if expected is not None: + assert self.insns == expected + for opname, count in counts.items(): + assert self.insns.get(opname, 0) == count + def Xtest_return_green(self): def f(): return 1 @@ -154,8 +163,10 @@ return x - y res = self.interpret(f, [1, 1, 2]) assert res == 3 + self.check_insns({"int_add": 1}) res = self.interpret(f, [0, 1, 2]) assert res == -1 + self.check_insns({"int_sub": 1}) def test_arith_plus_minus(self): def ll_plus_minus(encoded_insn, nb_insn, x, y): @@ -174,6 +185,7 @@ assert ll_plus_minus(0xA5A, 3, 32, 10) == 42 res = self.interpret(ll_plus_minus, [0xA5A, 3, 32, 10]) assert res == 42 + self.check_insns({'int_add': 2, 'int_sub': 1}) def test_red_switch(self): def f(x, y): From cfbolz at codespeak.net Sat Feb 2 18:52:04 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 18:52:04 +0100 (CET) Subject: [pypy-svn] r51219 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080202175204.23BD8168457@codespeak.net> Author: cfbolz Date: Sat Feb 2 18:52:03 2008 New Revision: 51219 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: cache the bytecode for a function Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Feb 2 18:52:03 2008 @@ -4,7 +4,7 @@ from pypy.jit.hintannotator.model import SomeLLAbstractConstant, OriginFlags from pypy.jit.hintannotator.model import originalconcretetype from pypy.jit.rainbow.bytecode import BytecodeWriter, label, tlabel, assemble -from pypy.jit.codegen.llgraph.rgenop import RGenOp +from pypy.jit.codegen.llgraph.rgenop import RGenOp as LLRGenOp from pypy.jit.rainbow.test.test_serializegraph import AbstractSerializationTest from pypy.jit.rainbow import bytecode from pypy.jit.timeshifter import rtimeshift, exception, rvalue @@ -32,7 +32,31 @@ class AbstractInterpretationTest(object): + RGenOp = LLRGenOp + + def setup_class(cls): + from pypy.jit.timeshifter.test.conftest import option + cls.on_llgraph = cls.RGenOp is LLRGenOp + cls._cache = {} + cls._cache_order = [] + + def teardown_class(cls): + del cls._cache + del cls._cache_order + def serialize(self, func, values, backendoptimize=False): + key = func, backendoptimize + try: + cache, argtypes = self._cache[key] + except KeyError: + pass + else: + self.__dict__.update(cache) + assert argtypes == getargtypes(self.rtyper.annotator, values) + return self.writer, self.jitcode, self.argcolors + + if len(self._cache_order) >= 3: + del self._cache[self._cache_order.pop(0)] # build the normal ll graphs for ll_function t = TranslationContext() a = t.buildannotator() @@ -58,12 +82,31 @@ t.view() graph2 = graphof(t, func) self.graph = graph2 - writer = BytecodeWriter(t, hannotator, RGenOp) + writer = BytecodeWriter(t, hannotator, self.RGenOp) jitcode = writer.make_bytecode(graph2) argcolors = [] + + # make residual functype + ha = self.hannotator + RESTYPE = originalconcretetype(hannotator.binding(graph2.getreturnvar())) + ARGS = [] + for var in graph2.getargs(): + # XXX ignoring virtualizables for now + binding = hannotator.binding(var) + if not binding.is_green(): + ARGS.append(originalconcretetype(binding)) + self.RESIDUAL_FUNCTYPE = lltype.FuncType(ARGS, RESTYPE) + for i, ll_val in enumerate(values): color = writer.varcolor(graph2.startblock.inputargs[i]) argcolors.append(color) + self.writer = writer + self.jitcode = jitcode + self.argcolors = argcolors + + cache = self.__dict__.copy() + self._cache[key] = cache, getargtypes(rtyper.annotator, values) + self._cache_order.append(key) return writer, jitcode, argcolors def interpret(self, ll_function, values, opt_consts=[], *args, **kwds): @@ -74,20 +117,8 @@ rtyper=writer.translator.annotator.base_translator.rtyper) edesc = exception.ExceptionDesc(hrtyper, False) rgenop = writer.RGenOp() - # make residual functype - FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) - ha = self.hannotator - RESTYPE = originalconcretetype(self.hannotator.binding(self.graph.getreturnvar())) - ARGS = [] - for var in self.graph.getargs(): - # XXX ignoring virtualizables for now - binding = self.hannotator.binding(var) - if not binding.is_green(): - ARGS.append(originalconcretetype(binding)) - FUNC = lltype.FuncType(ARGS, RESTYPE) - sigtoken = rgenop.sigToken(FUNC) + sigtoken = rgenop.sigToken(self.RESIDUAL_FUNCTYPE) builder, gv_generated, inputargs_gv = rgenop.newgraph(sigtoken, "generated") - print builder, builder.rgenop, rgenop builder.start_writing() jitstate = rtimeshift.JITState(builder, None, edesc.null_exc_type_box, @@ -122,7 +153,7 @@ if jitstate is not None: ll_finish_jitstate(jitstate, edesc, sigtoken) builder.end() - generated = gv_generated.revealconst(lltype.Ptr(FUNC)) + generated = gv_generated.revealconst(lltype.Ptr(self.RESIDUAL_FUNCTYPE)) graph = generated._obj.graph self.residual_graph = graph if conftest.option.view: From cfbolz at codespeak.net Sat Feb 2 19:25:13 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 19:25:13 +0100 (CET) Subject: [pypy-svn] r51220 - in pypy/branch/jit-refactoring/pypy/jit: rainbow/test timeshifter/test Message-ID: <20080202182513.5B1B2168425@codespeak.net> Author: cfbolz Date: Sat Feb 2 19:25:11 2008 New Revision: 51220 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py Log: start moving some of the test_timeshift tests over to run on the rainbow interpreter. Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Feb 2 19:25:11 2008 @@ -1,3 +1,4 @@ +import py from pypy.translator.translator import TranslationContext, graphof from pypy.jit.hintannotator.annotator import HintAnnotator from pypy.jit.hintannotator.policy import StopAtXPolicy, HintAnnotatorPolicy @@ -138,17 +139,21 @@ greenargs = [] redargs = [] residualargs = [] - i = 0 - for color, ll_val in zip(argcolors, values): + red_i = 0 + for i, (color, ll_val) in enumerate(zip(argcolors, values)): if color == "green": greenargs.append(writer.RGenOp.constPrebuiltGlobal(ll_val)) else: TYPE = lltype.typeOf(ll_val) kind = rgenop.kindToken(TYPE) boxcls = rvalue.ll_redboxcls(TYPE) - redargs.append(boxcls(kind, inputargs_gv[i])) + if i in opt_consts: + gv_arg = rgenop.genconst(ll_val) + else: + gv_arg = inputargs_gv[red_i] + redargs.append(boxcls(kind, gv_arg)) residualargs.append(ll_val) - i += 1 + red_i += 1 jitstate = writer.interpreter.run(jitstate, jitcode, greenargs, redargs) if jitstate is not None: ll_finish_jitstate(jitstate, edesc, sigtoken) @@ -169,22 +174,40 @@ for opname, count in counts.items(): assert self.insns.get(opname, 0) == count - def Xtest_return_green(self): - def f(): - return 1 - self.interpret(f, []) +class SimpleTests(AbstractInterpretationTest): + def test_simple_fixed(self): + py.test.skip("green return not working") + def ll_function(x, y): + return hint(x + y, concrete=True) + res = self.interpret(ll_function, [5, 7]) + assert res == 12 + self.check_insns({}) def test_very_simple(self): def f(x, y): return x + y res = self.interpret(f, [1, 2]) assert res == 3 + self.check_insns({"int_add": 1}) def test_convert_const_to_red(self): def f(x): return x + 1 res = self.interpret(f, [2]) assert res == 3 + self.check_insns({"int_add": 1}) + + def test_loop_convert_const_to_redbox(self): + def ll_function(x, y): + x = hint(x, concrete=True) + tot = 0 + while x: # conversion from green '0' to red 'tot' + tot += y + x -= 1 + return tot + res = self.interpret(ll_function, [7, 2]) + assert res == 14 + self.check_insns({'int_add': 7}) def test_green_switch(self): def f(green, x, y): @@ -199,24 +222,31 @@ assert res == -1 self.check_insns({"int_sub": 1}) - def test_arith_plus_minus(self): - def ll_plus_minus(encoded_insn, nb_insn, x, y): - acc = x - pc = 0 - hint(nb_insn, concrete=True) - while pc < nb_insn: - op = (encoded_insn >> (pc*4)) & 0xF - op = hint(op, concrete=True) - if op == 0xA: - acc += y - elif op == 0x5: - acc -= y - pc += 1 - return acc - assert ll_plus_minus(0xA5A, 3, 32, 10) == 42 - res = self.interpret(ll_plus_minus, [0xA5A, 3, 32, 10]) - assert res == 42 - self.check_insns({'int_add': 2, 'int_sub': 1}) + def test_simple_opt_const_propagation2(self): + def ll_function(x, y): + return x + y + res = self.interpret(ll_function, [5, 7], [0, 1]) + assert res == 12 + self.check_insns({}) + + def test_simple_opt_const_propagation1(self): + def ll_function(x): + return -x + res = self.interpret(ll_function, [5], [0]) + assert res == -5 + self.check_insns({}) + + def test_loop_folding(self): + def ll_function(x, y): + tot = 0 + x = hint(x, concrete=True) + while x: + tot += y + x -= 1 + return tot + res = self.interpret(ll_function, [7, 2], [0, 1]) + assert res == 14 + self.check_insns({}) def test_red_switch(self): def f(x, y): @@ -243,8 +273,25 @@ tot += y x -= 1 return tot - res = self.interpret(ll_function, [7, 2]) + res = self.interpret(ll_function, [7, 2], []) + assert res == 14 + self.check_insns(int_add = 2, + int_is_true = 2) + + res = self.interpret(ll_function, [7, 2], [0]) + assert res == 14 + self.check_insns(int_add = 2, + int_is_true = 1) + + res = self.interpret(ll_function, [7, 2], [1]) + assert res == 14 + self.check_insns(int_add = 1, + int_is_true = 2) + + res = self.interpret(ll_function, [7, 2], [0, 1]) assert res == 14 + self.check_insns(int_add = 1, + int_is_true = 1) def test_loop_merging2(self): def ll_function(x, y): @@ -259,5 +306,106 @@ res = self.interpret(ll_function, [7, 2]) assert res == 0 -class TestLLType(AbstractInterpretationTest): + def test_two_loops_merging(self): + def ll_function(x, y): + tot = 0 + while x: + tot += y + x -= 1 + while y: + tot += y + y -= 1 + return tot + res = self.interpret(ll_function, [7, 3], []) + assert res == 27 + self.check_insns(int_add = 3, + int_is_true = 3) + + def test_convert_greenvar_to_redvar(self): + def ll_function(x, y): + hint(x, concrete=True) + return x - y + res = self.interpret(ll_function, [70, 4], [0]) + assert res == 66 + self.check_insns(int_sub = 1) + res = self.interpret(ll_function, [70, 4], [0, 1]) + assert res == 66 + self.check_insns({}) + + def test_green_across_split(self): + def ll_function(x, y): + hint(x, concrete=True) + if y > 2: + z = x - y + else: + z = x + y + return z + res = self.interpret(ll_function, [70, 4], [0]) + assert res == 66 + self.check_insns(int_add = 1, + int_sub = 1) + + def test_merge_const_before_return(self): + def ll_function(x): + if x > 0: + y = 17 + else: + y = 22 + x -= 1 + y += 1 + return y+x + res = self.interpret(ll_function, [-70], []) + assert res == 23-71 + self.check_insns({'int_gt': 1, 'int_add': 2, 'int_sub': 2}) + + def test_merge_3_redconsts_before_return(self): + py.test.skip("XXX hint(variable=True) not implemented yet") + def ll_function(x): + if x > 2: + y = hint(54, variable=True) + elif x > 0: + y = hint(17, variable=True) + else: + y = hint(22, variable=True) + x -= 1 + y += 1 + return y+x + res = self.interpret(ll_function, [-70], []) + assert res == ll_function(-70) + res = self.interpret(ll_function, [1], []) + assert res == ll_function(1) + res = self.interpret(ll_function, [-70], []) + assert res == ll_function(-70) + + def test_merge_const_at_return(self): + py.test.skip("green return") + def ll_function(x): + if x > 0: + return 17 + else: + return 22 + res = self.interpret(ll_function, [-70], []) + assert res == 22 + self.check_insns({'int_gt': 1}) + + def test_arith_plus_minus(self): + def ll_plus_minus(encoded_insn, nb_insn, x, y): + acc = x + pc = 0 + while pc < nb_insn: + op = (encoded_insn >> (pc*4)) & 0xF + op = hint(op, concrete=True) + if op == 0xA: + acc += y + elif op == 0x5: + acc -= y + pc += 1 + return acc + assert ll_plus_minus(0xA5A, 3, 32, 10) == 42 + res = self.interpret(ll_plus_minus, [0xA5A, 3, 32, 10], [0, 1]) + assert res == 42 + self.check_insns({'int_add': 2, 'int_sub': 1}) + + +class TestLLType(SimpleTests): type_system = "lltype" Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py Sat Feb 2 19:25:11 2008 @@ -382,184 +382,6 @@ class BaseTestTimeshift(TimeshiftingTests): - def test_simple_fixed(self): - py.test.skip("green return not working") - def ll_function(x, y): - return hint(x + y, concrete=True) - res = self.timeshift(ll_function, [5, 7]) - assert res == 12 - self.check_insns({}) - - def test_very_simple(self): - def ll_function(x, y): - return x + y - res = self.timeshift(ll_function, [5, 7]) - assert res == 12 - self.check_insns({'int_add': 1}) - - def test_convert_const_to_redbox(self): - def ll_function(x, y): - x = hint(x, concrete=True) - tot = 0 - while x: # conversion from green '0' to red 'tot' - tot += y - x -= 1 - return tot - res = self.timeshift(ll_function, [7, 2]) - assert res == 14 - self.check_insns({'int_add': 7}) - - def test_simple_opt_const_propagation2(self): - def ll_function(x, y): - return x + y - res = self.timeshift(ll_function, [5, 7], [0, 1]) - assert res == 12 - self.check_insns({}) - - def test_simple_opt_const_propagation1(self): - def ll_function(x): - return -x - res = self.timeshift(ll_function, [5], [0]) - assert res == -5 - self.check_insns({}) - - def test_loop_folding(self): - def ll_function(x, y): - tot = 0 - x = hint(x, concrete=True) - while x: - tot += y - x -= 1 - return tot - res = self.timeshift(ll_function, [7, 2], [0, 1]) - assert res == 14 - self.check_insns({}) - - def test_loop_merging(self): - def ll_function(x, y): - tot = 0 - while x: - tot += y - x -= 1 - return tot - res = self.timeshift(ll_function, [7, 2], []) - assert res == 14 - self.check_insns(int_add = 2, - int_is_true = 2) - - res = self.timeshift(ll_function, [7, 2], [0]) - assert res == 14 - self.check_insns(int_add = 2, - int_is_true = 1) - - res = self.timeshift(ll_function, [7, 2], [1]) - assert res == 14 - self.check_insns(int_add = 1, - int_is_true = 2) - - res = self.timeshift(ll_function, [7, 2], [0, 1]) - assert res == 14 - self.check_insns(int_add = 1, - int_is_true = 1) - - def test_two_loops_merging(self): - def ll_function(x, y): - tot = 0 - while x: - tot += y - x -= 1 - while y: - tot += y - y -= 1 - return tot - res = self.timeshift(ll_function, [7, 3], []) - assert res == 27 - self.check_insns(int_add = 3, - int_is_true = 3) - - def test_convert_greenvar_to_redvar(self): - def ll_function(x, y): - hint(x, concrete=True) - return x - y - res = self.timeshift(ll_function, [70, 4], [0]) - assert res == 66 - self.check_insns(int_sub = 1) - res = self.timeshift(ll_function, [70, 4], [0, 1]) - assert res == 66 - self.check_insns({}) - - def test_green_across_split(self): - def ll_function(x, y): - hint(x, concrete=True) - if y > 2: - z = x - y - else: - z = x + y - return z - res = self.timeshift(ll_function, [70, 4], [0]) - assert res == 66 - self.check_insns(int_add = 1, - int_sub = 1) - - def test_merge_const_before_return(self): - def ll_function(x): - if x > 0: - y = 17 - else: - y = 22 - x -= 1 - y += 1 - return y+x - res = self.timeshift(ll_function, [-70], []) - assert res == 23-71 - self.check_insns({'int_gt': 1, 'int_add': 2, 'int_sub': 2}) - - def test_merge_3_redconsts_before_return(self): - def ll_function(x): - if x > 2: - y = hint(54, variable=True) - elif x > 0: - y = hint(17, variable=True) - else: - y = hint(22, variable=True) - x -= 1 - y += 1 - return y+x - res = self.timeshift(ll_function, [-70], []) - assert res == ll_function(-70) - res = self.timeshift(ll_function, [1], []) - assert res == ll_function(1) - res = self.timeshift(ll_function, [-70], []) - assert res == ll_function(-70) - - def test_merge_const_at_return(self): - py.test.skip("green return") - def ll_function(x): - if x > 0: - return 17 - else: - return 22 - res = self.timeshift(ll_function, [-70], []) - assert res == 22 - self.check_insns({'int_gt': 1}) - - def test_arith_plus_minus(self): - def ll_plus_minus(encoded_insn, nb_insn, x, y): - acc = x - pc = 0 - while pc < nb_insn: - op = (encoded_insn >> (pc*4)) & 0xF - op = hint(op, concrete=True) - if op == 0xA: - acc += y - elif op == 0x5: - acc -= y - pc += 1 - return acc - assert ll_plus_minus(0xA5A, 3, 32, 10) == 42 - res = self.timeshift(ll_plus_minus, [0xA5A, 3, 32, 10], [0, 1]) - assert res == 42 - self.check_insns({'int_add': 2, 'int_sub': 1}) def test_simple_struct(self): S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), From cfbolz at codespeak.net Sat Feb 2 20:13:42 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 2 Feb 2008 20:13:42 +0100 (CET) Subject: [pypy-svn] r51221 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080202191342.8273F168436@codespeak.net> Author: cfbolz Date: Sat Feb 2 20:13:40 2008 New Revision: 51221 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: support hint(variable=True). if only supporting hint(promote=True) were as easy Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Sat Feb 2 20:13:40 2008 @@ -418,7 +418,7 @@ block = self.current_block if (arg, block) in self.redvar_positions: # already converted - return self.redvar_positions[arg] + return self.redvar_positions[arg, block] self.emit("make_redbox") resultindex = self.register_redvar((arg, block)) argindex = self.green_position(arg) @@ -441,11 +441,13 @@ color = "red" return color - def register_redvar(self, arg): + def register_redvar(self, arg, where=-1): assert arg not in self.redvar_positions - self.redvar_positions[arg] = result = self.free_red[self.current_block] - self.free_red[self.current_block] += 1 - return result + if where == -1: + where = self.free_red[self.current_block] + self.free_red[self.current_block] += 1 + self.redvar_positions[arg] = where + return where def redvar_position(self, arg): return self.redvar_positions[arg] @@ -510,6 +512,14 @@ assert self.hannotator.binding(result).is_green() self.register_greenvar(result, self.green_position(arg)) return + if "variable" in hints: + assert not self.hannotator.binding(result).is_green() + if self.hannotator.binding(arg).is_green(): + resultindex = self.convert_to_red(arg) + self.register_redvar(result, resultindex) + else: + self.register_redvar(result, self.redvar_position(arg)) + return XXX Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sat Feb 2 20:13:40 2008 @@ -176,7 +176,7 @@ class SimpleTests(AbstractInterpretationTest): def test_simple_fixed(self): - py.test.skip("green return not working") + py.test.skip("green return") def ll_function(x, y): return hint(x + y, concrete=True) res = self.interpret(ll_function, [5, 7]) @@ -359,7 +359,6 @@ self.check_insns({'int_gt': 1, 'int_add': 2, 'int_sub': 2}) def test_merge_3_redconsts_before_return(self): - py.test.skip("XXX hint(variable=True) not implemented yet") def ll_function(x): if x > 2: y = hint(54, variable=True) From xoraxax at codespeak.net Sun Feb 3 11:59:57 2008 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 3 Feb 2008 11:59:57 +0100 (CET) Subject: [pypy-svn] r51224 - in pypy/dist/pypy: annotation jit/hintannotator tool/algo tool/algo/test translator/backendopt Message-ID: <20080203105957.8A537168407@codespeak.net> Author: xoraxax Date: Sun Feb 3 11:59:55 2008 New Revision: 51224 Modified: pypy/dist/pypy/annotation/description.py pypy/dist/pypy/annotation/specialize.py pypy/dist/pypy/jit/hintannotator/bookkeeper.py pypy/dist/pypy/tool/algo/test/test_unionfind.py pypy/dist/pypy/tool/algo/unionfind.py pypy/dist/pypy/translator/backendopt/malloc.py Log: Refactor UnionFind API. It calls absorb instead of update now. Modified: pypy/dist/pypy/annotation/description.py ============================================================================== --- pypy/dist/pypy/annotation/description.py (original) +++ pypy/dist/pypy/annotation/description.py Sun Feb 3 11:59:55 2008 @@ -26,6 +26,7 @@ for shape, table in other.calltables.items(): for row in table: self.calltable_add_row(shape, row) + absorb = update # UnionFind API def calltable_lookup_row(self, callshape, row): # this code looks up a table of which graph to @@ -63,6 +64,7 @@ self.descs.update(other.descs) self.read_locations.update(other.read_locations) self.attrs.update(other.attrs) + absorb = update # UnionFind API def get_s_value(self, attrname): try: @@ -100,6 +102,7 @@ self.descs.update(other.descs) self.read_locations.update(other.read_locations) self.s_value = unionof(self.s_value, other.s_value) + absorb = update # UnionFind API def get_s_value(self, attrname): return self.s_value Modified: pypy/dist/pypy/annotation/specialize.py ============================================================================== --- pypy/dist/pypy/annotation/specialize.py (original) +++ pypy/dist/pypy/annotation/specialize.py Sun Feb 3 11:59:55 2008 @@ -102,12 +102,10 @@ bookkeeper = self.funcdesc.bookkeeper bookkeeper.pending_specializations.append(self.finish) - def update(self, other): + def absorb(self, other): self.table.update(other.table) self.graph = None # just in case - - def cleanup(self): - self.do_not_process = True + other.do_not_process = True fieldnamecounter = 0 Modified: pypy/dist/pypy/jit/hintannotator/bookkeeper.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/bookkeeper.py (original) +++ pypy/dist/pypy/jit/hintannotator/bookkeeper.py Sun Feb 3 11:59:55 2008 @@ -110,6 +110,7 @@ def update(self, other): self.tsgraphs.update(other.tsgraphs) + absorb = update # UnionFind API class ImpurityAnalyzer(graphanalyze.GraphAnalyzer): Modified: pypy/dist/pypy/tool/algo/test/test_unionfind.py ============================================================================== --- pypy/dist/pypy/tool/algo/test/test_unionfind.py (original) +++ pypy/dist/pypy/tool/algo/test/test_unionfind.py Sun Feb 3 11:59:55 2008 @@ -8,11 +8,8 @@ state.append(self) self.obj = obj - def update(self, other): - pass - - def cleanup(self): - state.remove(self) + def absorb(self, other): + state.remove(other) uf = UnionFind(ReferencedByExternalState) uf.find(1) Modified: pypy/dist/pypy/tool/algo/unionfind.py ============================================================================== --- pypy/dist/pypy/tool/algo/unionfind.py (original) +++ pypy/dist/pypy/tool/algo/unionfind.py Sun Feb 3 11:59:55 2008 @@ -82,13 +82,11 @@ rep1, rep2, info1, info2, = rep2, rep1, info2, info1 if info1 is not None: - info1.update(info2) + info1.absorb(info2) self.link_to_parent[rep2] = rep1 del self.weight[rep2] - if hasattr(info2, "cleanup"): - info2.cleanup() del self.root_info[rep2] self.weight[rep1] = w Modified: pypy/dist/pypy/translator/backendopt/malloc.py ============================================================================== --- pypy/dist/pypy/translator/backendopt/malloc.py (original) +++ pypy/dist/pypy/translator/backendopt/malloc.py Sun Feb 3 11:59:55 2008 @@ -15,11 +15,12 @@ self.creationpoints = {} # set of ("type of creation point", ...) self.usepoints = {} # set of ("type of use point", ...) - def update(self, other): + def absorb(self, other): self.variables.update(other.variables) self.creationpoints.update(other.creationpoints) self.usepoints.update(other.usepoints) + class BaseMallocRemover(object): IDENTITY_OPS = ('same_as',) From cfbolz at codespeak.net Sun Feb 3 14:39:32 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 3 Feb 2008 14:39:32 +0100 (CET) Subject: [pypy-svn] r51225 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter timeshifter/test Message-ID: <20080203133932.3DB3A168414@codespeak.net> Author: cfbolz Date: Sun Feb 3 14:39:31 2008 New Revision: 51225 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py Log: some support for direct red calls in the jit (nothing fancy, no exceptions). Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Sun Feb 3 14:39:31 2008 @@ -3,6 +3,7 @@ from pypy.objspace.flow import model as flowmodel from pypy.rpython.lltypesystem import lltype from pypy.jit.hintannotator.model import originalconcretetype +from pypy.jit.hintannotator import model as hintmodel from pypy.jit.timeshifter import rtimeshift, rvalue from pypy.jit.timeshifter.greenkey import KeyDesc, empty_key, GreenKey @@ -19,14 +20,17 @@ green consts are negative indexes """ - def __init__(self, code, constants, typekinds, redboxclasses, keydescs, - num_mergepoints): + def __init__(self, name, code, constants, typekinds, redboxclasses, + keydescs, called_bytecodes, num_mergepoints, is_portal): + self.name = name self.code = code self.constants = constants self.typekinds = typekinds self.redboxclasses = redboxclasses self.keydescs = keydescs + self.called_bytecodes = called_bytecodes self.num_mergepoints = num_mergepoints + self.is_portal = is_portal def _freeze_(self): return True @@ -44,11 +48,10 @@ self.opname_to_index = {} self.jitstate = None self.queue = None - self.bytecode = None - self.pc = -1 self._add_implemented_opcodes() - def run(self, jitstate, bytecode, greenargs, redargs): + def run(self, jitstate, bytecode, greenargs, redargs, + start_bytecode_loop=True): self.jitstate = jitstate self.queue = rtimeshift.DispatchQueue(bytecode.num_mergepoints) rtimeshift.enter_frame(self.jitstate, self.queue) @@ -57,7 +60,8 @@ self.frame.bytecode = bytecode self.frame.local_boxes = redargs self.frame.local_green = greenargs - self.bytecode_loop() + if start_bytecode_loop: + self.bytecode_loop() return self.jitstate def bytecode_loop(self): @@ -70,14 +74,16 @@ assert result is None def dispatch(self): + is_portal = self.frame.bytecode.is_portal newjitstate = rtimeshift.dispatch_next(self.queue) resumepoint = rtimeshift.getresumepoint(newjitstate) self.newjitstate(newjitstate) if resumepoint == -1: - # XXX what about green returns? - newjitstate = rtimeshift.leave_graph_red(self.queue, is_portal=True) + newjitstate = rtimeshift.leave_graph_red( + self.queue, is_portal) self.newjitstate(newjitstate) - return STOP + if newjitstate is None or is_portal: + return STOP else: self.frame.pc = resumepoint @@ -192,6 +198,30 @@ if done: return self.dispatch() + def opimpl_red_direct_call(self): + greenargs = [] + num = self.load_2byte() + for i in range(num): + greenargs.append(self.get_greenarg()) + redargs = [] + num = self.load_2byte() + for i in range(num): + redargs.append(self.get_redarg()) + bytecodenum = self.load_2byte() + targetbytecode = self.frame.bytecode.called_bytecodes[bytecodenum] + self.run(self.jitstate, targetbytecode, greenargs, redargs, + start_bytecode_loop=False) + # this frame will be resumed later in the next bytecode, which is + # red_after_direct_call + + def opimpl_red_after_direct_call(self): + newjitstate = rtimeshift.collect_split( + self.jitstate, self.frame.pc, + self.frame.local_green) + assert newjitstate is self.jitstate + + + # ____________________________________________________________ # construction-time interface def _add_implemented_opcodes(self): @@ -244,21 +274,32 @@ class BytecodeWriter(object): - def __init__(self, t, hintannotator, RGenOp): + def __init__(self, t, hannotator, RGenOp): self.translator = t self.annotator = t.annotator - self.hannotator = hintannotator + self.hannotator = hannotator self.interpreter = JitInterpreter() self.RGenOp = RGenOp self.current_block = None - - def make_bytecode(self, graph): + self.raise_analyzer = hannotator.exceptiontransformer.raise_analyzer + self.all_graphs = {} # mapping graph to bytecode + self.unfinished_graphs = [] + + def can_raise(self, op): + return self.raise_analyzer.analyze(op) + + def make_bytecode(self, graph, is_portal=True): + if is_portal: + self.all_graphs[graph] = JitCode.__new__(JitCode) self.seen_blocks = {} self.assembler = [] self.constants = [] self.typekinds = [] self.redboxclasses = [] self.keydescs = [] + self.called_bytecodes = [] + self.num_mergepoints = 0 + self.is_portal = is_portal # mapping constant -> index in constants self.const_positions = {} # mapping blocks to True @@ -273,19 +314,31 @@ self.type_positions = {} # mapping tuple of green TYPES to index self.keydesc_positions = {} - - self.num_mergepoints = 0 + # mapping graphs to index + self.graph_positions = {} self.graph = graph self.entrymap = flowmodel.mkentrymap(graph) self.make_bytecode_block(graph.startblock) assert self.current_block is None - return JitCode(assemble(self.interpreter, *self.assembler), - self.constants, - self.typekinds, - self.redboxclasses, - self.keydescs, - self.num_mergepoints) + bytecode = self.all_graphs[graph] + bytecode.__init__(graph.name, + assemble(self.interpreter, *self.assembler), + self.constants, + self.typekinds, + self.redboxclasses, + self.keydescs, + self.called_bytecodes, + self.num_mergepoints, + self.is_portal) + if is_portal: + self.finish_all_graphs() + return bytecode + + def finish_all_graphs(self): + while self.unfinished_graphs: + graph = self.unfinished_graphs.pop() + self.make_bytecode(graph, is_portal=False) def make_bytecode_block(self, block, insert_goto=False): if block in self.seen_blocks: @@ -482,6 +535,18 @@ result = len(self.type_positions) self.type_positions[TYPE] = result return result + + def graph_position(self, graph): + if graph in self.graph_positions: + return self.graph_positions[graph] + bytecode = JitCode.__new__(JitCode) + index = len(self.called_bytecodes) + self.called_bytecodes.append(bytecode) + self.all_graphs[graph] = bytecode + self.graph_positions[graph] = index + self.unfinished_graphs.append(graph) + return index + def emit(self, stuff): assert stuff is not None @@ -501,6 +566,7 @@ reds.append(v) return reds, greens + # ____________________________________________________________ # operation special cases def serialize_op_hint(self, op): @@ -522,6 +588,108 @@ return XXX + def serialize_op_direct_call(self, op): + targets = dict(self.graphs_from(op)) + assert len(targets) == 1 + targetgraph, = targets.values() + kind, exc = self.guess_call_kind(op) + if kind == "red": + graphindex = self.graph_position(targetgraph) + args = targetgraph.getargs() + reds, greens = self.sort_by_color(op.args[1:], args) + result = [] + for color, args in [("green", greens), ("red", reds)]: + result.append(len(args)) + for v in args: + result.append(self.serialize_oparg(color, v)) + self.emit("red_direct_call") + for index in result: + self.emit(index) + self.emit(graphindex) + self.register_redvar(op.result) + self.emit("red_after_direct_call") + else: + XXX + + def serialize_op_indirect_call(self, op): + XXX + + # call handling + + def graphs_from(self, spaceop): + if spaceop.opname == 'direct_call': + c_func = spaceop.args[0] + fnobj = c_func.value._obj + graphs = [fnobj.graph] + args_v = spaceop.args[1:] + elif spaceop.opname == 'indirect_call': + graphs = spaceop.args[-1].value + if graphs is None: + return # cannot follow at all + args_v = spaceop.args[1:-1] + else: + raise AssertionError(spaceop.opname) + # if the graph - or all the called graphs - are marked as "don't + # follow", directly return None as a special case. (This is only + # an optimization for the indirect_call case.) + for graph in graphs: + if self.hannotator.policy.look_inside_graph(graph): + break + else: + return + for graph in graphs: + tsgraph = self.specialized_graph_of(graph, args_v, spaceop.result) + yield graph, tsgraph + + def guess_call_kind(self, spaceop): + if spaceop.opname == 'direct_call': + c_func = spaceop.args[0] + fnobj = c_func.value._obj + if hasattr(fnobj, 'jitcallkind'): + return fnobj.jitcallkind, None + if (hasattr(fnobj._callable, 'oopspec') and + self.hannotator.policy.oopspec): + if fnobj._callable.oopspec.startswith('vable.'): + return 'vable', None + hs_result = self.hannotator.binding(spaceop.result) + if (hs_result.is_green() and + hs_result.concretetype is not lltype.Void): + return 'green', self.can_raise(spaceop) + return 'oopspec', self.can_raise(spaceop) + if self.hannotator.bookkeeper.is_green_call(spaceop): + return 'green', None + withexc = self.can_raise(spaceop) + colors = {} + for graph, tsgraph in self.graphs_from(spaceop): + color = self.graph_calling_color(tsgraph) + colors[color] = tsgraph + if not colors: # cannot follow this call + return 'residual', withexc + assert len(colors) == 1, colors # buggy normalization? + return color, withexc + + def specialized_graph_of(self, graph, args_v, v_result): + bk = self.hannotator.bookkeeper + args_hs = [self.hannotator.binding(v) for v in args_v] + hs_result = self.hannotator.binding(v_result) + if isinstance(hs_result, hintmodel.SomeLLAbstractConstant): + fixed = hs_result.is_fixed() + else: + fixed = False + specialization_key = bk.specialization_key(fixed, args_hs) + special_graph = bk.get_graph_by_key(graph, specialization_key) + return special_graph + + def graph_calling_color(self, graph): + hs_res = self.hannotator.binding(graph.getreturnvar()) + if originalconcretetype(hs_res) is lltype.Void: + c = 'gray' + elif hs_res.is_green(): + c = 'yellow' + else: + c = 'red' + return c + class label(object): def __init__(self, name): Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sun Feb 3 14:39:31 2008 @@ -405,6 +405,91 @@ assert res == 42 self.check_insns({'int_add': 2, 'int_sub': 1}) + def test_call_simple(self): + def ll_add_one(x): + return x + 1 + def ll_function(y): + return ll_add_one(y) + res = self.interpret(ll_function, [5], []) + assert res == 6 + self.check_insns({'int_add': 1}) + + def test_call_2(self): + def ll_add_one(x): + return x + 1 + def ll_function(y): + return ll_add_one(y) + y + res = self.interpret(ll_function, [5], []) + assert res == 11 + self.check_insns({'int_add': 2}) + + def test_call_3(self): + def ll_add_one(x): + return x + 1 + def ll_two(x): + return ll_add_one(ll_add_one(x)) - x + def ll_function(y): + return ll_two(y) * y + res = self.interpret(ll_function, [5], []) + assert res == 10 + self.check_insns({'int_add': 2, 'int_sub': 1, 'int_mul': 1}) + + def test_call_4(self): + def ll_two(x): + if x > 0: + return x + 5 + else: + return x - 4 + def ll_function(y): + return ll_two(y) * y + + res = self.interpret(ll_function, [3], []) + assert res == 24 + self.check_insns({'int_gt': 1, 'int_add': 1, + 'int_sub': 1, 'int_mul': 1}) + + res = self.interpret(ll_function, [-3], []) + assert res == 21 + self.check_insns({'int_gt': 1, 'int_add': 1, + 'int_sub': 1, 'int_mul': 1}) + + def test_void_call(self): + py.test.skip("calls are WIP") + def ll_do_nothing(x): + pass + def ll_function(y): + ll_do_nothing(y) + return y + + res = self.interpret(ll_function, [3], []) + assert res == 3 + + def test_green_call(self): + py.test.skip("calls are WIP") + def ll_add_one(x): + return x+1 + def ll_function(y): + z = ll_add_one(y) + z = hint(z, concrete=True) + return hint(z, variable=True) + + res = self.interpret(ll_function, [3], [0]) + assert res == 4 + self.check_insns({}) + + def test_split_on_green_return(self): + py.test.skip("calls are WIP") + def ll_two(x): + if x > 0: + return 17 + else: + return 22 + def ll_function(x): + n = ll_two(x) + return hint(n+1, variable=True) + res = self.interpret(ll_function, [-70], []) + assert res == 23 + self.check_insns({'int_gt': 1}) class TestLLType(SimpleTests): type_system = "lltype" Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Sun Feb 3 14:39:31 2008 @@ -64,6 +64,8 @@ "red_return", 0) assert len(jitcode.constants) == 0 assert len(jitcode.typekinds) == 0 + assert jitcode.is_portal + assert len(jitcode.called_bytecodes) == 0 def test_constant(self): def f(x): @@ -78,6 +80,8 @@ assert len(jitcode.constants) == 1 assert len(jitcode.typekinds) == 1 assert len(jitcode.redboxclasses) == 1 + assert jitcode.is_portal + assert len(jitcode.called_bytecodes) == 0 def test_green_switch(self): def f(x, y, z): @@ -102,6 +106,8 @@ assert jitcode.code == expected assert len(jitcode.constants) == 0 assert len(jitcode.typekinds) == 0 + assert jitcode.is_portal + assert len(jitcode.called_bytecodes) == 0 def test_green_switch2(self): def f(x, y, z): @@ -134,6 +140,8 @@ assert jitcode.code == expected assert len(jitcode.constants) == 0 assert len(jitcode.typekinds) == 0 + assert jitcode.is_portal + assert len(jitcode.called_bytecodes) == 0 def test_merge(self): def f(x, y, z): @@ -169,6 +177,8 @@ assert jitcode.code == expected assert len(jitcode.constants) == 1 assert len(jitcode.typekinds) == 1 + assert jitcode.is_portal + assert len(jitcode.called_bytecodes) == 0 def test_loop(self): def f(x): @@ -197,6 +207,37 @@ "red_int_sub", 0, 3, "make_new_redvars", 2, 2, 4, "goto", tlabel("while")) + assert jitcode.is_portal + assert len(jitcode.called_bytecodes) == 0 + + def test_call(self): + def g(x): + return x + 1 + def f(x): + return g(x) * 2 + writer, jitcode = self.serialize(f, [int]) + assert jitcode.code == assemble(writer.interpreter, + "red_direct_call", 0, 1, 0, 0, + "red_after_direct_call", + "make_redbox", -1, 0, + "red_int_mul", 1, 2, + "make_new_redvars", 1, 3, + "make_new_greenvars", 0, + "red_return", 0) + assert jitcode.is_portal + assert len(jitcode.called_bytecodes) == 1 + called_jitcode = jitcode.called_bytecodes[0] + assert called_jitcode.code == assemble(writer.interpreter, + "make_redbox", -1, 0, + "red_int_add", 0, 1, + "make_new_redvars", 1, 2, + "make_new_greenvars", 0, + "red_return", 0) + assert not called_jitcode.is_portal + assert len(called_jitcode.called_bytecodes) == 0 + + + class TestLLType(AbstractSerializationTest): Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Sun Feb 3 14:39:31 2008 @@ -453,11 +453,10 @@ if gotexc: jitstate.residual_ll_exception(ll_evalue) -def collect_split(jitstate_chain, resumepoint, *greens_gv): +def collect_split(jitstate_chain, resumepoint, greens_gv): # YYY split to avoid over-specialization # assumes that the head of the jitstate_chain is ready for writing, # and all the other jitstates in the chain are paused - greens_gv = list(greens_gv) pending = jitstate_chain resuming = jitstate_chain.get_resuming() if resuming is not None and resuming.mergesleft == 0: @@ -467,7 +466,7 @@ pending = pending.next pending.greens.extend(greens_gv) if pending.returnbox is not None: - pending.frame.local_boxes.insert(0, getreturnbox(pending)) + pending.frame.local_boxes.append(getreturnbox(pending)) pending.next = None start_writing(pending, jitstate_chain) return pending @@ -478,7 +477,7 @@ pending = pending.next jitstate.greens.extend(greens_gv) # item 0 is the return value if jitstate.returnbox is not None: - jitstate.frame.local_boxes.insert(0, getreturnbox(jitstate)) + jitstate.frame.local_boxes.append(getreturnbox(jitstate)) jitstate.resumepoint = resumepoint if resuming is None: node = jitstate.promotion_path Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py Sun Feb 3 14:39:31 2008 @@ -762,88 +762,6 @@ assert res == 1 + 2 self.check_insns({'int_is_true': 1, 'int_add': 1}) - def test_call_simple(self): - def ll_add_one(x): - return x + 1 - def ll_function(y): - return ll_add_one(y) - res = self.timeshift(ll_function, [5], [], policy=P_NOVIRTUAL) - assert res == 6 - self.check_insns({'int_add': 1}) - - def test_call_2(self): - def ll_add_one(x): - return x + 1 - def ll_function(y): - return ll_add_one(y) + y - res = self.timeshift(ll_function, [5], [], policy=P_NOVIRTUAL) - assert res == 11 - self.check_insns({'int_add': 2}) - - def test_call_3(self): - def ll_add_one(x): - return x + 1 - def ll_two(x): - return ll_add_one(ll_add_one(x)) - x - def ll_function(y): - return ll_two(y) * y - res = self.timeshift(ll_function, [5], [], policy=P_NOVIRTUAL) - assert res == 10 - self.check_insns({'int_add': 2, 'int_sub': 1, 'int_mul': 1}) - - def test_call_4(self): - def ll_two(x): - if x > 0: - return x + 5 - else: - return x - 4 - def ll_function(y): - return ll_two(y) * y - - res = self.timeshift(ll_function, [3], [], policy=P_NOVIRTUAL) - assert res == 24 - self.check_insns({'int_gt': 1, 'int_add': 1, - 'int_sub': 1, 'int_mul': 1}) - - res = self.timeshift(ll_function, [-3], [], policy=P_NOVIRTUAL) - assert res == 21 - self.check_insns({'int_gt': 1, 'int_add': 1, - 'int_sub': 1, 'int_mul': 1}) - - def test_void_call(self): - def ll_do_nothing(x): - pass - def ll_function(y): - ll_do_nothing(y) - return y - - res = self.timeshift(ll_function, [3], [], policy=P_NOVIRTUAL) - assert res == 3 - - def test_green_call(self): - def ll_add_one(x): - return x+1 - def ll_function(y): - z = ll_add_one(y) - z = hint(z, concrete=True) - return hint(z, variable=True) - - res = self.timeshift(ll_function, [3], [0], policy=P_NOVIRTUAL) - assert res == 4 - self.check_insns({}) - - def test_split_on_green_return(self): - def ll_two(x): - if x > 0: - return 17 - else: - return 22 - def ll_function(x): - n = ll_two(x) - return hint(n+1, variable=True) - res = self.timeshift(ll_function, [-70], []) - assert res == 23 - self.check_insns({'int_gt': 1}) def test_green_with_side_effects(self): S = lltype.GcStruct('S', ('flag', lltype.Bool)) From cfbolz at codespeak.net Sun Feb 3 16:04:33 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 3 Feb 2008 16:04:33 +0100 (CET) Subject: [pypy-svn] r51226 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080203150433.8E62616840E@codespeak.net> Author: cfbolz Date: Sun Feb 3 16:04:28 2008 New Revision: 51226 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: support for gray calls Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Sun Feb 3 16:04:28 2008 @@ -21,7 +21,8 @@ """ def __init__(self, name, code, constants, typekinds, redboxclasses, - keydescs, called_bytecodes, num_mergepoints, is_portal): + keydescs, called_bytecodes, num_mergepoints, graph_color, + is_portal): self.name = name self.code = code self.constants = constants @@ -30,6 +31,7 @@ self.keydescs = keydescs self.called_bytecodes = called_bytecodes self.num_mergepoints = num_mergepoints + self.graph_color = graph_color self.is_portal = is_portal def _freeze_(self): @@ -75,12 +77,23 @@ def dispatch(self): is_portal = self.frame.bytecode.is_portal + graph_color = self.frame.bytecode.graph_color newjitstate = rtimeshift.dispatch_next(self.queue) resumepoint = rtimeshift.getresumepoint(newjitstate) self.newjitstate(newjitstate) if resumepoint == -1: - newjitstate = rtimeshift.leave_graph_red( - self.queue, is_portal) + if graph_color == "red": + newjitstate = rtimeshift.leave_graph_red( + self.queue, is_portal) + elif graph_color == "green": + XXX + elif graph_color == "gray": + assert not is_portal + newjitstate = rtimeshift.leave_graph_gray( + self.queue) + else: + assert 0, "unknown graph color %s" % (color, ) + self.newjitstate(newjitstate) if newjitstate is None or is_portal: return STOP @@ -160,6 +173,10 @@ rtimeshift.save_return(self.jitstate) return self.dispatch() + def opimpl_gray_return(self): + rtimeshift.save_return(self.jitstate) + return self.dispatch() + def opimpl_green_return(self): XXX @@ -299,6 +316,7 @@ self.keydescs = [] self.called_bytecodes = [] self.num_mergepoints = 0 + self.graph_color = self.graph_calling_color(graph) self.is_portal = is_portal # mapping constant -> index in constants self.const_positions = {} @@ -330,6 +348,7 @@ self.keydescs, self.called_bytecodes, self.num_mergepoints, + self.graph_color, self.is_portal) if is_portal: self.finish_all_graphs() @@ -369,10 +388,14 @@ if block.exits == (): returnvar, = block.inputargs color = self.varcolor(returnvar) - assert color == "red" # XXX green return values not supported yet - index = self.serialize_oparg(color, returnvar) - self.emit("%s_return" % color) - self.emit(index) + if color == "red": + index = self.serialize_oparg(color, returnvar) + self.emit("red_return") + self.emit(index) + elif originalconcretetype(returnvar) == lltype.Void: + self.emit("gray_return") + else: + XXX elif len(block.exits) == 1: link, = block.exits self.insert_renaming(link) @@ -593,7 +616,7 @@ assert len(targets) == 1 targetgraph, = targets.values() kind, exc = self.guess_call_kind(op) - if kind == "red": + if kind == "red" or kind == "gray": graphindex = self.graph_position(targetgraph) args = targetgraph.getargs() reds, greens = self.sort_by_color(op.args[1:], args) @@ -606,7 +629,8 @@ for index in result: self.emit(index) self.emit(graphindex) - self.register_redvar(op.result) + if kind == "red": + self.register_redvar(op.result) self.emit("red_after_direct_call") else: XXX Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sun Feb 3 16:04:28 2008 @@ -454,7 +454,6 @@ 'int_sub': 1, 'int_mul': 1}) def test_void_call(self): - py.test.skip("calls are WIP") def ll_do_nothing(x): pass def ll_function(y): From exarkun at codespeak.net Sun Feb 3 16:22:26 2008 From: exarkun at codespeak.net (exarkun at codespeak.net) Date: Sun, 3 Feb 2008 16:22:26 +0100 (CET) Subject: [pypy-svn] r51227 - pypy/dist/pypy/interpreter/pyparser/test Message-ID: <20080203152226.D465616841B@codespeak.net> Author: exarkun Date: Sun Feb 3 16:22:25 2008 New Revision: 51227 Modified: pypy/dist/pypy/interpreter/pyparser/test/test_samples.py Log: Order the parsing samples by filename before yielding them to the test runner; this makes it more likely that each sample will have the same test name each time the tests are run Modified: pypy/dist/pypy/interpreter/pyparser/test/test_samples.py ============================================================================== --- pypy/dist/pypy/interpreter/pyparser/test/test_samples.py (original) +++ pypy/dist/pypy/interpreter/pyparser/test/test_samples.py Sun Feb 3 16:22:25 2008 @@ -72,9 +72,15 @@ samples_dir = py.magic.autopath().dirpath("samples") for use_lookahead in (True, False): grammar.USE_LOOKAHEAD = use_lookahead - for path in samples_dir.listdir("*.py"): + sample_paths = samples_dir.listdir("*.py") + # Make it as likely as possible (without tons of effort) that each + # sample will have the same test name in each run. + sample_paths.sort() + for path in sample_paths: fname = path.basename if fname in SKIP_ALWAYS: + yield lambda: py.test.skip( + "%r is set to always skip." % (fname,)) continue if GRAMMAR_MISMATCH and fname in SKIP_IF_NOT_NATIVE: yield lambda: py.test.skip( From arigo at codespeak.net Sun Feb 3 19:09:38 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 3 Feb 2008 19:09:38 +0100 (CET) Subject: [pypy-svn] r51230 - pypy/branch/asmgcroot/pypy/translator/c/gcc/test Message-ID: <20080203180938.5BD111683F1@codespeak.net> Author: arigo Date: Sun Feb 3 19:09:36 2008 New Revision: 51230 Modified: pypy/branch/asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py Log: Check that the expected global labels are added. Modified: pypy/branch/asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py ============================================================================== --- pypy/branch/asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py (original) +++ pypy/branch/asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py Sun Feb 3 19:09:36 2008 @@ -52,6 +52,7 @@ print print path.basename lines = path.readlines() + expectedlines = lines[:] tracker = FunctionGcRootTracker(lines) tracker.is_main = tracker.funcname == 'main' table = tracker.computegcmaptable(verbose=sys.maxint) @@ -62,7 +63,7 @@ tabledict[entry[0]] = entry[1] # find the ";; expected" lines prevline = "" - for line in lines: + for i, line in enumerate(lines): match = r_expected.match(line) if match: expected = eval(match.group(1)) @@ -74,7 +75,10 @@ got = tabledict[label] assert got == expected seen[label] = True + expectedlines.insert(i-2, '\t.globl\t%s\n' % (label,)) + expectedlines.insert(i-1, '%s:\n' % (label,)) prevline = line assert len(seen) == len(tabledict), ( "computed table contains unexpected entries:\n%r" % [key for key in tabledict if key not in seen]) + assert lines == expectedlines From cfbolz at codespeak.net Sun Feb 3 20:29:50 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 3 Feb 2008 20:29:50 +0100 (CET) Subject: [pypy-svn] r51231 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080203192950.63829168406@codespeak.net> Author: cfbolz Date: Sun Feb 3 20:29:48 2008 New Revision: 51231 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Log: green calls Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Sun Feb 3 20:29:48 2008 @@ -22,7 +22,7 @@ def __init__(self, name, code, constants, typekinds, redboxclasses, keydescs, called_bytecodes, num_mergepoints, graph_color, - is_portal): + nonrainbow_functions, is_portal): self.name = name self.code = code self.constants = constants @@ -32,6 +32,7 @@ self.called_bytecodes = called_bytecodes self.num_mergepoints = num_mergepoints self.graph_color = graph_color + self.nonrainbow_functions = nonrainbow_functions self.is_portal = is_portal def _freeze_(self): @@ -126,6 +127,20 @@ return self.frame.bytecode.constants[~i] return self.frame.local_green[i] + def get_green_varargs(self): + greenargs = [] + num = self.load_2byte() + for i in range(num): + greenargs.append(self.get_greenarg()) + return greenargs + + def get_red_varargs(self): + redargs = [] + num = self.load_2byte() + for i in range(num): + redargs.append(self.get_redarg()) + return redargs + def get_redarg(self): return self.frame.local_boxes[self.load_2byte()] @@ -181,13 +196,7 @@ XXX def opimpl_make_new_redvars(self): - # an opcode with a variable number of args - # num_args arg_old_1 arg_new_1 ... - num = self.load_2byte() - newlocalboxes = [] - for i in range(num): - newlocalboxes.append(self.get_redarg()) - self.frame.local_boxes = newlocalboxes + self.frame.local_boxes = self.get_red_varargs() def opimpl_make_new_greenvars(self): # an opcode with a variable number of args @@ -216,14 +225,8 @@ return self.dispatch() def opimpl_red_direct_call(self): - greenargs = [] - num = self.load_2byte() - for i in range(num): - greenargs.append(self.get_greenarg()) - redargs = [] - num = self.load_2byte() - for i in range(num): - redargs.append(self.get_redarg()) + greenargs = self.get_green_varargs() + redargs = self.get_red_varargs() bytecodenum = self.load_2byte() targetbytecode = self.frame.bytecode.called_bytecodes[bytecodenum] self.run(self.jitstate, targetbytecode, greenargs, redargs, @@ -237,6 +240,12 @@ self.frame.local_green) assert newjitstate is self.jitstate + def opimpl_green_direct_call(self): + greenargs = self.get_green_varargs() + redargs = self.get_red_varargs() + index = self.load_2byte() + function = self.frame.bytecode.nonrainbow_functions[index] + function(self, greenargs, redargs) # ____________________________________________________________ # construction-time interface @@ -317,6 +326,7 @@ self.called_bytecodes = [] self.num_mergepoints = 0 self.graph_color = self.graph_calling_color(graph) + self.nonrainbow_functions = [] self.is_portal = is_portal # mapping constant -> index in constants self.const_positions = {} @@ -334,6 +344,8 @@ self.keydesc_positions = {} # mapping graphs to index self.graph_positions = {} + # mapping fnobjs to index + self.nonrainbow_positions = {} self.graph = graph self.entrymap = flowmodel.mkentrymap(graph) @@ -349,6 +361,7 @@ self.called_bytecodes, self.num_mergepoints, self.graph_color, + self.nonrainbow_functions, self.is_portal) if is_portal: self.finish_all_graphs() @@ -449,8 +462,7 @@ result.append(self.serialize_oparg(color, v)) self.emit("make_new_%svars" % (color, )) self.emit(len(args)) - for index in result: - self.emit(index) + self.emit(result) def serialize_op(self, op): specialcase = getattr(self, "serialize_op_%s" % (op.opname, ), None) @@ -461,8 +473,7 @@ for arg in op.args: args.append(self.serialize_oparg(color, arg)) self.serialize_opcode(color, op) - for index in args: - self.emit(index) + self.emit(args) if self.hannotator.binding(op.result).is_green(): self.register_greenvar(op.result) else: @@ -570,10 +581,35 @@ self.unfinished_graphs.append(graph) return index + def nonrainbow_position(self, fnptr): + fn = fnptr._obj + if fn in self.nonrainbow_positions: + return self.nonrainbow_positions[fn] + FUNCTYPE = lltype.typeOf(fn) + argiter = unrolling_iterable(enumerate(FUNCTYPE.ARGS)) + numargs = len(FUNCTYPE.ARGS) + def call_normal_function(interpreter, greenargs, redargs): + assert len(redargs) == 0 + assert len(greenargs) == numargs + args = () + for i, ARG in argiter: + genconst = greenargs[i] + arg = genconst.revealconst(ARG) + args += (arg, ) + rgenop = interpreter.jitstate.curbuilder.rgenop + result = rgenop.genconst(fnptr(*args)) + interpreter.green_result(result) + result = len(self.nonrainbow_functions) + self.nonrainbow_functions.append(call_normal_function) + self.nonrainbow_positions[fn] = result + return result def emit(self, stuff): assert stuff is not None - self.assembler.append(stuff) + if isinstance(stuff, list): + self.assembler.extend(stuff) + else: + self.assembler.append(stuff) def sort_by_color(self, vars, by_color_of_vars=None): reds = [] @@ -611,6 +647,16 @@ return XXX + def args_of_call(self, args, colored_as): + result = [] + reds, greens = self.sort_by_color(args, colored_as) + result = [] + for color, args in [("green", greens), ("red", reds)]: + result.append(len(args)) + for v in args: + result.append(self.serialize_oparg(color, v)) + return result + def serialize_op_direct_call(self, op): targets = dict(self.graphs_from(op)) assert len(targets) == 1 @@ -619,19 +665,21 @@ if kind == "red" or kind == "gray": graphindex = self.graph_position(targetgraph) args = targetgraph.getargs() - reds, greens = self.sort_by_color(op.args[1:], args) - result = [] - for color, args in [("green", greens), ("red", reds)]: - result.append(len(args)) - for v in args: - result.append(self.serialize_oparg(color, v)) + emitted_args = self.args_of_call(op.args[1:], args) self.emit("red_direct_call") - for index in result: - self.emit(index) + self.emit(emitted_args) self.emit(graphindex) if kind == "red": self.register_redvar(op.result) self.emit("red_after_direct_call") + elif kind == "green": + pos = self.nonrainbow_position(op.args[0].value) + args = targetgraph.getargs() + emitted_args = self.args_of_call(op.args[1:], args) + self.emit("green_direct_call") + self.emit(emitted_args) + self.emit(pos) + self.register_greenvar(op.result) else: XXX Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sun Feb 3 20:29:48 2008 @@ -464,7 +464,6 @@ assert res == 3 def test_green_call(self): - py.test.skip("calls are WIP") def ll_add_one(x): return x+1 def ll_function(y): Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Sun Feb 3 20:29:48 2008 @@ -236,8 +236,24 @@ assert not called_jitcode.is_portal assert len(called_jitcode.called_bytecodes) == 0 + def test_green_call(self): + def ll_add_one(x): + return x+1 + def ll_function(y): + z = ll_add_one(y) + z = hint(z, concrete=True) + return hint(z, variable=True) - + writer, jitcode = self.serialize(ll_function, [int]) + assert jitcode.code == assemble(writer.interpreter, + "green_direct_call", 1, 0, 0, 0, + "make_redbox", 1, 0, + "make_new_redvars", 1, 0, + "make_new_greenvars", 0, + "red_return", 0) + assert jitcode.is_portal + assert len(jitcode.called_bytecodes) == 0 + assert len(jitcode.nonrainbow_functions) == 1 class TestLLType(AbstractSerializationTest): From pypy-svn at codespeak.net Sun Feb 3 21:31:55 2008 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Sun, 3 Feb 2008 21:31:55 +0100 (CET) Subject: [pypy-svn] February 50% OFF Message-ID: <20080203143129.4972.qmail@ppp85-140-54-38.pppoe.mtu-net.ru> An HTML attachment was scrubbed... URL: From cfbolz at codespeak.net Mon Feb 4 02:45:08 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 4 Feb 2008 02:45:08 +0100 (CET) Subject: [pypy-svn] r51234 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter Message-ID: <20080204014508.0AC88168420@codespeak.net> Author: cfbolz Date: Mon Feb 4 02:45:07 2008 New Revision: 51234 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Log: first test about yellow calls is passing Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Mon Feb 4 02:45:07 2008 @@ -6,6 +6,7 @@ from pypy.jit.hintannotator import model as hintmodel from pypy.jit.timeshifter import rtimeshift, rvalue from pypy.jit.timeshifter.greenkey import KeyDesc, empty_key, GreenKey +from pypy.translator.backendopt.removenoops import remove_same_as class JitCode(object): """ @@ -70,6 +71,7 @@ def bytecode_loop(self): while 1: bytecode = self.load_2byte() + assert bytecode >= 0 result = self.opcode_implementations[bytecode](self) if result is STOP: return @@ -79,21 +81,23 @@ def dispatch(self): is_portal = self.frame.bytecode.is_portal graph_color = self.frame.bytecode.graph_color - newjitstate = rtimeshift.dispatch_next(self.queue) + queue = self.queue + newjitstate = rtimeshift.dispatch_next(queue) resumepoint = rtimeshift.getresumepoint(newjitstate) self.newjitstate(newjitstate) if resumepoint == -1: if graph_color == "red": newjitstate = rtimeshift.leave_graph_red( - self.queue, is_portal) + queue, is_portal) + elif graph_color == "yellow": + newjitstate = rtimeshift.leave_graph_yellow(queue) elif graph_color == "green": XXX elif graph_color == "gray": assert not is_portal - newjitstate = rtimeshift.leave_graph_gray( - self.queue) + newjitstate = rtimeshift.leave_graph_gray(queue) else: - assert 0, "unknown graph color %s" % (color, ) + assert 0, "unknown graph color %s" % (graph_color, ) self.newjitstate(newjitstate) if newjitstate is None or is_portal: @@ -152,8 +156,12 @@ def newjitstate(self, newjitstate): self.jitstate = newjitstate + self.queue = None if newjitstate is not None: - self.frame = newjitstate.frame + frame = newjitstate.frame + self.frame = frame + if frame is not None: + self.queue = frame.dispatchqueue else: self.frame = None @@ -192,8 +200,11 @@ rtimeshift.save_return(self.jitstate) return self.dispatch() - def opimpl_green_return(self): - XXX + def opimpl_yellow_return(self): + # save the greens to make the return value findable by collect_split + rtimeshift.save_greens(self.jitstate, self.frame.local_green) + rtimeshift.save_return(self.jitstate) + return self.dispatch() def opimpl_make_new_redvars(self): self.frame.local_boxes = self.get_red_varargs() @@ -247,6 +258,27 @@ function = self.frame.bytecode.nonrainbow_functions[index] function(self, greenargs, redargs) + def opimpl_yellow_direct_call(self): + greenargs = self.get_green_varargs() + redargs = self.get_red_varargs() + bytecodenum = self.load_2byte() + targetbytecode = self.frame.bytecode.called_bytecodes[bytecodenum] + self.run(self.jitstate, targetbytecode, greenargs, redargs, + start_bytecode_loop=False) + # this frame will be resumed later in the next bytecode, which is + # yellow_after_direct_call + + def opimpl_yellow_after_direct_call(self): + newjitstate = rtimeshift.collect_split( + self.jitstate, self.frame.pc, + self.frame.local_green) + assert newjitstate is self.jitstate + + def opimpl_yellow_retrieve_result(self): + # XXX all this jitstate.greens business is a bit messy + self.green_result(self.jitstate.greens[0]) + + # ____________________________________________________________ # construction-time interface @@ -315,6 +347,7 @@ return self.raise_analyzer.analyze(op) def make_bytecode(self, graph, is_portal=True): + remove_same_as(graph) if is_portal: self.all_graphs[graph] = JitCode.__new__(JitCode) self.seen_blocks = {} @@ -402,11 +435,11 @@ returnvar, = block.inputargs color = self.varcolor(returnvar) if color == "red": - index = self.serialize_oparg(color, returnvar) self.emit("red_return") - self.emit(index) elif originalconcretetype(returnvar) == lltype.Void: self.emit("gray_return") + elif color == "green": # really a yellow call # XXX use graphcolor + self.emit("yellow_return") else: XXX elif len(block.exits) == 1: @@ -680,6 +713,16 @@ self.emit(emitted_args) self.emit(pos) self.register_greenvar(op.result) + elif kind == "yellow": + graphindex = self.graph_position(targetgraph) + args = targetgraph.getargs() + emitted_args = self.args_of_call(op.args[1:], args) + self.emit("yellow_direct_call") + self.emit(emitted_args) + self.emit(graphindex) + self.emit("yellow_after_direct_call") + self.emit("yellow_retrieve_result") + self.register_greenvar(op.result) else: XXX @@ -785,7 +828,9 @@ result.append(chr(index & 0xff)) for arg in args: if isinstance(arg, str): - emit_2byte(interpreter.find_opcode(arg)) + opcode = interpreter.find_opcode(arg) + assert opcode >= 0, "unknown opcode %s" % (arg, ) + emit_2byte(opcode) elif isinstance(arg, int): emit_2byte(arg) elif isinstance(arg, label): @@ -797,6 +842,8 @@ for i in range(len(result)): b = result[i] if isinstance(b, tlabel): + for j in range(1, 4): + assert result[i + j] is None index = labelpos[b.name] result[i + 0] = chr((index >> 24) & 0xff) result[i + 1] = chr((index >> 16) & 0xff) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 4 02:45:07 2008 @@ -476,7 +476,6 @@ self.check_insns({}) def test_split_on_green_return(self): - py.test.skip("calls are WIP") def ll_two(x): if x > 0: return 17 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Mon Feb 4 02:45:07 2008 @@ -61,7 +61,7 @@ "red_int_add", 0, 1, "make_new_redvars", 1, 2, "make_new_greenvars", 0, - "red_return", 0) + "red_return") assert len(jitcode.constants) == 0 assert len(jitcode.typekinds) == 0 assert jitcode.is_portal @@ -76,7 +76,7 @@ "red_int_add", 0, 1, "make_new_redvars", 1, 2, "make_new_greenvars", 0, - "red_return", 0) + "red_return") assert len(jitcode.constants) == 1 assert len(jitcode.typekinds) == 1 assert len(jitcode.redboxclasses) == 1 @@ -97,7 +97,7 @@ "make_new_redvars", 1, 1, "make_new_greenvars", 0, label("return"), - "red_return", 0, + "red_return", label("true"), "make_new_redvars", 1, 0, "make_new_greenvars", 0, @@ -127,7 +127,7 @@ "make_new_redvars", 1, 2, "make_new_greenvars", 0, label("return"), - "red_return", 0, + "red_return", label("true"), "make_new_redvars", 2, 0, 1, "make_new_greenvars", 0, @@ -165,7 +165,7 @@ "red_int_add", 1, 0, "make_new_redvars", 1, 2, "make_new_greenvars", 0, - "red_return", 0, + "red_return", label("add"), "make_new_redvars", 2, 1, 2, "make_new_greenvars", 0, @@ -198,7 +198,7 @@ "red_goto_iftrue", 2, tlabel("body"), "make_new_redvars", 1, 0, "make_new_greenvars", 0, - "red_return", 0, + "red_return", label("body"), "make_new_redvars", 2, 0, 1, "make_new_greenvars", 0, @@ -223,7 +223,7 @@ "red_int_mul", 1, 2, "make_new_redvars", 1, 3, "make_new_greenvars", 0, - "red_return", 0) + "red_return") assert jitcode.is_portal assert len(jitcode.called_bytecodes) == 1 called_jitcode = jitcode.called_bytecodes[0] @@ -232,7 +232,7 @@ "red_int_add", 0, 1, "make_new_redvars", 1, 2, "make_new_greenvars", 0, - "red_return", 0) + "red_return") assert not called_jitcode.is_portal assert len(called_jitcode.called_bytecodes) == 0 @@ -250,11 +250,48 @@ "make_redbox", 1, 0, "make_new_redvars", 1, 0, "make_new_greenvars", 0, - "red_return", 0) + "red_return") assert jitcode.is_portal assert len(jitcode.called_bytecodes) == 0 assert len(jitcode.nonrainbow_functions) == 1 + def test_yellow_call(self): + def ll_two(x): + if x > 0: + return 17 + else: + return 22 + def ll_function(x): + n = ll_two(x) + return hint(n+1, variable=True) + writer, jitcode = self.serialize(ll_function, [int]) + assert jitcode.code == assemble(writer.interpreter, + "yellow_direct_call", 0, 1, 0, 0, + "yellow_after_direct_call", + "yellow_retrieve_result", + "green_int_add", 0, -1, + "make_redbox", 1, 0, + "make_new_redvars", 1, 1, + "make_new_greenvars", 0, + "red_return") + assert jitcode.is_portal + assert len(jitcode.called_bytecodes) == 1 + called_jitcode = jitcode.called_bytecodes[0] + assert called_jitcode.code == assemble(writer.interpreter, + "make_redbox", -1, 0, + "red_int_gt", 0, 1, + "red_goto_iftrue", 2, tlabel("true"), + "make_new_redvars", 0, + "make_new_greenvars", 1, -2, + label("return"), + "yellow_return", + label("true"), + "make_new_redvars", 0, + "make_new_greenvars", 1, -3, + "goto", tlabel("return") + ) + assert not called_jitcode.is_portal + assert len(called_jitcode.called_bytecodes) == 0 class TestLLType(AbstractSerializationTest): type_system = "lltype" Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Mon Feb 4 02:45:07 2008 @@ -527,7 +527,7 @@ assert None not in redboxes jitstate.frame.local_boxes = redboxes -def save_greens(jitstate, *greens_gv): +def save_greens(jitstate, greens_gv): jitstate.greens = list(greens_gv) def getlocalbox(jitstate, i): From cfbolz at codespeak.net Mon Feb 4 10:03:35 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 4 Feb 2008 10:03:35 +0100 (CET) Subject: [pypy-svn] r51235 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter/test Message-ID: <20080204090335.B71A816844D@codespeak.net> Author: cfbolz Date: Mon Feb 4 10:03:34 2008 New Revision: 51235 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py Log: move test_recursive_call over from test_timeshift and make it pass. Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Mon Feb 4 10:03:34 2008 @@ -100,7 +100,7 @@ assert 0, "unknown graph color %s" % (graph_color, ) self.newjitstate(newjitstate) - if newjitstate is None or is_portal: + if self.frame is None: return STOP else: self.frame.pc = resumepoint @@ -606,12 +606,15 @@ def graph_position(self, graph): if graph in self.graph_positions: return self.graph_positions[graph] - bytecode = JitCode.__new__(JitCode) + if graph in self.all_graphs: + bytecode = self.all_graphs[graph] + else: + bytecode = JitCode.__new__(JitCode) + self.all_graphs[graph] = bytecode + self.unfinished_graphs.append(graph) index = len(self.called_bytecodes) self.called_bytecodes.append(bytecode) - self.all_graphs[graph] = bytecode self.graph_positions[graph] = index - self.unfinished_graphs.append(graph) return index def nonrainbow_position(self, fnptr): Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 4 10:03:34 2008 @@ -488,5 +488,16 @@ assert res == 23 self.check_insns({'int_gt': 1}) + def test_recursive_call(self): + def ll_pseudo_factorial(n, fudge): + k = hint(n, concrete=True) + if n <= 0: + return 1 + return n * ll_pseudo_factorial(n - 1, fudge + n) - fudge + res = self.interpret(ll_pseudo_factorial, [4, 2], [0]) + expected = ll_pseudo_factorial(4, 2) + assert res == expected + + class TestLLType(SimpleTests): type_system = "lltype" Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py Mon Feb 4 10:03:34 2008 @@ -777,16 +777,6 @@ assert res == True self.check_insns({'setfield': 2, 'getfield': 1}) - def test_recursive_call(self): - def ll_pseudo_factorial(n, fudge): - k = hint(n, concrete=True) - if n <= 0: - return 1 - return n * ll_pseudo_factorial(n - 1, fudge + n) - fudge - res = self.timeshift(ll_pseudo_factorial, [4, 2], [0]) - expected = ll_pseudo_factorial(4, 2) - assert res == expected - def test_recursive_with_red_termination_condition(self): py.test.skip('Does not terminate') def ll_factorial(n): From cfbolz at codespeak.net Mon Feb 4 10:19:36 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 4 Feb 2008 10:19:36 +0100 (CET) Subject: [pypy-svn] r51236 - pypy/branch/jit-refactoring/pypy/jit/rainbow Message-ID: <20080204091936.48DA7168458@codespeak.net> Author: cfbolz Date: Mon Feb 4 10:19:35 2008 New Revision: 51236 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Log: convert some XXXs into asserts Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Mon Feb 4 10:19:35 2008 @@ -92,7 +92,7 @@ elif graph_color == "yellow": newjitstate = rtimeshift.leave_graph_yellow(queue) elif graph_color == "green": - XXX + assert 0, "green graphs shouldn't be seen by the rainbow interp" elif graph_color == "gray": assert not is_portal newjitstate = rtimeshift.leave_graph_gray(queue) @@ -433,15 +433,15 @@ def insert_exits(self, block): if block.exits == (): returnvar, = block.inputargs - color = self.varcolor(returnvar) + color = self.graph_calling_color(self.graph) if color == "red": self.emit("red_return") - elif originalconcretetype(returnvar) == lltype.Void: + elif color == "gray": self.emit("gray_return") - elif color == "green": # really a yellow call # XXX use graphcolor + elif color == "yellow": self.emit("yellow_return") else: - XXX + assert 0, "unknown graph calling color %s" % (color, ) elif len(block.exits) == 1: link, = block.exits self.insert_renaming(link) @@ -841,7 +841,7 @@ elif isinstance(arg, tlabel): result.extend((arg, None, None, None)) else: - XXX + assert "don't know how to emit %r" % (arg, ) for i in range(len(result)): b = result[i] if isinstance(b, tlabel): From cfbolz at codespeak.net Mon Feb 4 10:31:20 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 4 Feb 2008 10:31:20 +0100 (CET) Subject: [pypy-svn] r51237 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter Message-ID: <20080204093120.D318D16844A@codespeak.net> Author: cfbolz Date: Mon Feb 4 10:31:20 2008 New Revision: 51237 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/exception.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Log: clean up the Pseudo* stuff Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Mon Feb 4 10:31:20 2008 @@ -518,10 +518,10 @@ name = "%s_%s" % (color, opname) index = self.interpreter.find_opcode(name) if index == -1: - hop = PseudoHOP( - op, [self.hannotator.binding(arg) for arg in op.args], - self.hannotator.binding(op.result), self.RGenOp) - opdesc = rtimeshift.make_opdesc(hop) + opdesc = rtimeshift.make_opdesc( + self.RGenOp, opname, + [self.hannotator.binding(arg) for arg in op.args], + self.hannotator.binding(op.result), ) index = self.interpreter.make_opcode_implementation(color, opdesc) self.emit(name) @@ -853,18 +853,3 @@ result[i + 2] = chr((index >> 8) & 0xff) result[i + 3] = chr(index & 0xff) return "".join(result) - - - -# XXX too lazy to fix the interface of make_opdesc, ExceptionDesc -class PseudoHOP(object): - def __init__(self, op, args_s, s_result, RGenOp): - self.spaceop = op - self.args_s = args_s - self.s_result = s_result - self.rtyper = PseudoHRTyper(RGenOp=RGenOp) - -class PseudoHRTyper(object): - def __init__(self, **args): - self.__dict__.update(**args) - Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 4 10:31:20 2008 @@ -113,10 +113,10 @@ def interpret(self, ll_function, values, opt_consts=[], *args, **kwds): # XXX clean this mess up writer, jitcode, argcolors = self.serialize(ll_function, values) - hrtyper = bytecode.PseudoHRTyper(RGenOp=writer.RGenOp, - annotator=writer.translator.annotator, - rtyper=writer.translator.annotator.base_translator.rtyper) - edesc = exception.ExceptionDesc(hrtyper, False) + base_annotator = writer.translator.annotator + etrafo = base_annotator.exceptiontransformer + type_system = base_annotator.base_translator.rtyper.type_system.name + edesc = exception.ExceptionDesc(writer.RGenOp, etrafo, type_system, False) rgenop = writer.RGenOp() sigtoken = rgenop.sigToken(self.RESIDUAL_FUNCTYPE) builder, gv_generated, inputargs_gv = rgenop.newgraph(sigtoken, "generated") Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/exception.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/exception.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/exception.py Mon Feb 4 10:31:20 2008 @@ -4,9 +4,8 @@ class ExceptionDesc: - def __init__(self, hrtyper, lazy_exception_path): - RGenOp = hrtyper.RGenOp - self.etrafo = hrtyper.annotator.exceptiontransformer + def __init__(self, RGenOp, etrafo, type_system, lazy_exception_path): + self.etrafo = etrafo self.cexcdata = self.etrafo.cexcdata self.exc_data_ptr = self.cexcdata.value self.gv_excdata = RGenOp.constPrebuiltGlobal(self.exc_data_ptr) @@ -24,7 +23,7 @@ self.gv_null_exc_type = RGenOp.constPrebuiltGlobal(null_exc_type) self.gv_null_exc_value = RGenOp.constPrebuiltGlobal(null_exc_value) - if hrtyper.rtyper.type_system.name == 'lltypesystem': + if type_system == 'lltypesystem': self.null_exc_type_box = rvalue.PtrRedBox(self.exc_type_kind, self.gv_null_exc_type) self.null_exc_value_box = rvalue.PtrRedBox(self.exc_value_kind, Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Mon Feb 4 10:31:20 2008 @@ -64,11 +64,10 @@ _opdesc_cache = {} -def make_opdesc(hop): - hrtyper = hop.rtyper - op_key = (hrtyper.RGenOp, hop.spaceop.opname, - tuple([originalconcretetype(s_arg) for s_arg in hop.args_s]), - originalconcretetype(hop.s_result)) +def make_opdesc(RGenOp, opname, args_s, s_result): + op_key = (RGenOp, opname, + tuple([originalconcretetype(s_arg) for s_arg in args_s]), + originalconcretetype(s_result)) try: return _opdesc_cache[op_key] except KeyError: @@ -134,10 +133,6 @@ jitstate.greens.append(gv_raised) # for split_raisingop() return opdesc.redboxcls(opdesc.result_kind, genvar) -def ll_genmalloc_varsize(jitstate, contdesc, sizebox): - # the specialized by contdesc is not useful, unify paths - return genmalloc_varsize(jitstate, contdesc, sizebox) - def genmalloc_varsize(jitstate, contdesc, sizebox): gv_size = sizebox.getgenvar(jitstate) alloctoken = contdesc.varsizealloctoken @@ -599,10 +594,6 @@ def _freeze_(self): return True -def ll_gen_residual_call(jitstate, calldesc, funcbox): - # specialization is not useful here, we can unify the calldescs - return gen_residual_call(jitstate, calldesc, funcbox) - def gen_residual_call(jitstate, calldesc, funcbox): builder = jitstate.curbuilder gv_funcbox = funcbox.getgenvar(jitstate) @@ -797,10 +788,6 @@ def _freeze_(self): return True -def ll_promote(jitstate, promotebox, promotiondesc): - # the specialization by promotiondesc is not useful here, so unify paths - return promote(jitstate, promotebox, promotiondesc) - def promote(jitstate, promotebox, promotiondesc): builder = jitstate.curbuilder gv_switchvar = promotebox.getgenvar(jitstate) From antocuni at codespeak.net Mon Feb 4 10:45:09 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 4 Feb 2008 10:45:09 +0100 (CET) Subject: [pypy-svn] r51238 - in pypy/dist/pypy/translator/cli: . test Message-ID: <20080204094509.DFFBD168458@codespeak.net> Author: antocuni Date: Mon Feb 4 10:45:09 2008 New Revision: 51238 Modified: pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: add the ability to call a delegate object with the usual () notation Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Mon Feb 4 10:45:09 2008 @@ -3,6 +3,7 @@ from pypy.tool.pairtype import pair, pairtype from pypy.annotation.model import SomeObject, SomeInstance, SomeOOInstance, SomeInteger, s_None,\ s_ImpossibleValue, lltype_to_annotation, annotation_to_lltype, SomeChar, SomeString, SomePBC +from pypy.annotation.unaryop import immutablevalue from pypy.annotation.binaryop import _make_none_union from pypy.annotation import model as annmodel from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong @@ -56,6 +57,17 @@ def rtyper_makekey(self): return self.__class__, self.cli_class, self.meth_name +class __extend__(SomeOOInstance): + + def simple_call(self, *s_args): + from pypy.translator.cli.query import get_cli_class + DELEGATE = get_cli_class('System.Delegate')._INSTANCE + if ootype.isSubclass(self.ootype, DELEGATE): + s_invoke = self.getattr(immutablevalue('Invoke')) + return s_invoke.simple_call(*s_args) + else: + # cannot call a non-delegate + return SomeObject.simple_call(self, *s_args) class __extend__(pairtype(SomeOOInstance, SomeInteger)): def getitem((ooinst, index)): @@ -148,6 +160,19 @@ hop.exception_cannot_occur() return hop.genop('cli_arraylength', vlist, hop.r_result.lowleveltype) + def rtype_simple_call(self, hop): + TYPE = self.lowleveltype + _, meth = TYPE._lookup('Invoke') + assert isinstance(meth, ootype._overloaded_meth) + ARGS = tuple([repr.lowleveltype for repr in hop.args_r[1:]]) + desc = meth._get_desc('Invoke', ARGS) + cname = hop.inputconst(ootype.Void, desc) + vlist = hop.inputargs(self, *hop.args_r[1:]) + hop.exception_is_here() + return hop.genop("oosend", [cname]+vlist, + resulttype = hop.r_result.lowleveltype) + + ## OOType model class OverloadingResolver(ootype.OverloadingResolver): Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Mon Feb 4 10:45:09 2008 @@ -13,6 +13,21 @@ ArrayList = CLR.System.Collections.ArrayList OpCodes = System.Reflection.Emit.OpCodes DynamicMethod = System.Reflection.Emit.DynamicMethod +DelegateType = CLR.pypy.runtime.DelegateType_int__int_int +Utils = CLR.pypy.runtime.Utils + +# RPython function, used by test_dynamic_method and test_call_delegate +def build_fn(): + tInt = typeof(System.Int32) + args = init_array(System.Type, tInt, tInt) + meth = Utils.CreateDynamicMethod("add", tInt, args) + il = meth.GetILGenerator() + il.Emit(OpCodes.Ldarg_0) + il.Emit(OpCodes.Ldarg_1) + il.Emit(OpCodes.Add) + il.Emit(OpCodes.Ret) + myfunc = meth.CreateDelegate(typeof(DelegateType)) + return clidowncast(DelegateType, myfunc) class TestDotnetAnnotation(object): @@ -468,24 +483,19 @@ assert self.ll_to_string(res) == '42' def test_dynamic_method(self): - from pypy.rpython.ootypesystem import ootype - DelegateType = CLR.pypy.runtime.DelegateType_int__int_int - Utils = CLR.pypy.runtime.Utils def fn(): - tInt = typeof(System.Int32) - args = init_array(System.Type, tInt, tInt) - meth = Utils.CreateDynamicMethod("add", tInt, args) - il = meth.GetILGenerator() - il.Emit(OpCodes.Ldarg_0) - il.Emit(OpCodes.Ldarg_1) - il.Emit(OpCodes.Add) - il.Emit(OpCodes.Ret) - myfunc = meth.CreateDelegate(typeof(DelegateType)) - myfunc = clidowncast(DelegateType, myfunc) + myfunc = build_fn() return myfunc.Invoke(30, 12) res = self.interpret(fn, []) assert res == 42 + def test_call_delegate(self): + def fn(): + myfunc = build_fn() + return myfunc(30, 12) + res = self.interpret(fn, []) + assert res == 42 + class TestPythonnet(TestDotnetRtyping): # don't interpreter functions but execute them directly through pythonnet From arigo at codespeak.net Mon Feb 4 10:51:34 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 4 Feb 2008 10:51:34 +0100 (CET) Subject: [pypy-svn] r51239 - in pypy/branch/asmgcroot/pypy: rpython/memory/gctransform translator/c/gcc translator/c/gcc/test Message-ID: <20080204095134.B4821168457@codespeak.net> Author: arigo Date: Mon Feb 4 10:51:34 2008 New Revision: 51239 Added: pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track6.s Modified: pypy/branch/asmgcroot/pypy/rpython/memory/gctransform/asmgcroot.py pypy/branch/asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track0.s pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track1.s pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track2.s pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track3.s pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track4.s pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track5.s pypy/branch/asmgcroot/pypy/translator/c/gcc/trackgcroot.py Log: Refactor of the way the "location" integers are encoded and decoded, to allow both %esp-based and %ebp-based stack frame positions to be used. See the new track6.s test for why we need it. Simplifies some stuff too. Modified: pypy/branch/asmgcroot/pypy/rpython/memory/gctransform/asmgcroot.py ============================================================================== --- pypy/branch/asmgcroot/pypy/rpython/memory/gctransform/asmgcroot.py (original) +++ pypy/branch/asmgcroot/pypy/rpython/memory/gctransform/asmgcroot.py Mon Feb 4 10:51:34 2008 @@ -104,31 +104,24 @@ # # A "safe point" is the return address of a call. # The "shape" of a safe point is a list of integers - # as follows: - # - # * The size of the frame of the function containing the - # call (in theory it can be different from call to call). - # This size includes the return address (see picture + # that represent "locations". A "location" can be + # either in the stack or in a register. See + # getlocation() for the decoding of this integer. + # The locations stored in a "shape" are as follows: + # + # * The "location" of the return address. This is just + # after the end of the frame of 'callee'; it is the + # first word of the frame of 'caller' (see picture # below). # - # * Four integers that specify where the function saves + # * Four "locations" that specify where the function saves # each of the four callee-saved registers (%ebx, %esi, - # %edi, %ebp). This is a "location", see below. + # %edi, %ebp). # # * The number of live GC roots around the call. # # * For each GC root, an integer that specify where the - # GC pointer is stored. This is a "location", see below. - # - # A "location" can be either in the stack or in a register. - # If it is in the stack, it is specified as an integer offset - # from the current frame (so it is typically < 0, as seen - # in the picture below, though it can also be > 0 if the - # location is an input argument to the current function and - # thus lives at the bottom of the caller's frame). - # A "location" can also be in a register; in our context it - # can only be a callee-saved register. This is specified - # as a small odd-valued integer (1=%ebx, 3=%esi, etc.) + # GC pointer is stored. This is a "location" too. # # XXX the details are completely specific to X86!!! # a picture of the stack may help: @@ -159,19 +152,9 @@ llop.debug_fatalerror(lltype.Void, "cannot find gc roots!") return False # - # found! Now we can go to the caller_frame. + # found! Enumerate the GC roots in the caller frame # shape = item.address[1] - framesize = shape.signed[0] - if framesize == 0: - # frame size not known, use %ebp to find the frame address - caller_ebp = callee.regs_stored_at[INDEX_OF_EBP].address[0] - caller.frame_address = caller_ebp + 4 - else: - caller.frame_address = callee.frame_address + framesize - # - # enumerate the GC roots in the caller frame - # collect_stack_root = self.gcdata._gc_collect_stack_root gc = self.gc LIVELOCS = 1 + CALLEE_SAVED_REGS + 1 # index of the first gc root loc @@ -179,37 +162,53 @@ while livecount > 0: livecount -= 1 location = shape.signed[LIVELOCS + livecount] - addr = self.getlocation(callee, caller, location) + addr = self.getlocation(callee, location) if addr.address[0] != llmemory.NULL: collect_stack_root(gc, addr) # # track where the caller_frame saved the registers from its own # caller # - if shape.signed[1] == -1: # a marker that means "I'm the frame - return False # of the entry point, stop walking" + location = shape.signed[0] + caller.frame_address = self.getlocation(callee, location) + if not caller.frame_address: # marker that means "I'm the frame + return False # of the entry point, stop walking" reg = 0 while reg < CALLEE_SAVED_REGS: location = shape.signed[1+reg] - addr = self.getlocation(callee, caller, location) + addr = self.getlocation(callee, location) caller.regs_stored_at[reg] = addr reg += 1 return True - def getlocation(self, callee, caller, location): + def getlocation(self, callee, location): """Get the location in the 'caller' frame of a variable, based - on the integer 'location' that describes it. The 'callee' is - only used if the location is in a register that was saved in the - callee. + on the integer 'location' that describes it. All locations are + computed based on information saved by the 'callee'. """ - if location & 1: # register - reg = location >> 1 + kind = location & LOC_MASK + if kind == LOC_REG: # register + reg = location >> 2 ll_assert(0 <= reg < CALLEE_SAVED_REGS, "bad register location") return callee.regs_stored_at[reg] + elif kind == LOC_ESP_BASED: # in the caller stack frame at N(%esp) + offset = location & ~ LOC_MASK + ll_assert(offset >= 0, "bad %esp-based location") + esp_in_caller = callee.frame_address + 4 + return esp_in_caller + offset + elif kind == LOC_EBP_BASED: # in the caller stack frame at N(%ebp) + offset = location & ~ LOC_MASK + ebp_in_caller = callee.regs_stored_at[INDEX_OF_EBP].address[0] + return ebp_in_caller + offset else: - # in the stack frame - return caller.frame_address + location + return llmemory.NULL + +LOC_NOWHERE = 0 +LOC_REG = 1 +LOC_EBP_BASED = 2 +LOC_ESP_BASED = 3 +LOC_MASK = 0x03 # ____________________________________________________________ Modified: pypy/branch/asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py ============================================================================== --- pypy/branch/asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py (original) +++ pypy/branch/asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py Mon Feb 4 10:51:34 2008 @@ -1,5 +1,9 @@ import py import sys, re +from pypy.translator.c.gcc.trackgcroot import format_location +from pypy.translator.c.gcc.trackgcroot import format_callshape +from pypy.translator.c.gcc.trackgcroot import LOC_NOWHERE, LOC_REG +from pypy.translator.c.gcc.trackgcroot import LOC_EBP_BASED, LOC_ESP_BASED from pypy.translator.c.gcc.trackgcroot import GcRootTracker from pypy.translator.c.gcc.trackgcroot import FunctionGcRootTracker from StringIO import StringIO @@ -7,6 +11,31 @@ this_dir = py.path.local(__file__).dirpath() +def test_format_location(): + assert format_location(LOC_NOWHERE) == '?' + assert format_location(LOC_REG | (0<<2)) == '%ebx' + assert format_location(LOC_REG | (1<<2)) == '%esi' + assert format_location(LOC_REG | (2<<2)) == '%edi' + assert format_location(LOC_REG | (3<<2)) == '%ebp' + assert format_location(LOC_EBP_BASED + 0) == '(%ebp)' + assert format_location(LOC_EBP_BASED + 4) == '4(%ebp)' + assert format_location(LOC_EBP_BASED - 4) == '-4(%ebp)' + assert format_location(LOC_ESP_BASED + 0) == '(%esp)' + assert format_location(LOC_ESP_BASED + 4) == '4(%esp)' + assert format_location(LOC_ESP_BASED - 4) == '-4(%esp)' + +def test_format_callshape(): + expected = ('{4(%ebp) ' # position of the return address + '| 8(%ebp), 12(%ebp), 16(%ebp), 20(%ebp) ' # 4 saved regs + '| 24(%ebp), 28(%ebp)}') # GC roots + assert format_callshape((LOC_EBP_BASED+4, + LOC_EBP_BASED+8, + LOC_EBP_BASED+12, + LOC_EBP_BASED+16, + LOC_EBP_BASED+20, + LOC_EBP_BASED+24, + LOC_EBP_BASED+28)) == expected + def test_find_functions(): source = """\ \t.p2align 4,,15 @@ -46,7 +75,7 @@ yield check_computegcmaptable, path r_globallabel = re.compile(r"([\w]+)[:]") -r_expected = re.compile(r"\s*;;\s*expected\s*([(][-\d\s,+]+[)])") +r_expected = re.compile(r"\s*;;\s*expected\s+([{].+[}])") def check_computegcmaptable(path): print @@ -54,26 +83,24 @@ lines = path.readlines() expectedlines = lines[:] tracker = FunctionGcRootTracker(lines) - tracker.is_main = tracker.funcname == 'main' table = tracker.computegcmaptable(verbose=sys.maxint) tabledict = {} seen = {} for entry in table: - print entry + print '%s: %s' % (entry[0], format_callshape(entry[1])) tabledict[entry[0]] = entry[1] # find the ";; expected" lines prevline = "" for i, line in enumerate(lines): match = r_expected.match(line) if match: - expected = eval(match.group(1)) - assert isinstance(expected, tuple) + expected = match.group(1) prevmatch = r_globallabel.match(prevline) assert prevmatch, "the computed table is not complete" label = prevmatch.group(1) assert label in tabledict got = tabledict[label] - assert got == expected + assert format_callshape(got) == expected seen[label] = True expectedlines.insert(i-2, '\t.globl\t%s\n' % (label,)) expectedlines.insert(i-1, '%s:\n' % (label,)) Modified: pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track0.s ============================================================================== --- pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track0.s (original) +++ pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track0.s Mon Feb 4 10:51:34 2008 @@ -14,7 +14,7 @@ movl %ebp, %ebx movl $pypy_g_array_16, (%esp) call open - ;; expected (32, -16, -12, -8, -4) + ;; expected {28(%esp) | 12(%esp), 16(%esp), 20(%esp), 24(%esp) | } cmpl $-1, %eax movl %eax, %edi movl 32(%esp), %esi @@ -26,7 +26,7 @@ movl %esi, 4(%esp) movl %edi, (%esp) call read - ;; expected (32, -16, -12, -8, -4) + ;; expected {28(%esp) | 12(%esp), 16(%esp), 20(%esp), 24(%esp) | } testl %eax, %eax jle .L280 .L285: @@ -46,13 +46,13 @@ movl %eax, 8(%esp) movl %edi, (%esp) call read - ;; expected (32, -16, -12, -8, -4) + ;; expected {28(%esp) | 12(%esp), 16(%esp), 20(%esp), 24(%esp) | } testl %eax, %eax jg .L288 .L280: movl %edi, (%esp) call close - ;; expected (32, -16, -12, -8, -4) + ;; expected {28(%esp) | 12(%esp), 16(%esp), 20(%esp), 24(%esp) | } movl %ebx, %eax .L286: .L274: @@ -65,7 +65,7 @@ movl %ecx, 4(%esp) movl %esi, (%esp) call memset - ;; expected (32, -16, -12, -8, -4) + ;; expected {28(%esp) | 12(%esp), 16(%esp), 20(%esp), 24(%esp) | } .L270: addl $12, %esp popl %ebx Modified: pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track1.s ============================================================================== --- pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track1.s (original) +++ pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track1.s Mon Feb 4 10:51:34 2008 @@ -14,7 +14,7 @@ movl $0, 4(%esp) movl $pypy_g_array_16, (%esp) call open - ;; expected (80, 1, 3, 5, -4) + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } movl %eax, -12(%ebp) .loc 2 1522 0 cmpl $-1, -12(%ebp) @@ -55,7 +55,7 @@ movl -8(%ebp), %eax movl %eax, (%esp) call memset - ;; expected (80, 1, 3, 5, -4) + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } .loc 2 1545 0 jmp .L542 .L545: @@ -73,7 +73,7 @@ movl -12(%ebp), %eax movl %eax, (%esp) call close - ;; expected (80, 1, 3, 5, -4) + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } movl %eax, -36(%ebp) .loc 2 1556 0 movl -16(%ebp), %eax @@ -111,7 +111,7 @@ movl -12(%ebp), %eax movl %eax, (%esp) call read - ;; expected (80, 1, 3, 5, -4) + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } movl %eax, -48(%ebp) .loc 2 1575 0 movl -48(%ebp), %eax Modified: pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track2.s ============================================================================== --- pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track2.s (original) +++ pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track2.s Mon Feb 4 10:51:34 2008 @@ -54,7 +54,7 @@ movl %edx, 4(%esp) movl %esi, (%esp) call pypy_g_populate - ;; expected (32, -8, -4, 5, 7, 1) + ;; expected {28(%esp) | 20(%esp), 24(%esp), %edi, %ebp | %ebx} movl %ebx, %eax #APP /* GCROOT %eax */ @@ -73,7 +73,7 @@ .L1351: .L1352: call LL_stack_too_big - ;; expected (32, -8, -4, 5, 7, 1) + ;; expected {28(%esp) | 20(%esp), 24(%esp), %edi, %ebp | %ebx} testl %eax, %eax jne .L1418 .L1361: @@ -100,7 +100,7 @@ movl $24, %edx movl %edx, 4(%esp) call pypy_g_SemiSpaceGC_try_obtain_free_space - ;; expected (32, -8, -4, 5, 7, 1) + ;; expected {28(%esp) | 20(%esp), 24(%esp), %edi, %ebp | %ebx} movl pypy_g_ExcData, %edx xorl %ecx, %ecx testl %edx, %edx @@ -121,7 +121,7 @@ movl $24, %ecx movl %ecx, 4(%esp) call pypy_g_SemiSpaceGC_try_obtain_free_space - ;; expected (32, -8, -4, 5, 7, 1) + ;; expected {28(%esp) | 20(%esp), 24(%esp), %edi, %ebp | %ebx} movl pypy_g_ExcData, %edx xorl %ecx, %ecx testl %edx, %edx Modified: pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track3.s ============================================================================== --- pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track3.s (original) +++ pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track3.s Mon Feb 4 10:51:34 2008 @@ -56,7 +56,7 @@ movl %edx, 4(%esp) movl %esi, (%esp) call pypy_g_populate - ;; expected (32, -12, -8, 5, -4, 1) + ;; expected {4(%ebp) | -8(%ebp), -4(%ebp), %edi, (%ebp) | %ebx} movl %ebx, %eax #APP /* GCROOT %eax */ @@ -75,7 +75,7 @@ .L1350: .L1351: call LL_stack_too_big - ;; expected (32, -12, -8, 5, -4, 1) + ;; expected {4(%ebp) | -8(%ebp), -4(%ebp), %edi, (%ebp) | %ebx} testl %eax, %eax jne .L1417 .L1360: @@ -102,7 +102,7 @@ movl $24, %edx movl %edx, 4(%esp) call pypy_g_SemiSpaceGC_try_obtain_free_space - ;; expected (32, -12, -8, 5, -4, 1) + ;; expected {4(%ebp) | -8(%ebp), -4(%ebp), %edi, (%ebp) | %ebx} movl pypy_g_ExcData, %edx xorl %ecx, %ecx testl %edx, %edx @@ -123,7 +123,7 @@ movl $24, %ecx movl %ecx, 4(%esp) call pypy_g_SemiSpaceGC_try_obtain_free_space - ;; expected (32, -12, -8, 5, -4, 1) + ;; expected {4(%ebp) | -8(%ebp), -4(%ebp), %edi, (%ebp) | %ebx} movl pypy_g_ExcData, %edx xorl %ecx, %ecx testl %edx, %edx Modified: pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track4.s ============================================================================== --- pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track4.s (original) +++ pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track4.s Mon Feb 4 10:51:34 2008 @@ -7,11 +7,11 @@ movl %esp, %ebp pushl %edi subl $8, %esp - andl $-15, %esp + andl $-16, %esp movl %ebx, -8(%ebp) movl 8(%ebp), %edi call foobar - ;; expected (20, -12, 3, -8, -4, 5) + ;; expected {4(%ebp) | -8(%ebp), %esi, -4(%ebp), (%ebp) | %edi} .L1: cmpl $0, %eax je .L3 @@ -26,7 +26,7 @@ movl %esi, %ebx movl $nonsense, %esi call foobar - ;; expected (36, -12, 1, -8, -4, -28, -16) + ;; expected {4(%ebp) | -8(%ebp), %ebx, -4(%ebp), (%ebp) | -12(%ebp), 4(%esp)} addl %edi, %eax movl 4(%esp), %eax movl %ebx, %esi @@ -39,7 +39,7 @@ ;; end of inlined function .L3: call foobar - ;; expected (20, -12, 3, -8, -4, 5) + ;; expected {4(%ebp) | -8(%ebp), %esi, -4(%ebp), (%ebp) | %edi} #APP /* GCROOT %edi */ #NO_APP Modified: pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track5.s ============================================================================== --- pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track5.s (original) +++ pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track5.s Mon Feb 4 10:51:34 2008 @@ -15,11 +15,11 @@ movl %esi, 8(%esp) movl %esi, (%esp) call pypy_g_trace___trace_copy - ;; expected (32, -8, -4, 5, 7) + ;; expected {28(%esp) | 20(%esp), 24(%esp), %edi, %ebp | } movl %ebx, 4(%esp) movl %esi, (%esp) call pypy_g_SemiSpaceGC_get_size - ;; expected (32, -8, -4, 5, 7) + ;; expected {28(%esp) | 20(%esp), 24(%esp), %edi, %ebp | } addl %eax, %ebx .L1216: cmpl 12(%esi), %ebx @@ -36,15 +36,15 @@ movl %esi, 8(%esp) movl %esi, (%esp) call pypy_g_trace___trace_copy - ;; expected (32, -8, -4, 5, 7) + ;; expected {28(%esp) | 20(%esp), 24(%esp), %edi, %ebp | } movl %ebx, 4(%esp) movl %esi, (%esp) call pypy_g_SemiSpaceGC_get_size - ;; expected (32, -8, -4, 5, 7) + ;; expected {28(%esp) | 20(%esp), 24(%esp), %edi, %ebp | } addl %eax, %ebx .L1227: call RPyAbort - ;; expected (32, -8, -4, 5, 7) + ;; expected {28(%esp) | 20(%esp), 24(%esp), %edi, %ebp | } cmpl 12(%esi), %ebx jb .L1229 addl $20, %esp Added: pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track6.s ============================================================================== --- (empty file) +++ pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track6.s Mon Feb 4 10:51:34 2008 @@ -0,0 +1,26 @@ + .type main, @function +main: + ;; a minimal example showing what kind of code gcc + ;; can produce for main(): some local variable accesses + ;; are relative to %ebp, while others are relative to + ;; %esp, and the difference %ebp-%esp is not constant + ;; because of the 'andl' to align the stack + pushl %ebp + movl %esp, %ebp + subl $8, %esp + andl $-16, %esp + movl $globalptr1, -4(%ebp) + movl $globalptr2, (%esp) + pushl $0 + call foobar + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | -4(%ebp), 4(%esp)} + popl %eax +#APP + /* GCROOT -4(%ebp) */ + /* GCROOT (%esp) */ +#NO_APP + movl %ebp, %esp + popl %ebp + ret + + .size main, .-main Modified: pypy/branch/asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/asmgcroot/pypy/translator/c/gcc/trackgcroot.py Mon Feb 4 10:51:34 2008 @@ -30,6 +30,7 @@ self.gcmaptable = [] self.verbose = verbose self.seen_main = False + self.files_seen = 0 def dump(self, output): assert self.seen_main @@ -58,6 +59,11 @@ ret .size pypy_asm_stackwalk_init, .-pypy_asm_stackwalk_init """ + # XXX the two gcmap tables are a bit largish. They could easily + # be compressed by a factor of 4 or more. I suspect they also + # produce large linker tables which could be seriously reduced + # as well. A key observation is that in practice most functions + # seem to use exactly the same call shape for each call they contain. print >> output, '\t.data' print >> output, '\t.align\t4' print >> output, '\t.globl\t__gcmapstart' @@ -113,48 +119,37 @@ if in_function: lines = self.process_function(lines, entrypoint, filename) newfile.writelines(lines) + self.files_seen += 1 def process_function(self, lines, entrypoint, filename): - tracker = FunctionGcRootTracker(lines) - tracker.is_main = tracker.funcname == entrypoint + tracker = FunctionGcRootTracker(lines, filetag = self.files_seen) if self.verbose: print >> sys.stderr, '[trackgcroot:%s] %s' % (filename, tracker.funcname) table = tracker.computegcmaptable(self.verbose) if self.verbose > 1: for label, state in table: - print >> sys.stderr, label, '\t', state - if tracker.is_main: - fp = tracker.uses_frame_pointer - table = self.fixup_entrypoint_table(table, fp) + print >> sys.stderr, label, '\t', format_callshape(state) + if tracker.funcname == entrypoint: + table = self.fixup_entrypoint_table(table) self.gcmaptable.extend(table) return tracker.lines - def fixup_entrypoint_table(self, table, uses_frame_pointer): + def fixup_entrypoint_table(self, table): self.seen_main = True - # as an end marker, set the CALLEE_SAVE_REGISTERS locations to -1. - # this info is not useful because we don't go to main()'s caller. + # as an end marker, set all five initial entries of the callshapes + # to LOC_NOWHERE. This info is not useful anyway because we don't + # go to main()'s caller. newtable = [] - MARKERS = (-1, -1, -1, -1) + MARKERS = (LOC_NOWHERE,) * 5 for label, shape in table: - newtable.append((label, shape[:1] + MARKERS + shape[5:])) - table = newtable - - if uses_frame_pointer: - # the main() function may contain strange code that aligns the - # stack pointer to a multiple of 16, which messes up our framesize - # computation. So just for this function, we use a frame size - # of 0 to ask asmgcroot to read %ebp to find the frame pointer. - newtable = [] - for label, shape in table: - newtable.append((label, (0,) + shape[1:])) - table = newtable - return table + newtable.append((label, MARKERS + shape[5:])) + return newtable class FunctionGcRootTracker(object): - def __init__(self, lines): + def __init__(self, lines, filetag=0): match = r_functionstart.match(lines[0]) self.funcname = match.group(1) match = r_functionend.match(lines[-1]) @@ -163,7 +158,7 @@ self.lines = lines self.uses_frame_pointer = False self.r_localvar = r_localvarnofp - self.is_main = False + self.filetag = filetag def computegcmaptable(self, verbose=0): self.findlabels() @@ -182,26 +177,33 @@ return self.gettable() def gettable(self): - """Returns a list [(label_after_call, shape_tuple)] - where shape_tuple = (framesize, where_is_ebx_saved, ... - ..., where_is_ebp_saved, gcroot0, gcroot1...) + """Returns a list [(label_after_call, callshape_tuple)] + See format_callshape() for more details about callshape_tuple. """ table = [] for insn in self.list_call_insns(): if not hasattr(insn, 'framesize'): continue # calls that never end up reaching a RET - shape = [insn.framesize + 4] # accounts for the return address + if self.uses_frame_pointer: + retaddr = frameloc(LOC_EBP_BASED, 4) + else: + retaddr = frameloc(LOC_ESP_BASED, insn.framesize) + shape = [retaddr] # the first gcroots are always the ones corresponding to # the callee-saved registers for reg in CALLEE_SAVE_REGISTERS: shape.append(None) - for loc, tag in insn.gcroots.items(): - if not isinstance(loc, int): - # a special representation for a register location, - # as an odd-valued number - loc = CALLEE_SAVE_REGISTERS.index(loc) * 2 + 1 + gcroots = [] + for localvar, tag in insn.gcroots.items(): + if isinstance(localvar, LocalVar): + loc = localvar.getlocation(insn.framesize, + self.uses_frame_pointer) + else: + assert localvar in REG2LOC + loc = REG2LOC[localvar] + assert isinstance(loc, int) if tag is None: - shape.append(loc) + gcroots.append(loc) else: regindex = CALLEE_SAVE_REGISTERS.index(tag) shape[1 + regindex] = loc @@ -209,6 +211,8 @@ reg = CALLEE_SAVE_REGISTERS[shape.index(None) - 1] raise AssertionError("cannot track where register %s is saved" % (reg,)) + gcroots.sort() + shape.extend(gcroots) table.append((insn.global_label, tuple(shape))) return table @@ -281,6 +285,11 @@ return [insn for insn in self.insns if isinstance(insn, InsnCall)] def findframesize(self): + # the 'framesize' attached to an instruction is the number of bytes + # in the frame at this point. This doesn't count the return address + # which is the word immediately following the frame in memory. + # The 'framesize' is set to an odd value if it is only an estimate + # (see visit_andl()). def walker(insn, size_delta): check = deltas.setdefault(insn, size_delta) @@ -318,17 +327,21 @@ localvar = getattr(insn, name) match = r_localvar_esp.match(localvar) if match: + if localvar == '0(%esp)': # for pushl and popl, by + hint = None # default ebp addressing is + else: # a bit nicer + hint = 'esp' ofs_from_esp = int(match.group(1) or '0') localvar = ofs_from_esp - insn.framesize assert localvar != 0 # that's the return address - setattr(insn, name, localvar) + setattr(insn, name, LocalVar(localvar, hint=hint)) elif self.uses_frame_pointer: match = r_localvar_ebp.match(localvar) if match: ofs_from_ebp = int(match.group(1) or '0') localvar = ofs_from_ebp - 4 assert localvar != 0 # that's the return address - setattr(insn, name, localvar) + setattr(insn, name, LocalVar(localvar, hint='ebp')) def trackgcroots(self): @@ -346,7 +359,7 @@ def dump(self): for insn in self.insns: size = getattr(insn, 'framesize', '?') - print '%4s %s' % (size, insn) + print >> sys.stderr, '%4s %s' % (size, insn) def walk_instructions_backwards(self, walker, initial_insn, initial_state): pending = [] @@ -388,7 +401,7 @@ if label is None: k = call.lineno while 1: - label = '__gcmap_IN_%s_%d' % (self.funcname, k) + label = '__gcmap_IN%d_%s_%d' % (self.filetag, self.funcname, k) if label not in self.labels: break k += 1 @@ -433,7 +446,9 @@ target = match.group(2) if target == '%esp': count = match.group(1) - assert count.startswith('$') + if not count.startswith('$'): + # strange instruction - I've seen 'subl %eax, %esp' + return InsnCannotFollowEsp() return InsnStackAdjust(sign * int(count[1:])) elif self.r_localvar.match(target): return InsnSetLocal(target) @@ -471,15 +486,10 @@ target = match.group(2) if target == '%esp': # only for andl $-16, %esp used to align the stack in main(). - # If gcc compiled main() with a frame pointer, then it should use - # %ebp-relative addressing and not %esp-relative addressing - # and asmgcroot will read %ebp to find the frame. If main() - # is compiled without a frame pointer, the total frame size that - # we compute ends up being bogus but that's ok because gcc has - # to use %esp-relative addressing only and we don't need to walk - # to caller frames because it's main(). - assert self.is_main - return [] + # The exact amount of adjutment is not known yet, so we use + # an odd-valued estimate to make sure the real value is not used + # elsewhere by the FunctionGcRootTracker. + return InsnCannotFollowEsp() else: return self.binary_insn(line) @@ -493,8 +503,10 @@ if not match: framesize = None # strange instruction else: + if not self.uses_frame_pointer: + raise UnrecognizedOperation('epilogue without prologue') ofs_from_ebp = int(match.group(1) or '0') - assert ofs_from_ebp < 0 + assert ofs_from_ebp <= 0 framesize = 4 - ofs_from_ebp return InsnEpilogue(framesize) else: @@ -524,10 +536,10 @@ def visit_pushl(self, line): match = r_unaryinsn.match(line) source = match.group(1) - return [InsnStackAdjust(-4)] + self.insns_for_copy(source, '(%esp)') + return [InsnStackAdjust(-4)] + self.insns_for_copy(source, '0(%esp)') def _visit_pop(self, target): - return self.insns_for_copy('(%esp)', target) + [InsnStackAdjust(+4)] + return self.insns_for_copy('0(%esp)', target) + [InsnStackAdjust(+4)] def visit_popl(self, line): match = r_unaryinsn.match(line) @@ -629,6 +641,39 @@ pass somenewvalue = SomeNewValue() +class LocalVar(object): + # A local variable location at position 'ofs_from_frame_end', + # which is counted from the end of the stack frame (so it is always + # negative, unless it refers to arguments of the current function). + def __init__(self, ofs_from_frame_end, hint=None): + self.ofs_from_frame_end = ofs_from_frame_end + self.hint = hint + + def __repr__(self): + return '<%+d;%s>' % (self.ofs_from_frame_end, self.hint or 'e*p') + + def __hash__(self): + return hash(self.ofs_from_frame_end) + + def __cmp__(self, other): + if isinstance(other, LocalVar): + return cmp(self.ofs_from_frame_end, other.ofs_from_frame_end) + else: + return 1 + + def getlocation(self, framesize, uses_frame_pointer): + if (self.hint == 'esp' or not uses_frame_pointer + or self.ofs_from_frame_end % 2 != 0): + # try to use esp-relative addressing + ofs_from_esp = framesize + self.ofs_from_frame_end + if ofs_from_esp % 2 == 0: + return frameloc(LOC_ESP_BASED, ofs_from_esp) + # we can get an odd value if the framesize is marked as bogus + # by visit_andl() + assert uses_frame_pointer + ofs_from_ebp = self.ofs_from_frame_end + 4 + return frameloc(LOC_EBP_BASED, ofs_from_ebp) + class Insn(object): _args_ = [] @@ -659,7 +704,8 @@ self.arguments[reg] = somenewvalue def source_of(self, localvar, tag): if localvar not in self.arguments: - assert isinstance(localvar, int) and localvar > 0, ( + assert (isinstance(localvar, LocalVar) and + localvar.ofs_from_frame_end > 0), ( "must come from an argument to the function, got %r" % (localvar,)) self.arguments[localvar] = somenewvalue @@ -692,6 +738,10 @@ assert delta % 4 == 0 self.delta = delta +class InsnCannotFollowEsp(InsnStackAdjust): + def __init__(self): + self.delta = 7 # use an odd value as marker + class InsnStop(Insn): pass @@ -704,30 +754,34 @@ _args_ = ['lineno', 'gcroots'] def __init__(self, lineno): # 'gcroots' is a dict built by side-effect during the call to - # FunctionGcRootTracker.trackgcroots(). Its meaning is as follows: - # the keys are the location that contain gc roots (either register - # names like '%esi', or negative integer offsets relative to the end - # of the function frame). The value corresponding to a key is the - # "tag", which is None for a normal gc root, or else the name of a - # callee-saved register. In the latter case it means that this is - # only a gc root if the corresponding register in the caller was - # really containing a gc pointer. A typical example: + # FunctionGcRootTracker.trackgcroots(). Its meaning is as + # follows: the keys are the locations that contain gc roots + # (register names or LocalVar instances). The value + # corresponding to a key is the "tag", which is None for a + # normal gc root, or else the name of a callee-saved register. + # In the latter case it means that this is only a gc root if the + # corresponding register in the caller was really containing a + # gc pointer. A typical example: # - # InsnCall({'%ebp': '%ebp', -8: '%ebx', '%esi': None}) - # - # means that %esi is a gc root across this call; that %ebp is a - # gc root if it was in the caller (typically because %ebp is not - # modified at all in the current function); and that the word at 8 - # bytes before the end of the current stack frame is a gc root if - # %ebx was a gc root in the caller (typically because the current - # function saves and restores %ebx from there in the prologue and - # epilogue). + # InsnCall({LocalVar(-8)': None, + # '%esi': '%esi', + # LocalVar(-12)': '%ebx'}) # + # means that the value at -8 from the frame end is a gc root + # across this call; that %esi is a gc root if it was in the + # caller (typically because %esi is not modified at all in the + # current function); and that the value at -12 from the frame + # end is a gc root if %ebx was a gc root in the caller + # (typically because the current function saves and restores + # %ebx from there in the prologue and epilogue). self.gcroots = {} self.lineno = lineno def source_of(self, localvar, tag): - self.gcroots[localvar] = tag + tag1 = self.gcroots.setdefault(localvar, tag) + assert tag1 == tag, ( + "conflicting entries for InsnCall.gcroots[%s]:\n%r and %r" % ( + localvar, tag1, tag)) return localvar class InsnGCROOT(Insn): @@ -760,6 +814,66 @@ CALLEE_SAVE_REGISTERS_NOEBP = ['%ebx', '%esi', '%edi'] CALLEE_SAVE_REGISTERS = CALLEE_SAVE_REGISTERS_NOEBP + ['%ebp'] +LOC_NOWHERE = 0 +LOC_REG = 1 +LOC_EBP_BASED = 2 +LOC_ESP_BASED = 3 +LOC_MASK = 0x03 + +REG2LOC = {} +for _i, _reg in enumerate(CALLEE_SAVE_REGISTERS): + REG2LOC[_reg] = LOC_REG | (_i<<2) + +def frameloc(base, offset): + assert base in (LOC_EBP_BASED, LOC_ESP_BASED) + assert offset % 4 == 0 + return base | offset + +# __________ debugging output __________ + +def format_location(loc): + # A 'location' is a single number describing where a value is stored + # across a call. It can be in one of the CALLEE_SAVE_REGISTERS, or + # in the stack frame at an address relative to either %esp or %ebp. + # The last two bits of the location number are used to tell the cases + # apart; see format_location(). + kind = loc & LOC_MASK + if kind == LOC_NOWHERE: + return '?' + elif kind == LOC_REG: + reg = loc >> 2 + assert 0 <= reg <= 3 + return CALLEE_SAVE_REGISTERS[reg] + else: + if kind == LOC_EBP_BASED: + result = '(%ebp)' + else: + result = '(%esp)' + offset = loc & ~ LOC_MASK + if offset != 0: + result = str(offset) + result + return result + +def format_callshape(shape): + # A 'call shape' is a tuple of locations in the sense of format_location(). + # They describe where in a function frame interesting values are stored, + # when this function executes a 'call' instruction. + # + # shape[0] is the location that stores the fn's own return address + # (not the return address for the currently executing 'call') + # shape[1] is where the fn saved its own caller's %ebx value + # shape[2] is where the fn saved its own caller's %esi value + # shape[3] is where the fn saved its own caller's %edi value + # shape[4] is where the fn saved its own caller's %ebp value + # shape[>=5] are GC roots: where the fn has put its local GCPTR vars + # + assert isinstance(shape, tuple) + assert len(shape) >= 5 + result = [format_location(loc) for loc in shape] + return '{%s | %s | %s}' % (result[0], + ', '.join(result[1:5]), + ', '.join(result[5:])) + if __name__ == '__main__': if sys.argv and sys.argv[1] == '-v': From antocuni at codespeak.net Mon Feb 4 10:52:37 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 4 Feb 2008 10:52:37 +0100 (CET) Subject: [pypy-svn] r51240 - in pypy/dist/pypy: jit/codegen/cli jit/codegen/cli/test jit/codegen/test translator/cli translator/cli/test Message-ID: <20080204095237.F1FCC168457@codespeak.net> Author: antocuni Date: Mon Feb 4 10:52:37 2008 New Revision: 51240 Added: pypy/dist/pypy/jit/codegen/cli/ pypy/dist/pypy/jit/codegen/cli/__init__.py (contents, props changed) pypy/dist/pypy/jit/codegen/cli/rgenop.py (contents, props changed) pypy/dist/pypy/jit/codegen/cli/test/ pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py (contents, props changed) Modified: pypy/dist/pypy/jit/codegen/test/rgenop_tests.py pypy/dist/pypy/translator/cli/opcodes.py pypy/dist/pypy/translator/cli/test/runtest.py Log: first draft of the CLI backend for the JIT. Tests are disabled, since only test_adder_direct and test_adder_compile pass. Added: pypy/dist/pypy/jit/codegen/cli/__init__.py ============================================================================== Added: pypy/dist/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/jit/codegen/cli/rgenop.py Mon Feb 4 10:52:37 2008 @@ -0,0 +1,152 @@ +from pypy.tool.pairtype import extendabletype +from pypy.rpython.ootypesystem import ootype +from pypy.rlib.objectmodel import specialize +from pypy.jit.codegen.model import AbstractRGenOp, GenLabel, GenBuilder +from pypy.jit.codegen.model import GenVarOrConst, GenVar, GenConst, CodeGenSwitch +from pypy.translator.cli.dotnet import CLR, typeof, new_array, clidowncast +System = CLR.System +Utils = CLR.pypy.runtime.Utils +OpCodes = System.Reflection.Emit.OpCodes + +def token2clitype(tok): + if tok == '': + return typeof(System.Int32) + else: + assert False + +class __extend__(GenVarOrConst): + __metaclass__ = extendabletype + + def load(self, il): + raise NotImplementedError + + def store(self, il): + raise NotImplementedError + +class GenArgVar(GenVar): + def __init__(self, index): + self.index = index + # XXX maybe we need to store also the type? + + def load(self, il): + if self.index == 0: + il.Emit(OpCodes.Ldarg_0) + elif self.index == 1: + il.Emit(OpCodes.Ldarg_1) + elif self.index == 2: + il.Emit(OpCodes.Ldarg_2) + elif self.index == 3: + il.Emit(OpCodes.Ldarg_3) + else: + il.Emit(OpCodes.Ldarg, self.index) + + def store(self, il): + il.Emit(OpCodes.Starg, self.index) + + def __repr__(self): + return "GenArgVar(%d)" % self.index + +class GenLocalVar(GenVar): + def __init__(self, v): + self.v = v + + def load(self, il): + il.Emit(OpCodes.Ldloc, self.v) + + def store(self, il): + il.Emit(OpCodes.Stloc, self.v) + +class IntConst(GenConst): + + def __init__(self, value): + self.value = value + + @specialize.arg(1) + def revealconst(self, T): + assert T is ootype.Signed + return self.value + + def load(self, il): + il.Emit(OpCodes.Ldc_I4, self.value) + + def __repr__(self): + "NOT_RPYTHON" + return "const=%s" % self.value + +class ObjectConst(GenConst): + + def __init__(self, obj): + self.obj = obj + + @specialize.arg(1) + def revealconst(self, T): + DelegateType = CLR.pypy.runtime.DelegateType_int__int # XXX use T + return clidowncast(DelegateType, self.obj) + + +class RCliGenOp(AbstractRGenOp): + + def __init__(self): + self.meth = None + self.il = None + + @specialize.genconst(1) + def genconst(self, llvalue): + T = ootype.typeOf(llvalue) + if T is ootype.Signed: + return IntConst(llvalue) + else: + assert False, "XXX not implemented" + + @staticmethod + @specialize.memo() + def sigToken(FUNCTYPE): + """Return a token describing the signature of FUNCTYPE.""" + # XXX: the right thing to do would be to have a way to + # represent typeof(t) as a pbc + args = [repr(T) for T in FUNCTYPE.ARGS] + res = repr(FUNCTYPE.RESULT) + return args, res + + def newgraph(self, sigtoken, name): + argtoks, restok = sigtoken + args = new_array(System.Type, len(argtoks)) + for i in range(len(argtoks)): + args[i] = token2clitype(argtoks[i]) + res = token2clitype(restok) + builder = Builder(self, name, res, args) + return builder, builder.gv_entrypoint, builder.inputargs_gv[:] + + + +class Builder(GenBuilder): + + def __init__(self, rgenop, name, res, args): + self.rgenop = rgenop + self.meth = Utils.CreateDynamicMethod(name, res, args) + self.il = self.meth.GetILGenerator() + self.inputargs_gv = [] + for i in range(len(args)): + self.inputargs_gv.append(GenArgVar(i)) + self.gv_entrypoint = ObjectConst(None) # XXX? + + @specialize.arg(1) + def genop2(self, opname, gv_arg1, gv_arg2): + assert opname == 'int_add' + res = self.il.DeclareLocal(typeof(System.Int32)) + gv_res = GenLocalVar(res) + gv_arg1.load(self.il) + gv_arg2.load(self.il) + self.il.Emit(OpCodes.Add) + gv_res.store(self.il) + return gv_res + + def finish_and_return(self, sigtoken, gv_returnvar): + gv_returnvar.load(self.il) + self.il.Emit(OpCodes.Ret) + DelegateType = CLR.pypy.runtime.DelegateType_int__int # XXX use sigtoken + myfunc = self.meth.CreateDelegate(typeof(DelegateType)) + self.gv_entrypoint.obj = myfunc + + def end(self): + pass Added: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Mon Feb 4 10:52:37 2008 @@ -0,0 +1,28 @@ +import py +from pypy.rpython.ootypesystem import ootype +from pypy.jit.codegen.cli.rgenop import RCliGenOp +from pypy.jit.codegen.test.rgenop_tests import AbstractRGenOpTests, OOType +from pypy.translator.cli.test.runtest import compile_function + +# test disabled, only two pass +class xTestRCliGenop(AbstractRGenOpTests): + RGenOp = RCliGenOp + T = OOType + + # for the individual tests see + # ====> ../../test/rgenop_tests.py + + def getcompiled(self, fn, annotation, annotatorpolicy): + return compile_function(fn, annotation, + annotatorpolicy=annotatorpolicy, + nowrap=True) + + def cast(self, gv, nb_args): + "NOT_RPYTHON" + def fn(*args): + return gv.obj.Invoke(*args) + return fn + + def directtesthelper(self, FUNCTYPE, func): + py.test.skip('???') + Modified: pypy/dist/pypy/jit/codegen/test/rgenop_tests.py ============================================================================== --- pypy/dist/pypy/jit/codegen/test/rgenop_tests.py (original) +++ pypy/dist/pypy/jit/codegen/test/rgenop_tests.py Mon Feb 4 10:52:37 2008 @@ -3,6 +3,7 @@ from pypy.rlib.rarithmetic import intmask, r_uint from pypy.rlib.objectmodel import keepalive_until_here from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.ootypesystem import ootype from pypy.translator.c.test import test_boehm from ctypes import c_void_p, cast, CFUNCTYPE, c_int @@ -21,7 +22,20 @@ @staticmethod def Ptr(FUNC): return lltype.Ptr(FUNC) - + + +class OOType(object): + FUNC = ootype.StaticMethod([lltype.Signed], lltype.Signed) + FUNC2 = ootype.StaticMethod([lltype.Signed]*2, lltype.Signed) + FUNC3 = ootype.StaticMethod([lltype.Signed]*3, lltype.Signed) + FUNC5 = ootype.StaticMethod([lltype.Signed]*5, lltype.Signed) + FUNC27= ootype.StaticMethod([lltype.Signed]*27, lltype.Signed) + FUNC100 = ootype.StaticMethod([lltype.Signed]*100, lltype.Signed) + + @staticmethod + def Ptr(FUNC): + return FUNC + def make_adder(T, rgenop, n): # 'return x+n' Modified: pypy/dist/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/cli/opcodes.py (original) +++ pypy/dist/pypy/translator/cli/opcodes.py Mon Feb 4 10:52:37 2008 @@ -67,6 +67,7 @@ 'gc_set_max_heap_size': Ignore, 'resume_point': Ignore, 'debug_assert': Ignore, + 'keepalive': Ignore, # __________ numeric operations __________ Modified: pypy/dist/pypy/translator/cli/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/runtest.py (original) +++ pypy/dist/pypy/translator/cli/test/runtest.py Mon Feb 4 10:52:37 2008 @@ -136,15 +136,17 @@ def compile_function(func, annotation=[], graph=None, backendopt=True, - auto_raise_exc=False, exctrans=False): + auto_raise_exc=False, exctrans=False, + annotatorpolicy=None, nowrap=False): olddefs = patch_os() - gen = _build_gen(func, annotation, graph, backendopt, exctrans) + gen = _build_gen(func, annotation, graph, backendopt, exctrans, annotatorpolicy, nowrap) gen.generate_source() exe_name = gen.build_exe() unpatch_os(olddefs) # restore original values return CliFunctionWrapper(exe_name, func.__name__, auto_raise_exc) -def _build_gen(func, annotation, graph=None, backendopt=True, exctrans=False): +def _build_gen(func, annotation, graph=None, backendopt=True, exctrans=False, + annotatorpolicy=None, nowrap=False): try: func = func.im_func except AttributeError: @@ -152,12 +154,12 @@ t = TranslationContext() if graph is not None: graph.func = func - ann = t.buildannotator() + ann = t.buildannotator(policy=annotatorpolicy) inputcells = [ann.typeannotation(a) for a in annotation] ann.build_graph_types(graph, inputcells) t.graphs.insert(0, graph) else: - ann = t.buildannotator() + ann = t.buildannotator(policy=annotatorpolicy) ann.build_types(func, annotation) if getoption('view'): @@ -178,7 +180,7 @@ else: tmpdir = udir - return GenCli(tmpdir, t, TestEntryPoint(main_graph, True), exctrans=exctrans) + return GenCli(tmpdir, t, TestEntryPoint(main_graph, not nowrap), exctrans=exctrans) class CliFunctionWrapper(object): def __init__(self, exe_name, name=None, auto_raise_exc=False): From arigo at codespeak.net Mon Feb 4 10:57:37 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 4 Feb 2008 10:57:37 +0100 (CET) Subject: [pypy-svn] r51241 - pypy/branch/asmgcroot/pypy/translator/c/gcc Message-ID: <20080204095737.7B33E168457@codespeak.net> Author: arigo Date: Mon Feb 4 10:57:36 2008 New Revision: 51241 Modified: pypy/branch/asmgcroot/pypy/translator/c/gcc/trackgcroot.py Log: Workaround. Modified: pypy/branch/asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/asmgcroot/pypy/translator/c/gcc/trackgcroot.py Mon Feb 4 10:57:36 2008 @@ -704,10 +704,18 @@ self.arguments[reg] = somenewvalue def source_of(self, localvar, tag): if localvar not in self.arguments: - assert (isinstance(localvar, LocalVar) and - localvar.ofs_from_frame_end > 0), ( - "must come from an argument to the function, got %r" % - (localvar,)) + if localvar in ('%eax', '%edx', '%ecx'): + # xxx this might show a bug in trackgcroot.py failing to + # figure out which instruction stored a value in these + # registers. However, this case also occurs when the + # the function's calling convention was optimized by gcc: + # the 3 registers above are then used to pass arguments + pass + else: + assert (isinstance(localvar, LocalVar) and + localvar.ofs_from_frame_end > 0), ( + "must come from an argument to the function, got %r" % + (localvar,)) self.arguments[localvar] = somenewvalue return self.arguments[localvar] From arigo at codespeak.net Mon Feb 4 12:19:19 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 4 Feb 2008 12:19:19 +0100 (CET) Subject: [pypy-svn] r51243 - in pypy/branch/asmgcroot/pypy/translator/c/gcc: . test Message-ID: <20080204111919.E468516845A@codespeak.net> Author: arigo Date: Mon Feb 4 12:19:19 2008 New Revision: 51243 Added: pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track7.s Modified: pypy/branch/asmgcroot/pypy/translator/c/gcc/trackgcroot.py Log: 'cmovCOND' instructions. Added: pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track7.s ============================================================================== --- (empty file) +++ pypy/branch/asmgcroot/pypy/translator/c/gcc/test/track7.s Mon Feb 4 12:19:19 2008 @@ -0,0 +1,18 @@ + .type main, @function +main: + ;; cmovCOND tests. + pushl %ebx + movl 12(%esp), %ebx + cmove 16(%esp), %ebx + cmovge 20(%esp), %ebx + movl 24(%esp), %eax + cmovs %eax, %ebx + call foobar + ;; expected {4(%esp) | (%esp), %esi, %edi, %ebp | %ebx} +#APP + /* GCROOT %ebx */ +#NO_APP + popl %ebx + ret + + .size main, .-main Modified: pypy/branch/asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/asmgcroot/pypy/translator/c/gcc/trackgcroot.py Mon Feb 4 12:19:19 2008 @@ -480,6 +480,22 @@ visit_xorl = binary_insn # used in "xor reg, reg" to create a NULL GC ptr visit_orl = binary_insn + visit_cmove = binary_insn + visit_cmovne = binary_insn + visit_cmovg = binary_insn + visit_cmovge = binary_insn + visit_cmovl = binary_insn + visit_cmovle = binary_insn + visit_cmova = binary_insn + visit_cmovae = binary_insn + visit_cmovb = binary_insn + visit_cmovbe = binary_insn + visit_cmovp = binary_insn + visit_cmovnp = binary_insn + visit_cmovs = binary_insn + visit_cmovns = binary_insn + visit_cmovo = binary_insn + visit_cmovno = binary_insn def visit_andl(self, line): match = r_binaryinsn.match(line) From cfbolz at codespeak.net Mon Feb 4 12:25:33 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 4 Feb 2008 12:25:33 +0100 (CET) Subject: [pypy-svn] r51244 - in pypy/branch/jit-refactoring/pypy/jit: rainbow/test timeshifter/test Message-ID: <20080204112533.764EA16803C@codespeak.net> Author: cfbolz Date: Mon Feb 4 12:25:32 2008 New Revision: 51244 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py Log: moving struct and array tests over to test_interpreter.py. skip them all Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 4 12:25:32 2008 @@ -499,5 +499,417 @@ assert res == expected + def test_simple_struct(self): + py.test.skip("arrays and structs are not working") + S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), + ('world', lltype.Signed), + hints={'immutable': True}) + + def ll_function(s): + return s.hello * s.world + + def struct_S(string): + items = string.split(',') + assert len(items) == 2 + s1 = lltype.malloc(S) + s1.hello = int(items[0]) + s1.world = int(items[1]) + return s1 + ll_function.convert_arguments = [struct_S] + + res = self.interpret(ll_function, ["6,7"], []) + assert res == 42 + self.check_insns({'getfield': 2, 'int_mul': 1}) + res = self.interpret(ll_function, ["8,9"], [0]) + assert res == 72 + self.check_insns({}) + + def test_simple_array(self): + py.test.skip("arrays and structs are not working") + A = lltype.GcArray(lltype.Signed, + hints={'immutable': True}) + def ll_function(a): + return a[0] * a[1] + + def int_array(string): + items = [int(x) for x in string.split(',')] + n = len(items) + a1 = lltype.malloc(A, n) + for i in range(n): + a1[i] = items[i] + return a1 + ll_function.convert_arguments = [int_array] + + res = self.interpret(ll_function, ["6,7"], []) + assert res == 42 + self.check_insns({'getarrayitem': 2, 'int_mul': 1}) + res = self.interpret(ll_function, ["8,3"], [0]) + assert res == 24 + self.check_insns({}) + + + + def test_simple_struct_malloc(self): + py.test.skip("blue containers: to be reimplemented") + S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), + ('world', lltype.Signed)) + def ll_function(x): + s = lltype.malloc(S) + s.hello = x + return s.hello + s.world + + res = self.interpret(ll_function, [3], []) + assert res == 3 + self.check_insns({'int_add': 1}) + + res = self.interpret(ll_function, [3], [0]) + assert res == 3 + self.check_insns({}) + + def test_inlined_substructure(self): + py.test.skip("blue containers: to be reimplemented") + S = lltype.GcStruct('S', ('n', lltype.Signed)) + T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) + def ll_function(k): + t = lltype.malloc(T) + t.s.n = k + l = t.s.n + return l + res = self.interpret(ll_function, [7], []) + assert res == 7 + self.check_insns({}) + + res = self.interpret(ll_function, [7], [0]) + assert res == 7 + self.check_insns({}) + + def test_degenerated_before_return(self): + py.test.skip("arrays and structs are not working") + S = lltype.GcStruct('S', ('n', lltype.Signed)) + T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) + + def ll_function(flag): + t = lltype.malloc(T) + t.s.n = 3 + s = lltype.malloc(S) + s.n = 4 + if flag: + s = t.s + s.n += 1 + return s.n * t.s.n + res = self.interpret(ll_function, [0], []) + assert res == 5 * 3 + res = self.interpret(ll_function, [1], []) + assert res == 4 * 4 + + def test_degenerated_before_return_2(self): + py.test.skip("arrays and structs are not working") + S = lltype.GcStruct('S', ('n', lltype.Signed)) + T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) + + def ll_function(flag): + t = lltype.malloc(T) + t.s.n = 3 + s = lltype.malloc(S) + s.n = 4 + if flag: + pass + else: + s = t.s + s.n += 1 + return s.n * t.s.n + res = self.interpret(ll_function, [1], []) + assert res == 5 * 3 + res = self.interpret(ll_function, [0], []) + assert res == 4 * 4 + + def test_degenerated_at_return(self): + S = lltype.GcStruct('S', ('n', lltype.Signed)) + T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) + class Result: + def convert(self, s): + self.s = s + return str(s.n) + glob_result = Result() + + def ll_function(flag): + t = lltype.malloc(T) + t.n = 3.25 + t.s.n = 3 + s = lltype.malloc(S) + s.n = 4 + if flag: + s = t.s + return s + ll_function.convert_result = glob_result.convert + + res = self.interpret(ll_function, [0], []) + assert res == "4" + if self.__class__ in (TestLLType, TestOOType): + assert lltype.parentlink(glob_result.s._obj) == (None, None) + res = self.interpret(ll_function, [1], []) + assert res == "3" + if self.__class__ in (TestLLType, TestOOType): + parent, parentindex = lltype.parentlink(glob_result.s._obj) + assert parentindex == 's' + assert parent.n == 3.25 + + def test_degenerated_via_substructure(self): + py.test.skip("arrays and structs are not working") + S = lltype.GcStruct('S', ('n', lltype.Signed)) + T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) + + def ll_function(flag): + t = lltype.malloc(T) + t.s.n = 3 + s = lltype.malloc(S) + s.n = 7 + if flag: + pass + else: + s = t.s + t.s.n += 1 + return s.n * t.s.n + res = self.interpret(ll_function, [1], []) + assert res == 7 * 4 + res = self.interpret(ll_function, [0], []) + assert res == 4 * 4 + + def test_degenerate_with_voids(self): + py.test.skip("arrays and structs are not working") + S = lltype.GcStruct('S', ('y', lltype.Void), + ('x', lltype.Signed)) + def ll_function(): + s = lltype.malloc(S) + s.x = 123 + return s + ll_function.convert_result = lambda s: str(s.x) + res = self.interpret(ll_function, [], [], policy=P_NOVIRTUAL) + assert res == "123" + + def test_plus_minus_all_inlined(self): + py.test.skip("arrays and structs are not working") + def ll_plus_minus(s, x, y): + acc = x + n = len(s) + pc = 0 + while pc < n: + op = s[pc] + op = hint(op, concrete=True) + if op == '+': + acc += y + elif op == '-': + acc -= y + pc += 1 + return acc + ll_plus_minus.convert_arguments = [LLSupport.to_rstr, int, int] + res = self.interpret(ll_plus_minus, ["+-+", 0, 2], [0], inline=100000) + assert res == ll_plus_minus("+-+", 0, 2) + self.check_insns({'int_add': 2, 'int_sub': 1}) + + def test_red_virtual_container(self): + py.test.skip("arrays and structs are not working") + # this checks that red boxes are able to be virtualized dynamically by + # the compiler (the P_NOVIRTUAL policy prevents the hint-annotator from + # marking variables in blue) + S = lltype.GcStruct('S', ('n', lltype.Signed)) + def ll_function(n): + s = lltype.malloc(S) + s.n = n + return s.n + res = self.interpret(ll_function, [42], [], policy=P_NOVIRTUAL) + assert res == 42 + self.check_insns({}) + + + def test_setarrayitem(self): + py.test.skip("arrays and structs are not working") + A = lltype.GcArray(lltype.Signed) + a = lltype.malloc(A, 2, immortal=True) + def ll_function(): + a[0] = 1 + a[1] = 2 + return a[0]+a[1] + + res = self.interpret(ll_function, [], [], policy=P_NOVIRTUAL) + assert res == 3 + + def test_red_array(self): + py.test.skip("arrays and structs are not working") + A = lltype.GcArray(lltype.Signed) + def ll_function(x, y, n): + a = lltype.malloc(A, 2) + a[0] = x + a[1] = y + return a[n]*len(a) + + res = self.interpret(ll_function, [21, -21, 0], [], + policy=P_NOVIRTUAL) + assert res == 42 + self.check_insns({'malloc_varsize': 1, + 'setarrayitem': 2, 'getarrayitem': 1, + 'getarraysize': 1, 'int_mul': 1}) + + res = self.interpret(ll_function, [21, -21, 1], [], + policy=P_NOVIRTUAL) + assert res == -42 + self.check_insns({'malloc_varsize': 1, + 'setarrayitem': 2, 'getarrayitem': 1, + 'getarraysize': 1, 'int_mul': 1}) + + def test_red_struct_array(self): + py.test.skip("arrays and structs are not working") + S = lltype.Struct('s', ('x', lltype.Signed)) + A = lltype.GcArray(S) + def ll_function(x, y, n): + a = lltype.malloc(A, 2) + a[0].x = x + a[1].x = y + return a[n].x*len(a) + + res = self.interpret(ll_function, [21, -21, 0], [], + policy=P_NOVIRTUAL) + assert res == 42 + self.check_insns({'malloc_varsize': 1, + 'setinteriorfield': 2, 'getinteriorfield': 1, + 'getarraysize': 1, 'int_mul': 1}) + + res = self.interpret(ll_function, [21, -21, 1], [], + policy=P_NOVIRTUAL) + assert res == -42 + self.check_insns({'malloc_varsize': 1, + 'setinteriorfield': 2, 'getinteriorfield': 1, + 'getarraysize': 1, 'int_mul': 1}) + + + def test_red_varsized_struct(self): + py.test.skip("arrays and structs are not working") + A = lltype.Array(lltype.Signed) + S = lltype.GcStruct('S', ('foo', lltype.Signed), ('a', A)) + def ll_function(x, y, n): + s = lltype.malloc(S, 3) + s.foo = len(s.a)-1 + s.a[0] = x + s.a[1] = y + return s.a[n]*s.foo + + res = self.interpret(ll_function, [21, -21, 0], [], + policy=P_NOVIRTUAL) + assert res == 42 + self.check_insns(malloc_varsize=1, + getinteriorarraysize=1) + + res = self.interpret(ll_function, [21, -21, 1], [], + policy=P_NOVIRTUAL) + assert res == -42 + self.check_insns(malloc_varsize=1, + getinteriorarraysize=1) + + def test_array_of_voids(self): + py.test.skip("arrays and structs are not working") + A = lltype.GcArray(lltype.Void) + def ll_function(n): + a = lltype.malloc(A, 3) + a[1] = None + b = a[n] + res = a, b + keepalive_until_here(b) # to keep getarrayitem around + return res + ll_function.convert_result = lambda x: str(len(x.item0)) + + res = self.interpret(ll_function, [2], [], policy=P_NOVIRTUAL) + assert res == "3" + + def test_red_propagate(self): + py.test.skip("arrays and structs are not working") + S = lltype.GcStruct('S', ('n', lltype.Signed)) + def ll_function(n, k): + s = lltype.malloc(S) + s.n = n + if k < 0: + return -123 + return s.n * k + res = self.interpret(ll_function, [3, 8], [], policy=P_NOVIRTUAL) + assert res == 24 + self.check_insns({'int_lt': 1, 'int_mul': 1}) + + def test_red_subcontainer(self): + py.test.skip("arrays and structs are not working") + S = lltype.GcStruct('S', ('n', lltype.Signed)) + T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) + def ll_function(k): + t = lltype.malloc(T) + s = t.s + s.n = k + if k < 0: + return -123 + result = s.n * (k-1) + keepalive_until_here(t) + return result + res = self.interpret(ll_function, [7], [], policy=P_NOVIRTUAL) + assert res == 42 + self.check_insns({'int_lt': 1, 'int_mul': 1, 'int_sub': 1}) + + + def test_red_subcontainer_cast(self): + py.test.skip("arrays and structs are not working") + S = lltype.GcStruct('S', ('n', lltype.Signed)) + T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) + def ll_function(k): + t = lltype.malloc(T) + s = lltype.cast_pointer(lltype.Ptr(S), t) + s.n = k + if k < 0: + return -123 + result = s.n * (k-1) + keepalive_until_here(t) + return result + res = self.interpret(ll_function, [7], [], policy=P_NOVIRTUAL) + assert res == 42 + self.check_insns({'int_lt': 1, 'int_mul': 1, 'int_sub': 1}) + + + def test_merge_structures(self): + py.test.skip("arrays and structs are not working") + S = lltype.GcStruct('S', ('n', lltype.Signed)) + T = lltype.GcStruct('T', ('s', lltype.Ptr(S)), ('n', lltype.Signed)) + + def ll_function(flag): + if flag: + s = lltype.malloc(S) + s.n = 1 + t = lltype.malloc(T) + t.s = s + t.n = 2 + else: + s = lltype.malloc(S) + s.n = 5 + t = lltype.malloc(T) + t.s = s + t.n = 6 + return t.n + t.s.n + res = self.interpret(ll_function, [0], [], policy=P_NOVIRTUAL) + assert res == 5 + 6 + self.check_insns({'int_is_true': 1, 'int_add': 1}) + res = self.interpret(ll_function, [1], [], policy=P_NOVIRTUAL) + assert res == 1 + 2 + self.check_insns({'int_is_true': 1, 'int_add': 1}) + + + def test_green_with_side_effects(self): + py.test.skip("arrays and structs are not working") + S = lltype.GcStruct('S', ('flag', lltype.Bool)) + s = lltype.malloc(S) + s.flag = False + def ll_set_flag(s): + s.flag = True + def ll_function(): + s.flag = False + ll_set_flag(s) + return s.flag + res = self.interpret(ll_function, [], []) + assert res == True + self.check_insns({'setfield': 2, 'getfield': 1}) + class TestLLType(SimpleTests): type_system = "lltype" Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py Mon Feb 4 12:25:32 2008 @@ -383,399 +383,6 @@ class BaseTestTimeshift(TimeshiftingTests): - def test_simple_struct(self): - S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), - ('world', lltype.Signed), - hints={'immutable': True}) - - def ll_function(s): - return s.hello * s.world - - def struct_S(string): - items = string.split(',') - assert len(items) == 2 - s1 = lltype.malloc(S) - s1.hello = int(items[0]) - s1.world = int(items[1]) - return s1 - ll_function.convert_arguments = [struct_S] - - res = self.timeshift(ll_function, ["6,7"], []) - assert res == 42 - self.check_insns({'getfield': 2, 'int_mul': 1}) - res = self.timeshift(ll_function, ["8,9"], [0]) - assert res == 72 - self.check_insns({}) - - def test_simple_array(self): - A = lltype.GcArray(lltype.Signed, - hints={'immutable': True}) - def ll_function(a): - return a[0] * a[1] - - def int_array(string): - items = [int(x) for x in string.split(',')] - n = len(items) - a1 = lltype.malloc(A, n) - for i in range(n): - a1[i] = items[i] - return a1 - ll_function.convert_arguments = [int_array] - - res = self.timeshift(ll_function, ["6,7"], []) - assert res == 42 - self.check_insns({'getarrayitem': 2, 'int_mul': 1}) - res = self.timeshift(ll_function, ["8,3"], [0]) - assert res == 24 - self.check_insns({}) - - - - def test_simple_struct_malloc(self): - py.test.skip("blue containers: to be reimplemented") - S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), - ('world', lltype.Signed)) - def ll_function(x): - s = lltype.malloc(S) - s.hello = x - return s.hello + s.world - - res = self.timeshift(ll_function, [3], []) - assert res == 3 - self.check_insns({'int_add': 1}) - - res = self.timeshift(ll_function, [3], [0]) - assert res == 3 - self.check_insns({}) - - def test_inlined_substructure(self): - py.test.skip("blue containers: to be reimplemented") - S = lltype.GcStruct('S', ('n', lltype.Signed)) - T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) - def ll_function(k): - t = lltype.malloc(T) - t.s.n = k - l = t.s.n - return l - res = self.timeshift(ll_function, [7], []) - assert res == 7 - self.check_insns({}) - - res = self.timeshift(ll_function, [7], [0]) - assert res == 7 - self.check_insns({}) - - def test_degenerated_before_return(self): - S = lltype.GcStruct('S', ('n', lltype.Signed)) - T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) - - def ll_function(flag): - t = lltype.malloc(T) - t.s.n = 3 - s = lltype.malloc(S) - s.n = 4 - if flag: - s = t.s - s.n += 1 - return s.n * t.s.n - res = self.timeshift(ll_function, [0], []) - assert res == 5 * 3 - res = self.timeshift(ll_function, [1], []) - assert res == 4 * 4 - - def test_degenerated_before_return_2(self): - S = lltype.GcStruct('S', ('n', lltype.Signed)) - T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) - - def ll_function(flag): - t = lltype.malloc(T) - t.s.n = 3 - s = lltype.malloc(S) - s.n = 4 - if flag: - pass - else: - s = t.s - s.n += 1 - return s.n * t.s.n - res = self.timeshift(ll_function, [1], []) - assert res == 5 * 3 - res = self.timeshift(ll_function, [0], []) - assert res == 4 * 4 - - def test_degenerated_at_return(self): - S = lltype.GcStruct('S', ('n', lltype.Signed)) - T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) - class Result: - def convert(self, s): - self.s = s - return str(s.n) - glob_result = Result() - - def ll_function(flag): - t = lltype.malloc(T) - t.n = 3.25 - t.s.n = 3 - s = lltype.malloc(S) - s.n = 4 - if flag: - s = t.s - return s - ll_function.convert_result = glob_result.convert - - res = self.timeshift(ll_function, [0], []) - assert res == "4" - if self.__class__ in (TestLLType, TestOOType): - assert lltype.parentlink(glob_result.s._obj) == (None, None) - res = self.timeshift(ll_function, [1], []) - assert res == "3" - if self.__class__ in (TestLLType, TestOOType): - parent, parentindex = lltype.parentlink(glob_result.s._obj) - assert parentindex == 's' - assert parent.n == 3.25 - - def test_degenerated_via_substructure(self): - S = lltype.GcStruct('S', ('n', lltype.Signed)) - T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) - - def ll_function(flag): - t = lltype.malloc(T) - t.s.n = 3 - s = lltype.malloc(S) - s.n = 7 - if flag: - pass - else: - s = t.s - t.s.n += 1 - return s.n * t.s.n - res = self.timeshift(ll_function, [1], []) - assert res == 7 * 4 - res = self.timeshift(ll_function, [0], []) - assert res == 4 * 4 - - def test_degenerate_with_voids(self): - S = lltype.GcStruct('S', ('y', lltype.Void), - ('x', lltype.Signed)) - def ll_function(): - s = lltype.malloc(S) - s.x = 123 - return s - ll_function.convert_result = lambda s: str(s.x) - res = self.timeshift(ll_function, [], [], policy=P_NOVIRTUAL) - assert res == "123" - - def test_plus_minus_all_inlined(self): - def ll_plus_minus(s, x, y): - acc = x - n = len(s) - pc = 0 - while pc < n: - op = s[pc] - op = hint(op, concrete=True) - if op == '+': - acc += y - elif op == '-': - acc -= y - pc += 1 - return acc - ll_plus_minus.convert_arguments = [LLSupport.to_rstr, int, int] - res = self.timeshift(ll_plus_minus, ["+-+", 0, 2], [0], inline=100000) - assert res == ll_plus_minus("+-+", 0, 2) - self.check_insns({'int_add': 2, 'int_sub': 1}) - - def test_red_virtual_container(self): - # this checks that red boxes are able to be virtualized dynamically by - # the compiler (the P_NOVIRTUAL policy prevents the hint-annotator from - # marking variables in blue) - S = lltype.GcStruct('S', ('n', lltype.Signed)) - def ll_function(n): - s = lltype.malloc(S) - s.n = n - return s.n - res = self.timeshift(ll_function, [42], [], policy=P_NOVIRTUAL) - assert res == 42 - self.check_insns({}) - - - def test_setarrayitem(self): - A = lltype.GcArray(lltype.Signed) - a = lltype.malloc(A, 2, immortal=True) - def ll_function(): - a[0] = 1 - a[1] = 2 - return a[0]+a[1] - - res = self.timeshift(ll_function, [], [], policy=P_NOVIRTUAL) - assert res == 3 - - def test_red_array(self): - A = lltype.GcArray(lltype.Signed) - def ll_function(x, y, n): - a = lltype.malloc(A, 2) - a[0] = x - a[1] = y - return a[n]*len(a) - - res = self.timeshift(ll_function, [21, -21, 0], [], - policy=P_NOVIRTUAL) - assert res == 42 - self.check_insns({'malloc_varsize': 1, - 'setarrayitem': 2, 'getarrayitem': 1, - 'getarraysize': 1, 'int_mul': 1}) - - res = self.timeshift(ll_function, [21, -21, 1], [], - policy=P_NOVIRTUAL) - assert res == -42 - self.check_insns({'malloc_varsize': 1, - 'setarrayitem': 2, 'getarrayitem': 1, - 'getarraysize': 1, 'int_mul': 1}) - - def test_red_struct_array(self): - S = lltype.Struct('s', ('x', lltype.Signed)) - A = lltype.GcArray(S) - def ll_function(x, y, n): - a = lltype.malloc(A, 2) - a[0].x = x - a[1].x = y - return a[n].x*len(a) - - res = self.timeshift(ll_function, [21, -21, 0], [], - policy=P_NOVIRTUAL) - assert res == 42 - self.check_insns({'malloc_varsize': 1, - 'setinteriorfield': 2, 'getinteriorfield': 1, - 'getarraysize': 1, 'int_mul': 1}) - - res = self.timeshift(ll_function, [21, -21, 1], [], - policy=P_NOVIRTUAL) - assert res == -42 - self.check_insns({'malloc_varsize': 1, - 'setinteriorfield': 2, 'getinteriorfield': 1, - 'getarraysize': 1, 'int_mul': 1}) - - - def test_red_varsized_struct(self): - A = lltype.Array(lltype.Signed) - S = lltype.GcStruct('S', ('foo', lltype.Signed), ('a', A)) - def ll_function(x, y, n): - s = lltype.malloc(S, 3) - s.foo = len(s.a)-1 - s.a[0] = x - s.a[1] = y - return s.a[n]*s.foo - - res = self.timeshift(ll_function, [21, -21, 0], [], - policy=P_NOVIRTUAL) - assert res == 42 - self.check_insns(malloc_varsize=1, - getinteriorarraysize=1) - - res = self.timeshift(ll_function, [21, -21, 1], [], - policy=P_NOVIRTUAL) - assert res == -42 - self.check_insns(malloc_varsize=1, - getinteriorarraysize=1) - - def test_array_of_voids(self): - A = lltype.GcArray(lltype.Void) - def ll_function(n): - a = lltype.malloc(A, 3) - a[1] = None - b = a[n] - res = a, b - keepalive_until_here(b) # to keep getarrayitem around - return res - ll_function.convert_result = lambda x: str(len(x.item0)) - - res = self.timeshift(ll_function, [2], [], policy=P_NOVIRTUAL) - assert res == "3" - - def test_red_propagate(self): - S = lltype.GcStruct('S', ('n', lltype.Signed)) - def ll_function(n, k): - s = lltype.malloc(S) - s.n = n - if k < 0: - return -123 - return s.n * k - res = self.timeshift(ll_function, [3, 8], [], policy=P_NOVIRTUAL) - assert res == 24 - self.check_insns({'int_lt': 1, 'int_mul': 1}) - - def test_red_subcontainer(self): - S = lltype.GcStruct('S', ('n', lltype.Signed)) - T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) - def ll_function(k): - t = lltype.malloc(T) - s = t.s - s.n = k - if k < 0: - return -123 - result = s.n * (k-1) - keepalive_until_here(t) - return result - res = self.timeshift(ll_function, [7], [], policy=P_NOVIRTUAL) - assert res == 42 - self.check_insns({'int_lt': 1, 'int_mul': 1, 'int_sub': 1}) - - - def test_red_subcontainer_cast(self): - S = lltype.GcStruct('S', ('n', lltype.Signed)) - T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) - def ll_function(k): - t = lltype.malloc(T) - s = lltype.cast_pointer(lltype.Ptr(S), t) - s.n = k - if k < 0: - return -123 - result = s.n * (k-1) - keepalive_until_here(t) - return result - res = self.timeshift(ll_function, [7], [], policy=P_NOVIRTUAL) - assert res == 42 - self.check_insns({'int_lt': 1, 'int_mul': 1, 'int_sub': 1}) - - - def test_merge_structures(self): - S = lltype.GcStruct('S', ('n', lltype.Signed)) - T = lltype.GcStruct('T', ('s', lltype.Ptr(S)), ('n', lltype.Signed)) - - def ll_function(flag): - if flag: - s = lltype.malloc(S) - s.n = 1 - t = lltype.malloc(T) - t.s = s - t.n = 2 - else: - s = lltype.malloc(S) - s.n = 5 - t = lltype.malloc(T) - t.s = s - t.n = 6 - return t.n + t.s.n - res = self.timeshift(ll_function, [0], [], policy=P_NOVIRTUAL) - assert res == 5 + 6 - self.check_insns({'int_is_true': 1, 'int_add': 1}) - res = self.timeshift(ll_function, [1], [], policy=P_NOVIRTUAL) - assert res == 1 + 2 - self.check_insns({'int_is_true': 1, 'int_add': 1}) - - - def test_green_with_side_effects(self): - S = lltype.GcStruct('S', ('flag', lltype.Bool)) - s = lltype.malloc(S) - s.flag = False - def ll_set_flag(s): - s.flag = True - def ll_function(): - s.flag = False - ll_set_flag(s) - return s.flag - res = self.timeshift(ll_function, [], []) - assert res == True - self.check_insns({'setfield': 2, 'getfield': 1}) def test_recursive_with_red_termination_condition(self): py.test.skip('Does not terminate') From cfbolz at codespeak.net Mon Feb 4 13:06:09 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 4 Feb 2008 13:06:09 +0100 (CET) Subject: [pypy-svn] r51245 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080204120609.800651684D8@codespeak.net> Author: cfbolz Date: Mon Feb 4 13:06:08 2008 New Revision: 51245 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: whoopsie: missing skip, wrong indentation?! Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 4 13:06:08 2008 @@ -46,6 +46,10 @@ del cls._cache_order def serialize(self, func, values, backendoptimize=False): + if hasattr(func, 'convert_arguments'): + assert len(func.convert_arguments) == len(values) + values = [decoder(value) for decoder, value in zip( + func.convert_arguments, values)] key = func, backendoptimize try: cache, argtypes = self._cache[key] @@ -504,7 +508,7 @@ S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), ('world', lltype.Signed), hints={'immutable': True}) - + def ll_function(s): return s.hello * s.world @@ -624,6 +628,7 @@ assert res == 4 * 4 def test_degenerated_at_return(self): + py.test.skip("arrays and structs are not working") S = lltype.GcStruct('S', ('n', lltype.Signed)) T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) class Result: @@ -736,74 +741,74 @@ def test_red_array(self): py.test.skip("arrays and structs are not working") - A = lltype.GcArray(lltype.Signed) - def ll_function(x, y, n): - a = lltype.malloc(A, 2) - a[0] = x - a[1] = y - return a[n]*len(a) - - res = self.interpret(ll_function, [21, -21, 0], [], - policy=P_NOVIRTUAL) - assert res == 42 - self.check_insns({'malloc_varsize': 1, - 'setarrayitem': 2, 'getarrayitem': 1, - 'getarraysize': 1, 'int_mul': 1}) - - res = self.interpret(ll_function, [21, -21, 1], [], - policy=P_NOVIRTUAL) - assert res == -42 - self.check_insns({'malloc_varsize': 1, - 'setarrayitem': 2, 'getarrayitem': 1, - 'getarraysize': 1, 'int_mul': 1}) + A = lltype.GcArray(lltype.Signed) + def ll_function(x, y, n): + a = lltype.malloc(A, 2) + a[0] = x + a[1] = y + return a[n]*len(a) + + res = self.interpret(ll_function, [21, -21, 0], [], + policy=P_NOVIRTUAL) + assert res == 42 + self.check_insns({'malloc_varsize': 1, + 'setarrayitem': 2, 'getarrayitem': 1, + 'getarraysize': 1, 'int_mul': 1}) + + res = self.interpret(ll_function, [21, -21, 1], [], + policy=P_NOVIRTUAL) + assert res == -42 + self.check_insns({'malloc_varsize': 1, + 'setarrayitem': 2, 'getarrayitem': 1, + 'getarraysize': 1, 'int_mul': 1}) def test_red_struct_array(self): py.test.skip("arrays and structs are not working") - S = lltype.Struct('s', ('x', lltype.Signed)) - A = lltype.GcArray(S) - def ll_function(x, y, n): - a = lltype.malloc(A, 2) - a[0].x = x - a[1].x = y - return a[n].x*len(a) - - res = self.interpret(ll_function, [21, -21, 0], [], - policy=P_NOVIRTUAL) - assert res == 42 - self.check_insns({'malloc_varsize': 1, - 'setinteriorfield': 2, 'getinteriorfield': 1, - 'getarraysize': 1, 'int_mul': 1}) - - res = self.interpret(ll_function, [21, -21, 1], [], - policy=P_NOVIRTUAL) - assert res == -42 - self.check_insns({'malloc_varsize': 1, - 'setinteriorfield': 2, 'getinteriorfield': 1, - 'getarraysize': 1, 'int_mul': 1}) + S = lltype.Struct('s', ('x', lltype.Signed)) + A = lltype.GcArray(S) + def ll_function(x, y, n): + a = lltype.malloc(A, 2) + a[0].x = x + a[1].x = y + return a[n].x*len(a) + + res = self.interpret(ll_function, [21, -21, 0], [], + policy=P_NOVIRTUAL) + assert res == 42 + self.check_insns({'malloc_varsize': 1, + 'setinteriorfield': 2, 'getinteriorfield': 1, + 'getarraysize': 1, 'int_mul': 1}) + + res = self.interpret(ll_function, [21, -21, 1], [], + policy=P_NOVIRTUAL) + assert res == -42 + self.check_insns({'malloc_varsize': 1, + 'setinteriorfield': 2, 'getinteriorfield': 1, + 'getarraysize': 1, 'int_mul': 1}) def test_red_varsized_struct(self): py.test.skip("arrays and structs are not working") - A = lltype.Array(lltype.Signed) - S = lltype.GcStruct('S', ('foo', lltype.Signed), ('a', A)) - def ll_function(x, y, n): - s = lltype.malloc(S, 3) - s.foo = len(s.a)-1 - s.a[0] = x - s.a[1] = y - return s.a[n]*s.foo - - res = self.interpret(ll_function, [21, -21, 0], [], - policy=P_NOVIRTUAL) - assert res == 42 - self.check_insns(malloc_varsize=1, - getinteriorarraysize=1) - - res = self.interpret(ll_function, [21, -21, 1], [], - policy=P_NOVIRTUAL) - assert res == -42 - self.check_insns(malloc_varsize=1, - getinteriorarraysize=1) + A = lltype.Array(lltype.Signed) + S = lltype.GcStruct('S', ('foo', lltype.Signed), ('a', A)) + def ll_function(x, y, n): + s = lltype.malloc(S, 3) + s.foo = len(s.a)-1 + s.a[0] = x + s.a[1] = y + return s.a[n]*s.foo + + res = self.interpret(ll_function, [21, -21, 0], [], + policy=P_NOVIRTUAL) + assert res == 42 + self.check_insns(malloc_varsize=1, + getinteriorarraysize=1) + + res = self.interpret(ll_function, [21, -21, 1], [], + policy=P_NOVIRTUAL) + assert res == -42 + self.check_insns(malloc_varsize=1, + getinteriorarraysize=1) def test_array_of_voids(self): py.test.skip("arrays and structs are not working") From arigo at codespeak.net Mon Feb 4 13:10:22 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 4 Feb 2008 13:10:22 +0100 (CET) Subject: [pypy-svn] r51246 - pypy/branch/asmgcroot/pypy/translator/c/gcc Message-ID: <20080204121022.B68471684DB@codespeak.net> Author: arigo Date: Mon Feb 4 13:10:22 2008 New Revision: 51246 Modified: pypy/branch/asmgcroot/pypy/translator/c/gcc/trackgcroot.py Log: Trying to fix the handling of main(). Modified: pypy/branch/asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/asmgcroot/pypy/translator/c/gcc/trackgcroot.py Mon Feb 4 13:10:22 2008 @@ -123,6 +123,7 @@ def process_function(self, lines, entrypoint, filename): tracker = FunctionGcRootTracker(lines, filetag = self.files_seen) + tracker.is_main = tracker.funcname == entrypoint if self.verbose: print >> sys.stderr, '[trackgcroot:%s] %s' % (filename, tracker.funcname) @@ -130,22 +131,10 @@ if self.verbose > 1: for label, state in table: print >> sys.stderr, label, '\t', format_callshape(state) - if tracker.funcname == entrypoint: - table = self.fixup_entrypoint_table(table) self.gcmaptable.extend(table) + self.seen_main |= tracker.is_main return tracker.lines - def fixup_entrypoint_table(self, table): - self.seen_main = True - # as an end marker, set all five initial entries of the callshapes - # to LOC_NOWHERE. This info is not useful anyway because we don't - # go to main()'s caller. - newtable = [] - MARKERS = (LOC_NOWHERE,) * 5 - for label, shape in table: - newtable.append((label, MARKERS + shape[5:])) - return newtable - class FunctionGcRootTracker(object): @@ -159,6 +148,7 @@ self.uses_frame_pointer = False self.r_localvar = r_localvarnofp self.filetag = filetag + self.is_main = False def computegcmaptable(self, verbose=0): self.findlabels() @@ -184,7 +174,9 @@ for insn in self.list_call_insns(): if not hasattr(insn, 'framesize'): continue # calls that never end up reaching a RET - if self.uses_frame_pointer: + if self.is_main: + retaddr = LOC_NOWHERE # end marker for asmgcroot.py + elif self.uses_frame_pointer: retaddr = frameloc(LOC_EBP_BASED, 4) else: retaddr = frameloc(LOC_ESP_BASED, insn.framesize) @@ -192,7 +184,7 @@ # the first gcroots are always the ones corresponding to # the callee-saved registers for reg in CALLEE_SAVE_REGISTERS: - shape.append(None) + shape.append(LOC_NOWHERE) gcroots = [] for localvar, tag in insn.gcroots.items(): if isinstance(localvar, LocalVar): @@ -207,8 +199,8 @@ else: regindex = CALLEE_SAVE_REGISTERS.index(tag) shape[1 + regindex] = loc - if None in shape: - reg = CALLEE_SAVE_REGISTERS[shape.index(None) - 1] + if LOC_NOWHERE in shape and not self.is_main: + reg = CALLEE_SAVE_REGISTERS[shape.index(LOC_NOWHERE) - 1] raise AssertionError("cannot track where register %s is saved" % (reg,)) gcroots.sort() @@ -577,7 +569,7 @@ return self._visit_epilogue() + self._visit_pop('%ebp') def visit_ret(self, line): - return InsnRet() + return InsnRet(self.is_main) def visit_jmp(self, line): match = r_jmp_switch.match(line) @@ -598,7 +590,8 @@ return InsnStop() if r_unaryinsn_star.match(line): # that looks like an indirect tail-call. - return InsnRet() # tail-calls are equivalent to RET for us + # tail-calls are equivalent to RET for us + return InsnRet(self.is_main) try: self.conditional_jump(line) except KeyError: @@ -606,7 +599,8 @@ match = r_unaryinsn.match(line) target = match.group(1) assert not target.startswith('.') - return InsnRet() # tail-calls are equivalent to RET for us + # tail-calls are equivalent to RET for us + return InsnRet(self.is_main) return InsnStop() def register_jump_to(self, label): @@ -771,8 +765,13 @@ class InsnRet(InsnStop): framesize = 0 + def __init__(self, is_main): + self.is_main = is_main def requestgcroots(self): - return dict(zip(CALLEE_SAVE_REGISTERS, CALLEE_SAVE_REGISTERS)) + if self.is_main: # no need to track the value of these registers in + return {} # the caller function if we are the main() + else: + return dict(zip(CALLEE_SAVE_REGISTERS, CALLEE_SAVE_REGISTERS)) class InsnCall(Insn): _args_ = ['lineno', 'gcroots'] From arigo at codespeak.net Mon Feb 4 13:11:11 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 4 Feb 2008 13:11:11 +0100 (CET) Subject: [pypy-svn] r51247 - pypy/branch/asmgcroot/pypy/translator/c/gcc Message-ID: <20080204121111.EA3081684E0@codespeak.net> Author: arigo Date: Mon Feb 4 13:11:11 2008 New Revision: 51247 Modified: pypy/branch/asmgcroot/pypy/translator/c/gcc/trackgcroot.py Log: Wrong direction. Doesn't matter really, but it gives confusing debugging outputs. Modified: pypy/branch/asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/asmgcroot/pypy/translator/c/gcc/trackgcroot.py Mon Feb 4 13:11:11 2008 @@ -758,7 +758,7 @@ class InsnCannotFollowEsp(InsnStackAdjust): def __init__(self): - self.delta = 7 # use an odd value as marker + self.delta = -7 # use an odd value as marker class InsnStop(Insn): pass From cfbolz at codespeak.net Mon Feb 4 13:18:51 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 4 Feb 2008 13:18:51 +0100 (CET) Subject: [pypy-svn] r51248 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080204121851.20CE51684EA@codespeak.net> Author: cfbolz Date: Mon Feb 4 13:18:51 2008 New Revision: 51248 Added: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py - copied, changed from r51237, pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (contents, props changed) Removed: pypy/branch/jit-refactoring/pypy/jit/rainbow/bytecode.py Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Log: split the codewriter into its own file Added: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- (empty file) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Mon Feb 4 13:18:51 2008 @@ -0,0 +1,327 @@ +from pypy.rlib.rarithmetic import intmask +from pypy.rlib.unroll import unrolling_iterable +from pypy.jit.timeshifter import rtimeshift +from pypy.jit.timeshifter.greenkey import empty_key, GreenKey +from pypy.rpython.lltypesystem import lltype + +class JitCode(object): + """ + normal operations have the following format: + 2 byte - operation + n * 2 byte - arguments + + for nonvoid results the result is appended to the varlist + + red vars are just indexes + green vars are positive indexes + green consts are negative indexes + """ + + def __init__(self, name, code, constants, typekinds, redboxclasses, + keydescs, called_bytecodes, num_mergepoints, graph_color, + nonrainbow_functions, is_portal): + self.name = name + self.code = code + self.constants = constants + self.typekinds = typekinds + self.redboxclasses = redboxclasses + self.keydescs = keydescs + self.called_bytecodes = called_bytecodes + self.num_mergepoints = num_mergepoints + self.graph_color = graph_color + self.nonrainbow_functions = nonrainbow_functions + self.is_portal = is_portal + + def _freeze_(self): + return True + +SIGN_EXTEND2 = 1 << 15 + +class STOP(object): + pass +STOP = STOP() + +class JitInterpreter(object): + def __init__(self): + self.opcode_implementations = [] + self.opcode_descs = [] + self.opname_to_index = {} + self.jitstate = None + self.queue = None + self._add_implemented_opcodes() + + def run(self, jitstate, bytecode, greenargs, redargs, + start_bytecode_loop=True): + self.jitstate = jitstate + self.queue = rtimeshift.DispatchQueue(bytecode.num_mergepoints) + rtimeshift.enter_frame(self.jitstate, self.queue) + self.frame = self.jitstate.frame + self.frame.pc = 0 + self.frame.bytecode = bytecode + self.frame.local_boxes = redargs + self.frame.local_green = greenargs + if start_bytecode_loop: + self.bytecode_loop() + return self.jitstate + + def bytecode_loop(self): + while 1: + bytecode = self.load_2byte() + assert bytecode >= 0 + result = self.opcode_implementations[bytecode](self) + if result is STOP: + return + else: + assert result is None + + def dispatch(self): + is_portal = self.frame.bytecode.is_portal + graph_color = self.frame.bytecode.graph_color + queue = self.queue + newjitstate = rtimeshift.dispatch_next(queue) + resumepoint = rtimeshift.getresumepoint(newjitstate) + self.newjitstate(newjitstate) + if resumepoint == -1: + if graph_color == "red": + newjitstate = rtimeshift.leave_graph_red( + queue, is_portal) + elif graph_color == "yellow": + newjitstate = rtimeshift.leave_graph_yellow(queue) + elif graph_color == "green": + assert 0, "green graphs shouldn't be seen by the rainbow interp" + elif graph_color == "gray": + assert not is_portal + newjitstate = rtimeshift.leave_graph_gray(queue) + else: + assert 0, "unknown graph color %s" % (graph_color, ) + + self.newjitstate(newjitstate) + if self.frame is None: + return STOP + else: + self.frame.pc = resumepoint + + # operation helper functions + + def load_2byte(self): + pc = self.frame.pc + assert pc >= 0 + result = ((ord(self.frame.bytecode.code[pc]) << 8) | + ord(self.frame.bytecode.code[pc + 1])) + self.frame.pc = pc + 2 + return intmask((result ^ SIGN_EXTEND2) - SIGN_EXTEND2) + + def load_4byte(self): + pc = self.frame.pc + assert pc >= 0 + result = ((ord(self.frame.bytecode.code[pc + 0]) << 24) | + (ord(self.frame.bytecode.code[pc + 1]) << 16) | + (ord(self.frame.bytecode.code[pc + 2]) << 8) | + (ord(self.frame.bytecode.code[pc + 3]) << 0)) + self.frame.pc = pc + 4 + return intmask(result) + + def get_greenarg(self): + i = self.load_2byte() + if i < 0: + return self.frame.bytecode.constants[~i] + return self.frame.local_green[i] + + def get_green_varargs(self): + greenargs = [] + num = self.load_2byte() + for i in range(num): + greenargs.append(self.get_greenarg()) + return greenargs + + def get_red_varargs(self): + redargs = [] + num = self.load_2byte() + for i in range(num): + redargs.append(self.get_redarg()) + return redargs + + def get_redarg(self): + return self.frame.local_boxes[self.load_2byte()] + + def red_result(self, box): + self.frame.local_boxes.append(box) + + def green_result(self, gv): + self.frame.local_green.append(gv) + + def newjitstate(self, newjitstate): + self.jitstate = newjitstate + self.queue = None + if newjitstate is not None: + frame = newjitstate.frame + self.frame = frame + if frame is not None: + self.queue = frame.dispatchqueue + else: + self.frame = None + + # operation implementations + def opimpl_make_redbox(self): + genconst = self.get_greenarg() + typeindex = self.load_2byte() + kind = self.frame.bytecode.typekinds[typeindex] + redboxcls = self.frame.bytecode.redboxclasses[typeindex] + self.red_result(redboxcls(kind, genconst)) + + def opimpl_goto(self): + target = self.load_4byte() + self.frame.pc = target + + def opimpl_green_goto_iftrue(self): + genconst = self.get_greenarg() + target = self.load_4byte() + arg = genconst.revealconst(lltype.Bool) + if arg: + self.frame.pc = target + + def opimpl_red_goto_iftrue(self): + switchbox = self.get_redarg() + target = self.load_4byte() + # XXX not sure about passing no green vars + descision = rtimeshift.split(self.jitstate, switchbox, self.frame.pc) + if descision: + self.frame.pc = target + + def opimpl_red_return(self): + rtimeshift.save_return(self.jitstate) + return self.dispatch() + + def opimpl_gray_return(self): + rtimeshift.save_return(self.jitstate) + return self.dispatch() + + def opimpl_yellow_return(self): + # save the greens to make the return value findable by collect_split + rtimeshift.save_greens(self.jitstate, self.frame.local_green) + rtimeshift.save_return(self.jitstate) + return self.dispatch() + + def opimpl_make_new_redvars(self): + self.frame.local_boxes = self.get_red_varargs() + + def opimpl_make_new_greenvars(self): + # an opcode with a variable number of args + # num_args arg_old_1 arg_new_1 ... + num = self.load_2byte() + if num == 0 and len(self.frame.local_green) == 0: + # fast (very common) case + return + newgreens = [] + for i in range(num): + newgreens.append(self.get_greenarg()) + self.frame.local_green = newgreens + + def opimpl_merge(self): + mergepointnum = self.load_2byte() + keydescnum = self.load_2byte() + if keydescnum == -1: + key = empty_key + else: + keydesc = self.frame.bytecode.keydescs[keydescnum] + key = GreenKey(self.frame.local_green[:keydesc.nb_vals], keydesc) + states_dic = self.queue.local_caches[mergepointnum] + done = rtimeshift.retrieve_jitstate_for_merge(states_dic, self.jitstate, + key, None) + if done: + return self.dispatch() + + def opimpl_red_direct_call(self): + greenargs = self.get_green_varargs() + redargs = self.get_red_varargs() + bytecodenum = self.load_2byte() + targetbytecode = self.frame.bytecode.called_bytecodes[bytecodenum] + self.run(self.jitstate, targetbytecode, greenargs, redargs, + start_bytecode_loop=False) + # this frame will be resumed later in the next bytecode, which is + # red_after_direct_call + + def opimpl_red_after_direct_call(self): + newjitstate = rtimeshift.collect_split( + self.jitstate, self.frame.pc, + self.frame.local_green) + assert newjitstate is self.jitstate + + def opimpl_green_direct_call(self): + greenargs = self.get_green_varargs() + redargs = self.get_red_varargs() + index = self.load_2byte() + function = self.frame.bytecode.nonrainbow_functions[index] + function(self, greenargs, redargs) + + def opimpl_yellow_direct_call(self): + greenargs = self.get_green_varargs() + redargs = self.get_red_varargs() + bytecodenum = self.load_2byte() + targetbytecode = self.frame.bytecode.called_bytecodes[bytecodenum] + self.run(self.jitstate, targetbytecode, greenargs, redargs, + start_bytecode_loop=False) + # this frame will be resumed later in the next bytecode, which is + # yellow_after_direct_call + + def opimpl_yellow_after_direct_call(self): + newjitstate = rtimeshift.collect_split( + self.jitstate, self.frame.pc, + self.frame.local_green) + assert newjitstate is self.jitstate + + def opimpl_yellow_retrieve_result(self): + # XXX all this jitstate.greens business is a bit messy + self.green_result(self.jitstate.greens[0]) + + + # ____________________________________________________________ + # construction-time interface + + def _add_implemented_opcodes(self): + for name in dir(self): + if not name.startswith("opimpl_"): + continue + opname = name[len("opimpl_"):] + self.opname_to_index[opname] = len(self.opcode_implementations) + self.opcode_implementations.append(getattr(self, name).im_func) + self.opcode_descs.append(None) + + def find_opcode(self, name): + return self.opname_to_index.get(name, -1) + + def make_opcode_implementation(self, color, opdesc): + numargs = unrolling_iterable(range(opdesc.nb_args)) + if color == "green": + def implementation(self): + args = (opdesc.RESULT, ) + for i in numargs: + genconst = self.get_greenarg() + arg = genconst.revealconst(opdesc.ARGS[i]) + args += (arg, ) + rgenop = self.jitstate.curbuilder.rgenop + result = rgenop.genconst(opdesc.llop(*args)) + self.green_result(result) + elif color == "red": + if opdesc.nb_args == 1: + impl = rtimeshift.ll_gen1 + elif opdesc.nb_args == 2: + impl = rtimeshift.ll_gen2 + else: + XXX + def implementation(self): + args = (opdesc, self.jitstate, ) + for i in numargs: + args += (self.get_redarg(), ) + result = impl(*args) + self.red_result(result) + else: + assert 0, "unknown color" + implementation.func_name = "opimpl_%s_%s" % (color, opdesc.opname) + opname = "%s_%s" % (color, opdesc.opname) + index = self.opname_to_index[opname] = len(self.opcode_implementations) + self.opcode_implementations.append(implementation) + self.opcode_descs.append(opdesc) + return index + + Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 4 13:18:51 2008 @@ -4,7 +4,7 @@ from pypy.jit.hintannotator.policy import StopAtXPolicy, HintAnnotatorPolicy from pypy.jit.hintannotator.model import SomeLLAbstractConstant, OriginFlags from pypy.jit.hintannotator.model import originalconcretetype -from pypy.jit.rainbow.bytecode import BytecodeWriter, label, tlabel, assemble +from pypy.jit.rainbow.codewriter import BytecodeWriter, label, tlabel, assemble from pypy.jit.codegen.llgraph.rgenop import RGenOp as LLRGenOp from pypy.jit.rainbow.test.test_serializegraph import AbstractSerializationTest from pypy.jit.rainbow import bytecode Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Mon Feb 4 13:18:51 2008 @@ -2,8 +2,8 @@ from pypy.jit.hintannotator.annotator import HintAnnotator from pypy.jit.hintannotator.policy import StopAtXPolicy, HintAnnotatorPolicy from pypy.jit.hintannotator.model import SomeLLAbstractConstant, OriginFlags -from pypy.jit.rainbow.bytecode import BytecodeWriter, label, tlabel, assemble from pypy.jit.codegen.llgraph.rgenop import RGenOp +from pypy.jit.rainbow.codewriter import BytecodeWriter, label, tlabel, assemble from pypy.rlib.jit import hint from pypy import conftest From cfbolz at codespeak.net Mon Feb 4 13:47:02 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 4 Feb 2008 13:47:02 +0100 (CET) Subject: [pypy-svn] r51249 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080204124702.66A961684E9@codespeak.net> Author: cfbolz Date: Mon Feb 4 13:47:01 2008 New Revision: 51249 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: put the exceptiondesc into the interpreter and the codewriter Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 4 13:47:01 2008 @@ -3,7 +3,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.jit.hintannotator.model import originalconcretetype from pypy.jit.hintannotator import model as hintmodel -from pypy.jit.timeshifter import rtimeshift, rvalue +from pypy.jit.timeshifter import rtimeshift, rvalue, exception from pypy.jit.timeshifter.greenkey import KeyDesc from pypy.jit.rainbow.interpreter import JitCode, JitInterpreter from pypy.translator.backendopt.removenoops import remove_same_as @@ -16,7 +16,11 @@ self.translator = t self.annotator = t.annotator self.hannotator = hannotator - self.interpreter = JitInterpreter() + etrafo = hannotator.exceptiontransformer + type_system = hannotator.base_translator.rtyper.type_system.name + self.exceptiondesc = exception.ExceptionDesc( + RGenOp, etrafo, type_system, False) + self.interpreter = JitInterpreter(self.exceptiondesc) self.RGenOp = RGenOp self.current_block = None self.raise_analyzer = hannotator.exceptiontransformer.raise_analyzer @@ -412,6 +416,9 @@ def serialize_op_indirect_call(self, op): XXX + def serialize_op_getfield(self, op): + XXX + # call handling def graphs_from(self, spaceop): Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Mon Feb 4 13:47:01 2008 @@ -42,7 +42,8 @@ STOP = STOP() class JitInterpreter(object): - def __init__(self): + def __init__(self, exceptiondesc): + self.exceptiondesc = exceptiondesc self.opcode_implementations = [] self.opcode_descs = [] self.opname_to_index = {} Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 4 13:47:01 2008 @@ -8,7 +8,7 @@ from pypy.jit.codegen.llgraph.rgenop import RGenOp as LLRGenOp from pypy.jit.rainbow.test.test_serializegraph import AbstractSerializationTest from pypy.jit.rainbow import bytecode -from pypy.jit.timeshifter import rtimeshift, exception, rvalue +from pypy.jit.timeshifter import rtimeshift, rvalue from pypy.rpython.lltypesystem import lltype, rstr from pypy.rpython.llinterp import LLInterpreter from pypy.annotation import model as annmodel @@ -117,17 +117,13 @@ def interpret(self, ll_function, values, opt_consts=[], *args, **kwds): # XXX clean this mess up writer, jitcode, argcolors = self.serialize(ll_function, values) - base_annotator = writer.translator.annotator - etrafo = base_annotator.exceptiontransformer - type_system = base_annotator.base_translator.rtyper.type_system.name - edesc = exception.ExceptionDesc(writer.RGenOp, etrafo, type_system, False) rgenop = writer.RGenOp() sigtoken = rgenop.sigToken(self.RESIDUAL_FUNCTYPE) builder, gv_generated, inputargs_gv = rgenop.newgraph(sigtoken, "generated") builder.start_writing() jitstate = rtimeshift.JITState(builder, None, - edesc.null_exc_type_box, - edesc.null_exc_value_box) + writer.exceptiondesc.null_exc_type_box, + writer.exceptiondesc.null_exc_value_box) def ll_finish_jitstate(jitstate, exceptiondesc, graphsigtoken): returnbox = rtimeshift.getreturnbox(jitstate) gv_ret = returnbox.getgenvar(jitstate) @@ -160,7 +156,8 @@ red_i += 1 jitstate = writer.interpreter.run(jitstate, jitcode, greenargs, redargs) if jitstate is not None: - ll_finish_jitstate(jitstate, edesc, sigtoken) + ll_finish_jitstate(jitstate, writer.interpreter.exceptiondesc, + sigtoken) builder.end() generated = gv_generated.revealconst(lltype.Ptr(self.RESIDUAL_FUNCTYPE)) graph = generated._obj.graph From arigo at codespeak.net Mon Feb 4 14:06:02 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 4 Feb 2008 14:06:02 +0100 (CET) Subject: [pypy-svn] r51250 - pypy/dist/pypy/config/test Message-ID: <20080204130602.975DE16842D@codespeak.net> Author: arigo Date: Mon Feb 4 14:05:55 2008 New Revision: 51250 Added: pypy/dist/pypy/config/test/test_pypyoption.py.merge.tmp - copied, changed from r51249, pypy/dist/pypy/config/test/test_pypyoption.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/asmgcroot/pypy/config/test/test_pypyoption.py revisions 50007 to 51249: ------------------------------------------------------------------------ r50987 | cfbolz | 2008-01-24 19:34:28 +0100 (Thu, 24 Jan 2008) | 2 lines unify the various options for chosing a GC root finding algorithm ------------------------------------------------------------------------ r50261 | arigo | 2008-01-02 16:31:17 +0100 (Wed, 02 Jan 2008) | 3 lines Branch the branch, to play with a more flexible implementation of trackgcroot.py. ------------------------------------------------------------------------ r50009 | arigo | 2007-12-22 16:16:48 +0100 (Sat, 22 Dec 2007) | 2 lines A branch in which to try out the llvm.gcroot() operation. ------------------------------------------------------------------------ From arigo at codespeak.net Mon Feb 4 14:06:03 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 4 Feb 2008 14:06:03 +0100 (CET) Subject: [pypy-svn] r51251 - pypy/dist/pypy/translator/c Message-ID: <20080204130603.CC99416842D@codespeak.net> Author: arigo Date: Mon Feb 4 14:06:01 2008 New Revision: 51251 Added: pypy/dist/pypy/translator/c/genc.py.merge.tmp - copied, changed from r51249, pypy/dist/pypy/translator/c/genc.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/asmgcroot/pypy/translator/c/genc.py revisions 50007 to 51249: ------------------------------------------------------------------------ r50987 | cfbolz | 2008-01-24 19:34:28 +0100 (Thu, 24 Jan 2008) | 2 lines unify the various options for chosing a GC root finding algorithm ------------------------------------------------------------------------ r50928 | arigo | 2008-01-23 17:01:00 +0100 (Wed, 23 Jan 2008) | 2 lines Fix this logic. ------------------------------------------------------------------------ r50925 | arigo | 2008-01-23 16:19:43 +0100 (Wed, 23 Jan 2008) | 3 lines Put logic and tests for --asmgcroot and trackgcroot.py into their own subdirectory 'gcc'. Some tests fail. ------------------------------------------------------------------------ r50266 | arigo | 2008-01-02 17:52:49 +0100 (Wed, 02 Jan 2008) | 2 lines Another in-progress check-in. ------------------------------------------------------------------------ r50261 | arigo | 2008-01-02 16:31:17 +0100 (Wed, 02 Jan 2008) | 3 lines Branch the branch, to play with a more flexible implementation of trackgcroot.py. ------------------------------------------------------------------------ r50146 | arigo | 2007-12-27 20:12:08 +0100 (Thu, 27 Dec 2007) | 3 lines Tweak the Makefile generation so that it still works in the non-llvmgcroot case. ------------------------------------------------------------------------ r50141 | arigo | 2007-12-27 18:12:05 +0100 (Thu, 27 Dec 2007) | 9 lines targetgcbench works now via genc using the --llvmgcroot logic. Only for 386-compatible Linux boxes. To try it out, use translate.py --llvmgcroot --backend=c --gc=xxx --text targetxxx and use the generated Makefile to process the result. ------------------------------------------------------------------------ r50137 | arigo | 2007-12-27 16:10:26 +0100 (Thu, 27 Dec 2007) | 2 lines Fooling around with a '.s' file parser. ------------------------------------------------------------------------ r50010 | arigo | 2007-12-22 16:18:43 +0100 (Sat, 22 Dec 2007) | 4 lines In-progress: the llvmgcroot GC transformer, with its custom StackRootIterator that walks the tables produced by the llvm GC plugin. ------------------------------------------------------------------------ r50009 | arigo | 2007-12-22 16:16:48 +0100 (Sat, 22 Dec 2007) | 2 lines A branch in which to try out the llvm.gcroot() operation. ------------------------------------------------------------------------ From arigo at codespeak.net Mon Feb 4 14:06:14 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 4 Feb 2008 14:06:14 +0100 (CET) Subject: [pypy-svn] r51252 - pypy/dist/pypy/translator/c Message-ID: <20080204130614.BB89B1684EC@codespeak.net> Author: arigo Date: Mon Feb 4 14:06:08 2008 New Revision: 51252 Added: pypy/dist/pypy/translator/c/funcgen.py.merge.tmp - copied, changed from r51249, pypy/dist/pypy/translator/c/funcgen.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/asmgcroot/pypy/translator/c/funcgen.py revisions 50007 to 51249: ------------------------------------------------------------------------ r50266 | arigo | 2008-01-02 17:52:49 +0100 (Wed, 02 Jan 2008) | 2 lines Another in-progress check-in. ------------------------------------------------------------------------ r50261 | arigo | 2008-01-02 16:31:17 +0100 (Wed, 02 Jan 2008) | 3 lines Branch the branch, to play with a more flexible implementation of trackgcroot.py. ------------------------------------------------------------------------ r50137 | arigo | 2007-12-27 16:10:26 +0100 (Thu, 27 Dec 2007) | 2 lines Fooling around with a '.s' file parser. ------------------------------------------------------------------------ r50009 | arigo | 2007-12-22 16:16:48 +0100 (Sat, 22 Dec 2007) | 2 lines A branch in which to try out the llvm.gcroot() operation. ------------------------------------------------------------------------ From arigo at codespeak.net Mon Feb 4 14:06:26 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 4 Feb 2008 14:06:26 +0100 (CET) Subject: [pypy-svn] r51253 - pypy/dist/pypy/rpython/memory/test Message-ID: <20080204130626.350A1168430@codespeak.net> Author: arigo Date: Mon Feb 4 14:06:24 2008 New Revision: 51253 Added: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py.merge.tmp - copied, changed from r51249, pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/asmgcroot/pypy/rpython/memory/test/test_transformed_gc.py revisions 50007 to 51249: ------------------------------------------------------------------------ r50988 | cfbolz | 2008-01-24 19:50:34 +0100 (Thu, 24 Jan 2008) | 2 lines missed some places ------------------------------------------------------------------------ r50261 | arigo | 2008-01-02 16:31:17 +0100 (Wed, 02 Jan 2008) | 3 lines Branch the branch, to play with a more flexible implementation of trackgcroot.py. ------------------------------------------------------------------------ r50009 | arigo | 2007-12-22 16:16:48 +0100 (Sat, 22 Dec 2007) | 2 lines A branch in which to try out the llvm.gcroot() operation. ------------------------------------------------------------------------ From arigo at codespeak.net Mon Feb 4 14:06:34 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 4 Feb 2008 14:06:34 +0100 (CET) Subject: [pypy-svn] r51254 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20080204130634.6C4B6168430@codespeak.net> Author: arigo Date: Mon Feb 4 14:06:30 2008 New Revision: 51254 Added: pypy/dist/pypy/rpython/memory/gc/generation.py.merge.tmp - copied, changed from r51249, pypy/dist/pypy/rpython/memory/gc/generation.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/asmgcroot/pypy/rpython/memory/gc/generation.py revisions 50007 to 51249: ------------------------------------------------------------------------ r50288 | arigo | 2008-01-03 15:17:10 +0100 (Thu, 03 Jan 2008) | 2 lines Yet Another intermediate check-in... Thinking about changing the approach. ------------------------------------------------------------------------ r50261 | arigo | 2008-01-02 16:31:17 +0100 (Wed, 02 Jan 2008) | 3 lines Branch the branch, to play with a more flexible implementation of trackgcroot.py. ------------------------------------------------------------------------ r50009 | arigo | 2007-12-22 16:16:48 +0100 (Sat, 22 Dec 2007) | 2 lines A branch in which to try out the llvm.gcroot() operation. ------------------------------------------------------------------------ From arigo at codespeak.net Mon Feb 4 14:06:44 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 4 Feb 2008 14:06:44 +0100 (CET) Subject: [pypy-svn] r51255 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20080204130644.F2BE01684EE@codespeak.net> Author: arigo Date: Mon Feb 4 14:06:38 2008 New Revision: 51255 Added: pypy/dist/pypy/rpython/memory/gc/base.py.merge.tmp - copied, changed from r51249, pypy/dist/pypy/rpython/memory/gc/base.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/asmgcroot/pypy/rpython/memory/gc/base.py revisions 50007 to 51249: ------------------------------------------------------------------------ r50288 | arigo | 2008-01-03 15:17:10 +0100 (Thu, 03 Jan 2008) | 2 lines Yet Another intermediate check-in... Thinking about changing the approach. ------------------------------------------------------------------------ r50261 | arigo | 2008-01-02 16:31:17 +0100 (Wed, 02 Jan 2008) | 3 lines Branch the branch, to play with a more flexible implementation of trackgcroot.py. ------------------------------------------------------------------------ r50009 | arigo | 2007-12-22 16:16:48 +0100 (Sat, 22 Dec 2007) | 2 lines A branch in which to try out the llvm.gcroot() operation. ------------------------------------------------------------------------ From arigo at codespeak.net Mon Feb 4 14:08:53 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 4 Feb 2008 14:08:53 +0100 (CET) Subject: [pypy-svn] r51256 - in pypy/dist/pypy: config config/test doc/config rpython/lltypesystem rpython/memory rpython/memory/gc rpython/memory/gctransform rpython/memory/test translator/c translator/c/gcc translator/c/src translator/c/test translator/llvm Message-ID: <20080204130853.AC32A1684E1@codespeak.net> Author: arigo Date: Mon Feb 4 14:08:52 2008 New Revision: 51256 Added: pypy/dist/pypy/config/config.py - copied unchanged from r51255, pypy/branch/asmgcroot/pypy/config/config.py pypy/dist/pypy/config/test/test_config.py - copied unchanged from r51255, pypy/branch/asmgcroot/pypy/config/test/test_config.py pypy/dist/pypy/config/test/test_pypyoption.py - copied unchanged from r51255, pypy/dist/pypy/config/test/test_pypyoption.py.merge.tmp pypy/dist/pypy/config/translationoption.py - copied unchanged from r51255, pypy/branch/asmgcroot/pypy/config/translationoption.py pypy/dist/pypy/doc/config/translation.gcrootfinder.txt - copied unchanged from r51255, pypy/branch/asmgcroot/pypy/doc/config/translation.gcrootfinder.txt pypy/dist/pypy/rpython/lltypesystem/lloperation.py - copied unchanged from r51255, pypy/branch/asmgcroot/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/rpython/memory/gc/base.py - copied unchanged from r51255, pypy/dist/pypy/rpython/memory/gc/base.py.merge.tmp pypy/dist/pypy/rpython/memory/gc/generation.py - copied unchanged from r51255, pypy/dist/pypy/rpython/memory/gc/generation.py.merge.tmp pypy/dist/pypy/rpython/memory/gc/semispace.py - copied unchanged from r51255, pypy/branch/asmgcroot/pypy/rpython/memory/gc/semispace.py pypy/dist/pypy/rpython/memory/gctransform/ - copied from r51255, pypy/branch/asmgcroot/pypy/rpython/memory/gctransform/ pypy/dist/pypy/rpython/memory/gcwrapper.py - copied unchanged from r51255, pypy/branch/asmgcroot/pypy/rpython/memory/gcwrapper.py pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py - copied unchanged from r51255, pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py.merge.tmp pypy/dist/pypy/translator/c/funcgen.py - copied unchanged from r51255, pypy/dist/pypy/translator/c/funcgen.py.merge.tmp pypy/dist/pypy/translator/c/gc.py - copied unchanged from r51255, pypy/branch/asmgcroot/pypy/translator/c/gc.py pypy/dist/pypy/translator/c/gcc/ - copied from r51255, pypy/branch/asmgcroot/pypy/translator/c/gcc/ pypy/dist/pypy/translator/c/genc.py - copied unchanged from r51255, pypy/dist/pypy/translator/c/genc.py.merge.tmp pypy/dist/pypy/translator/c/src/mem.h - copied unchanged from r51255, pypy/branch/asmgcroot/pypy/translator/c/src/mem.h pypy/dist/pypy/translator/c/test/ - copied from r51255, pypy/branch/asmgcroot/pypy/translator/c/test/ pypy/dist/pypy/translator/llvm/ - copied from r51255, pypy/branch/asmgcroot/pypy/translator/llvm/ Removed: pypy/dist/pypy/config/test/test_pypyoption.py.merge.tmp pypy/dist/pypy/rpython/memory/gc/base.py.merge.tmp pypy/dist/pypy/rpython/memory/gc/generation.py.merge.tmp pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py.merge.tmp pypy/dist/pypy/translator/c/funcgen.py.merge.tmp pypy/dist/pypy/translator/c/genc.py.merge.tmp Log: Merging the asmgcroot branch, adding the --gcrootfinder translation option with a new choice: "asmgcc", only for gcc on Intel IA32. From arigo at codespeak.net Mon Feb 4 14:09:58 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 4 Feb 2008 14:09:58 +0100 (CET) Subject: [pypy-svn] r51257 - pypy/branch/asmgcroot Message-ID: <20080204130958.ED6991684E1@codespeak.net> Author: arigo Date: Mon Feb 4 14:09:57 2008 New Revision: 51257 Removed: pypy/branch/asmgcroot/ Log: Branch merged. From arigo at codespeak.net Mon Feb 4 14:22:46 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 4 Feb 2008 14:22:46 +0100 (CET) Subject: [pypy-svn] r51258 - pypy/dist/pypy/translator/c Message-ID: <20080204132246.8F771168417@codespeak.net> Author: arigo Date: Mon Feb 4 14:22:45 2008 New Revision: 51258 Modified: pypy/dist/pypy/translator/c/genc.py Log: Use "make profopt" instead of "make" when profiling-based optimizations are enabled. Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Mon Feb 4 14:22:45 2008 @@ -306,7 +306,11 @@ compiler = self.getccompiler() if self.config.translation.gcrootfinder == "asmgcc": # as we are gcc-only anyway, let's just use the Makefile. - cmdline = "make -C '%s'" % (self.targetdir,) + if compiler.profbased: + target = 'profopt' + else: + target = '' # default target + cmdline = "make -C '%s' %s" % (self.targetdir, target) err = os.system(cmdline) if err != 0: raise OSError("failed (see output): " + cmdline) From arigo at codespeak.net Mon Feb 4 15:42:48 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 4 Feb 2008 15:42:48 +0100 (CET) Subject: [pypy-svn] r51259 - pypy/dist/pypy/translator/c Message-ID: <20080204144248.2CD031684E2@codespeak.net> Author: arigo Date: Mon Feb 4 15:42:46 2008 New Revision: 51259 Modified: pypy/dist/pypy/translator/c/genc.py Log: Tweak the Makefile a bit more for -asmgcc-faassen translations of PyPy. Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Mon Feb 4 15:42:46 2008 @@ -379,6 +379,8 @@ f = targetdir.join('Makefile').open('w') print >> f, '# automatically generated Makefile' print >> f + print >> f, 'PYPYDIR =', autopath.pypydir + print >> f print >> f, 'TARGET =', py.path.local(compiler.outputfilename).basename print >> f write_list(cfiles, 'SOURCES =') @@ -389,15 +391,17 @@ else: write_list(ofiles, 'OBJECTS =') print >> f - if self.config.translation.gcrootfinder == "asmgcc": - print >> f, 'TRACKGCROOT="%s"' % (os.path.join(autopath.this_dir, - 'gcc', 'trackgcroot.py'),) - print >> f + def makerel(path): + rel = py.path.local(path).relto(py.path.local(autopath.pypydir)) + if rel: + return os.path.join('$(PYPYDIR)', rel) + else: + return path args = ['-l'+libname for libname in self.eci.libraries] print >> f, 'LIBS =', ' '.join(args) - args = ['-L'+path for path in self.eci.library_dirs] + args = ['-L'+makerel(path) for path in self.eci.library_dirs] print >> f, 'LIBDIRS =', ' '.join(args) - args = ['-I'+path for path in self.eci.include_dirs] + args = ['-I'+makerel(path) for path in self.eci.include_dirs] write_list(args, 'INCLUDEDIRS =') print >> f print >> f, 'CFLAGS =', ' '.join(compiler.compile_extra) @@ -832,7 +836,7 @@ \t$(CC) $(CFLAGS) -o $@ -S $< $(INCLUDEDIRS) gcmaptable.s: $(ASMFILES) -\t$(TRACKGCROOT) $(ASMFILES) > $@ || (rm -f $@ && exit 1) +\t$(PYPYDIR)/translator/c/gcc/trackgcroot.py $(ASMFILES) > $@ || (rm -f $@ && exit 1) clean: \trm -f $(OBJECTS) $(TARGET) @@ -857,7 +861,7 @@ profopt: \t$(MAKE) CFLAGS="-fprofile-generate $(CFLAGS)" LDFLAGS="-fprofile-generate $(LDFLAGS)" -\t./$(TARGET) $(PROFOPT) +\tcd $(PYPYDIR)/translator/goal && $(abspath $(TARGET)) $(PROFOPT) \trm -f $(OBJECTS) $(TARGET) \t$(MAKE) CFLAGS="-fprofile-use $(CFLAGS)" LDFLAGS="-fprofile-use $(LDFLAGS)" ''' From cfbolz at codespeak.net Mon Feb 4 16:47:15 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 4 Feb 2008 16:47:15 +0100 (CET) Subject: [pypy-svn] r51260 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter Message-ID: <20080204154715.6F6B91684FA@codespeak.net> Author: cfbolz Date: Mon Feb 4 16:47:13 2008 New Revision: 51260 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rcontainer.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Log: the first very very simple struct test passes Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 4 16:47:13 2008 @@ -3,7 +3,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.jit.hintannotator.model import originalconcretetype from pypy.jit.hintannotator import model as hintmodel -from pypy.jit.timeshifter import rtimeshift, rvalue, exception +from pypy.jit.timeshifter import rtimeshift, rvalue, rcontainer, exception from pypy.jit.timeshifter.greenkey import KeyDesc from pypy.jit.rainbow.interpreter import JitCode, JitInterpreter from pypy.translator.backendopt.removenoops import remove_same_as @@ -40,6 +40,8 @@ self.typekinds = [] self.redboxclasses = [] self.keydescs = [] + self.structtypedescs = [] + self.fielddescs = [] self.called_bytecodes = [] self.num_mergepoints = 0 self.graph_color = self.graph_calling_color(graph) @@ -57,8 +59,12 @@ self.free_green = {} # mapping TYPE to index self.type_positions = {} - # mapping tuple of green TYPES to index + # mapping tuples of green TYPES to index self.keydesc_positions = {} + # mapping STRUCTS to index + self.structtypedesc_positions = {} + # mapping tuples of STRUCT, name to index + self.fielddesc_positions = {} # mapping graphs to index self.graph_positions = {} # mapping fnobjs to index @@ -75,6 +81,8 @@ self.typekinds, self.redboxclasses, self.keydescs, + self.structtypedescs, + self.fielddescs, self.called_bytecodes, self.num_mergepoints, self.graph_color, @@ -179,7 +187,7 @@ result.append(self.serialize_oparg(color, v)) self.emit("make_new_%svars" % (color, )) self.emit(len(args)) - self.emit(result) + self.emit(*result) def serialize_op(self, op): specialcase = getattr(self, "serialize_op_%s" % (op.opname, ), None) @@ -190,7 +198,7 @@ for arg in op.args: args.append(self.serialize_oparg(color, arg)) self.serialize_opcode(color, op) - self.emit(args) + self.emit(*args) if self.hannotator.binding(op.result).is_green(): self.register_greenvar(op.result) else: @@ -287,6 +295,28 @@ self.type_positions[TYPE] = result return result + def structtypedesc_position(self, TYPE): + if TYPE in self.structtypedesc_positions: + return self.structtypedesc_positions[TYPE] + self.structtypedescs.append( + rcontainer.StructTypeDesc(self.RGenOp, TYPE)) + result = len(self.structtypedesc_positions) + self.structtypedesc_positions[TYPE] = result + return result + + def fielddesc_position(self, TYPE, fieldname): + if (fieldname, TYPE) in self.fielddesc_positions: + return self.fielddesc_positions[fieldname, TYPE] + structtypedesc = rcontainer.StructTypeDesc(self.RGenOp, TYPE) + fielddesc = structtypedesc.getfielddesc(fieldname) + if fielddesc is None: + self.fielddesc_positions[fieldname, TYPE] = -1 + return -1 + result = len(self.fielddescs) + self.fielddescs.append(fielddesc) + self.fielddesc_positions[fieldname, TYPE] = result + return result + def graph_position(self, graph): if graph in self.graph_positions: return self.graph_positions[graph] @@ -324,12 +354,11 @@ self.nonrainbow_positions[fn] = result return result - def emit(self, stuff): + def emit(self, *stuff): assert stuff is not None - if isinstance(stuff, list): - self.assembler.extend(stuff) - else: - self.assembler.append(stuff) + for x in stuff: + assert not isinstance(x, list) + self.assembler.append(x) def sort_by_color(self, vars, by_color_of_vars=None): reds = [] @@ -417,7 +446,37 @@ XXX def serialize_op_getfield(self, op): - XXX + assert self.opcolor(op) == "red" + args = op.args + if args[0] == self.exceptiondesc.cexcdata: + # reading one of the exception boxes (exc_type or exc_value) + fieldname = args[1].value + if fieldname == 'exc_type': + self.emit("read_exctype") + elif fieldname == 'exc_value': + self.emit("read_excvalue") + else: + raise Exception("getfield(exc_data, %r)" % (fieldname,)) + self.register_redvar(op.result) + return + + # virtualizable access read + PTRTYPE = args[0].concretetype + if PTRTYPE.TO._hints.get('virtualizable', False): + XXX + + # non virtual case + index = self.serialize_oparg("red", args[0]) + fieldname = args[1].value + s_struct = self.hannotator.binding(args[0]) + deepfrozen = s_struct.deepfrozen + + fielddescindex = self.fielddesc_position(PTRTYPE.TO, fieldname) + if fielddescindex == -1: # Void field + return + self.emit("red_getfield", index, fielddescindex, deepfrozen) + self.register_redvar(op.result) + # call handling @@ -521,6 +580,8 @@ opcode = interpreter.find_opcode(arg) assert opcode >= 0, "unknown opcode %s" % (arg, ) emit_2byte(opcode) + elif isinstance(arg, bool): + result.append(chr(int(arg))) elif isinstance(arg, int): emit_2byte(arg) elif isinstance(arg, label): Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Mon Feb 4 16:47:13 2008 @@ -18,7 +18,8 @@ """ def __init__(self, name, code, constants, typekinds, redboxclasses, - keydescs, called_bytecodes, num_mergepoints, graph_color, + keydescs, structtypedescs, fielddescs, called_bytecodes, + num_mergepoints, graph_color, nonrainbow_functions, is_portal): self.name = name self.code = code @@ -26,6 +27,8 @@ self.typekinds = typekinds self.redboxclasses = redboxclasses self.keydescs = keydescs + self.structtypedescs = structtypedescs + self.fielddescs = fielddescs self.called_bytecodes = called_bytecodes self.num_mergepoints = num_mergepoints self.graph_color = graph_color @@ -103,6 +106,12 @@ self.frame.pc = resumepoint # operation helper functions + def load_byte(self): + pc = self.frame.pc + assert pc >= 0 + result = ord(self.frame.bytecode.code[pc]) + self.frame.pc = pc + 1 + return result def load_2byte(self): pc = self.frame.pc @@ -122,6 +131,9 @@ self.frame.pc = pc + 4 return intmask(result) + def load_bool(self): + return bool(self.load_byte()) + def get_greenarg(self): i = self.load_2byte() if i < 0: @@ -276,6 +288,25 @@ self.green_result(self.jitstate.greens[0]) + # exceptions + + def opimpl_read_exctype(self): + XXX + + def opimpl_read_excvalue(self): + XXX + + # structs and arrays + + def opimpl_red_getfield(self): + structbox = self.get_redarg() + fielddesc = self.frame.bytecode.fielddescs[self.load_2byte()] + deepfrozen = self.load_bool() + resbox = rtimeshift.gengetfield(self.jitstate, deepfrozen, fielddesc, + structbox) + self.red_result(resbox) + + # ____________________________________________________________ # construction-time interface Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 4 16:47:13 2008 @@ -46,10 +46,6 @@ del cls._cache_order def serialize(self, func, values, backendoptimize=False): - if hasattr(func, 'convert_arguments'): - assert len(func.convert_arguments) == len(values) - values = [decoder(value) for decoder, value in zip( - func.convert_arguments, values)] key = func, backendoptimize try: cache, argtypes = self._cache[key] @@ -115,7 +111,10 @@ return writer, jitcode, argcolors def interpret(self, ll_function, values, opt_consts=[], *args, **kwds): - # XXX clean this mess up + if hasattr(ll_function, 'convert_arguments'): + assert len(ll_function.convert_arguments) == len(values) + values = [decoder(value) for decoder, value in zip( + ll_function.convert_arguments, values)] writer, jitcode, argcolors = self.serialize(ll_function, values) rgenop = writer.RGenOp() sigtoken = rgenop.sigToken(self.RESIDUAL_FUNCTYPE) @@ -501,7 +500,6 @@ def test_simple_struct(self): - py.test.skip("arrays and structs are not working") S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), ('world', lltype.Signed), hints={'immutable': True}) Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rcontainer.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rcontainer.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rcontainer.py Mon Feb 4 16:47:13 2008 @@ -70,14 +70,13 @@ firstsubstructdesc = None materialize = None - def __new__(cls, hrtyper, TYPE): + def __new__(cls, RGenOp, TYPE): if TYPE._hints.get('virtualizable', False): return object.__new__(VirtualizableStructTypeDesc) else: return object.__new__(StructTypeDesc) - def __init__(self, hrtyper, TYPE): - RGenOp = hrtyper.RGenOp + def __init__(self, RGenOp, TYPE): self.TYPE = TYPE self.PTRTYPE = lltype.Ptr(TYPE) self.ptrkind = RGenOp.kindToken(self.PTRTYPE) @@ -93,7 +92,7 @@ self.null = self.PTRTYPE._defl() self.gv_null = RGenOp.constPrebuiltGlobal(self.null) - self._compute_fielddescs(hrtyper) + self._compute_fielddescs(RGenOp) if self.immutable and self.noidentity: self._define_materialize() @@ -102,8 +101,7 @@ self._define_devirtualize() - def _compute_fielddescs(self, hrtyper): - RGenOp = hrtyper.RGenOp + def _compute_fielddescs(self, RGenOp): TYPE = self.TYPE innermostdesc = self fielddescs = [] @@ -112,10 +110,10 @@ FIELDTYPE = getattr(TYPE, name) if isinstance(FIELDTYPE, lltype.ContainerType): if isinstance(FIELDTYPE, lltype.Array): - self.arrayfielddesc = ArrayFieldDesc(hrtyper, FIELDTYPE) + self.arrayfielddesc = ArrayFieldDesc(RGenOp, FIELDTYPE) self.varsizealloctoken = RGenOp.varsizeAllocToken(TYPE) continue - substructdesc = StructTypeDesc(hrtyper, FIELDTYPE) + substructdesc = StructTypeDesc(RGenOp, FIELDTYPE) assert name == TYPE._names[0], ( "unsupported: inlined substructures not as first field") fielddescs.extend(substructdesc.fielddescs) @@ -126,7 +124,7 @@ if FIELDTYPE is lltype.Void: desc = None else: - desc = StructFieldDesc(hrtyper, self.PTRTYPE, name, index) + desc = StructFieldDesc(RGenOp, self.PTRTYPE, name, index) fielddescs.append(desc) fielddesc_by_name[name] = desc @@ -212,8 +210,8 @@ get_rti set_rti """.split() - def __init__(self, hrtyper, TYPE): - RGenOp = hrtyper.RGenOp + def __init__(self, RGenOp, TYPE): + XXX StructTypeDesc.__init__(self, hrtyper, TYPE) ACCESS = self.TYPE.ACCESS redirected_fields = ACCESS.redirected_fields @@ -231,8 +229,6 @@ TOPPTR = self.access_desc.PTRTYPE self.s_structtype = annmodel.lltype_to_annotation(TOPPTR) - annhelper = hrtyper.annhelper - self.my_redirected_getsetters_untouched = {} self.my_redirected_getsetters_touched = {} self.my_redirected_names = my_redirected_names = [] @@ -264,6 +260,7 @@ pass def _define_getset_field_ptr(self, hrtyper, fielddesc, j): + XXX annhelper = hrtyper.annhelper s_lltype = annmodel.lltype_to_annotation(fielddesc.RESTYPE) @@ -326,6 +323,7 @@ def _define_access_is_null(self, hrtyper): + XXX RGenOp = hrtyper.RGenOp annhelper = hrtyper.annhelper def access_is_null(struc): @@ -353,7 +351,7 @@ class InteriorDesc(object): __metaclass__ = cachedtype - def __init__(self, hrtyper, TOPCONTAINER, path): + def __init__(self, RGenOp, TOPCONTAINER, path): self.TOPCONTAINER = TOPCONTAINER self.path = path PTRTYPE = lltype.Ptr(TOPCONTAINER) @@ -362,10 +360,10 @@ for offset in path: LASTCONTAINER = TYPE if offset is None: # array substruct - fielddescs.append(ArrayFieldDesc(hrtyper, TYPE)) + fielddescs.append(ArrayFieldDesc(RGenOp, TYPE)) TYPE = TYPE.OF else: - fielddescs.append(NamedFieldDesc(hrtyper, lltype.Ptr(TYPE), + fielddescs.append(NamedFieldDesc(RGenOp, lltype.Ptr(TYPE), offset)) TYPE = getattr(TYPE, offset) unroll_path = unrolling_iterable(path) @@ -426,7 +424,7 @@ else: assert isinstance(TYPE, lltype.Array) - arrayfielddesc = ArrayFieldDesc(hrtyper, TYPE) + arrayfielddesc = ArrayFieldDesc(RGenOp, TYPE) getinterior_all = make_interior_getter(fielddescs) def gengetinteriorarraysize(jitstate, argbox, *indexboxes): @@ -504,8 +502,7 @@ gcref = False fieldnonnull = False - def __init__(self, hrtyper, PTRTYPE, RESTYPE): - RGenOp = hrtyper.RGenOp + def __init__(self, RGenOp, PTRTYPE, RESTYPE): self.PTRTYPE = PTRTYPE T = None if isinstance(RESTYPE, lltype.ContainerType): @@ -534,7 +531,7 @@ pass # no redboxcls at all else: if self.virtualizable: - self.structdesc = StructTypeDesc(hrtyper, T) + self.structdesc = StructTypeDesc(RGenOp, T) self.redboxcls = rvalue.ll_redboxcls(RESTYPE) self.immutable = PTRTYPE.TO._hints.get('immutable', False) @@ -561,11 +558,17 @@ class NamedFieldDesc(FieldDesc): - def __init__(self, hrtyper, PTRTYPE, name): - FieldDesc.__init__(self, hrtyper, PTRTYPE, getattr(PTRTYPE.TO, name)) + def __init__(self, RGenOp, PTRTYPE, name): + FieldDesc.__init__(self, RGenOp, PTRTYPE, getattr(PTRTYPE.TO, name)) T = self.PTRTYPE.TO self.fieldname = name - self.fieldtoken = hrtyper.RGenOp.fieldToken(T, name) + self.fieldtoken = RGenOp.fieldToken(T, name) + def getfield_if_non_null(jitstate, genvar): + ptr = genvar.revealconst(PTRTYPE) + if ptr: + res = getattr(ptr, name) + return rvalue.ll_gv_fromvalue(jitstate, res) + self.getfield_if_non_null = getfield_if_non_null def compact_repr(self): # goes in ll helper names return "Fld_%s_in_%s" % (self.fieldname, self.PTRTYPE._short_name()) @@ -586,17 +589,16 @@ class StructFieldDesc(NamedFieldDesc): - def __init__(self, hrtyper, PTRTYPE, name, index): - NamedFieldDesc.__init__(self, hrtyper, PTRTYPE, name) + def __init__(self, RGenOp, PTRTYPE, name, index): + NamedFieldDesc.__init__(self, RGenOp, PTRTYPE, name) self.fieldindex = index class ArrayFieldDesc(FieldDesc): allow_void = True - def __init__(self, hrtyper, TYPE): + def __init__(self, RGenOp, TYPE): assert isinstance(TYPE, lltype.Array) - FieldDesc.__init__(self, hrtyper, lltype.Ptr(TYPE), TYPE.OF) - RGenOp = hrtyper.RGenOp + FieldDesc.__init__(self, RGenOp, lltype.Ptr(TYPE), TYPE.OF) self.arraytoken = RGenOp.arrayToken(TYPE) self.varsizealloctoken = RGenOp.varsizeAllocToken(TYPE) self.indexkind = RGenOp.kindToken(lltype.Signed) Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Mon Feb 4 16:47:13 2008 @@ -140,15 +140,15 @@ # XXX MemoryError handling return rvalue.PtrRedBox(contdesc.ptrkind, genvar, known_nonzero=True) -def ll_gengetfield(jitstate, deepfrozen, fielddesc, argbox): +def gengetfield(jitstate, deepfrozen, fielddesc, argbox): if (fielddesc.immutable or deepfrozen) and argbox.is_constant(): - ptr = rvalue.ll_getvalue(argbox, fielddesc.PTRTYPE) - if ptr: # else don't constant-fold the segfault... - res = getattr(ptr, fielddesc.fieldname) - return rvalue.ll_fromvalue(jitstate, res) + resgv = fielddesc.getfield_if_non_null( + jitstate, argbox.getgenvar(jitstate)) + if resgv is not None: + return fielddesc.makebox(jitstate, resgv) return argbox.op_getfield(jitstate, fielddesc) -def ll_gensetfield(jitstate, fielddesc, destbox, valuebox): +def gensetfield(jitstate, fielddesc, destbox, valuebox): destbox.op_setfield(jitstate, fielddesc, valuebox) def ll_gengetsubstruct(jitstate, fielddesc, argbox): From cfbolz at codespeak.net Mon Feb 4 21:10:18 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 4 Feb 2008 21:10:18 +0100 (CET) Subject: [pypy-svn] r51266 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter Message-ID: <20080204201018.600BB168515@codespeak.net> Author: cfbolz Date: Mon Feb 4 21:10:16 2008 New Revision: 51266 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Log: simple virtual container test passes Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 4 21:10:16 2008 @@ -144,7 +144,21 @@ linkfalse, linktrue = linktrue, linkfalse color = self.varcolor(block.exitswitch) index = self.serialize_oparg(color, block.exitswitch) - self.emit("%s_goto_iftrue" % color) + srcopname, srcargs = self.trace_back_bool_var( + block, block.exitswitch) + if color == "red" and srcopname is not None: + if srcopname == 'ptr_nonzero': + reverse = False + split_extras = srcargs + elif srcopname == 'ptr_iszero': + reverse = True + split_extras = srcargs + ptrindex = self.serialize_oparg("red", srcargs[0]) + self.emit("red_goto_ifptrnonzero") + self.emit(reverse) + self.emit(ptrindex) + else: + self.emit("%s_goto_iftrue" % color) self.emit(index) self.emit(tlabel(linktrue)) self.insert_renaming(linkfalse) @@ -192,7 +206,10 @@ def serialize_op(self, op): specialcase = getattr(self, "serialize_op_%s" % (op.opname, ), None) if specialcase is not None: - return specialcase(op) + try: + return specialcase(op) + except NotImplementedError: + pass color = self.opcolor(op) args = [] for arg in op.args: @@ -445,6 +462,14 @@ def serialize_op_indirect_call(self, op): XXX + def serialize_op_malloc(self, op): + index = self.structtypedesc_position(op.args[0].value) + self.emit("red_malloc", index) + self.register_redvar(op.result) + + def serialize_op_zero_gc_pointers_inside(self, op): + pass # XXX is that right? + def serialize_op_getfield(self, op): assert self.opcolor(op) == "red" args = op.args @@ -478,6 +503,33 @@ self.register_redvar(op.result) + def serialize_op_setfield(self, op): + args = op.args + PTRTYPE = args[0].concretetype + VALUETYPE = args[2].concretetype + if VALUETYPE is lltype.Void: + return + if args[0] == self.exceptiondesc.cexcdata: + # reading one of the exception boxes (exc_type or exc_value) + fieldname = args[1].value + val = self.serialize_oparg("red", args[2]) + if fieldname == 'exc_type': + self.emit("write_exctype", val) + elif fieldname == 'exc_value': + self.emit("write_excvalue", val) + else: + raise Exception("getfield(exc_data, %r)" % (fieldname,)) + return + # non virtual case + destboxindex = self.serialize_oparg("red", args[0]) + valboxindex = self.serialize_oparg("red", args[2]) + fieldname = args[1].value + fielddescindex = self.fielddesc_position(PTRTYPE.TO, fieldname) + if fielddescindex == -1: # Void field + return + self.emit("red_setfield", destboxindex, fielddescindex, valboxindex) + + # call handling def graphs_from(self, spaceop): @@ -505,6 +557,29 @@ tsgraph = self.specialized_graph_of(graph, args_v, spaceop.result) yield graph, tsgraph + def trace_back_bool_var(self, block, v): + """Return the (opname, arguments) that created the exitswitch of + the block. The opname is None if not found. + """ + inverted = False + for i in range(len(block.operations)-1, -1, -1): + op = block.operations[i] + if op.result is v: + if op.opname == 'bool_not': + inverted = not inverted + [v] = op.args + elif op.opname == 'same_as': + [v] = op.args + else: + opname = op.opname + opargs = op.args + if inverted: + opname = {'ptr_nonzero': 'ptr_iszero', + 'ptr_iszero' : 'ptr_nonzero'}.get(opname) + return opname, opargs # found + # not found, comes from earlier block - give up + return None, None + def guess_call_kind(self, spaceop): if spaceop.opname == 'direct_call': c_func = spaceop.args[0] Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Mon Feb 4 21:10:16 2008 @@ -1,6 +1,6 @@ from pypy.rlib.rarithmetic import intmask from pypy.rlib.unroll import unrolling_iterable -from pypy.jit.timeshifter import rtimeshift +from pypy.jit.timeshifter import rtimeshift, rcontainer from pypy.jit.timeshifter.greenkey import empty_key, GreenKey from pypy.rpython.lltypesystem import lltype @@ -201,6 +201,35 @@ if descision: self.frame.pc = target + def opimpl_red_goto_ifptrnonzero(self): + reverse = self.load_bool() + ptrbox = self.get_redarg() + switchbox = self.get_redarg() + target = self.load_4byte() + # XXX not sure about passing no green vars + descision = rtimeshift.split_ptr_nonzero(self.jitstate, switchbox, + self.frame.pc, ptrbox, reverse) + if descision: + self.frame.pc = target + + def opimpl_red_ptr_nonzero(self, reverse=False): + ptrbox = self.get_redarg() + resultbox = rtimeshift.genptrnonzero(self.jitstate, ptrbox, reverse) + self.red_result(resultbox) + + def opimpl_red_ptr_iszero(self): + self.opimpl_red_ptr_nonzero(reverse=True) + + def opimpl_red_ptr_eq(self, reverse=False): + ptrbox1 = self.get_redarg() + ptrbox2 = self.get_redarg() + resultbox = rtimeshift.genptreq(self.jitstate, ptrbox1, + ptrbox2, reverse) + self.red_result(resultbox) + + def opimpl_red_ptr_ne(self): + self.opimpl_red_ptr_eq(reverse=True) + def opimpl_red_return(self): rtimeshift.save_return(self.jitstate) return self.dispatch() @@ -296,8 +325,21 @@ def opimpl_read_excvalue(self): XXX + def opimpl_write_exctype(self): + typebox = self.get_redarg() + rtimeshift.setexctypebox(self.jitstate, typebox) + + def opimpl_write_excvalue(self): + valuebox = self.get_redarg() + rtimeshift.setexcvaluebox(self.jitstate, valuebox) + # structs and arrays + def opimpl_red_malloc(self): + structtypedesc = self.frame.bytecode.structtypedescs[self.load_2byte()] + redbox = rcontainer.create(self.jitstate, structtypedesc) + self.red_result(redbox) + def opimpl_red_getfield(self): structbox = self.get_redarg() fielddesc = self.frame.bytecode.fielddescs[self.load_2byte()] @@ -306,6 +348,12 @@ structbox) self.red_result(resbox) + def opimpl_red_setfield(self): + destbox = self.get_redarg() + fielddesc = self.frame.bytecode.fielddescs[self.load_2byte()] + valuebox = self.get_redarg() + resbox = rtimeshift.gensetfield(self.jitstate, fielddesc, destbox, + valuebox) # ____________________________________________________________ # construction-time interface Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 4 21:10:16 2008 @@ -708,7 +708,6 @@ self.check_insns({'int_add': 2, 'int_sub': 1}) def test_red_virtual_container(self): - py.test.skip("arrays and structs are not working") # this checks that red boxes are able to be virtualized dynamically by # the compiler (the P_NOVIRTUAL policy prevents the hint-annotator from # marking variables in blue) @@ -717,7 +716,7 @@ s = lltype.malloc(S) s.n = n return s.n - res = self.interpret(ll_function, [42], [], policy=P_NOVIRTUAL) + res = self.interpret(ll_function, [42], []) assert res == 42 self.check_insns({}) Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Mon Feb 4 21:10:16 2008 @@ -222,7 +222,7 @@ return interiordesc.gengetinteriorarraysize(jitstate, argbox, *indexboxes) -def ll_genptrnonzero(jitstate, argbox, reverse): +def genptrnonzero(jitstate, argbox, reverse): if argbox.is_constant(): addr = rvalue.ll_getvalue(argbox, llmemory.Address) return rvalue.ll_fromvalue(jitstate, bool(addr) ^ reverse) @@ -237,7 +237,7 @@ gv_res = builder.genop_ptr_nonzero(argbox.kind, gv_addr) return rvalue.IntRedBox(builder.rgenop.kindToken(lltype.Bool), gv_res) -def ll_genptreq(jitstate, argbox0, argbox1, reverse): +def genptreq(jitstate, argbox0, argbox1, reverse): builder = jitstate.curbuilder if argbox0.is_constant() and argbox1.is_constant(): addr0 = rvalue.ll_getvalue(argbox0, llmemory.Address) From cfbolz at codespeak.net Mon Feb 4 21:20:51 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 4 Feb 2008 21:20:51 +0100 (CET) Subject: [pypy-svn] r51267 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080204202051.333D5168508@codespeak.net> Author: cfbolz Date: Mon Feb 4 21:20:50 2008 New Revision: 51267 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: another passing test. reminder: always run _all_ tests. Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 4 21:20:50 2008 @@ -144,15 +144,16 @@ linkfalse, linktrue = linktrue, linkfalse color = self.varcolor(block.exitswitch) index = self.serialize_oparg(color, block.exitswitch) - srcopname, srcargs = self.trace_back_bool_var( - block, block.exitswitch) - if color == "red" and srcopname is not None: - if srcopname == 'ptr_nonzero': - reverse = False - split_extras = srcargs - elif srcopname == 'ptr_iszero': - reverse = True - split_extras = srcargs + reverse = None + if color == "red": + srcopname, srcargs = self.trace_back_bool_var( + block, block.exitswitch) + if srcopname is not None: + if srcopname == 'ptr_nonzero': + reverse = False + elif srcopname == 'ptr_iszero': + reverse = True + if reverse is not None: ptrindex = self.serialize_oparg("red", srcargs[0]) self.emit("red_goto_ifptrnonzero") self.emit(reverse) @@ -433,7 +434,7 @@ args = targetgraph.getargs() emitted_args = self.args_of_call(op.args[1:], args) self.emit("red_direct_call") - self.emit(emitted_args) + self.emit(*emitted_args) self.emit(graphindex) if kind == "red": self.register_redvar(op.result) @@ -443,7 +444,7 @@ args = targetgraph.getargs() emitted_args = self.args_of_call(op.args[1:], args) self.emit("green_direct_call") - self.emit(emitted_args) + self.emit(*emitted_args) self.emit(pos) self.register_greenvar(op.result) elif kind == "yellow": @@ -451,7 +452,7 @@ args = targetgraph.getargs() emitted_args = self.args_of_call(op.args[1:], args) self.emit("yellow_direct_call") - self.emit(emitted_args) + self.emit(*emitted_args) self.emit(graphindex) self.emit("yellow_after_direct_call") self.emit("yellow_retrieve_result") Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 4 21:20:50 2008 @@ -820,7 +820,6 @@ assert res == "3" def test_red_propagate(self): - py.test.skip("arrays and structs are not working") S = lltype.GcStruct('S', ('n', lltype.Signed)) def ll_function(n, k): s = lltype.malloc(S) @@ -828,7 +827,7 @@ if k < 0: return -123 return s.n * k - res = self.interpret(ll_function, [3, 8], [], policy=P_NOVIRTUAL) + res = self.interpret(ll_function, [3, 8], []) assert res == 24 self.check_insns({'int_lt': 1, 'int_mul': 1}) From cfbolz at codespeak.net Mon Feb 4 21:35:07 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 4 Feb 2008 21:35:07 +0100 (CET) Subject: [pypy-svn] r51268 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080204203507.0B75316851C@codespeak.net> Author: cfbolz Date: Mon Feb 4 21:35:06 2008 New Revision: 51268 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: this one passes out of the box Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 4 21:35:06 2008 @@ -868,7 +868,6 @@ def test_merge_structures(self): - py.test.skip("arrays and structs are not working") S = lltype.GcStruct('S', ('n', lltype.Signed)) T = lltype.GcStruct('T', ('s', lltype.Ptr(S)), ('n', lltype.Signed)) @@ -886,10 +885,10 @@ t.s = s t.n = 6 return t.n + t.s.n - res = self.interpret(ll_function, [0], [], policy=P_NOVIRTUAL) + res = self.interpret(ll_function, [0], []) assert res == 5 + 6 self.check_insns({'int_is_true': 1, 'int_add': 1}) - res = self.interpret(ll_function, [1], [], policy=P_NOVIRTUAL) + res = self.interpret(ll_function, [1], []) assert res == 1 + 2 self.check_insns({'int_is_true': 1, 'int_add': 1}) From cfbolz at codespeak.net Mon Feb 4 21:36:57 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 4 Feb 2008 21:36:57 +0100 (CET) Subject: [pypy-svn] r51269 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080204203657.135C516851C@codespeak.net> Author: cfbolz Date: Mon Feb 4 21:36:56 2008 New Revision: 51269 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: another passing one Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 4 21:36:56 2008 @@ -894,7 +894,6 @@ def test_green_with_side_effects(self): - py.test.skip("arrays and structs are not working") S = lltype.GcStruct('S', ('flag', lltype.Bool)) s = lltype.malloc(S) s.flag = False From cfbolz at codespeak.net Mon Feb 4 21:57:29 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 4 Feb 2008 21:57:29 +0100 (CET) Subject: [pypy-svn] r51270 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080204205729.7A81C16851E@codespeak.net> Author: cfbolz Date: Mon Feb 4 21:57:28 2008 New Revision: 51270 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: nex Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 4 21:57:28 2008 @@ -471,6 +471,17 @@ def serialize_op_zero_gc_pointers_inside(self, op): pass # XXX is that right? + def serialize_op_cast_pointer(self, op): + color = self.varcolor(op.result) + assert color == self.varcolor(op.args[0]) + if color == "green": + self.register_greenvar(op.result, self.green_position(op.args[0])) + else: + self.register_redvar(op.result, self.redvar_position(op.args[0])) + + def serialize_op_keepalive(self, op): + pass + def serialize_op_getfield(self, op): assert self.opcolor(op) == "red" args = op.args Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 4 21:57:28 2008 @@ -14,6 +14,7 @@ from pypy.annotation import model as annmodel from pypy.objspace.flow.model import summary from pypy.rlib.jit import hint +from pypy.rlib.objectmodel import keepalive_until_here from pypy import conftest def getargtypes(annotator, values): @@ -850,7 +851,6 @@ def test_red_subcontainer_cast(self): - py.test.skip("arrays and structs are not working") S = lltype.GcStruct('S', ('n', lltype.Signed)) T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) def ll_function(k): @@ -862,7 +862,7 @@ result = s.n * (k-1) keepalive_until_here(t) return result - res = self.interpret(ll_function, [7], [], policy=P_NOVIRTUAL) + res = self.interpret(ll_function, [7], []) assert res == 42 self.check_insns({'int_lt': 1, 'int_mul': 1, 'int_sub': 1}) From cfbolz at codespeak.net Mon Feb 4 22:11:08 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 4 Feb 2008 22:11:08 +0100 (CET) Subject: [pypy-svn] r51271 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080204211108.348AB168445@codespeak.net> Author: cfbolz Date: Mon Feb 4 22:11:06 2008 New Revision: 51271 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: another one Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 4 22:11:06 2008 @@ -833,7 +833,6 @@ self.check_insns({'int_lt': 1, 'int_mul': 1}) def test_red_subcontainer(self): - py.test.skip("arrays and structs are not working") S = lltype.GcStruct('S', ('n', lltype.Signed)) T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) def ll_function(k): @@ -845,7 +844,7 @@ result = s.n * (k-1) keepalive_until_here(t) return result - res = self.interpret(ll_function, [7], [], policy=P_NOVIRTUAL) + res = self.interpret(ll_function, [7], []) assert res == 42 self.check_insns({'int_lt': 1, 'int_mul': 1, 'int_sub': 1}) From cfbolz at codespeak.net Mon Feb 4 22:17:26 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 4 Feb 2008 22:17:26 +0100 (CET) Subject: [pypy-svn] r51272 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080204211726.492921684CB@codespeak.net> Author: cfbolz Date: Mon Feb 4 22:17:26 2008 New Revision: 51272 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: this one works Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 4 22:17:26 2008 @@ -677,7 +677,6 @@ assert res == 4 * 4 def test_degenerate_with_voids(self): - py.test.skip("arrays and structs are not working") S = lltype.GcStruct('S', ('y', lltype.Void), ('x', lltype.Signed)) def ll_function(): @@ -685,8 +684,8 @@ s.x = 123 return s ll_function.convert_result = lambda s: str(s.x) - res = self.interpret(ll_function, [], [], policy=P_NOVIRTUAL) - assert res == "123" + res = self.interpret(ll_function, [], []) + assert res.x == 123 def test_plus_minus_all_inlined(self): py.test.skip("arrays and structs are not working") From cfbolz at codespeak.net Mon Feb 4 22:34:10 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 4 Feb 2008 22:34:10 +0100 (CET) Subject: [pypy-svn] r51273 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080204213410.4F28916851C@codespeak.net> Author: cfbolz Date: Mon Feb 4 22:34:09 2008 New Revision: 51273 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: those all pass Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 4 22:34:09 2008 @@ -584,7 +584,6 @@ self.check_insns({}) def test_degenerated_before_return(self): - py.test.skip("arrays and structs are not working") S = lltype.GcStruct('S', ('n', lltype.Signed)) T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) @@ -603,7 +602,6 @@ assert res == 4 * 4 def test_degenerated_before_return_2(self): - py.test.skip("arrays and structs are not working") S = lltype.GcStruct('S', ('n', lltype.Signed)) T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) @@ -656,7 +654,6 @@ assert parent.n == 3.25 def test_degenerated_via_substructure(self): - py.test.skip("arrays and structs are not working") S = lltype.GcStruct('S', ('n', lltype.Signed)) T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) From cfbolz at codespeak.net Mon Feb 4 22:38:28 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 4 Feb 2008 22:38:28 +0100 (CET) Subject: [pypy-svn] r51274 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080204213828.9209F16851F@codespeak.net> Author: cfbolz Date: Mon Feb 4 22:38:28 2008 New Revision: 51274 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: passes Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 4 22:38:28 2008 @@ -622,7 +622,6 @@ assert res == 4 * 4 def test_degenerated_at_return(self): - py.test.skip("arrays and structs are not working") S = lltype.GcStruct('S', ('n', lltype.Signed)) T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) class Result: @@ -643,15 +642,9 @@ ll_function.convert_result = glob_result.convert res = self.interpret(ll_function, [0], []) - assert res == "4" - if self.__class__ in (TestLLType, TestOOType): - assert lltype.parentlink(glob_result.s._obj) == (None, None) + assert res.n == 4 res = self.interpret(ll_function, [1], []) - assert res == "3" - if self.__class__ in (TestLLType, TestOOType): - parent, parentindex = lltype.parentlink(glob_result.s._obj) - assert parentindex == 's' - assert parent.n == 3.25 + assert res.n == 3 def test_degenerated_via_substructure(self): S = lltype.GcStruct('S', ('n', lltype.Signed)) From antocuni at codespeak.net Mon Feb 4 23:30:50 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 4 Feb 2008 23:30:50 +0100 (CET) Subject: [pypy-svn] r51275 - in pypy/dist/pypy: jit/codegen/cli jit/codegen/cli/test jit/codegen/test translator/c/test Message-ID: <20080204223050.0AEC5168512@codespeak.net> Author: antocuni Date: Mon Feb 4 23:30:49 2008 New Revision: 51275 Added: pypy/dist/pypy/jit/codegen/cli/operation.py (contents, props changed) Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py pypy/dist/pypy/jit/codegen/test/rgenop_tests.py pypy/dist/pypy/translator/c/test/test_boehm.py Log: more tests pass Added: pypy/dist/pypy/jit/codegen/cli/operation.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/jit/codegen/cli/operation.py Mon Feb 4 23:30:49 2008 @@ -0,0 +1,57 @@ +from pypy.translator.cli.dotnet import CLR +OpCodes = CLR.System.Reflection.Emit.OpCodes + +class Operation: + restype = None + _gv_res = None + + def gv_res(self): + from pypy.jit.codegen.cli.rgenop import GenLocalVar + if self._gv_res is None: + # if restype is None, assume it's the same as the first arg + t = self.restype or self.gv_x.getCliType() + loc = self.il.DeclareLocal(t) + self._gv_res = GenLocalVar(loc) + return self._gv_res + + def emit(self): + raise NotImplementedError + + +class UnaryOp(Operation): + def __init__(self, il, gv_x): + self.il = il + self.gv_x = gv_x + + +class SameAs(UnaryOp): + def emit(self): + gv_res = self.gv_res() + self.gv_x.load(self.il) + self.gv_res().store(self.il) + + +class BinaryOp(Operation): + def __init__(self, il, gv_x, gv_y): + self.il = il + self.gv_x = gv_x + self.gv_y = gv_y + + def emit(self): + self.gv_x.load(self.il) + self.gv_y.load(self.il) + self.il.Emit(self.getOpCode()) + self.gv_res().store(self.il) + + def getOpCode(self): + raise NotImplementedError + + +class Add(BinaryOp): + def getOpCode(self): + return OpCodes.Add + + +class Sub(BinaryOp): + def getOpCode(self): + return OpCodes.Sub Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/rgenop.py Mon Feb 4 23:30:49 2008 @@ -1,8 +1,9 @@ from pypy.tool.pairtype import extendabletype from pypy.rpython.ootypesystem import ootype from pypy.rlib.objectmodel import specialize -from pypy.jit.codegen.model import AbstractRGenOp, GenLabel, GenBuilder +from pypy.jit.codegen.model import AbstractRGenOp, GenBuilder from pypy.jit.codegen.model import GenVarOrConst, GenVar, GenConst, CodeGenSwitch +from pypy.jit.codegen.cli import operation as ops from pypy.translator.cli.dotnet import CLR, typeof, new_array, clidowncast System = CLR.System Utils = CLR.pypy.runtime.Utils @@ -14,8 +15,19 @@ else: assert False +def sigtoken2clitype(tok): + if tok == ([''], ''): + return typeof(CLR.pypy.runtime.DelegateType_int__int) + elif tok == (['', ''], ''): + return typeof(CLR.pypy.runtime.DelegateType_int__int_int) + else: + assert False + class __extend__(GenVarOrConst): __metaclass__ = extendabletype + + def getCliType(self): + raise NotImplementedError def load(self, il): raise NotImplementedError @@ -24,9 +36,12 @@ raise NotImplementedError class GenArgVar(GenVar): - def __init__(self, index): + def __init__(self, index, cliType): self.index = index - # XXX maybe we need to store also the type? + self.cliType = cliType + + def getCliType(self): + return self.cliType def load(self, il): if self.index == 0: @@ -50,12 +65,16 @@ def __init__(self, v): self.v = v + def getCliType(self): + return self.v.get_LocalType() + def load(self, il): il.Emit(OpCodes.Ldloc, self.v) def store(self, il): il.Emit(OpCodes.Stloc, self.v) + class IntConst(GenConst): def __init__(self, value): @@ -70,9 +89,11 @@ il.Emit(OpCodes.Ldc_I4, self.value) def __repr__(self): - "NOT_RPYTHON" return "const=%s" % self.value +SM_INT__INT = ootype.StaticMethod([ootype.Signed], ootype.Signed) +SM_INT__INT_INT = ootype.StaticMethod([ootype.Signed, ootype.Signed], ootype.Signed) + class ObjectConst(GenConst): def __init__(self, obj): @@ -80,9 +101,16 @@ @specialize.arg(1) def revealconst(self, T): - DelegateType = CLR.pypy.runtime.DelegateType_int__int # XXX use T - return clidowncast(DelegateType, self.obj) - + # XXX: probably you can't mix StaticMethod and others OOTypes + if T == SM_INT__INT: + DelegateType = CLR.pypy.runtime.DelegateType_int__int + return clidowncast(DelegateType, self.obj) + elif T == SM_INT__INT_INT: + DelegateType = CLR.pypy.runtime.DelegateType_int__int_int + return clidowncast(DelegateType, self.obj) + else: + assert isinstance(T, ootype.OOType) + return ootype.oodowncast(T, self.obj) class RCliGenOp(AbstractRGenOp): @@ -95,6 +123,8 @@ T = ootype.typeOf(llvalue) if T is ootype.Signed: return IntConst(llvalue) + elif isinstance(T, ootype.OOType): + return ObjectConst(llvalue) else: assert False, "XXX not implemented" @@ -104,10 +134,15 @@ """Return a token describing the signature of FUNCTYPE.""" # XXX: the right thing to do would be to have a way to # represent typeof(t) as a pbc - args = [repr(T) for T in FUNCTYPE.ARGS] - res = repr(FUNCTYPE.RESULT) + args = [RCliGenOp.kindToken(T) for T in FUNCTYPE.ARGS] + res = RCliGenOp.kindToken(FUNCTYPE.RESULT) return args, res + @staticmethod + @specialize.memo() + def kindToken(T): + return repr(T) + def newgraph(self, sigtoken, name): argtoks, restok = sigtoken args = new_array(System.Type, len(argtoks)) @@ -127,26 +162,35 @@ self.il = self.meth.GetILGenerator() self.inputargs_gv = [] for i in range(len(args)): - self.inputargs_gv.append(GenArgVar(i)) + self.inputargs_gv.append(GenArgVar(i, args[i])) self.gv_entrypoint = ObjectConst(None) # XXX? @specialize.arg(1) def genop2(self, opname, gv_arg1, gv_arg2): - assert opname == 'int_add' - res = self.il.DeclareLocal(typeof(System.Int32)) - gv_res = GenLocalVar(res) - gv_arg1.load(self.il) - gv_arg2.load(self.il) - self.il.Emit(OpCodes.Add) - gv_res.store(self.il) - return gv_res + if opname == 'int_add': + op = ops.Add(self.il, gv_arg1, gv_arg2) + elif opname == 'int_sub': + op = ops.Sub(self.il, gv_arg1, gv_arg2) + else: + assert False + op.emit() + return op.gv_res() def finish_and_return(self, sigtoken, gv_returnvar): gv_returnvar.load(self.il) self.il.Emit(OpCodes.Ret) - DelegateType = CLR.pypy.runtime.DelegateType_int__int # XXX use sigtoken - myfunc = self.meth.CreateDelegate(typeof(DelegateType)) + delegate_type = sigtoken2clitype(sigtoken) + myfunc = self.meth.CreateDelegate(delegate_type) self.gv_entrypoint.obj = myfunc def end(self): pass + + def enter_next_block(self, kinds, args_gv): + for i in range(len(args_gv)): + op = ops.SameAs(self.il, args_gv[i]) + op.emit() + args_gv[i] = op.gv_res() + lbl = self.il.DefineLabel() + self.il.MarkLabel(lbl) + return lbl Modified: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Mon Feb 4 23:30:49 2008 @@ -4,8 +4,23 @@ from pypy.jit.codegen.test.rgenop_tests import AbstractRGenOpTests, OOType from pypy.translator.cli.test.runtest import compile_function -# test disabled, only two pass -class xTestRCliGenop(AbstractRGenOpTests): +passing = set() +def fn(): + prefixes = [ + 'test_adder', + 'test_dummy', + 'test_hide_and_reveal', + 'test_hide_and_reveal_p', + ] + + for p in prefixes: + passing.add(p) + passing.add(p + '_direct') + passing.add(p + '_compile') +fn() +del fn + +class TestRCliGenop(AbstractRGenOpTests): RGenOp = RCliGenOp T = OOType @@ -26,3 +41,10 @@ def directtesthelper(self, FUNCTYPE, func): py.test.skip('???') + def __getattribute__(self, name): + if name.startswith('test_') and name not in passing: + def fn(): + py.test.skip("doesn't work yet") + return fn + else: + return object.__getattribute__(self, name) Modified: pypy/dist/pypy/jit/codegen/test/rgenop_tests.py ============================================================================== --- pypy/dist/pypy/jit/codegen/test/rgenop_tests.py (original) +++ pypy/dist/pypy/jit/codegen/test/rgenop_tests.py Mon Feb 4 23:30:49 2008 @@ -20,8 +20,17 @@ FUNC100 = lltype.FuncType([lltype.Signed]*100, lltype.Signed) @staticmethod - def Ptr(FUNC): - return lltype.Ptr(FUNC) + def Ptr(T): + return lltype.Ptr(T) + + @staticmethod + def Struct(name, *fields): + S = lltype.GcStruct(name, *fields) + return S + + @staticmethod + def malloc(S): + return lltype.malloc(S) class OOType(object): @@ -33,8 +42,17 @@ FUNC100 = ootype.StaticMethod([lltype.Signed]*100, lltype.Signed) @staticmethod - def Ptr(FUNC): - return FUNC + def Ptr(T): + return T + + @staticmethod + def Struct(name, *fields): + I = ootype.Instance(name, ootype.ROOT, dict(fields)) + return I + + @staticmethod + def malloc(I): + return ootype.new(I) def make_adder(T, rgenop, n): @@ -944,11 +962,11 @@ def test_hide_and_reveal_p(self): RGenOp = self.RGenOp - S = lltype.GcStruct('s', ('x', lltype.Signed)) + S = self.T.Struct('s', ('x', lltype.Signed)) S_PTR = self.T.Ptr(S) - s1 = lltype.malloc(S) + s1 = self.T.malloc(S) s1.x = 8111 - s2 = lltype.malloc(S) + s2 = self.T.malloc(S) s2.x = 8222 def hide_and_reveal_p(n): rgenop = RGenOp() Modified: pypy/dist/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_boehm.py (original) +++ pypy/dist/pypy/translator/c/test/test_boehm.py Mon Feb 4 23:30:49 2008 @@ -9,7 +9,7 @@ if not check_boehm_presence(): py.test.skip("Boehm GC not present") -class AbstractGCTestClass: +class AbstractGCTestClass(object): gcpolicy = "boehm" stacklessgc = False From antocuni at codespeak.net Mon Feb 4 23:47:51 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 4 Feb 2008 23:47:51 +0100 (CET) Subject: [pypy-svn] r51276 - pypy/dist/pypy/translator/cli/test Message-ID: <20080204224751.AC4C2168501@codespeak.net> Author: antocuni Date: Mon Feb 4 23:47:51 2008 New Revision: 51276 Modified: pypy/dist/pypy/translator/cli/test/runtest.py pypy/dist/pypy/translator/cli/test/test_runtest.py Log: don't fail when having more than 8 arguments Modified: pypy/dist/pypy/translator/cli/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/runtest.py (original) +++ pypy/dist/pypy/translator/cli/test/runtest.py Mon Feb 4 23:47:51 2008 @@ -67,7 +67,7 @@ # convert string arguments to their true type for i, arg in enumerate(self.graph.getargs()): ilasm.opcode('ldarg.0') - ilasm.opcode('ldc.i4.%d' % i) + ilasm.opcode('ldc.i4 %d' % i) ilasm.opcode('ldelem.ref') arg_type, arg_var = self.cts.llvar_to_cts(arg) ilasm.call('%s class [mscorlib]System.Convert::%s(string)' % Modified: pypy/dist/pypy/translator/cli/test/test_runtest.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_runtest.py (original) +++ pypy/dist/pypy/translator/cli/test/test_runtest.py Mon Feb 4 23:47:51 2008 @@ -9,3 +9,10 @@ raise ValueError f = self._compile(fn, [], auto_raise_exc=True) py.test.raises(ValueError, f) + + def test_big_arglist(self): + def fn(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9): + return a0 + res = self.interpret(fn, [42]*10) + assert res == 42 + From cfbolz at codespeak.net Mon Feb 4 23:51:26 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 4 Feb 2008 23:51:26 +0100 (CET) Subject: [pypy-svn] r51277 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter Message-ID: <20080204225126.8630B1684FD@codespeak.net> Author: cfbolz Date: Mon Feb 4 23:51:25 2008 New Revision: 51277 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rcontainer.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Log: first getarrayitem test passing Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 4 23:51:25 2008 @@ -42,6 +42,7 @@ self.keydescs = [] self.structtypedescs = [] self.fielddescs = [] + self.arrayfielddescs = [] self.called_bytecodes = [] self.num_mergepoints = 0 self.graph_color = self.graph_calling_color(graph) @@ -65,6 +66,8 @@ self.structtypedesc_positions = {} # mapping tuples of STRUCT, name to index self.fielddesc_positions = {} + # mapping ARRAYS to index + self.arrayfielddesc_positions = {} # mapping graphs to index self.graph_positions = {} # mapping fnobjs to index @@ -83,6 +86,7 @@ self.keydescs, self.structtypedescs, self.fielddescs, + self.arrayfielddescs, self.called_bytecodes, self.num_mergepoints, self.graph_color, @@ -335,6 +339,15 @@ self.fielddesc_positions[fieldname, TYPE] = result return result + def arrayfielddesc_position(self, TYPE): + if TYPE in self.fielddesc_positions: + return self.fielddesc_positions[TYPE] + arrayfielddesc = rcontainer.ArrayFieldDesc(self.RGenOp, TYPE) + result = len(self.arrayfielddescs) + self.arrayfielddescs.append(arrayfielddesc) + self.arrayfielddesc_positions[TYPE] = result + return result + def graph_position(self, graph): if graph in self.graph_positions: return self.graph_positions[graph] @@ -541,6 +554,18 @@ return self.emit("red_setfield", destboxindex, fielddescindex, valboxindex) + def serialize_op_getarrayitem(self, op): + arrayvar, indexvar = op.args + PTRTYPE = arrayvar.concretetype + if PTRTYPE.TO.OF is lltype.Void: + return + deepfrozen = self.hannotator.binding(arrayvar).deepfrozen + fielddescindex = self.arrayfielddesc_position(PTRTYPE.TO) + arrayindex = self.serialize_oparg("red", arrayvar) + index = self.serialize_oparg("red", indexvar) + self.emit("red_getarrayitem", arrayindex, fielddescindex, index, + deepfrozen) + self.register_redvar(op.result) # call handling Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Mon Feb 4 23:51:25 2008 @@ -18,8 +18,8 @@ """ def __init__(self, name, code, constants, typekinds, redboxclasses, - keydescs, structtypedescs, fielddescs, called_bytecodes, - num_mergepoints, graph_color, + keydescs, structtypedescs, fielddescs, arrayfielddescs, + called_bytecodes, num_mergepoints, graph_color, nonrainbow_functions, is_portal): self.name = name self.code = code @@ -29,6 +29,7 @@ self.keydescs = keydescs self.structtypedescs = structtypedescs self.fielddescs = fielddescs + self.arrayfielddescs = arrayfielddescs self.called_bytecodes = called_bytecodes self.num_mergepoints = num_mergepoints self.graph_color = graph_color @@ -355,6 +356,14 @@ resbox = rtimeshift.gensetfield(self.jitstate, fielddesc, destbox, valuebox) + def opimpl_red_getarrayitem(self): + arraybox = self.get_redarg() + fielddesc = self.frame.bytecode.arrayfielddescs[self.load_2byte()] + indexbox = self.get_redarg() + deepfrozen = self.load_bool() + resbox = rtimeshift.gengetarrayitem(self.jitstate, deepfrozen, fielddesc, + arraybox, indexbox) + self.red_result(resbox) # ____________________________________________________________ # construction-time interface Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 4 23:51:25 2008 @@ -525,7 +525,6 @@ self.check_insns({}) def test_simple_array(self): - py.test.skip("arrays and structs are not working") A = lltype.GcArray(lltype.Signed, hints={'immutable': True}) def ll_function(a): Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rcontainer.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rcontainer.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rcontainer.py Mon Feb 4 23:51:25 2008 @@ -603,6 +603,13 @@ self.varsizealloctoken = RGenOp.varsizeAllocToken(TYPE) self.indexkind = RGenOp.kindToken(lltype.Signed) + def getarrayitem_if_non_null(jitstate, genvar, gv_index): + array = genvar.revealconst(self.PTRTYPE) + index = gv_index.revealconst(lltype.Signed) + if array and 0 <= index < len(array): # else don't constant-fold + res = array[index] + return rvalue.ll_gv_fromvalue(jitstate, res) + self.getarrayitem_if_non_null = getarrayitem_if_non_null # ____________________________________________________________ class FrozenVirtualStruct(FrozenContainer): Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Mon Feb 4 23:51:25 2008 @@ -159,14 +159,14 @@ return rvalue.ll_fromvalue(jitstate, res) return argbox.op_getsubstruct(jitstate, fielddesc) -def ll_gengetarrayitem(jitstate, deepfrozen, fielddesc, argbox, indexbox): +def gengetarrayitem(jitstate, deepfrozen, fielddesc, argbox, indexbox): if ((fielddesc.immutable or deepfrozen) and argbox.is_constant() and indexbox.is_constant()): - array = rvalue.ll_getvalue(argbox, fielddesc.PTRTYPE) - index = rvalue.ll_getvalue(indexbox, lltype.Signed) - if array and 0 <= index < len(array): # else don't constant-fold - res = array[index] - return rvalue.ll_fromvalue(jitstate, res) + resgv = fielddesc.getarrayitem_if_non_null( + jitstate, argbox.getgenvar(jitstate), + indexbox.getgenvar(jitstate)) + if resgv is not None: + return fielddesc.makebox(jitstate, resgv) genvar = jitstate.curbuilder.genop_getarrayitem( fielddesc.arraytoken, argbox.getgenvar(jitstate), @@ -197,7 +197,7 @@ valuebox.getgenvar(jitstate) ) -def ll_gengetarraysize(jitstate, fielddesc, argbox): +def gengetarraysize(jitstate, fielddesc, argbox): if argbox.is_constant(): array = rvalue.ll_getvalue(argbox, fielddesc.PTRTYPE) if array: # else don't constant-fold the segfault... From cfbolz at codespeak.net Tue Feb 5 00:03:08 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 5 Feb 2008 00:03:08 +0100 (CET) Subject: [pypy-svn] r51279 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080204230308.33283168512@codespeak.net> Author: cfbolz Date: Tue Feb 5 00:03:07 2008 New Revision: 51279 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: setarrayitem works too Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Tue Feb 5 00:03:07 2008 @@ -567,6 +567,22 @@ deepfrozen) self.register_redvar(op.result) + def serialize_op_setarrayitem(self, op): + args = op.args + PTRTYPE = args[0].concretetype + VALUETYPE = PTRTYPE.TO.OF + if VALUETYPE is lltype.Void: + return + destboxindex = self.serialize_oparg("red", args[0]) + indexboxindex = self.serialize_oparg("red", args[1]) + valboxindex = self.serialize_oparg("red", args[2]) + fieldname = args[1].value + fielddescindex = self.arrayfielddesc_position(PTRTYPE.TO) + if fielddescindex == -1: # Void field + return + self.emit("red_setarrayitem", destboxindex, fielddescindex, + indexboxindex, valboxindex) + # call handling def graphs_from(self, spaceop): Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Tue Feb 5 00:03:07 2008 @@ -364,6 +364,15 @@ resbox = rtimeshift.gengetarrayitem(self.jitstate, deepfrozen, fielddesc, arraybox, indexbox) self.red_result(resbox) + + + def opimpl_red_setarrayitem(self): + destbox = self.get_redarg() + fielddesc = self.frame.bytecode.arrayfielddescs[self.load_2byte()] + indexbox = self.get_redarg() + valuebox = self.get_redarg() + resbox = rtimeshift.gensetarrayitem(self.jitstate, fielddesc, destbox, + indexbox, valuebox) # ____________________________________________________________ # construction-time interface Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Tue Feb 5 00:03:07 2008 @@ -546,8 +546,6 @@ assert res == 24 self.check_insns({}) - - def test_simple_struct_malloc(self): py.test.skip("blue containers: to be reimplemented") S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), @@ -711,7 +709,6 @@ def test_setarrayitem(self): - py.test.skip("arrays and structs are not working") A = lltype.GcArray(lltype.Signed) a = lltype.malloc(A, 2, immortal=True) def ll_function(): @@ -719,7 +716,7 @@ a[1] = 2 return a[0]+a[1] - res = self.interpret(ll_function, [], [], policy=P_NOVIRTUAL) + res = self.interpret(ll_function, [], []) assert res == 3 def test_red_array(self): From cfbolz at codespeak.net Tue Feb 5 00:23:28 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 5 Feb 2008 00:23:28 +0100 (CET) Subject: [pypy-svn] r51280 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter Message-ID: <20080204232328.B615E16850C@codespeak.net> Author: cfbolz Date: Tue Feb 5 00:23:28 2008 New Revision: 51280 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Log: start supporting malloc_varsize Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Tue Feb 5 00:23:28 2008 @@ -481,6 +481,21 @@ self.emit("red_malloc", index) self.register_redvar(op.result) + def serialize_op_malloc_varsize(self, op): + + PTRTYPE = op.result.concretetype + TYPE = PTRTYPE.TO + v_size = op.args[2] + sizeindex = self.serialize_oparg("red", v_size) + if isinstance(TYPE, lltype.Struct): + index = self.structtypedesc_position(op.args[0].value) + self.emit("red_malloc_varsize_struct") + else: + index = self.arrayfielddesc_position(TYPE) + self.emit("red_malloc_varsize_array") + self.emit(index, sizeindex) + self.register_redvar(op.result) + def serialize_op_zero_gc_pointers_inside(self, op): pass # XXX is that right? Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Tue Feb 5 00:23:28 2008 @@ -341,6 +341,20 @@ redbox = rcontainer.create(self.jitstate, structtypedesc) self.red_result(redbox) + def opimpl_red_malloc_varsize_struct(self): + structtypedesc = self.frame.bytecode.structtypedescs[self.load_2byte()] + sizebox = self.get_redarg() + redbox = rcontainer.create_varsize(self.jitstate, structtypedesc, + sizebox) + self.red_result(redbox) + + def opimpl_red_malloc_varsize_array(self): + arraytypedesc = self.frame.bytecode.arrayfielddescs[self.load_2byte()] + sizebox = self.get_redarg() + redbox = rtimeshift.genmalloc_varsize(self.jitstate, arraytypedesc, + sizebox) + self.red_result(redbox) + def opimpl_red_getfield(self): structbox = self.get_redarg() fielddesc = self.frame.bytecode.fielddescs[self.load_2byte()] @@ -365,7 +379,6 @@ arraybox, indexbox) self.red_result(resbox) - def opimpl_red_setarrayitem(self): destbox = self.get_redarg() fielddesc = self.frame.bytecode.arrayfielddescs[self.load_2byte()] @@ -373,6 +386,7 @@ valuebox = self.get_redarg() resbox = rtimeshift.gensetarrayitem(self.jitstate, fielddesc, destbox, indexbox, valuebox) + # ____________________________________________________________ # construction-time interface Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Tue Feb 5 00:23:28 2008 @@ -791,7 +791,6 @@ getinteriorarraysize=1) def test_array_of_voids(self): - py.test.skip("arrays and structs are not working") A = lltype.GcArray(lltype.Void) def ll_function(n): a = lltype.malloc(A, 3) @@ -800,10 +799,9 @@ res = a, b keepalive_until_here(b) # to keep getarrayitem around return res - ll_function.convert_result = lambda x: str(len(x.item0)) - res = self.interpret(ll_function, [2], [], policy=P_NOVIRTUAL) - assert res == "3" + res = self.interpret(ll_function, [2], []) + assert len(res.item0) == 3 def test_red_propagate(self): S = lltype.GcStruct('S', ('n', lltype.Signed)) Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Tue Feb 5 00:23:28 2008 @@ -189,7 +189,7 @@ return fielddesc.makebox(jitstate, genvar) -def ll_gensetarrayitem(jitstate, fielddesc, destbox, indexbox, valuebox): +def gensetarrayitem(jitstate, fielddesc, destbox, indexbox, valuebox): genvar = jitstate.curbuilder.genop_setarrayitem( fielddesc.arraytoken, destbox.getgenvar(jitstate), From cfbolz at codespeak.net Tue Feb 5 01:47:50 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 5 Feb 2008 01:47:50 +0100 (CET) Subject: [pypy-svn] r51282 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter Message-ID: <20080205004750.A1C05168439@codespeak.net> Author: cfbolz Date: Tue Feb 5 01:47:50 2008 New Revision: 51282 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/hrtyper.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rcontainer.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Log: support for getarraysize Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Tue Feb 5 01:47:50 2008 @@ -43,6 +43,7 @@ self.structtypedescs = [] self.fielddescs = [] self.arrayfielddescs = [] + self.interiordescs = [] self.called_bytecodes = [] self.num_mergepoints = 0 self.graph_color = self.graph_calling_color(graph) @@ -87,6 +88,7 @@ self.structtypedescs, self.fielddescs, self.arrayfielddescs, + self.interiordescs, self.called_bytecodes, self.num_mergepoints, self.graph_color, @@ -384,6 +386,33 @@ self.nonrainbow_functions.append(call_normal_function) self.nonrainbow_positions[fn] = result return result + + def interiordesc(self, op, PTRTYPE, nb_offsets): + path = [] + CONTAINER = PTRTYPE.TO + indices_v = [] + for i in range(1, 1 + nb_offsets): + varg = op.args[i] + T = varg.concretetype + if T is lltype.Void: + fieldname = varg.value + CONTAINER = getattr(CONTAINER, fieldname) + path.append(fieldname) + else: + assert T is lltype.Signed + CONTAINER = CONTAINER.OF + path.append(None) # placeholder for 'array index' + indices_v.append(varg) + if CONTAINER is lltype.Void: # Void field + return -1, None + else: + key = (PTRTYPE.TO, tuple(path)) + if key in self.interiordesc_positions: + return self.interiordesc_positions[key] + desc = rcontainer.InteriorDesc(self.RGenOp, PTRTYPE.TO, tuple(path)), + result = len(self.interiordescs) + self.interiordescs.append(desc) + return (result, indices_v) def emit(self, *stuff): assert stuff is not None @@ -482,7 +511,6 @@ self.register_redvar(op.result) def serialize_op_malloc_varsize(self, op): - PTRTYPE = op.result.concretetype TYPE = PTRTYPE.TO v_size = op.args[2] @@ -598,6 +626,16 @@ self.emit("red_setarrayitem", destboxindex, fielddescindex, indexboxindex, valboxindex) + def serialize_op_getarraysize(self, op): + arrayvar, = op.args + PTRTYPE = arrayvar.concretetype + if PTRTYPE.TO.OF is lltype.Void: + return + fielddescindex = self.arrayfielddesc_position(PTRTYPE.TO) + arrayindex = self.serialize_oparg("red", arrayvar) + self.emit("red_getarraysize", arrayindex, fielddescindex) + self.register_redvar(op.result) + # call handling def graphs_from(self, spaceop): Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Tue Feb 5 01:47:50 2008 @@ -19,7 +19,7 @@ def __init__(self, name, code, constants, typekinds, redboxclasses, keydescs, structtypedescs, fielddescs, arrayfielddescs, - called_bytecodes, num_mergepoints, graph_color, + interiordescs, called_bytecodes, num_mergepoints, graph_color, nonrainbow_functions, is_portal): self.name = name self.code = code @@ -30,6 +30,7 @@ self.structtypedescs = structtypedescs self.fielddescs = fielddescs self.arrayfielddescs = arrayfielddescs + self.interiordescs = interiordescs self.called_bytecodes = called_bytecodes self.num_mergepoints = num_mergepoints self.graph_color = graph_color @@ -387,6 +388,12 @@ resbox = rtimeshift.gensetarrayitem(self.jitstate, fielddesc, destbox, indexbox, valuebox) + def opimpl_red_getarraysize(self): + arraybox = self.get_redarg() + fielddesc = self.frame.bytecode.arrayfielddescs[self.load_2byte()] + resbox = rtimeshift.gengetarraysize(self.jitstate, fielddesc, arraybox) + self.red_result(resbox) + # ____________________________________________________________ # construction-time interface Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Tue Feb 5 01:47:50 2008 @@ -546,6 +546,28 @@ assert res == 24 self.check_insns({}) + def test_arraysize(self): + A = lltype.GcArray(lltype.Signed) + def ll_function(a): + return len(a) + + def int_array(string): + items = [int(x) for x in string.split(',')] + n = len(items) + a1 = lltype.malloc(A, n) + for i in range(n): + a1[i] = items[i] + return a1 + ll_function.convert_arguments = [int_array] + + res = self.interpret(ll_function, ["6,7"], []) + assert res == 2 + self.check_insns({'getarraysize': 1}) + res = self.interpret(ll_function, ["8,3,3,4,5"], [0]) + assert res == 5 + self.check_insns({}) + + def test_simple_struct_malloc(self): py.test.skip("blue containers: to be reimplemented") S = lltype.GcStruct('helloworld', ('hello', lltype.Signed), Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/hrtyper.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/hrtyper.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/hrtyper.py Tue Feb 5 01:47:50 2008 @@ -880,27 +880,6 @@ ts.s_RedBox) - def _getinteriordesc(self, hop, PTRTYPE, nb_offsets): - path = [] - CONTAINER = PTRTYPE.TO - indices_v = [] - for i in range(1, 1 + nb_offsets): - T = originalconcretetype(hop.args_s[i]) - if T is lltype.Void: - fieldname = hop.args_v[i].value - CONTAINER = getattr(CONTAINER, fieldname) - path.append(fieldname) - else: - assert T is lltype.Signed - CONTAINER = CONTAINER.OF - path.append(None) # placeholder for 'array index' - v_index = hop.inputarg(self.getredrepr(lltype.Signed), arg=i) - indices_v.append(v_index) - if CONTAINER is lltype.Void: # Void field - return None, None - else: - return (rcontainer.InteriorDesc(self, PTRTYPE.TO, tuple(path)), - indices_v) def translate_op_getinteriorfield(self, hop): ts = self Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rcontainer.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rcontainer.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rcontainer.py Tue Feb 5 01:47:50 2008 @@ -610,6 +610,12 @@ res = array[index] return rvalue.ll_gv_fromvalue(jitstate, res) self.getarrayitem_if_non_null = getarrayitem_if_non_null + def getarraysize_if_non_null(jitstate, genvar): + array = genvar.revealconst(self.PTRTYPE) + if array: # else don't constant-fold + res = len(array) + return rvalue.ll_gv_fromvalue(jitstate, res) + self.getarraysize_if_non_null = getarraysize_if_non_null # ____________________________________________________________ class FrozenVirtualStruct(FrozenContainer): Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Tue Feb 5 01:47:50 2008 @@ -199,10 +199,10 @@ def gengetarraysize(jitstate, fielddesc, argbox): if argbox.is_constant(): - array = rvalue.ll_getvalue(argbox, fielddesc.PTRTYPE) - if array: # else don't constant-fold the segfault... - res = len(array) - return rvalue.ll_fromvalue(jitstate, res) + resgv = fielddesc.getarraysize_if_non_null( + jitstate, argbox.getgenvar(jitstate)) + if resgv is not None: + return fielddesc.makebox(jitstate, resgv) genvar = jitstate.curbuilder.genop_getarraysize( fielddesc.arraytoken, argbox.getgenvar(jitstate)) From cfbolz at codespeak.net Tue Feb 5 12:04:09 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 5 Feb 2008 12:04:09 +0100 (CET) Subject: [pypy-svn] r51284 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter Message-ID: <20080205110409.91DA01683F1@codespeak.net> Author: cfbolz Date: Tue Feb 5 12:04:07 2008 New Revision: 51284 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rcontainer.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Log: support for red get/setinteriorfield and getinteriorarraysize Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Tue Feb 5 12:04:07 2008 @@ -69,6 +69,8 @@ self.fielddesc_positions = {} # mapping ARRAYS to index self.arrayfielddesc_positions = {} + # mapping (TYPE, path) to index + self.interiordesc_positions = {} # mapping graphs to index self.graph_positions = {} # mapping fnobjs to index @@ -409,7 +411,7 @@ key = (PTRTYPE.TO, tuple(path)) if key in self.interiordesc_positions: return self.interiordesc_positions[key] - desc = rcontainer.InteriorDesc(self.RGenOp, PTRTYPE.TO, tuple(path)), + desc = rcontainer.InteriorDesc(self.RGenOp, PTRTYPE.TO, tuple(path)) result = len(self.interiordescs) self.interiordescs.append(desc) return (result, indices_v) @@ -466,6 +468,9 @@ result.append(self.serialize_oparg(color, v)) return result + def serialize_op_debug_assert(self, op): + pass + def serialize_op_direct_call(self, op): targets = dict(self.graphs_from(op)) assert len(targets) == 1 @@ -636,6 +641,67 @@ self.emit("red_getarraysize", arrayindex, fielddescindex) self.register_redvar(op.result) + def serialize_op_getinteriorfield(self, op): + structvar = op.args[0] + PTRTYPE = structvar.concretetype + # no virtualizable access read here + assert not PTRTYPE.TO._hints.get('virtualizable', False) + + # non virtual case + interiordescindex, indices_v = self.interiordesc( + op, PTRTYPE, len(op.args) - 1) + if interiordescindex == -1: # Void field + return None + structindex = self.serialize_oparg("red", structvar) + deepfrozen = self.hannotator.binding(structvar).deepfrozen + indexes = [] + for arg in indices_v: + indexes.append(self.serialize_oparg("red", arg)) + self.emit("red_getinteriorfield", structindex, interiordescindex, + deepfrozen) + self.emit(len(indexes)) + self.emit(*indexes) + self.register_redvar(op.result) + + def serialize_op_setinteriorfield(self, op): + structvar = op.args[0] + valuevar = op.args[-1] + PTRTYPE = structvar.concretetype + # non virtual case + interiordescindex, indices_v = self.interiordesc( + op, PTRTYPE, len(op.args) - 2) + structindex = self.serialize_oparg("red", structvar) + indexes = [] + for arg in indices_v: + indexes.append(self.serialize_oparg("red", arg)) + valueindex = self.serialize_oparg("red", valuevar) + self.emit("red_setinteriorfield", structindex, interiordescindex) + self.emit(len(indexes)) + self.emit(*indexes) + self.emit(valueindex) + + def serialize_op_getinteriorarraysize(self, op): + structvar = op.args[0] + PTRTYPE = structvar.concretetype + color = self.opcolor(op) + # non virtual case + interiordescindex, indices_v = self.interiordesc( + op, PTRTYPE, len(op.args) - 1) + assert interiordescindex != -1 + structindex = self.serialize_oparg(color, structvar) + indexes = [] + for arg in indices_v: + indexes.append(self.serialize_oparg(color, arg)) + self.emit("%s_getinteriorarraysize" % color, structindex, + interiordescindex) + self.emit(len(indexes)) + self.emit(*indexes) + if color == "red": + self.register_redvar(op.result) + else: + self.register_greenvar(op.result) + + # call handling def graphs_from(self, spaceop): Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Tue Feb 5 12:04:07 2008 @@ -394,6 +394,37 @@ resbox = rtimeshift.gengetarraysize(self.jitstate, fielddesc, arraybox) self.red_result(resbox) + def opimpl_red_getinteriorfield(self): + structbox = self.get_redarg() + interiordesc = self.frame.bytecode.interiordescs[self.load_2byte()] + deepfrozen = self.load_bool() + indexboxes = self.get_red_varargs() + resultbox = interiordesc.gengetinteriorfield(self.jitstate, deepfrozen, + structbox, indexboxes) + self.red_result(resultbox) + + def opimpl_red_setinteriorfield(self): + destbox = self.get_redarg() + interiordesc = self.frame.bytecode.interiordescs[self.load_2byte()] + indexboxes = self.get_red_varargs() + valuebox = self.get_redarg() + interiordesc.gensetinteriorfield(self.jitstate, destbox, valuebox, indexboxes) + + def opimpl_red_getinteriorarraysize(self): + arraybox = self.get_redarg() + interiordesc = self.frame.bytecode.interiordescs[self.load_2byte()] + indexboxes = self.get_red_varargs() + resultbox = interiordesc.gengetinteriorarraysize( + self.jitstate, arraybox, indexboxes) + self.red_result(resultbox) + + def opimpl_green_getinteriorarraysize(self): + arraygenconst = self.get_greenarg() + interiordesc = self.frame.bytecode.interiordescs[self.load_2byte()] + indexgenconsts = self.get_green_varargs() + resultbox = interiordesc.gengetinteriorarraysize( + self.jitstate, arraybox, indexboxes) + self.red_result(resultbox) # ____________________________________________________________ # construction-time interface Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Tue Feb 5 12:04:07 2008 @@ -11,6 +11,7 @@ from pypy.jit.timeshifter import rtimeshift, rvalue from pypy.rpython.lltypesystem import lltype, rstr from pypy.rpython.llinterp import LLInterpreter +from pypy.rpython.module.support import LLSupport from pypy.annotation import model as annmodel from pypy.objspace.flow.model import summary from pypy.rlib.jit import hint @@ -712,7 +713,7 @@ pc += 1 return acc ll_plus_minus.convert_arguments = [LLSupport.to_rstr, int, int] - res = self.interpret(ll_plus_minus, ["+-+", 0, 2], [0], inline=100000) + res = self.interpret(ll_plus_minus, ["+-+", 0, 2], [0]) assert res == ll_plus_minus("+-+", 0, 2) self.check_insns({'int_add': 2, 'int_sub': 1}) @@ -742,7 +743,6 @@ assert res == 3 def test_red_array(self): - py.test.skip("arrays and structs are not working") A = lltype.GcArray(lltype.Signed) def ll_function(x, y, n): a = lltype.malloc(A, 2) @@ -750,22 +750,19 @@ a[1] = y return a[n]*len(a) - res = self.interpret(ll_function, [21, -21, 0], [], - policy=P_NOVIRTUAL) + res = self.interpret(ll_function, [21, -21, 0], []) assert res == 42 self.check_insns({'malloc_varsize': 1, 'setarrayitem': 2, 'getarrayitem': 1, 'getarraysize': 1, 'int_mul': 1}) - res = self.interpret(ll_function, [21, -21, 1], [], - policy=P_NOVIRTUAL) + res = self.interpret(ll_function, [21, -21, 1], []) assert res == -42 self.check_insns({'malloc_varsize': 1, 'setarrayitem': 2, 'getarrayitem': 1, 'getarraysize': 1, 'int_mul': 1}) def test_red_struct_array(self): - py.test.skip("arrays and structs are not working") S = lltype.Struct('s', ('x', lltype.Signed)) A = lltype.GcArray(S) def ll_function(x, y, n): @@ -774,15 +771,13 @@ a[1].x = y return a[n].x*len(a) - res = self.interpret(ll_function, [21, -21, 0], [], - policy=P_NOVIRTUAL) + res = self.interpret(ll_function, [21, -21, 0], []) assert res == 42 self.check_insns({'malloc_varsize': 1, 'setinteriorfield': 2, 'getinteriorfield': 1, 'getarraysize': 1, 'int_mul': 1}) - res = self.interpret(ll_function, [21, -21, 1], [], - policy=P_NOVIRTUAL) + res = self.interpret(ll_function, [21, -21, 1], []) assert res == -42 self.check_insns({'malloc_varsize': 1, 'setinteriorfield': 2, 'getinteriorfield': 1, @@ -790,7 +785,6 @@ def test_red_varsized_struct(self): - py.test.skip("arrays and structs are not working") A = lltype.Array(lltype.Signed) S = lltype.GcStruct('S', ('foo', lltype.Signed), ('a', A)) def ll_function(x, y, n): @@ -800,14 +794,12 @@ s.a[1] = y return s.a[n]*s.foo - res = self.interpret(ll_function, [21, -21, 0], [], - policy=P_NOVIRTUAL) + res = self.interpret(ll_function, [21, -21, 0], []) assert res == 42 self.check_insns(malloc_varsize=1, getinteriorarraysize=1) - res = self.interpret(ll_function, [21, -21, 1], [], - policy=P_NOVIRTUAL) + res = self.interpret(ll_function, [21, -21, 1], []) assert res == -42 self.check_insns(malloc_varsize=1, getinteriorarraysize=1) Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rcontainer.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rcontainer.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rcontainer.py Tue Feb 5 12:04:07 2008 @@ -375,7 +375,7 @@ immutable = LASTCONTAINER._hints.get('immutable', False) getinterior_initial = make_interior_getter(fielddescs[:-1]) - def gengetinteriorfield(jitstate, deepfrozen, argbox, *indexboxes): + def gengetinteriorfield(jitstate, deepfrozen, argbox, indexboxes): if (immutable or deepfrozen) and argbox.is_constant(): ptr = rvalue.ll_getvalue(argbox, PTRTYPE) if ptr: # else don't constant-fold the segfault... @@ -395,7 +395,7 @@ # constant-folding: success assert i == len(indexboxes) return rvalue.ll_fromvalue(jitstate, ptr) - argbox = getinterior_initial(jitstate, argbox, *indexboxes) + argbox = getinterior_initial(jitstate, argbox, indexboxes) if lastoffset is None: # getarrayitem indexbox = indexboxes[-1] genvar = jitstate.curbuilder.genop_getarrayitem( @@ -406,8 +406,8 @@ else: # getfield return argbox.op_getfield(jitstate, lastfielddesc) - def gensetinteriorfield(jitstate, destbox, valuebox, *indexboxes): - destbox = getinterior_initial(jitstate, destbox, *indexboxes) + def gensetinteriorfield(jitstate, destbox, valuebox, indexboxes): + destbox = getinterior_initial(jitstate, destbox, indexboxes) if lastoffset is None: # setarrayitem indexbox = indexboxes[-1] genvar = jitstate.curbuilder.genop_setarrayitem( @@ -427,7 +427,7 @@ arrayfielddesc = ArrayFieldDesc(RGenOp, TYPE) getinterior_all = make_interior_getter(fielddescs) - def gengetinteriorarraysize(jitstate, argbox, *indexboxes): + def gengetinteriorarraysize(jitstate, argbox, indexboxes): if argbox.is_constant(): ptr = rvalue.ll_getvalue(argbox, PTRTYPE) if ptr: # else don't constant-fold the segfault... @@ -447,7 +447,7 @@ # constant-folding: success assert i == len(indexboxes) return rvalue.ll_fromvalue(jitstate, len(ptr)) - argbox = getinterior_all(jitstate, argbox, *indexboxes) + argbox = getinterior_all(jitstate, argbox, indexboxes) genvar = jitstate.curbuilder.genop_getarraysize( arrayfielddesc.arraytoken, argbox.getgenvar(jitstate)) @@ -460,7 +460,7 @@ def make_interior_getter(fielddescs, _cache={}): - # returns a 'getinterior(jitstate, argbox, *indexboxes)' function + # returns a 'getinterior(jitstate, argbox, indexboxes)' function key = tuple(fielddescs) try: return _cache[key] @@ -469,7 +469,7 @@ (fielddesc, isinstance(fielddesc, ArrayFieldDesc)) for fielddesc in fielddescs]) - def getinterior(jitstate, argbox, *indexboxes): + def getinterior(jitstate, argbox, indexboxes): i = 0 for fielddesc, is_array in unroll_fielddescs: if is_array: # array substruct Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Tue Feb 5 12:04:07 2008 @@ -209,17 +209,6 @@ return rvalue.IntRedBox(fielddesc.indexkind, genvar) -def ll_gengetinteriorfield(jitstate, deepfrozen, interiordesc, - argbox, *indexboxes): - return interiordesc.gengetinteriorfield(jitstate, deepfrozen, - argbox, *indexboxes) - -def ll_gensetinteriorfield(jitstate, interiordesc, destbox, - valuebox, *indexboxes): - interiordesc.gensetinteriorfield(jitstate, destbox, valuebox, *indexboxes) - -def ll_gengetinteriorarraysize(jitstate, interiordesc, argbox, *indexboxes): - return interiordesc.gengetinteriorarraysize(jitstate, argbox, *indexboxes) def genptrnonzero(jitstate, argbox, reverse): From arigo at codespeak.net Tue Feb 5 13:27:47 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 5 Feb 2008 13:27:47 +0100 (CET) Subject: [pypy-svn] r51286 - pypy/dist/pypy/doc/config Message-ID: <20080205122747.B97C1168445@codespeak.net> Author: arigo Date: Tue Feb 5 13:27:45 2008 New Revision: 51286 Removed: pypy/dist/pypy/doc/config/translation.stacklessgc.txt Modified: pypy/dist/pypy/doc/config/translation.gcrootfinder.txt Log: Remove doc for --stacklessgc, which was generalized as --gcrootfinder. Modified: pypy/dist/pypy/doc/config/translation.gcrootfinder.txt ============================================================================== --- pypy/dist/pypy/doc/config/translation.gcrootfinder.txt (original) +++ pypy/dist/pypy/doc/config/translation.gcrootfinder.txt Tue Feb 5 13:27:45 2008 @@ -5,7 +5,9 @@ - use a shadow stack (XXX link to paper), e.g. explicitely maintaining a stack of roots - - use stackless to find roots by unwinding the stack + - use stackless to find roots by unwinding the stack. Requires + :config:`translation.stackless`. Note that this turned out to + be slower than just using a shadow stack. - use GCC and i386 specific assembler hackery to find the roots on the stack. This is fastest but platform specific. From arigo at codespeak.net Tue Feb 5 13:29:50 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 5 Feb 2008 13:29:50 +0100 (CET) Subject: [pypy-svn] r51287 - pypy/dist/pypy/rpython/lltypesystem/test Message-ID: <20080205122950.71973168445@codespeak.net> Author: arigo Date: Tue Feb 5 13:29:49 2008 New Revision: 51287 Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_lloperation.py Log: Fix test. Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_lloperation.py Tue Feb 5 13:29:49 2008 @@ -50,6 +50,6 @@ for opname, llop in LL_OPERATIONS.items(): if llop.canfold: continue - if opname.startswith('gc_x_'): + if opname.startswith('gc_x_') or opname.startswith('llvm_'): continue # ignore experimental stuff assert opname in LL_INTERP_OPERATIONS From arigo at codespeak.net Tue Feb 5 13:53:11 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 5 Feb 2008 13:53:11 +0100 (CET) Subject: [pypy-svn] r51288 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20080205125311.0662716840B@codespeak.net> Author: arigo Date: Tue Feb 5 13:53:11 2008 New Revision: 51288 Modified: pypy/dist/pypy/rpython/memory/gc/marksweep.py Log: Upgrade MarkSweepGC to the new way to invoke the root stack walker. Modified: pypy/dist/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/dist/pypy/rpython/memory/gc/marksweep.py Tue Feb 5 13:53:11 2008 @@ -38,7 +38,7 @@ POOLNODE.become(lltype.Struct('gc_pool_node', ('linkedlist', HDRPTR), ('nextnode', POOLNODEPTR))) - def __init__(self, AddressLinkedList, start_heap_size=4096, get_roots=None): + def __init__(self, AddressLinkedList, start_heap_size=4096): self.heap_usage = 0 # at the end of the latest collection self.bytes_malloced = 0 # since the latest collection self.bytes_malloced_threshold = start_heap_size @@ -49,7 +49,6 @@ # these are usually only the small bits of memory that make a # weakref object self.objects_with_weak_pointers = lltype.nullptr(self.HDR) - self.get_roots = get_roots self.gcheaderbuilder = GCHeaderBuilder(self.HDR) # pools, for x_swap_pool(): # 'curpool' is the current pool, lazily allocated (i.e. NULL means @@ -222,20 +221,18 @@ llop.debug_print(lltype.Void, 'collecting...') start_time = time.time() self.collect_in_progress = True - roots = self.get_roots() size_gc_header = self.gcheaderbuilder.size_gc_header ## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, ## size_gc_header) # push the roots on the mark stack objects = self.AddressLinkedList() # mark stack - while 1: - curr = roots.pop() - if curr == NULL: - break - # roots is a list of addresses to addresses: - objects.append(curr.address[0]) - free_non_gc_object(roots) + self._mark_stack = objects + self.root_walker.walk_roots( + MarkSweepGC._mark_root, # stack roots + MarkSweepGC._mark_root, # static in prebuilt non-gc structures + MarkSweepGC._mark_root) # static in prebuilt gc objects + # from this point onwards, no more mallocs should be possible old_malloced = self.bytes_malloced self.bytes_malloced = 0 @@ -454,6 +451,18 @@ hdr = next self.collect_in_progress = False + def _mark_root(self, root): # 'root' is the address of the GCPTR + gcobjectaddr = root.address[0] + self._mark_stack.append(gcobjectaddr) + + def _mark_root_and_clear_bit(self, root): + gcobjectaddr = root.address[0] + self._mark_stack.append(gcobjectaddr) + size_gc_header = self.gcheaderbuilder.size_gc_header + gc_info = gcobjectaddr - size_gc_header + hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) + hdr.typeid = hdr.typeid & (~1) + STAT_HEAP_USAGE = 0 STAT_BYTES_MALLOCED = 1 STATISTICS_NUMBERS = 2 @@ -685,25 +694,20 @@ if DEBUG_PRINT: llop.debug_print(lltype.Void, 'collecting...') start_time = time.time() - roots = self.get_roots() size_gc_header = self.gcheaderbuilder.size_gc_header ## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, ## size_gc_header) # push the roots on the mark stack objects = self.AddressLinkedList() # mark stack - while 1: - curr = roots.pop() - if curr == NULL: - break - # roots is a list of addresses to addresses: - objects.append(curr.address[0]) - # the last sweep did not clear the mark bit of static roots, - # since they are not in the malloced_objects list - gc_info = curr.address[0] - size_gc_header - hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - hdr.typeid = hdr.typeid & (~1) - free_non_gc_object(roots) + self._mark_stack = objects + # the last sweep did not clear the mark bit of static roots, + # since they are not in the malloced_objects list + self.root_walker.walk_roots( + MarkSweepGC._mark_root_and_clear_bit, # stack roots + MarkSweepGC._mark_root_and_clear_bit, # static in prebuilt non-gc + MarkSweepGC._mark_root_and_clear_bit) # static in prebuilt gc + # from this point onwards, no more mallocs should be possible old_malloced = self.bytes_malloced self.bytes_malloced = 0 @@ -839,8 +843,8 @@ _alloc_flavor_ = "raw" COLLECT_EVERY = 2000 - def __init__(self, AddressLinkedList, start_heap_size=4096, get_roots=None): - MarkSweepGC.__init__(self, AddressLinkedList, start_heap_size, get_roots) + def __init__(self, AddressLinkedList, start_heap_size=4096): + MarkSweepGC.__init__(self, AddressLinkedList, start_heap_size) self.count_mallocs = 0 def write_malloc_statistics(self, typeid, size, result, varsize): @@ -1012,20 +1016,18 @@ self.count_mallocs = 0 start_time = time.time() self.collect_in_progress = True - roots = self.get_roots() size_gc_header = self.gcheaderbuilder.size_gc_header ## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, ## size_gc_header) # push the roots on the mark stack objects = self.AddressLinkedList() # mark stack - while 1: - curr = roots.pop() - if curr == NULL: - break - # roots is a list of addresses to addresses: - objects.append(curr.address[0]) - free_non_gc_object(roots) + self._mark_stack = objects + self.root_walker.walk_roots( + MarkSweepGC._mark_root, # stack roots + MarkSweepGC._mark_root, # static in prebuilt non-gc structures + MarkSweepGC._mark_root) # static in prebuilt gc objects + # from this point onwards, no more mallocs should be possible old_malloced = self.bytes_malloced self.bytes_malloced = 0 From arigo at codespeak.net Tue Feb 5 14:25:32 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 5 Feb 2008 14:25:32 +0100 (CET) Subject: [pypy-svn] r51289 - pypy/dist/pypy/translator/llvm Message-ID: <20080205132532.4E00B168427@codespeak.net> Author: arigo Date: Tue Feb 5 14:25:28 2008 New Revision: 51289 Modified: pypy/dist/pypy/translator/llvm/funcnode.py Log: Only generate the "gc" annotation when the flag is defined. Modified: pypy/dist/pypy/translator/llvm/funcnode.py ============================================================================== --- pypy/dist/pypy/translator/llvm/funcnode.py (original) +++ pypy/dist/pypy/translator/llvm/funcnode.py Tue Feb 5 14:25:28 2008 @@ -124,7 +124,10 @@ def getdecl(self): returntype, ref, args = self.getdecl_parts() - return '%s %s(%s) gc "gcrootsingle"' % (returntype, ref, ", ".join(args)) + annotations = '' + if self.db.genllvm.config.translation.gcrootfinder == "llvmgc": + annotations += ' gc "gcrootsingle"' + return '%s %s(%s)%s' % (returntype, ref, ", ".join(args), annotations) # ______________________________________________________________________ # helpers for block writers From arigo at codespeak.net Tue Feb 5 14:26:12 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 5 Feb 2008 14:26:12 +0100 (CET) Subject: [pypy-svn] r51290 - pypy/dist/pypy/translator/llvm Message-ID: <20080205132612.19923168426@codespeak.net> Author: arigo Date: Tue Feb 5 14:26:11 2008 New Revision: 51290 Modified: pypy/dist/pypy/translator/llvm/node.py Log: Revert r50061 as it breaks tests. Modified: pypy/dist/pypy/translator/llvm/node.py ============================================================================== --- pypy/dist/pypy/translator/llvm/node.py (original) +++ pypy/dist/pypy/translator/llvm/node.py Tue Feb 5 14:26:11 2008 @@ -1,16 +1,29 @@ from pypy.rpython.lltypesystem import lltype -from pypy.translator.gensupp import NameManager - -NODE_NAMES = NameManager() class Node(object): __slots__ = "name".split() prefix = '%' + nodename_count = {} + def mangle(self, name): + if name not in self.nodename_count: + self.nodename_count[name] = 1 + return name + else: + result = '%s_%d' % (name, self.nodename_count[name]) + self.nodename_count[name] += 1 + # this ensures (a) doesn exist yet, and (b) adds it to the + # dictionary just to prevent some function called xxx_42() and clashing + return self.mangle(result) + def make_name(self, name=''): " helper for creating names" - name = self.prefix + NODE_NAMES.uniquename(name) + name = self.prefix + name + name = self.mangle(name) + if " " in name or "<" in name: + name = '"%s"' % name + self.name = name def setup(self): From fijal at codespeak.net Tue Feb 5 15:31:01 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 5 Feb 2008 15:31:01 +0100 (CET) Subject: [pypy-svn] r51291 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080205143101.DED2116842B@codespeak.net> Author: fijal Date: Tue Feb 5 15:31:00 2008 New Revision: 51291 Modified: pypy/dist/pypy/lib/app_test/ctypes/test_functions.py Log: This test passes out of the box. Modified: pypy/dist/pypy/lib/app_test/ctypes/test_functions.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_functions.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_functions.py Tue Feb 5 15:31:00 2008 @@ -306,7 +306,6 @@ def test_errors(self): raises(AttributeError, getattr, dll, "_xxx_yyy") - py.test.skip("in_dll not supported") raises(ValueError, c_int.in_dll, dll, "_xxx_yyy") def test_byval(self): From fijal at codespeak.net Tue Feb 5 15:39:02 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 5 Feb 2008 15:39:02 +0100 (CET) Subject: [pypy-svn] r51293 - pypy/dist/pypy/doc/discussion Message-ID: <20080205143902.37E601683F7@codespeak.net> Author: fijal Date: Tue Feb 5 15:39:01 2008 New Revision: 51293 Added: pypy/dist/pypy/doc/discussion/ctypes_todo.txt (contents, props changed) Log: Some ctypes-related todo Added: pypy/dist/pypy/doc/discussion/ctypes_todo.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/discussion/ctypes_todo.txt Tue Feb 5 15:39:01 2008 @@ -0,0 +1,31 @@ +Few ctypes-related todo points: + +* Write down missing parts and port all tests, eventually adding + additional tests. + + - for unions and structs, late assignement of _fields_ is somewhat buggy. + Tests about behavior of getattr working properly on instances + are missing or not full. Some tests are skipped because I didn't understand + details. + + - restype being a function is not working. + + - there are features, which we don't support like buffer() and + array() protocols. + + - raw structure/arrays as function arguments/result types are not supported + (even on _rawffi level) + + - not all combinations of arrays/structures inlined in other + arrays/structures work. + + - unicode is not implemented (all the conversion functions and friends) + +* keepalive is not there (everything is kept alive forever basically) + +* as all stuff is applevel, we cannot have it really fast right now. + +* we shall at least try to approach ctypes from the point of the jit + backends (at least on platforms that we support). The thing is that + we need a lot broader support of jit backends for different argument + passing in order to do it. From fijal at codespeak.net Tue Feb 5 16:56:33 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 5 Feb 2008 16:56:33 +0100 (CET) Subject: [pypy-svn] r51297 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080205155633.6D043168454@codespeak.net> Author: fijal Date: Tue Feb 5 16:56:32 2008 New Revision: 51297 Added: pypy/dist/pypy/lib/app_test/ctypes/test_unicode.py (contents, props changed) Log: Port this test, not working. Added: pypy/dist/pypy/lib/app_test/ctypes/test_unicode.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lib/app_test/ctypes/test_unicode.py Tue Feb 5 16:56:32 2008 @@ -0,0 +1,114 @@ +# coding: latin-1 +import ctypes +import py +py.test.skip("Unsupported") + +try: + ctypes.c_wchar +except AttributeError: + pass +else: + def setup_module(mod): + import conftest + dll = ctypes.CDLL(str(conftest.sofile)) + mod.wcslen = dll.my_wcslen + mod.wcslen.argtypes = [ctypes.c_wchar_p] + mod.func = dll._testfunc_p_p + + class TestUnicode: + def setup_method(self, method): + self.prev_conv_mode = ctypes.set_conversion_mode("ascii", "strict") + + def teardown_method(self, method): + ctypes.set_conversion_mode(*self.prev_conv_mode) + + def test_ascii_strict(self): + ctypes.set_conversion_mode("ascii", "strict") + # no conversions take place with unicode arguments + assert wcslen(u"abc") == 3 + assert wcslen(u"ab\u2070") == 3 + # string args are converted + assert wcslen("abc") == 3 + raises(ctypes.ArgumentError, wcslen, "ab?") + + def test_ascii_replace(self): + ctypes.set_conversion_mode("ascii", "replace") + assert wcslen(u"abc") == 3 + assert wcslen(u"ab\u2070") == 3 + assert wcslen("abc") == 3 + assert wcslen("ab?") == 3 + + def test_ascii_ignore(self): + ctypes.set_conversion_mode("ascii", "ignore") + assert wcslen(u"abc") == 3 + assert wcslen(u"ab\u2070") == 3 + # ignore error mode skips non-ascii characters + assert wcslen("abc") == 3 + assert wcslen("????") == 0 + + def test_latin1_strict(self): + ctypes.set_conversion_mode("latin-1", "strict") + assert wcslen(u"abc") == 3 + assert wcslen(u"ab\u2070") == 3 + assert wcslen("abc") == 3 + assert wcslen("????") == 4 + + def test_buffers(self): + ctypes.set_conversion_mode("ascii", "strict") + buf = ctypes.create_unicode_buffer("abc") + assert len(buf) == 3+1 + + ctypes.set_conversion_mode("ascii", "replace") + buf = ctypes.create_unicode_buffer("ab???") + assert buf[:] == u"ab\uFFFD\uFFFD\uFFFD\0" + + ctypes.set_conversion_mode("ascii", "ignore") + buf = ctypes.create_unicode_buffer("ab???") + # is that correct? not sure. But with 'ignore', you get what you pay for.. + assert buf[:] == u"ab\0\0\0\0" + + class TestString(TestUnicode): + def setup_method(self): + self.prev_conv_mode = ctypes.set_conversion_mode("ascii", "strict") + func.argtypes = [ctypes.c_char_p] + func.restype = ctypes.c_char_p + + def teardown_method(self): + ctypes.set_conversion_mode(*self.prev_conv_mode) + func.argtypes = None + func.restype = ctypes.c_int + + def test_ascii_replace(self): + ctypes.set_conversion_mode("ascii", "strict") + assert func("abc") == "abc" + assert func(u"abc") == "abc" + raises(ctypes.ArgumentError, func, u"ab?") + + def test_ascii_ignore(self): + ctypes.set_conversion_mode("ascii", "ignore") + assert func("abc") == "abc" + assert func(u"abc") == "abc" + assert func(u"????") == "" + + def test_ascii_replace(self): + ctypes.set_conversion_mode("ascii", "replace") + assert func("abc") == "abc" + assert func(u"abc") == "abc" + assert func(u"????") == "????" + + def test_buffers(self): + ctypes.set_conversion_mode("ascii", "strict") + buf = ctypes.create_string_buffer(u"abc") + assert len(buf) == 3+1 + + ctypes.set_conversion_mode("ascii", "replace") + buf = ctypes.create_string_buffer(u"ab???") + assert buf[:] == "ab???\0" + + ctypes.set_conversion_mode("ascii", "ignore") + buf = ctypes.create_string_buffer(u"ab???") + # is that correct? not sure. But with 'ignore', you get what you pay for.. + assert buf[:] == "ab\0\0\0\0" + +if __name__ == '__main__': + unittest.main() From fijal at codespeak.net Tue Feb 5 18:54:59 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 5 Feb 2008 18:54:59 +0100 (CET) Subject: [pypy-svn] r51299 - pypy/dist/pypy/doc/discussion Message-ID: <20080205175459.21694168457@codespeak.net> Author: fijal Date: Tue Feb 5 18:54:57 2008 New Revision: 51299 Modified: pypy/dist/pypy/doc/discussion/ctypes_todo.txt Log: Update Modified: pypy/dist/pypy/doc/discussion/ctypes_todo.txt ============================================================================== --- pypy/dist/pypy/doc/discussion/ctypes_todo.txt (original) +++ pypy/dist/pypy/doc/discussion/ctypes_todo.txt Tue Feb 5 18:54:57 2008 @@ -21,6 +21,10 @@ - unicode is not implemented (all the conversion functions and friends) + - bitfields are not implemented + + + * keepalive is not there (everything is kept alive forever basically) * as all stuff is applevel, we cannot have it really fast right now. From fijal at codespeak.net Tue Feb 5 18:55:31 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 5 Feb 2008 18:55:31 +0100 (CET) Subject: [pypy-svn] r51300 - pypy/dist/pypy/module/__builtin__ Message-ID: <20080205175531.CD5711684C1@codespeak.net> Author: fijal Date: Tue Feb 5 18:55:30 2008 New Revision: 51300 Modified: pypy/dist/pypy/module/__builtin__/app_buffer.py Log: Kill deprecated comment Modified: pypy/dist/pypy/module/__builtin__/app_buffer.py ============================================================================== --- pypy/dist/pypy/module/__builtin__/app_buffer.py (original) +++ pypy/dist/pypy/module/__builtin__/app_buffer.py Tue Feb 5 18:55:30 2008 @@ -1,4 +1,4 @@ -# Might probably be deprecated in Python at some point. + import sys class buffer(object): From vinogradov at codespeak.net Tue Feb 5 19:59:11 2008 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Tue, 5 Feb 2008 19:59:11 +0100 (CET) Subject: [pypy-svn] r51301 - pypy/branch/ghop-ropes-classes/pypy/rlib/test Message-ID: <20080205185911.4D8B016842C@codespeak.net> Author: vinogradov Date: Tue Feb 5 19:59:08 2008 New Revision: 51301 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Log: Add test for find method Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Tue Feb 5 19:59:08 2008 @@ -74,7 +74,14 @@ assert s2.lower() == s1 s3 = self.const("Welcome to PyPy World") assert s3.swapcase() == self.const("wELCOME TO pYpY wORLD") + assert s1.title() == self.const("Hello Python") + def test_str_find(self): + s1 = self.const("Welcome to PyPy world") + assert s1.find("Py") == 11 + assert s1.find("dd") == -1 + assert s1.find("py") == -1 + class AbstractRopeTest(object): def test_add_long(self): s1 = self.const("a") From vinogradov at codespeak.net Tue Feb 5 20:01:18 2008 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Tue, 5 Feb 2008 20:01:18 +0100 (CET) Subject: [pypy-svn] r51302 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20080205190118.19159168448@codespeak.net> Author: vinogradov Date: Tue Feb 5 20:01:17 2008 New Revision: 51302 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: Add test for find method Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Tue Feb 5 20:01:17 2008 @@ -101,7 +101,10 @@ def getchar(self,index): return self._node.getchar(index) - + + def find(self, sub): + return -1 + class RopeUnicodeIterator(object): def __init__(self, node): self._iter = rope.ItemIterator(node) From vinogradov at codespeak.net Tue Feb 5 20:13:51 2008 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Tue, 5 Feb 2008 20:13:51 +0100 (CET) Subject: [pypy-svn] r51303 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20080205191351.362E816844B@codespeak.net> Author: vinogradov Date: Tue Feb 5 20:13:50 2008 New Revision: 51303 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: Implement basic string search Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Tue Feb 5 20:13:50 2008 @@ -49,6 +49,13 @@ return self.__class__(rope.getslice(self._node, start, stop, step)) raise NotImplementedError("Index type not known.") + def find(self, sub): + if isinstance(sub, str): + data = RopeString(sub) + if isinstance(sub, unicode): + data = RopeUnicode(sub) + + return rope.find(self._node, data._node) class RopeStringIterator(object): def __init__(self, node): @@ -102,9 +109,6 @@ def getchar(self,index): return self._node.getchar(index) - def find(self, sub): - return -1 - class RopeUnicodeIterator(object): def __init__(self, node): self._iter = rope.ItemIterator(node) From vinogradov at codespeak.net Tue Feb 5 20:19:31 2008 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Tue, 5 Feb 2008 20:19:31 +0100 (CET) Subject: [pypy-svn] r51304 - pypy/branch/ghop-ropes-classes/pypy/rlib/test Message-ID: <20080205191931.579D916844B@codespeak.net> Author: vinogradov Date: Tue Feb 5 20:19:30 2008 New Revision: 51304 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Log: Add more tests to rope find method Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Tue Feb 5 20:19:30 2008 @@ -81,6 +81,9 @@ assert s1.find("Py") == 11 assert s1.find("dd") == -1 assert s1.find("py") == -1 + assert s1.find("Welcome to PyPy world") == 0 + assert s1.find(s1) == 0 + py.test.raises(TypeError, s1.find, 4) class AbstractRopeTest(object): def test_add_long(self): From vinogradov at codespeak.net Wed Feb 6 07:47:26 2008 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Wed, 6 Feb 2008 07:47:26 +0100 (CET) Subject: [pypy-svn] r51310 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20080206064726.4B021168450@codespeak.net> Author: vinogradov Date: Wed Feb 6 07:47:23 2008 New Revision: 51310 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: Update implementation of find method Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Wed Feb 6 07:47:23 2008 @@ -50,12 +50,19 @@ raise NotImplementedError("Index type not known.") def find(self, sub): + data = None + if isinstance(sub, str): data = RopeString(sub) if isinstance(sub, unicode): data = RopeUnicode(sub) - - return rope.find(self._node, data._node) + if isinstance(sub, RopeBaseString): + data = sub + + if data: + return rope.find(self._node, data._node) + else: + raise TypeError("expected a character buffer object") class RopeStringIterator(object): def __init__(self, node): From antocuni at codespeak.net Wed Feb 6 16:25:57 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 6 Feb 2008 16:25:57 +0100 (CET) Subject: [pypy-svn] r51312 - in pypy/dist/pypy/translator/cli: . test Message-ID: <20080206152557.247FC1684C0@codespeak.net> Author: antocuni Date: Wed Feb 6 16:25:55 2008 New Revision: 51312 Modified: pypy/dist/pypy/translator/cli/class_.py pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: we can't use null as default value for valuetypes Modified: pypy/dist/pypy/translator/cli/class_.py ============================================================================== --- pypy/dist/pypy/translator/cli/class_.py (original) +++ pypy/dist/pypy/translator/cli/class_.py Wed Feb 6 16:25:55 2008 @@ -131,6 +131,8 @@ default_values = self.INSTANCE._fields.copy() default_values.update(self.INSTANCE._overridden_defaults) for f_name, (F_TYPE, f_default) in default_values.iteritems(): + if getattr(F_TYPE, '_is_value_type', False): + continue # we can't set it to null INSTANCE_DEF, _ = self.INSTANCE._lookup_field(f_name) cts_type = self.cts.lltype_to_cts(F_TYPE) f_name = self.cts.escape_name(f_name) Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Wed Feb 6 16:25:55 2008 @@ -496,6 +496,16 @@ res = self.interpret(fn, []) assert res == 42 + def test_valuetype_field(self): + class Foo: + def __init__(self, x): + self.x = x + + def fn(): + f = Foo(OpCodes.Add) + return f + self.interpret(fn, []) + class TestPythonnet(TestDotnetRtyping): # don't interpreter functions but execute them directly through pythonnet From antocuni at codespeak.net Wed Feb 6 16:32:19 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 6 Feb 2008 16:32:19 +0100 (CET) Subject: [pypy-svn] r51313 - in pypy/dist/pypy: jit/codegen/cli jit/codegen/cli/test translator/cli/src Message-ID: <20080206153219.D06DC1684D1@codespeak.net> Author: antocuni Date: Wed Feb 6 16:32:19 2008 New Revision: 51313 Modified: pypy/dist/pypy/jit/codegen/cli/operation.py pypy/dist/pypy/jit/codegen/cli/rgenop.py pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py pypy/dist/pypy/translator/cli/src/pypylib.cs Log: - test_largedummy would work out of the box, if only .maxstack were big enough to fit it. - implemented branching Modified: pypy/dist/pypy/jit/codegen/cli/operation.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/operation.py (original) +++ pypy/dist/pypy/jit/codegen/cli/operation.py Wed Feb 6 16:32:19 2008 @@ -1,29 +1,71 @@ -from pypy.translator.cli.dotnet import CLR -OpCodes = CLR.System.Reflection.Emit.OpCodes +from pypy.rpython.ootypesystem import ootype +from pypy.translator.cli.dotnet import CLR, typeof +System = CLR.System +OpCodes = System.Reflection.Emit.OpCodes class Operation: - restype = None _gv_res = None + def restype(self): + return self.gv_x.getCliType() + def gv_res(self): from pypy.jit.codegen.cli.rgenop import GenLocalVar if self._gv_res is None: - # if restype is None, assume it's the same as the first arg - t = self.restype or self.gv_x.getCliType() - loc = self.il.DeclareLocal(t) - self._gv_res = GenLocalVar(loc) + restype = self.restype() + if restype is not None: + loc = self.il.DeclareLocal(restype) + self._gv_res = GenLocalVar(loc) return self._gv_res def emit(self): raise NotImplementedError +class Branch(Operation): + + def __init__(self, il, label): + self.il = il + self.label = label + + def emit(self): + self.il.emit(OpCodes.Br, self.label) + + class UnaryOp(Operation): def __init__(self, il, gv_x): self.il = il self.gv_x = gv_x +class AbstractBranchIf(UnaryOp): + + def __init__(self, il, gv_x, label): + self.il = il + self.gv_x = gv_x + self.label = label + + def restype(self): + return None + + def emit(self): + self.il.emit(self.getOpCode(), self.label) + + def getOpCode(self): + return OpCodes.Brtrue + + +class BrFalse(AbstractBranchIf): + + def getOpCode(self): + return OpCodes.Brfalse + +class BrTrue(AbstractBranchIf): + + def getOpCode(self): + return OpCodes.Brtrue + + class SameAs(UnaryOp): def emit(self): gv_res = self.gv_res() @@ -55,3 +97,10 @@ class Sub(BinaryOp): def getOpCode(self): return OpCodes.Sub + +class Gt(BinaryOp): + def restype(self): + return typeof(System.Boolean) + + def getOpCode(self): + return OpCodes.Cgt Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/rgenop.py Wed Feb 6 16:32:19 2008 @@ -20,6 +20,8 @@ return typeof(CLR.pypy.runtime.DelegateType_int__int) elif tok == (['', ''], ''): return typeof(CLR.pypy.runtime.DelegateType_int__int_int) + elif tok == ([''] * 100, ''): + return typeof(CLR.pypy.runtime.DelegateType_int__int_100) else: assert False @@ -85,6 +87,9 @@ assert T is ootype.Signed return self.value + def getCliType(self): + return typeof(System.Int32) + def load(self, il): il.Emit(OpCodes.Ldc_I4, self.value) @@ -93,6 +98,7 @@ SM_INT__INT = ootype.StaticMethod([ootype.Signed], ootype.Signed) SM_INT__INT_INT = ootype.StaticMethod([ootype.Signed, ootype.Signed], ootype.Signed) +SM_INT__INT_100 = ootype.StaticMethod([ootype.Signed] * 100, ootype.Signed) class ObjectConst(GenConst): @@ -108,6 +114,9 @@ elif T == SM_INT__INT_INT: DelegateType = CLR.pypy.runtime.DelegateType_int__int_int return clidowncast(DelegateType, self.obj) + elif T == SM_INT__INT_100: + DelegateType = CLR.pypy.runtime.DelegateType_int__int_100 + return clidowncast(DelegateType, self.obj) else: assert isinstance(T, ootype.OOType) return ootype.oodowncast(T, self.obj) @@ -149,14 +158,14 @@ for i in range(len(argtoks)): args[i] = token2clitype(argtoks[i]) res = token2clitype(restok) - builder = Builder(self, name, res, args) + builder = Builder(self, name, res, args, sigtoken) return builder, builder.gv_entrypoint, builder.inputargs_gv[:] class Builder(GenBuilder): - def __init__(self, rgenop, name, res, args): + def __init__(self, rgenop, name, res, args, sigtoken): self.rgenop = rgenop self.meth = Utils.CreateDynamicMethod(name, res, args) self.il = self.meth.GetILGenerator() @@ -164,6 +173,8 @@ for i in range(len(args)): self.inputargs_gv.append(GenArgVar(i, args[i])) self.gv_entrypoint = ObjectConst(None) # XXX? + self.sigtoken = sigtoken + self.isOpen = False @specialize.arg(1) def genop2(self, opname, gv_arg1, gv_arg2): @@ -171,26 +182,59 @@ op = ops.Add(self.il, gv_arg1, gv_arg2) elif opname == 'int_sub': op = ops.Sub(self.il, gv_arg1, gv_arg2) + elif opname == 'int_gt': + op = ops.Gt(self.il, gv_arg1, gv_arg2) else: assert False - op.emit() + self.emit(op) return op.gv_res() + def emit(self, op): + op.emit() + + def start_writing(self): + self.isOpen = True + def finish_and_return(self, sigtoken, gv_returnvar): gv_returnvar.load(self.il) self.il.Emit(OpCodes.Ret) - delegate_type = sigtoken2clitype(sigtoken) - myfunc = self.meth.CreateDelegate(delegate_type) - self.gv_entrypoint.obj = myfunc + self.isOpen = False def end(self): - pass + delegate_type = sigtoken2clitype(self.sigtoken) + myfunc = self.meth.CreateDelegate(delegate_type) + self.gv_entrypoint.obj = myfunc def enter_next_block(self, kinds, args_gv): for i in range(len(args_gv)): op = ops.SameAs(self.il, args_gv[i]) op.emit() args_gv[i] = op.gv_res() - lbl = self.il.DefineLabel() - self.il.MarkLabel(lbl) - return lbl + label = self.il.DefineLabel() + self.il.MarkLabel(label) + return label + + def _jump_if(self, gv_condition, opcode): + label = self.il.DefineLabel() + gv_condition.load(self.il) + self.il.Emit(opcode, label) + return BranchBuilder(self, label) + + def jump_if_false(self, gv_condition, args_for_jump_gv): + return self._jump_if(gv_condition, OpCodes.Brfalse) + + def jump_if_true(self, gv_condition, args_for_jump_gv): + return self._jump_if(gv_condition, OpCodes.Brtrue) + +class BranchBuilder(Builder): + + def __init__(self, parent, label): + self.parent = parent + self.label = label + self.il = parent.il + self.isOpen = False + + def start_writing(self): + assert not self.parent.isOpen + self.isOpen = True + self.il.MarkLabel(self.label) Modified: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Wed Feb 6 16:32:19 2008 @@ -11,6 +11,8 @@ 'test_dummy', 'test_hide_and_reveal', 'test_hide_and_reveal_p', + 'test_largedummy_direct', # _compile works if we set a higher maxstack + 'test_branching', ] for p in prefixes: Modified: pypy/dist/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/dist/pypy/translator/cli/src/pypylib.cs Wed Feb 6 16:32:19 2008 @@ -55,6 +55,7 @@ { public delegate int DelegateType_int__int(int a); public delegate int DelegateType_int__int_int(int a, int b); + public delegate int DelegateType_int__int_100(int a0, int a1, int a2, int a3, int a4, int a5, int a6, int a7, int a8, int a9, int a10, int a11, int a12, int a13, int a14, int a15, int a16, int a17, int a18, int a19, int a20, int a21, int a22, int a23, int a24, int a25, int a26, int a27, int a28, int a29, int a30, int a31, int a32, int a33, int a34, int a35, int a36, int a37, int a38, int a39, int a40, int a41, int a42, int a43, int a44, int a45, int a46, int a47, int a48, int a49, int a50, int a51, int a52, int a53, int a54, int a55, int a56, int a57, int a58, int a59, int a60, int a61, int a62, int a63, int a64, int a65, int a66, int a67, int a68, int a69, int a70, int a71, int a72, int a73, int a74, int a75, int a76, int a77, int a78, int a79, int a80, int a81, int a82, int a83, int a84, int a85, int a86, int a87, int a88, int a89, int a90, int a91, int a92, int a93, int a94, int a95, int a96, int a97, int a98, int a99); public class Utils { From vinogradov at codespeak.net Thu Feb 7 19:20:45 2008 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Thu, 7 Feb 2008 19:20:45 +0100 (CET) Subject: [pypy-svn] r51320 - pypy/branch/ghop-ropes-classes/pypy/rlib/test Message-ID: <20080207182045.08FDA168519@codespeak.net> Author: vinogradov Date: Thu Feb 7 19:20:44 2008 New Revision: 51320 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Log: Add tests to basic index functionality Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Thu Feb 7 19:20:44 2008 @@ -85,6 +85,15 @@ assert s1.find(s1) == 0 py.test.raises(TypeError, s1.find, 4) + def test_str_index(self): + s1 = self.const("Welcome to PyPy world") + assert s1.index("Py") == 11 + py.test.raises(ValueError, s1.index, "dd") + py.test.raises(ValueError, s1.index, "py") + assert s1.index("Welcome to PyPy world") == 0 + assert s1.index(s1) == 0 + py.test.raises(TypeError, s1.index, 4) + class AbstractRopeTest(object): def test_add_long(self): s1 = self.const("a") From vinogradov at codespeak.net Thu Feb 7 19:24:53 2008 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Thu, 7 Feb 2008 19:24:53 +0100 (CET) Subject: [pypy-svn] r51321 - pypy/branch/ghop-ropes-classes/pypy/rlib Message-ID: <20080207182453.07B9B168534@codespeak.net> Author: vinogradov Date: Thu Feb 7 19:24:53 2008 New Revision: 51321 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Log: Basic Implementation of index method Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/ropewrapper.py Thu Feb 7 19:24:53 2008 @@ -64,6 +64,25 @@ else: raise TypeError("expected a character buffer object") + def index(self, sub, start=None, stop=None): + data = None + + if isinstance(sub, str): + data = RopeString(sub) + if isinstance(sub, unicode): + data = RopeUnicode(sub) + if isinstance(sub, RopeBaseString): + data = sub + + if data: + result = rope.find(self._node, data._node) + if result >= 0: + return result + else: + raise ValueError("substring not found") + else: + raise TypeError("expected a character buffer object") + class RopeStringIterator(object): def __init__(self, node): self._iter = rope.ItemIterator(node) From vinogradov at codespeak.net Thu Feb 7 19:30:30 2008 From: vinogradov at codespeak.net (vinogradov at codespeak.net) Date: Thu, 7 Feb 2008 19:30:30 +0100 (CET) Subject: [pypy-svn] r51322 - pypy/branch/ghop-ropes-classes/pypy/rlib/test Message-ID: <20080207183030.1CEAD168541@codespeak.net> Author: vinogradov Date: Thu Feb 7 19:30:29 2008 New Revision: 51322 Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Log: Add some range tests for find and index Modified: pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py ============================================================================== --- pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py (original) +++ pypy/branch/ghop-ropes-classes/pypy/rlib/test/test_ropewrapper.py Thu Feb 7 19:30:29 2008 @@ -84,6 +84,8 @@ assert s1.find("Welcome to PyPy world") == 0 assert s1.find(s1) == 0 py.test.raises(TypeError, s1.find, 4) + assert s1.find("o", 5, 10) == 9 + assert s1.find("o", 10) == 17 def test_str_index(self): s1 = self.const("Welcome to PyPy world") @@ -93,6 +95,8 @@ assert s1.index("Welcome to PyPy world") == 0 assert s1.index(s1) == 0 py.test.raises(TypeError, s1.index, 4) + assert s1.index("o", 5, 10) == 9 + assert s1.index("o", 10) == 17 class AbstractRopeTest(object): def test_add_long(self): From lamby at codespeak.net Thu Feb 7 19:46:38 2008 From: lamby at codespeak.net (lamby at codespeak.net) Date: Thu, 7 Feb 2008 19:46:38 +0100 (CET) Subject: [pypy-svn] r51323 - in pypy/build/debian: . debian debian/manpages debian/patches debian/scripts Message-ID: <20080207184638.96FC21684D2@codespeak.net> Author: lamby Date: Thu Feb 7 19:46:37 2008 New Revision: 51323 Added: pypy/build/debian/ pypy/build/debian/debian/ pypy/build/debian/debian/Makefile.in pypy/build/debian/debian/changelog pypy/build/debian/debian/compat pypy/build/debian/debian/configure.py pypy/build/debian/debian/control pypy/build/debian/debian/copyright pypy/build/debian/debian/manpages/ pypy/build/debian/debian/manpages/jscompile.1 pypy/build/debian/debian/manpages/py.py.1 pypy/build/debian/debian/manpages/pypy-stackless.1 pypy/build/debian/debian/manpages/pypy-translate.1 pypy/build/debian/debian/manpages/pypy.1 pypy/build/debian/debian/patches/ pypy/build/debian/debian/patches/01-package-specific-temp-prefix pypy/build/debian/debian/patches/02-mips-support pypy/build/debian/debian/patches/series pypy/build/debian/debian/pypy-dev.dirs pypy/build/debian/debian/pypy-dev.install pypy/build/debian/debian/pypy-dev.links pypy/build/debian/debian/pypy-dev.manpages pypy/build/debian/debian/pypy-dev.override pypy/build/debian/debian/pypy-dev.postinst pypy/build/debian/debian/pypy-dev.prerm pypy/build/debian/debian/pypy-doc.docs pypy/build/debian/debian/pypy-lib.dirs pypy/build/debian/debian/pypy-lib.install pypy/build/debian/debian/pypy-lib.links pypy/build/debian/debian/pypy-lib.override pypy/build/debian/debian/pypy-logic.install pypy/build/debian/debian/pypy-stackless.install pypy/build/debian/debian/pypy-stackless.manpages pypy/build/debian/debian/pypy.install pypy/build/debian/debian/pypy.manpages pypy/build/debian/debian/rules (contents, props changed) pypy/build/debian/debian/scripts/ pypy/build/debian/debian/scripts/jscompile pypy/build/debian/debian/scripts/py.py pypy/build/debian/debian/scripts/pypy-translate Log: Import Debian packaging for building the (upcoming) 1.0.0-svn51288-1 package. Added: pypy/build/debian/debian/Makefile.in ============================================================================== --- (empty file) +++ pypy/build/debian/debian/Makefile.in Thu Feb 7 19:46:37 2008 @@ -0,0 +1,44 @@ +TARGET=pypy/translator/goal/targetpypystandalone + +TRANSLATE=pypy/translator/goal/translate.py +TRANSLATEOPTS=--text --batch -b c --source --thread +TARGETOPTS=%(TARGETOPTS)s +TARGETOPTS_STACKLESS=%(TARGETOPTS_STACKLESS)s +PREFIX=%(PREFIX)s +OUTPUTDIR=debian/tmp/usr/bin +STACKLESS=--stackless +LOGIC=-o logic + +BINARIES=bin/pypy bin/pypy-stackless + +all: $(BINARIES) + +.NOTPARALLEL: $(BINARIES) + +bin/pypy: export TMPDIR = $(CURDIR)/build-default +bin/pypy: + @rm -rf $(TMPDIR) + mkdir $(TMPDIR) + $(TRANSLATE) $(TRANSLATEOPTS) $(TARGET) $(TARGETOPTS) + make -C $(TMPDIR)/debian-pypy-usession-0/testing_1 + install -D $(TMPDIR)/debian-pypy-usession-0/testing_1/testing_1 $@ + +bin/pypy-stackless: export TMPDIR = $(CURDIR)/build-stackless +bin/pypy-stackless: + @rm -rf $(TMPDIR) + mkdir $(TMPDIR) + $(TRANSLATE) $(TRANSLATEOPTS) $(STACKLESS) $(TARGET) $(TARGETOPTS_STACKLESS) + make -C $(TMPDIR)/debian-pypy-usession-0/testing_1 + install -D $(TMPDIR)/debian-pypy-usession-0/testing_1/testing_1 $@ + +clean: + rm -rf bin + rm -rf build-* + rm -rf pypy/_cache + rm -rf py/c-extension/greenlet/build + rm -f py/c-extension/greenlet/greenlet.so + find . -name "*.pyc" | xargs rm -f + +install: all + install -d $(PREFIX)/bin + install $(BINARIES) $(PREFIX)/bin Added: pypy/build/debian/debian/changelog ============================================================================== --- (empty file) +++ pypy/build/debian/debian/changelog Thu Feb 7 19:46:37 2008 @@ -0,0 +1,82 @@ +pypy (1.0.0-svn51288-1) unstable; urgency=low + + * New upstream release. (Closes: #459118) + * New maintainer. (Closes: #455734) + * Bump debhelper compatibility version. + * Refresh and tidy package descriptions. + * Tidy debian/configure.py, debian/rules, Lintian overrides. + * Add pypy(1), py.py(1), pypy-translate(1), jscompile(1) and + pypy-stackless(1) manpages. + * Add get-orig-source target in debian/rules. + * Use quilt to manage Thiemo Seufer's MIPS and MIPSEL patch. + * Use PyPy-specific temporary directory prefixes. (Closes: #452851) + + -- Chris Lamb Tue, 05 Feb 2008 13:26:11 +0000 + +pypy (1.0.0-svn51091-1) unstable; urgency=low + + * New upstream svn snapshot, fixing 64bit build (closes: #462008) + + -- Alexandre Fayolle Mon, 28 Jan 2008 09:40:48 +0100 + +pypy (1.0.0-svn50146-2) unstable; urgency=low + + * Added missing build dependency on zlib1g-dev (closes: #460696) + + -- Alexandre Fayolle Mon, 14 Jan 2008 18:15:32 +0100 + +pypy (1.0.0-svn50146-1) unstable; urgency=low + + * Acknowledge NMU (closes: #431197) + * Apply Yaroslav Hachenko's patch to use more recent svn snapshot, and + enable threading support (closes: #458953) + * Apply Thiemo Seufer's patch to enable mips and mipsel support + (closes: #459520) + * Added build dep on procps (closes: #444652) + * Bumped up standards to 3.7.3 + * Note that pypy is currently under ITA: I lack time to properly follow + upstream developments, and all patches and NMU's are welcome. If you'd + like to adopt pypy, just send me an email to notify me and jump for + it. Offers to co-maintain are welcome too. Thanks. + -- Alexandre Fayolle Mon, 14 Jan 2008 09:46:11 +0100 + +pypy (1.0.0-3.1) unstable; urgency=low + + * Non-maintainer upload. + * Disable build of upstream removed logic object space. Remove pypy-logic + package. (closes: #442060) + * Move build output away from /tmp. (closes: #452850) + * Move compile step out of the main translation. This reduces the memory + footprint by one third. + + -- Bastian Blank Tue, 27 Nov 2007 12:08:06 +0000 + +pypy (1.0.0-3) unstable; urgency=low + + * Use more conservative options for platforms for which support is uncertain + * Enable check of available memory during configuration + * Packages are once again arch: any + + -- Alexandre.Fayolle Wed, 18 Jul 2007 12:57:28 +0200 + +pypy (1.0.0-2) unstable; urgency=low + + * Updated debian/copyright (Closes: #423603) + * Removed support for archs not supported upstream (Closes: #426265, #423725) + * Adds missing conftest.py (Closes: #428539) + + -- Alexandre.Fayolle Thu, 28 Jun 2007 12:25:53 +0200 + +pypy (1.0.0-1) unstable; urgency=low + + * Fixed debian/copyright and reupload (Closes: #326892) + * New upstream release + + -- Alexandre Fayolle Fri, 13 Apr 2007 09:33:09 +0200 + +pypy (0.99.0-1) experimental; urgency=low + + * Initial release (Closes: #326892) + + -- Alexandre Fayolle Mon, 19 Feb 2007 10:47:33 +0100 + Added: pypy/build/debian/debian/compat ============================================================================== --- (empty file) +++ pypy/build/debian/debian/compat Thu Feb 7 19:46:37 2008 @@ -0,0 +1 @@ +6 Added: pypy/build/debian/debian/configure.py ============================================================================== --- (empty file) +++ pypy/build/debian/debian/configure.py Thu Feb 7 19:46:37 2008 @@ -0,0 +1,70 @@ +import getopt +import sys +import os + +MEMTHRESHOLD = 1024 + +LOWMEM = """\ +The translation of pypy requires a large amount of physical +memory. Our tests show that on a system with less than %dMB of RAM, +the translation process will cause heavy thrashing of the hard drive and take days to complete. """ % MEMTHRESHOLD + +def check_memory(exit=False): + stream = os.popen('free -mo') + try: + for line in stream: + if line.startswith('Mem:'): + memory = int(line.split()[1]) + print "Detected %d MB of memory" % memory + if memory < MEMTHRESHOLD and exit: + print LOWMEM + sys.exit('Aborting: not enough memory available') + finally: + stream.close() + + +def gen_make(make_in, build_arch, host_arch, prefix): + options = {'default': {'TARGETOPTS': '--allworkingmodules', + 'TARGETOPTS_STACKLESS': '--allworkingmodules', + 'TARGETOPTS_LOGIC': '--allworkingmodules', + }, + + 'i386': {'TARGETOPTS': '--allworkingmodules --faassen', + 'TARGETOPTS_STACKLESS': '--allworkingmodules --faassen', + 'TARGETOPTS_LOGIC': '--allworkingmodules', + }, + + 'amd64':{'TARGETOPTS': '--allworkingmodules --faassen', + 'TARGETOPTS_STACKLESS': '--allworkingmodules', + 'TARGETOPTS_LOGIC': '--allworkingmodules', + }, + } + substitutions = options.get(build_arch, options['default']) + substitutions['PREFIX'] = prefix + return make_in % substitutions + +def run(): + check_memory() + build_arch = None + host_arch = None + prefix= None + usage = 'USAGE: configure.py --build=BUILD_ARCH --host=HOST_ARCH --prefix=PREFIX Makefile.in' + opts, args = getopt.getopt(sys.argv[1:], '', ['build=', 'host=', 'prefix=']) + for opt, value in opts: + if opt == '--build': + build_arch = value + elif opt == '--host': + host_arch = value + elif opt == '--prefix': + prefix = value + assert len(args) == 1 and build_arch and host_arch and prefix, usage + + infile = open(args[0]) + make_in = infile.read() + infile.close() + makefile = open('Makefile', 'w') + makefile.write(gen_make(make_in, build_arch, host_arch, prefix)) + makefile.close() + +if __name__ == '__main__': + run() Added: pypy/build/debian/debian/control ============================================================================== --- (empty file) +++ pypy/build/debian/debian/control Thu Feb 7 19:46:37 2008 @@ -0,0 +1,68 @@ +Source: pypy +Section: python +Priority: extra +Maintainer: Chris Lamb +Build-Depends: debhelper (>= 6), python-dev, libgc-dev, libbz2-dev, python-ctypes, libreadline5-dev, procps, zlib1g-dev, quilt +Standards-Version: 3.7.3 +Vcs-Git: git://git.chris-lamb.co.uk/pkg-pypy.git +Vcs-Browser: http://git.chris-lamb.co.uk/?p=pkg-pypy.git +Homepage: http://codespeak.net/pypy/ + +Package: pypy-dev +Architecture: all +Depends: ${shlibs:Depends}, ${misc:Depends}, python-codespeak-lib (>= 0.9.0), pypy-lib (>= ${source:Version}), python, python-ctypes +Recommends: pypy-doc, python-pygame, graphviz, python-dev, gcc, libgc-dev, llvm-cfe, mono-gmcs, spidermonkey-bin, jasmin-sable, libreadline5-dev +Suggests: gcl-dev +Description: the Python in python interpreter, interpreted version + The PyPy project aims to provide a Python implementation of the Python + interpreter. + . + This package provides the interpreted version of PyPy, which requires + a Python interpreter to run. It is able to translate itself using + several backends. To do this, you will need to install some of the + Recommended packages, as well as upgrade your machine with lots of RAM. + . + Install this package if you want to translate RPython programs, or to + build a custom translated version of the PyPy interpreter. + +Package: pypy +Architecture: any +Depends: ${shlibs:Depends}, ${misc:Depends}, pypy-lib +Description: the Python in Python interpreter, C backend translation + The PyPy project aims to provide a Python implementation of the Python + interpreter. + . + This package provides the PyPy interpreter compiled using the C backend + and no special additionnal functionality. As such it can be used as another + implementation of the Python language. See pypy-stackless for an enhanced + version. + +Package: pypy-stackless +Architecture: any +Depends: ${shlibs:Depends}, ${misc:Depends}, pypy-lib +Description: the python in python interpreter, C backend translation + The PyPy project aims to provide a Python implementation of the Python + interpreter. + . + This package provides the PyPy interpreter compiled using the C backend, with + stackless functionality. + +Package: pypy-doc +Architecture: all +Section: doc +Description: Documentation for PyPy + The PyPy project aims to provide a Python implementation of the Python + interpreter. + . + This package provides documentation for the PyPy interpreter, the translation + process, etc. You will definitely need it if you intend to play with PyPy. + +Package: pypy-lib +Architecture: all +Recommends: pypy | pypy-dev +Description: standard Python library for PyPy + The PyPy project aims to provide a Python implementation of the Python + interpreter. + . + This package provides a version of the standard Python library suitable for + use with the PyPy interpreter, both interpreted and translated. Added: pypy/build/debian/debian/copyright ============================================================================== --- (empty file) +++ pypy/build/debian/debian/copyright Thu Feb 7 19:46:37 2008 @@ -0,0 +1,199 @@ +This package was debianized by Alexandre Fayolle on +Mon, 8 Jan 2007 19:13:59 +0100. + +It was downloaded from . + +Upstream Authors: + + Armin Rigo + Samuele Pedroni + Michael Hudson + Carl Friedrich Bolz + Christian Tismer + Holger Krekel + Eric van Riet Paap + Antonio Cuni + Anders Chrigstrom + Maciek Fijalkowski + Richard Emslie + Aurelien Campeas + Anders Lehmann + Niklaus Haldimann + Seo Sanghyeon + Lawrence Oluyede + Alex Martelli + Ludovic Aubry + Adrien Di Mascio + Stephan Diehl + Guido Wesdorp + Stefan Schwarzer + Tomek Meka + Patrick Maupin + Leonardo Santagada + Bob Ippolito + Laura Creighton + Jacob Hallen + Marius Gedminas + Niko Matsakis + Amaury Forgeot d Arc + Guido van Rossum + Valentino Volonghi + Alexander Schremmer + Alexandre Fayolle + Wanja Saatkamp + Gerald Klix + Eugene Oden + Dinu Gherman + Guenter Jantzen + Ben Young + Nicolas Chauvat + Michael Twomey + Rocco Moretti + Simon Burton + Boris Feigin + Olivier Dormond + Gintautas Miliauskas + Stuart Williams + Jens-Uwe Mager + Brian Dorsey + Jonathan David Riehl + Anders Qvist + Beatrice During + Andreas Friedge + Alan McIntyre + Bert Freudenberg + + Heinrich-Heine University, Germany + Open End AB (formerly AB Strakt), Sweden + merlinux GmbH, Germany + tismerysoft GmbH, Germany + Logilab Paris, France + DFKI GmbH, Germany + Impara, Germany + Change Maker, Sweden + +License: + + Copyright (C) 2003-2007 + + Except when otherwise stated (look for LICENSE files or information at + the beginning of each file) the files in the 'pypy' directory are each + copyrighted by one or more of the following people and organizations: + + Armin Rigo + Samuele Pedroni + Holger Krekel + Christian Tismer + Michael Hudson + Carl Friedrich Bolz + Eric van Riet Paap + Richard Emslie + Anders Chrigstrom + Niklaus Haldimann + Antonio Cuni + Maciek Fijalkowski + Aur?lien Camp?as + Seo Sanghyeon + Alex Martelli + Anders Lehmann + Stephan Diehl + Patrick Maupin + Ludovic Aubry + Bob Ippolito + Adrien Di Mascio + Jacob Hallen + Laura Creighton + Marius Gedminas + Amaury Forgeot d Arc + Boris Feigin + Valentino Volonghi + Bert Freudenberg + Andrew Thompson + Jonathan David Riehl + Amaury Forgeot D Arc + Alexandre Fayolle + Guido van Rossum + + Heinrich-Heine University, Germany + AB Strakt, Sweden + merlinux GmbH, Germany + tismerysoft GmbH, Germany + Logilab Paris, France + DFKI GmbH, Germany + Impara, Germany + Change Maker, Sweden + + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + + +License for 'lib-python/2.4.1' and 'lib-python/2.4.1-modified' +============================================================== + +Except when otherwise stated (look for LICENSE files or +copyright/license information at the beginning of each file) the files +in the 'lib-python/2.4.1' and 'lib-python/2.4.1-modified' directories +are all copyrighted by the Python Software Foundation and licensed under +the Python Software License of which you can find a copy here: +http://www.python.org/doc/Copyright.html + +License for files in pypy/modules/unicodedata +============================================= + +The following files are from the website of The Unicode Consortium +at . For the terms of use of these files, see +. + + CompositionExclusions-3.2.0.txt + CompositionExclusions-4.1.0.txt + CompositionExclusions-5.0.0.txt + EastAsianWidth-3.2.0.txt + EastAsianWidth-4.1.0.txt + EastAsianWidth-5.0.0.txt + UnicodeData-3.2.0.txt + UnicodeData-4.1.0.txt + UnicodeData-5.0.0.txt + +The following files are derived from files from the above website. The same +terms of use apply: + + UnihanNumeric-3.2.0.txt + UnihanNumeric-4.1.0.txt + UnihanNumeric-5.0.0.txt + +License for pyrex +================= + +Copyright stuff: Pyrex is free of restrictions. You may use, redistribute, +modify and distribute modified versions. + +The latest version of Pyrex can be found here: + + + +Greg Ewing, Computer Science Dept, +--------------------------------------+ +University of Canterbury, | A citizen of NewZealandCorp, a | +Christchurch, New Zealand | wholly-owned subsidiary of USA Inc. | +greg at cosc.canterbury.ac.nz + +The Debian packaging is (C) 2007, 2008 Alexandre Fayolle +, Sylvain Th?nault + and Chris Lamb and is +licensed under the GPL, see `/usr/share/common-licenses/GPL'. Added: pypy/build/debian/debian/manpages/jscompile.1 ============================================================================== --- (empty file) +++ pypy/build/debian/debian/manpages/jscompile.1 Thu Feb 7 19:46:37 2008 @@ -0,0 +1,39 @@ +.TH JSCOMPILE 1 "January 27, 2008" +.SH NAME +jscompile \- RPython to JavaScript compiler. +.SH SYNOPSIS +.B jscompile +.RI module_to_compile +.RI [options] +.SH DESCRIPTION +\fBjscompile\fP compiles RPython programs to JavaScript using the PyPy +translation toolchain. +.SH OPTIONS +.TP +.B \-\-help, -h +Show help message and exit. +.TP +.B \-\-view +View flow graphs. +.TP +.B \-\-no-view +Unset option set by --view. [default] +.TP +.B \--pdb +Use debugger. +.TP +.B \-\-no-pdb +Unset option set by --pdb. [default] +.TP +.B \-\-output=OUTPUT +File to save results. [default: output.js] +.SH SEE ALSO +.BR pypy (1), +.BR pypy-stackless (1), +.BR py.py (1), +.BR pypy-translate (1) +.SH AUTHORS +\fBPyPy\fP was written by the members of the PyPy project . +.PP +This manual page was written by Chris Lamb , +for the Debian project (but may be used by others). Added: pypy/build/debian/debian/manpages/py.py.1 ============================================================================== --- (empty file) +++ pypy/build/debian/debian/manpages/py.py.1 Thu Feb 7 19:46:37 2008 @@ -0,0 +1,23 @@ +.TH PY.PY 1 "January 27, 2008" +.SH NAME +py.py \- PyPy Python interpreter running on top of Python. +.SH SYNOPSIS +.B py.py +.RI [options] +.SH DESCRIPTION +\fBpy.py\fP is a Python interpreter running on top of Python. +.SH OPTIONS +\fBpy.py\fP has a very large number of options which are changing +often. +.TP +To see a full list, please see the PyPy homepage. +.SH SEE ALSO +.BR pypy (1), +.BR pypy-stackless (1), +.BR pypy-translate (1), +.BR jscompile (1) +.SH AUTHORS +\fBPyPy\fP was written by the members of the PyPy project . +.PP +This manual page was written by Chris Lamb , +for the Debian project (but may be used by others). Added: pypy/build/debian/debian/manpages/pypy-stackless.1 ============================================================================== --- (empty file) +++ pypy/build/debian/debian/manpages/pypy-stackless.1 Thu Feb 7 19:46:37 2008 @@ -0,0 +1,47 @@ +.TH PYPY-STACKLESS 1 "January 27, 2008" +.SH NAME +pypy-stackless \- PyPy Python interpreter compiled using the stackless C backend. +.SH SYNOPSIS +.B pypy-stackless +.RI [options] +.SH DESCRIPTION +\fBpypy-stackless\fP is a PyPy interpreter compiled using the C backend, with +stackless functionality. +.SH OPTIONS +.TP +.B \-i +Inspect interactively after running the specified script. +.TP +.B \-O +Dummy optimization flag for compatibility with CPython. +.TP +.B \-c CMD +program passed in as CMD (terminates option list). +.TP +.B \-S +Do not "import site" on initialization. +.TP +.B \-u +Unbuffered binary stdout and stderr. +.TP +.B \-h, \-\-help +Show this help message and exit. +.TP +.B \-m +Library module to be run as a script (terminates option list). +.TP +.B \-\-version +Print the PyPy version. +.TP +.B \-\-info +Print translation information about this PyPy executable. +.SH SEE ALSO +.BR pypy (1), +.BR py.py (1), +.BR pypy-translate (1), +.BR jscompile (1) +.SH AUTHORS +\fBPyPy\fP was written by the members of the PyPy project . +.PP +This manual page was written by Chris Lamb , +for the Debian project (but may be used by others). Added: pypy/build/debian/debian/manpages/pypy-translate.1 ============================================================================== --- (empty file) +++ pypy/build/debian/debian/manpages/pypy-translate.1 Thu Feb 7 19:46:37 2008 @@ -0,0 +1,21 @@ +.TH PYPY-TRANSLATE 1 "January 27, 2008" +.SH NAME +pypy-translate \- PyPy translation tool. +.SH SYNOPSIS +.B pypy-translate +.RI [options] +.SH OPTIONS +\fBpypy-translate\fP has a very large number of options which are changing +often. +.TP +To see the full list, please see the documentation. +.SH SEE ALSO +.BR pypy (1), +.BR pypy-stackless (1), +.BR py.py (1), +.BR jscompile (1) +.SH AUTHORS +\fBPyPy\fP was written by the members of the PyPy project . +.PP +This manual page was written by Chris Lamb , +for the Debian project (but may be used by others). Added: pypy/build/debian/debian/manpages/pypy.1 ============================================================================== --- (empty file) +++ pypy/build/debian/debian/manpages/pypy.1 Thu Feb 7 19:46:37 2008 @@ -0,0 +1,46 @@ +.TH PYPY 1 "January 27, 2008" +.SH NAME +pypy \- PyPy Python interpreter compiled using the C backend. +.SH SYNOPSIS +.B pypy +.RI [options] +.SH DESCRIPTION +\fBpypy\fP is a Python interpreter compiled using the C backend. +.SH OPTIONS +.TP +.B \-i +Inspect interactively after running the specified script. +.TP +.B \-O +Dummy optimization flag for compatibility with CPython. +.TP +.B \-c CMD +program passed in as CMD (terminates option list). +.TP +.B \-S +Do not "import site" on initialization. +.TP +.B \-u +Unbuffered binary stdout and stderr. +.TP +.B \-h, \-\-help +Show this help message and exit. +.TP +.B \-m +Library module to be run as a script (terminates option list). +.TP +.B \-\-version +Print the PyPy version. +.TP +.B \-\-info +Print translation information about this PyPy executable. +.SH SEE ALSO +.BR pypy-stackless (1), +.BR py.py (1), +.BR pypy-translate (1), +.BR jscompile (1) +.SH AUTHORS +\fBPyPy\fP was written by the members of the PyPy project . +.PP +This manual page was written by Chris Lamb , +for the Debian project (but may be used by others). Added: pypy/build/debian/debian/patches/01-package-specific-temp-prefix ============================================================================== --- (empty file) +++ pypy/build/debian/debian/patches/01-package-specific-temp-prefix Thu Feb 7 19:46:37 2008 @@ -0,0 +1,19 @@ +Use a PyPy-specific tempororary directory so that unrelated dirs are never +removed. This fixes Debian bug #452851. + + -- Chris Lamb Sun, 27 Jan 2008 18:27:57 +0000 + +=============================================================================== +diff -urNad pypy-svn.orig/py/path/local/local.py pypy-svn/py/path/local/local.py +--- pypy-svn.orig/py/path/local/local.py 2008-01-27 06:40:27.000000000 +0000 ++++ pypy-svn/py/path/local/local.py 2008-01-27 18:26:16.000000000 +0000 +@@ -587,6 +587,9 @@ + if keep is true directories with a number less than (maxnum-keep) + will be removed. + """ ++ ++ prefix = 'debian-pypy-' + prefix ++ + if rootdir is None: + rootdir = cls.get_temproot() + Added: pypy/build/debian/debian/patches/02-mips-support ============================================================================== --- (empty file) +++ pypy/build/debian/debian/patches/02-mips-support Thu Feb 7 19:46:37 2008 @@ -0,0 +1,76 @@ +Add MIPS and MIPSEL support. + +This patch was originally written by Thiemo Seufer and +was submitted via Debian bug #459520. + + -- Chris Lamb Mon, 28 Jan 2008 05:39:13 +0000 + + +--- pypy-1.0.0-svn50146.orig/py/c-extension/greenlet/slp_platformselect.h ++++ pypy-1.0.0-svn50146/py/c-extension/greenlet/slp_platformselect.h +@@ -18,4 +18,6 @@ + #include "switch_s390_unix.h" /* Linux/S390 */ + #elif defined(__GNUC__) && defined(__s390x__) && defined(__linux__) + #include "switch_s390_unix.h" /* Linux/S390 zSeries (identical) */ ++#elif defined(__GNUC__) && defined(__mips__) && defined(__linux__) ++#include "switch_mips_unix.h" /* Linux/MIPS */ + #endif +--- pypy-1.0.0-svn50146.orig/py/c-extension/greenlet/switch_mips_unix.h ++++ pypy-1.0.0-svn50146/py/c-extension/greenlet/switch_mips_unix.h +@@ -0,0 +1,56 @@ ++/* ++ * this is the internal transfer function. ++ * ++ * HISTORY ++ * 05-Jan-08 Thiemo Seufer ++ * Ported from ppc. ++ */ ++ ++#define STACK_REFPLUS 1 ++ ++#ifdef SLP_EVAL ++ ++#define STACK_MAGIC 0 ++ ++#ifdef __mips64 ++#define REGS_TO_SAVE "$16", "$17", "$18", "$19", "$20", "$21", "$22", \ ++ "$23", "$28", "$30" ++#else ++#define REGS_TO_SAVE "$16", "$17", "$18", "$19", "$20", "$21", "$22", \ ++ "$23", "$30" ++#endif ++static int ++slp_switch(void) ++{ ++ register int *stackref, stsizediff; ++ __asm__ __volatile__ ("" : : : REGS_TO_SAVE); ++ __asm__ ("move %0, $29" : "=r" (stackref) : ); ++ { ++ SLP_SAVE_STATE(stackref, stsizediff); ++ __asm__ __volatile__ ( ++#ifdef __mips64 ++ "daddu $29, %0\n" ++#else ++ "addu $29, %0\n" ++#endif ++ : /* no outputs */ ++ : "r" (stsizediff) ++ ); ++ SLP_RESTORE_STATE(); ++ } ++ __asm__ __volatile__ ("" : : : REGS_TO_SAVE); ++ return 0; ++} ++ ++#endif ++ ++/* ++ * further self-processing support ++ */ ++ ++/* ++ * if you want to add self-inspection tools, place them ++ * here. See the x86_msvc for the necessary defines. ++ * These features are highly experimental und not ++ * essential yet. ++ */ Added: pypy/build/debian/debian/patches/series ============================================================================== --- (empty file) +++ pypy/build/debian/debian/patches/series Thu Feb 7 19:46:37 2008 @@ -0,0 +1,2 @@ +01-package-specific-temp-prefix +02-mips-support Added: pypy/build/debian/debian/pypy-dev.dirs ============================================================================== --- (empty file) +++ pypy/build/debian/debian/pypy-dev.dirs Thu Feb 7 19:46:37 2008 @@ -0,0 +1,2 @@ +var/cache/pypy/_cache +usr/share/lintian/overrides Added: pypy/build/debian/debian/pypy-dev.install ============================================================================== --- (empty file) +++ pypy/build/debian/debian/pypy-dev.install Thu Feb 7 19:46:37 2008 @@ -0,0 +1,17 @@ +pypy/annotation usr/share/pypy-1.0/pypy +pypy/bin usr/share/pypy-1.0/pypy +pypy/config usr/share/pypy-1.0/pypy +pypy/__init__.py usr/share/pypy-1.0/pypy +pypy/interpreter usr/share/pypy-1.0/pypy +pypy/jit usr/share/pypy-1.0/pypy +pypy/lang usr/share/pypy-1.0/pypy +pypy/module usr/share/pypy-1.0/pypy +pypy/objspace usr/share/pypy-1.0/pypy +pypy/rlib usr/share/pypy-1.0/pypy +pypy/rpython usr/share/pypy-1.0/pypy +pypy/tool usr/share/pypy-1.0/pypy +pypy/translator usr/share/pypy-1.0/pypy +pypy/conftest.py usr/share/pypy-1.0/pypy +debian/scripts/py.py usr/bin +debian/scripts/pypy-translate usr/bin +debian/scripts/jscompile usr/bin Added: pypy/build/debian/debian/pypy-dev.links ============================================================================== --- (empty file) +++ pypy/build/debian/debian/pypy-dev.links Thu Feb 7 19:46:37 2008 @@ -0,0 +1 @@ +/var/cache/pypy/_cache /usr/share/pypy-1.0/pypy/_cache Added: pypy/build/debian/debian/pypy-dev.manpages ============================================================================== --- (empty file) +++ pypy/build/debian/debian/pypy-dev.manpages Thu Feb 7 19:46:37 2008 @@ -0,0 +1,3 @@ +debian/manpages/py.py.1 +debian/manpages/pypy-translate.1 +debian/manpages/jscompile.1 Added: pypy/build/debian/debian/pypy-dev.override ============================================================================== --- (empty file) +++ pypy/build/debian/debian/pypy-dev.override Thu Feb 7 19:46:37 2008 @@ -0,0 +1,5 @@ +# Gentoo specific file. We will keep it. +pypy-dev: unusual-interpreter ./usr/share/pypy-0.9/pypy/tool/build/bin/gentoo_init.d #!/sbin/runscript + +# This is the name of the interpreted interpreter. Renaming it to py makes no sense +pypy-dev: script-with-language-extension usr/bin/py.py Added: pypy/build/debian/debian/pypy-dev.postinst ============================================================================== --- (empty file) +++ pypy/build/debian/debian/pypy-dev.postinst Thu Feb 7 19:46:37 2008 @@ -0,0 +1,13 @@ +#!/bin/sh + +set -e + +case "$1" in + configure) + chmod o+rwxt /var/cache/pypy/_cache + ;; +esac + +#DEBHELPER# + +exit 0 Added: pypy/build/debian/debian/pypy-dev.prerm ============================================================================== --- (empty file) +++ pypy/build/debian/debian/pypy-dev.prerm Thu Feb 7 19:46:37 2008 @@ -0,0 +1,13 @@ +#!/bin/sh + +set -e + +case "$1" in + remove) + rm -rf /var/cache/pypy/_cache/* + ;; +esac + +#DEBHELPER# + +exit 0 Added: pypy/build/debian/debian/pypy-doc.docs ============================================================================== --- (empty file) +++ pypy/build/debian/debian/pypy-doc.docs Thu Feb 7 19:46:37 2008 @@ -0,0 +1 @@ +pypy/doc/* Added: pypy/build/debian/debian/pypy-lib.dirs ============================================================================== --- (empty file) +++ pypy/build/debian/debian/pypy-lib.dirs Thu Feb 7 19:46:37 2008 @@ -0,0 +1 @@ +usr/share/lintian/overrides Added: pypy/build/debian/debian/pypy-lib.install ============================================================================== --- (empty file) +++ pypy/build/debian/debian/pypy-lib.install Thu Feb 7 19:46:37 2008 @@ -0,0 +1,2 @@ +lib-python usr/share/pypy-1.0 +pypy/lib usr/share/pypy-1.0/pypy Added: pypy/build/debian/debian/pypy-lib.links ============================================================================== --- (empty file) +++ pypy/build/debian/debian/pypy-lib.links Thu Feb 7 19:46:37 2008 @@ -0,0 +1 @@ +usr/share/pypy-1.0/pypy/lib usr/share/pypy-1.0/lib Added: pypy/build/debian/debian/pypy-lib.override ============================================================================== --- (empty file) +++ pypy/build/debian/debian/pypy-lib.override Thu Feb 7 19:46:37 2008 @@ -0,0 +1,2 @@ +# The package is meant to be used by PyPy, and therefore has no dependency on Python. +pypy-lib: python-script-but-no-python-dep Added: pypy/build/debian/debian/pypy-logic.install ============================================================================== --- (empty file) +++ pypy/build/debian/debian/pypy-logic.install Thu Feb 7 19:46:37 2008 @@ -0,0 +1 @@ +debian/tmp/usr/bin/pypy-logic usr/bin Added: pypy/build/debian/debian/pypy-stackless.install ============================================================================== --- (empty file) +++ pypy/build/debian/debian/pypy-stackless.install Thu Feb 7 19:46:37 2008 @@ -0,0 +1 @@ +debian/tmp/usr/bin/pypy-stackless usr/bin Added: pypy/build/debian/debian/pypy-stackless.manpages ============================================================================== --- (empty file) +++ pypy/build/debian/debian/pypy-stackless.manpages Thu Feb 7 19:46:37 2008 @@ -0,0 +1 @@ +debian/manpages/pypy-stackless.1 Added: pypy/build/debian/debian/pypy.install ============================================================================== --- (empty file) +++ pypy/build/debian/debian/pypy.install Thu Feb 7 19:46:37 2008 @@ -0,0 +1 @@ +debian/tmp/usr/bin/pypy usr/bin Added: pypy/build/debian/debian/pypy.manpages ============================================================================== --- (empty file) +++ pypy/build/debian/debian/pypy.manpages Thu Feb 7 19:46:37 2008 @@ -0,0 +1 @@ +debian/manpages/pypy.1 Added: pypy/build/debian/debian/rules ============================================================================== --- (empty file) +++ pypy/build/debian/debian/rules Thu Feb 7 19:46:37 2008 @@ -0,0 +1,126 @@ +#!/usr/bin/make -f +# -*- makefile -*- +# Uncomment this to turn on verbose mode. +#export DH_VERBOSE=1 + +include /usr/share/quilt/quilt.make + +# This has to be exported to make some magic below work. +export DH_OPTIONS + +DEB_BUILD_ARCH := $(shell dpkg-architecture -qDEB_BUILD_ARCH) +DEB_HOST_ARCH := $(shell dpkg-architecture -qDEB_HOST_ARCH) + +VERSION=1.0 + +get-orig-source: + set -e ; \ + svn co http://codespeak.net/svn/pypy/dist pypy-svn ; \ + REVISION=`svnversion pypy-svn/` ; \ + find pypy-svn/ -depth -type d -name ".svn" | xargs rm -rf ; \ + tar cfz pypy_1.0.0-svn$$REVISION.orig.tar.gz pypy-svn/ ; \ + rm -rf pypy-svn/ + +Makefile: debian/Makefile.in + python debian/configure.py --build=$(DEB_BUILD_ARCH) --host=$(DEB_HOST_ARCH) --prefix=debian/tmp/usr debian/Makefile.in + +build: patch build-arch build-indep + +build-arch: build-arch-stamp +build-arch-stamp: Makefile + $(MAKE) all + touch $@ + +build-indep: build-indep-stamp +build-indep-stamp: + touch $@ + +clean: unpatch + dh_testdir + dh_testroot + rm -f build-arch-stamp build-indep-stamp + if test -e Makefile ; then $(MAKE) clean || true ; fi + rm -f Makefile + dh_clean + +install: install-indep install-arch +install-indep: + dh_testdir + dh_testroot + dh_clean -k -i + dh_installdirs -i + cp debian/pypy-lib.override debian/pypy-lib/usr/share/lintian/overrides/pypy-lib + cp debian/pypy-dev.override debian/pypy-dev/usr/share/lintian/overrides/pypy-dev + dh_install -i + find debian/pypy-lib -name CVS | xargs rm -rf + find debian/pypy-lib -name "*.pyc" | xargs rm -rf + find debian/pypy-dev -name "*.pyc" | xargs rm -rf + find debian/pypy-dev/usr/share/pypy-*/pypy/translator/llvm \ + debian/pypy-lib/usr/share/pypy-*/lib-python \ + debian/pypy-dev/usr/share/pypy-*/pypy/lang/js/test/ecma \ + -type f | xargs chmod a-x + for f in `find debian/pypy-lib debian/pypy-dev -type f` ; do \ + if head -n 1 $$f | grep -q '^#!.*python' ; then \ + chmod a+x $$f ; \ + fi ; \ + done + chmod a+x debian/pypy-lib/usr/share/pypy-*/lib-python/2.4.1/plat-*/regen + for f in debian/pypy-lib/usr/share/pypy-$(VERSION)/lib-python/2.4.1/test/test_largefile.py \ + debian/pypy-lib/usr/share/pypy-$(VERSION)/lib-python/2.4.1/test/test_largefile.py \ + debian/pypy-lib/usr/share/pypy-$(VERSION)/lib-python/2.4.1/cgi.py \ + debian/pypy-lib/usr/share/pypy-$(VERSION)/lib-python/2.4.1/bsddb/dbshelve.py \ + debian/pypy-dev/usr/share/pypy-$(VERSION)/pypy/translator/microbench/pybench/pybench.py \ + debian/pypy-dev/usr/share/pypy-$(VERSION)/pypy/translator/microbench/pybench/Setup.py \ + debian/pypy-dev/usr/share/pypy-$(VERSION)/pypy/translator/microbench/pybench/Setup.py \ + debian/pypy-dev/usr/share/pypy-$(VERSION)/pypy/translator/microbench/pybench/platform.py \ + debian/pypy-lib/usr/share/pypy-$(VERSION)/pypy/lib/app_test/test_binascii.py \ + debian/pypy-lib/usr/share/pypy-$(VERSION)/pypy/lib/test2/pickledtasklet.py \ + ; do \ + echo "#!/usr/bin/python" > debian/tmpfile ; \ + tail -n +2 $$f >> debian/tmpfile ; \ + mv debian/tmpfile $$f ; \ + chmod a+x $$f ; \ + done + rm debian/pypy-dev/usr/share/pypy-$(VERSION)/pypy/module/unicodedata/LICENSE + rm debian/pypy-dev/usr/share/pypy-$(VERSION)/pypy/lang/js/commit + rm debian/pypy-lib/usr/share/pypy-$(VERSION)/pypy/lib/pyrepl/.cvsignore + rmdir debian/pypy-dev/usr/share/pypy-$(VERSION)/pypy/translator/js/examples/bnb/data/images/ + +install-arch: + dh_testdir + dh_testroot + dh_clean -k -s + dh_installdirs -s + $(MAKE) install + dh_install -s + +# Must not depend on anything. This is to be called by binary-arch/binary-indep +# in another 'make' thread. +binary-common: + dh_testdir + dh_testroot + dh_installchangelogs + dh_installdocs + dh_installexamples + dh_installman --language=C + dh_link + dh_strip + dh_compress + dh_fixperms + dh_makeshlibs + dh_installdeb + dh_shlibdeps + dh_gencontrol + dh_md5sums + dh_builddeb + +# Build architecture independent packages using the common target. +binary-indep: build-indep install-indep + $(MAKE) -f debian/rules DH_OPTIONS=-i binary-common + +# Build architecture dependent packages using the common target. +binary-arch: build-arch install-arch + $(MAKE) -f debian/rules DH_OPTIONS=-s binary-common + +binary: binary-arch binary-indep +.PHONY: build clean binary-indep binary-arch binary install install-indep install-arch patch unpatch get-orig-source Added: pypy/build/debian/debian/scripts/jscompile ============================================================================== --- (empty file) +++ pypy/build/debian/debian/scripts/jscompile Thu Feb 7 19:46:37 2008 @@ -0,0 +1,7 @@ +#!/usr/bin/python + +import sys +sys.path.append('/usr/share/pypy-1.0') +sys.path.append('/usr/share/pypy-1.0/pypy/bin') +sys.path.remove('/usr/bin') +execfile('/usr/share/pypy-1.0/pypy/bin/jscompile.py') Added: pypy/build/debian/debian/scripts/py.py ============================================================================== --- (empty file) +++ pypy/build/debian/debian/scripts/py.py Thu Feb 7 19:46:37 2008 @@ -0,0 +1,5 @@ +#!/usr/bin/python +import sys +sys.path.insert(0, '/usr/share/pypy-1.0') +sys.path.remove('/usr/bin') +execfile('/usr/share/pypy-1.0/pypy/bin/py.py') Added: pypy/build/debian/debian/scripts/pypy-translate ============================================================================== --- (empty file) +++ pypy/build/debian/debian/scripts/pypy-translate Thu Feb 7 19:46:37 2008 @@ -0,0 +1,3 @@ +#!/bin/sh + +/usr/share/pypy-1.0/pypy/translator/goal/translate.py "$@" From antocuni at codespeak.net Fri Feb 8 15:07:02 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 8 Feb 2008 15:07:02 +0100 (CET) Subject: [pypy-svn] r51334 - in pypy/dist/pypy: jit/codegen/cli translator/cli Message-ID: <20080208140702.4896A168410@codespeak.net> Author: antocuni Date: Fri Feb 8 15:07:00 2008 New Revision: 51334 Modified: pypy/dist/pypy/jit/codegen/cli/operation.py pypy/dist/pypy/jit/codegen/cli/rgenop.py pypy/dist/pypy/translator/cli/opcodes.py Log: the cli jit and non jit backends now share the descriptions of the various opcodes. Not all of those are yet supported by the jit backend though. Modified: pypy/dist/pypy/jit/codegen/cli/operation.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/operation.py (original) +++ pypy/dist/pypy/jit/codegen/cli/operation.py Fri Feb 8 15:07:00 2008 @@ -1,5 +1,8 @@ +import py +from pypy.rlib.objectmodel import specialize from pypy.rpython.ootypesystem import ootype from pypy.translator.cli.dotnet import CLR, typeof +from pypy.translator.cli import opcodes as cli_opcodes System = CLR.System OpCodes = System.Reflection.Emit.OpCodes @@ -21,15 +24,11 @@ def emit(self): raise NotImplementedError + def pushAllArgs(self): + raise NotImplementedError -class Branch(Operation): - - def __init__(self, il, label): - self.il = il - self.label = label - - def emit(self): - self.il.emit(OpCodes.Br, self.label) + def storeResult(self): + self.gv_res().store(self.il) class UnaryOp(Operation): @@ -37,41 +36,8 @@ self.il = il self.gv_x = gv_x - -class AbstractBranchIf(UnaryOp): - - def __init__(self, il, gv_x, label): - self.il = il - self.gv_x = gv_x - self.label = label - - def restype(self): - return None - - def emit(self): - self.il.emit(self.getOpCode(), self.label) - - def getOpCode(self): - return OpCodes.Brtrue - - -class BrFalse(AbstractBranchIf): - - def getOpCode(self): - return OpCodes.Brfalse - -class BrTrue(AbstractBranchIf): - - def getOpCode(self): - return OpCodes.Brtrue - - -class SameAs(UnaryOp): - def emit(self): - gv_res = self.gv_res() + def pushAllArgs(self): self.gv_x.load(self.il) - self.gv_res().store(self.il) - class BinaryOp(Operation): def __init__(self, il, gv_x, gv_y): @@ -79,28 +45,77 @@ self.gv_x = gv_x self.gv_y = gv_y - def emit(self): + def pushAllArgs(self): self.gv_x.load(self.il) self.gv_y.load(self.il) + + def emit(self): + self.pushAllArgs() self.il.Emit(self.getOpCode()) - self.gv_res().store(self.il) + self.storeResult() def getOpCode(self): raise NotImplementedError -class Add(BinaryOp): - def getOpCode(self): - return OpCodes.Add +class SameAs(UnaryOp): + def emit(self): + gv_res = self.gv_res() + self.gv_x.load(self.il) + self.gv_res().store(self.il) -class Sub(BinaryOp): - def getOpCode(self): - return OpCodes.Sub +def opcode2attrname(opcode): + parts = map(str.capitalize, opcode.split('.')) + return '_'.join(parts) + +def is_comparison(opname): + suffixes = '_lt _le _eq _ne _gt _ge'.split() + for suffix in suffixes: + if opname.endswith(suffix): + return True + return False -class Gt(BinaryOp): +def fillops(ops, baseclass): + # monkey-patch boolean operations def restype(self): return typeof(System.Boolean) - def getOpCode(self): - return OpCodes.Cgt + out = {} + for opname, value in ops.iteritems(): + if isinstance(value, str): + attrname = opcode2attrname(value) + source = py.code.Source(""" + class %(opname)s (%(baseclass)s): + def getOpCode(self): + return OpCodes.%(attrname)s + """ % locals()) + code = source.compile() + exec code in globals(), out + if is_comparison(opname): + out[opname].restype = restype + elif value is cli_opcodes.DoNothing: + out[opname] = SameAs + else: + pass # XXX: handle remaining ops + return out + +UNARYOPS = fillops(cli_opcodes.unary_ops, "UnaryOp") +BINARYOPS = fillops(cli_opcodes.binary_ops, "BinaryOp") + + at specialize.memo() +def getopclass1(opname): + try: + return UNARYOPS[opname] + except KeyError: + raise MissingBackendOperation(opname) + + at specialize.memo() +def getopclass2(opname): + try: + return BINARYOPS[opname] + except KeyError: + raise MissingBackendOperation(opname) + +class MissingBackendOperation(Exception): + pass Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/rgenop.py Fri Feb 8 15:07:00 2008 @@ -178,14 +178,8 @@ @specialize.arg(1) def genop2(self, opname, gv_arg1, gv_arg2): - if opname == 'int_add': - op = ops.Add(self.il, gv_arg1, gv_arg2) - elif opname == 'int_sub': - op = ops.Sub(self.il, gv_arg1, gv_arg2) - elif opname == 'int_gt': - op = ops.Gt(self.il, gv_arg1, gv_arg2) - else: - assert False + opcls = ops.getopclass2(opname) + op = opcls(self.il, gv_arg1, gv_arg2) self.emit(op) return op.gv_res() Modified: pypy/dist/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/cli/opcodes.py (original) +++ pypy/dist/pypy/translator/cli/opcodes.py Fri Feb 8 15:07:00 2008 @@ -26,9 +26,8 @@ mapping = [('[mscorlib]System.DivideByZeroException', 'exceptions.ZeroDivisionError')] return [MapException(op, mapping)] - -opcodes = { - # __________ object oriented operations __________ +# __________ object oriented & misc operations __________ +misc_ops = { 'new': [New], 'runtimenew': [RuntimeNew], 'oosetfield': [SetField], @@ -57,7 +56,6 @@ 'ooparse_float': [PushAllArgs, 'call float64 [pypylib]pypy.runtime.Utils::OOParseFloat(string)'], 'oonewcustomdict': [NewCustomDict], - 'same_as': DoNothing, 'hint': [PushArg(0), StoreResult], 'direct_call': [Call], 'indirect_call': [IndirectCall], @@ -68,11 +66,65 @@ 'resume_point': Ignore, 'debug_assert': Ignore, 'keepalive': Ignore, + 'is_early_constant': [PushPrimitive(ootype.Bool, False)], + } - # __________ numeric operations __________ +# __________ numeric operations __________ +unary_ops = { + 'same_as': DoNothing, + 'bool_not': [PushAllArgs]+Not, + 'int_is_true': [PushAllArgs, 'ldc.i4.0', 'cgt.un'], + 'int_neg': 'neg', + 'int_neg_ovf': _check_ovf(['ldc.i4.0', PushAllArgs, 'sub.ovf', StoreResult]), + 'int_abs': _abs('int32'), + 'int_abs_ovf': _check_ovf(_abs('int32')), + 'int_invert': 'not', + + 'uint_is_true': [PushAllArgs, 'ldc.i4.0', 'cgt.un'], + 'uint_invert': 'not', + + 'float_is_true': [PushAllArgs, 'ldc.r8 0', 'ceq']+Not, + 'float_neg': 'neg', + 'float_abs': _abs('float64'), + + 'llong_is_true': [PushAllArgs, 'ldc.i8 0', 'cgt.un'], + 'llong_neg': 'neg', + 'llong_neg_ovf': _check_ovf(['ldc.i8 0', PushAllArgs, 'sub.ovf', StoreResult]), + 'llong_abs': _abs('int64'), + 'llong_abs_ovf': _check_ovf(_abs('int64')), + 'llong_invert': 'not', + + 'ullong_is_true': [PushAllArgs, 'ldc.i8 0', 'cgt.un'], + 'ullong_invert': 'not', + + # when casting from bool we want that every truth value is casted + # to 1: we can't simply DoNothing, because the CLI stack could + # contains a truth value not equal to 1, so we should use the !=0 + # trick. + 'cast_bool_to_int': [PushAllArgs, 'ldc.i4.0', 'ceq']+Not, + 'cast_bool_to_uint': [PushAllArgs, 'ldc.i4.0', 'ceq']+Not, + 'cast_bool_to_float': [PushAllArgs, 'ldc.i4 0', 'ceq']+Not+['conv.r8'], + 'cast_char_to_int': DoNothing, + 'cast_unichar_to_int': DoNothing, + 'cast_int_to_char': DoNothing, + 'cast_int_to_unichar': DoNothing, + 'cast_int_to_uint': DoNothing, + 'cast_int_to_float': 'conv.r8', + 'cast_int_to_longlong': 'conv.i8', + 'cast_uint_to_int': DoNothing, + 'cast_uint_to_float': [PushAllArgs, 'conv.u8', 'conv.r8'], + 'cast_float_to_int': 'conv.i4', + 'cast_float_to_uint': 'conv.u4', + 'cast_longlong_to_float': 'conv.r8', + 'cast_float_to_longlong': 'conv.i8', + 'cast_primitive': [PushAllArgs, CastPrimitive], + 'truncate_longlong_to_int': 'conv.i4', + } + +binary_ops = { 'char_lt': 'clt', 'char_le': _not('cgt'), 'char_eq': 'ceq', @@ -83,13 +135,6 @@ 'unichar_eq': 'ceq', 'unichar_ne': _not('ceq'), - 'int_is_true': [PushAllArgs, 'ldc.i4.0', 'cgt.un'], - 'int_neg': 'neg', - 'int_neg_ovf': _check_ovf(['ldc.i4.0', PushAllArgs, 'sub.ovf', StoreResult]), - 'int_abs': _abs('int32'), - 'int_abs_ovf': _check_ovf(_abs('int32')), - 'int_invert': 'not', - 'int_add': 'add', 'int_sub': 'sub', 'int_mul': 'mul', @@ -133,9 +178,6 @@ 'int_mod_ovf_zer': _check_zer('rem'), 'int_mod_zer': _check_zer('rem'), - 'uint_is_true': [PushAllArgs, 'ldc.i4.0', 'cgt.un'], - 'uint_invert': 'not', - 'uint_add': 'add', 'uint_sub': 'sub', 'uint_mul': 'mul', @@ -155,10 +197,6 @@ 'uint_rshift': 'shr.un', 'uint_xor': 'xor', - 'float_is_true': [PushAllArgs, 'ldc.r8 0', 'ceq']+Not, - 'float_neg': 'neg', - 'float_abs': _abs('float64'), - 'float_add': 'add', 'float_sub': 'sub', 'float_mul': 'mul', @@ -169,13 +207,6 @@ 'float_ne': _not('ceq'), 'float_gt': 'cgt', 'float_ge': _not('clt'), - - 'llong_is_true': [PushAllArgs, 'ldc.i8 0', 'cgt.un'], - 'llong_neg': 'neg', - 'llong_neg_ovf': _check_ovf(['ldc.i8 0', PushAllArgs, 'sub.ovf', StoreResult]), - 'llong_abs': _abs('int64'), - 'llong_abs_ovf': _check_ovf(_abs('int64')), - 'llong_invert': 'not', 'llong_add': 'add', 'llong_sub': 'sub', @@ -198,9 +229,6 @@ 'llong_rshift': [PushAllArgs, 'conv.i4', 'shr'], 'llong_xor': 'xor', - 'ullong_is_true': [PushAllArgs, 'ldc.i8 0', 'cgt.un'], - 'ullong_invert': 'not', - 'ullong_add': 'add', 'ullong_sub': 'sub', 'ullong_mul': 'mul', @@ -216,32 +244,12 @@ 'ullong_ge': _not('clt.un'), 'ullong_lshift': [PushAllArgs, 'conv.u4', 'shl'], 'ullong_rshift': [PushAllArgs, 'conv.i4', 'shr'], - - # when casting from bool we want that every truth value is casted - # to 1: we can't simply DoNothing, because the CLI stack could - # contains a truth value not equal to 1, so we should use the !=0 - # trick. - 'cast_bool_to_int': [PushAllArgs, 'ldc.i4.0', 'ceq']+Not, - 'cast_bool_to_uint': [PushAllArgs, 'ldc.i4.0', 'ceq']+Not, - 'cast_bool_to_float': [PushAllArgs, 'ldc.i4 0', 'ceq']+Not+['conv.r8'], - 'cast_char_to_int': DoNothing, - 'cast_unichar_to_int': DoNothing, - 'cast_int_to_char': DoNothing, - 'cast_int_to_unichar': DoNothing, - 'cast_int_to_uint': DoNothing, - 'cast_int_to_float': 'conv.r8', - 'cast_int_to_longlong': 'conv.i8', - 'cast_uint_to_int': DoNothing, - 'cast_uint_to_float': [PushAllArgs, 'conv.u8', 'conv.r8'], - 'cast_float_to_int': 'conv.i4', - 'cast_float_to_uint': 'conv.u4', - 'cast_longlong_to_float': 'conv.r8', - 'cast_float_to_longlong': 'conv.i8', - 'cast_primitive': [PushAllArgs, CastPrimitive], - 'truncate_longlong_to_int': 'conv.i4', - 'is_early_constant': [PushPrimitive(ootype.Bool, False)] } +opcodes = misc_ops.copy() +opcodes.update(unary_ops) +opcodes.update(binary_ops) + for key, value in opcodes.iteritems(): if type(value) is str: value = InstructionList([PushAllArgs, value, StoreResult]) @@ -252,4 +260,3 @@ opcodes[key] = value - From antocuni at codespeak.net Fri Feb 8 16:17:22 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 8 Feb 2008 16:17:22 +0100 (CET) Subject: [pypy-svn] r51338 - in pypy/dist/pypy/jit/codegen/cli: . test Message-ID: <20080208151722.68E43168539@codespeak.net> Author: antocuni Date: Fri Feb 8 16:17:21 2008 New Revision: 51338 Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Log: test_goto_* pass Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/rgenop.py Fri Feb 8 16:17:21 2008 @@ -1,7 +1,7 @@ from pypy.tool.pairtype import extendabletype from pypy.rpython.ootypesystem import ootype from pypy.rlib.objectmodel import specialize -from pypy.jit.codegen.model import AbstractRGenOp, GenBuilder +from pypy.jit.codegen.model import AbstractRGenOp, GenBuilder, GenLabel from pypy.jit.codegen.model import GenVarOrConst, GenVar, GenConst, CodeGenSwitch from pypy.jit.codegen.cli import operation as ops from pypy.translator.cli.dotnet import CLR, typeof, new_array, clidowncast @@ -121,6 +121,12 @@ assert isinstance(T, ootype.OOType) return ootype.oodowncast(T, self.obj) +class Label(GenLabel): + def __init__(self, label, inputargs_gv): + self.label = label + self.inputargs_gv = inputargs_gv + + class RCliGenOp(AbstractRGenOp): def __init__(self): @@ -194,6 +200,15 @@ self.il.Emit(OpCodes.Ret) self.isOpen = False + def finish_and_goto(self, outputargs_gv, target): + inputargs_gv = target.inputargs_gv + assert len(inputargs_gv) == len(outputargs_gv) + for i in range(len(outputargs_gv)): + outputargs_gv[i].load(self.il) + inputargs_gv[i].store(self.il) + self.il.Emit(OpCodes.Br, target.label) + self.isOpen = False + def end(self): delegate_type = sigtoken2clitype(self.sigtoken) myfunc = self.meth.CreateDelegate(delegate_type) @@ -206,7 +221,7 @@ args_gv[i] = op.gv_res() label = self.il.DefineLabel() self.il.MarkLabel(label) - return label + return Label(label, args_gv) def _jump_if(self, gv_condition, opcode): label = self.il.DefineLabel() @@ -232,3 +247,10 @@ assert not self.parent.isOpen self.isOpen = True self.il.MarkLabel(self.label) + + @specialize.arg(1) + def genop2(self, opname, gv_arg1, gv_arg2): + # XXX: this only serves to mask a bug in gencli which I don't + # feel like fixing now. Try to uncomment this and run + # test_goto_compile to see why it fails + return Builder.genop2(self, opname, gv_arg1, gv_arg2) Modified: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Fri Feb 8 16:17:21 2008 @@ -13,6 +13,7 @@ 'test_hide_and_reveal_p', 'test_largedummy_direct', # _compile works if we set a higher maxstack 'test_branching', + 'test_goto', ] for p in prefixes: From antocuni at codespeak.net Fri Feb 8 16:54:22 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 8 Feb 2008 16:54:22 +0100 (CET) Subject: [pypy-svn] r51339 - pypy/dist/pypy/translator/cli Message-ID: <20080208155422.DDEF916853D@codespeak.net> Author: antocuni Date: Fri Feb 8 16:54:21 2008 New Revision: 51339 Modified: pypy/dist/pypy/translator/cli/opcodes.py Log: remove integer truediv Modified: pypy/dist/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/cli/opcodes.py (original) +++ pypy/dist/pypy/translator/cli/opcodes.py Fri Feb 8 16:54:21 2008 @@ -182,7 +182,6 @@ 'uint_sub': 'sub', 'uint_mul': 'mul', 'uint_div': 'div.un', - 'uint_truediv': None, # TODO 'uint_floordiv': 'div.un', 'uint_mod': 'rem.un', 'uint_lt': 'clt.un', @@ -212,7 +211,6 @@ 'llong_sub': 'sub', 'llong_mul': 'mul', 'llong_div': 'div', - 'llong_truediv': None, # TODO 'llong_floordiv': 'div', 'llong_floordiv_zer': _check_zer('div'), 'llong_mod': 'rem', @@ -233,7 +231,6 @@ 'ullong_sub': 'sub', 'ullong_mul': 'mul', 'ullong_div': 'div.un', - 'ullong_truediv': None, # TODO 'ullong_floordiv': 'div.un', 'ullong_mod': 'rem.un', 'ullong_lt': 'clt.un', From arigo at codespeak.net Fri Feb 8 19:52:06 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 8 Feb 2008 19:52:06 +0100 (CET) Subject: [pypy-svn] r51345 - pypy/dist/pypy/doc/discussion Message-ID: <20080208185206.E550616852B@codespeak.net> Author: arigo Date: Fri Feb 8 19:52:06 2008 New Revision: 51345 Modified: pypy/dist/pypy/doc/discussion/jit-refactoring-plan.txt Log: Discussion: "Plan B" control flow for the JIT. Modified: pypy/dist/pypy/doc/discussion/jit-refactoring-plan.txt ============================================================================== --- pypy/dist/pypy/doc/discussion/jit-refactoring-plan.txt (original) +++ pypy/dist/pypy/doc/discussion/jit-refactoring-plan.txt Fri Feb 8 19:52:06 2008 @@ -26,3 +26,158 @@ - interpreter is manually written in a stackless style: jitstates have linked lists of frames anyway already + +"Plan B" Control Flow +--------------------- + +A few notes about a refactoring that I was thinking about for after the +rainbow interpreter works nicely. Let's use the Python interpreter as a +motivating example. + +* hot path: for a frequently-executed Python opcode in a certain + function in the user program, the "hot path" is the path in the + interpreter that is generally followed for this particular opcode. + Opposite: "cold path". + +The current approach tends to produce far too much machine code at +run-time - many cold paths give machine code. So let's see what would +be involved in producing as little machine code as possible (maybe +that's the opposite extreme and some middle path would be better). +While we're at it let's include the question of how to produce machine +code for only the relevant parts of the user program. + +Hints +++++++++++++++++ + +We'd replace portals and global merge points with the following variant: +two hints, "jit_can_enter" and "jit_can_leave", which are where the +execution can go from interpreter to JITted and back. The idea is that +"jit_can_leave" is present at the beginning of the main interpreter loop +-- i.e. it is a global merge point. + +The other hint, "jit_can_enter", is the place where some lightweight +profiling occurs in order to know if we should enter the JIT. It's +important to not have one "jit_can_enter" for each opcode -- that's a +too heavy slow-down for regularly interpreted code (but it would be +correct too). A probably reasonable idea is to put it in the opcodes +that close loops (JUMP_ABSOLUTE, CONTINUE). This would make the regular +Python interpreter try to start JITting the Python-level loops that are +often executed. (In time, the JIT should follow calls too, so that +means that the functions called by loops also get JITted.) + +If the profiling in "jit_can_enter" finds out we should start JITting, +it calls the JIT, which compiles and executes some machine code, which +makes the current function frame progress, maybe to its end or not, but +at least to an opcode boundary; so when the call done by "jit_can_enter" +returns the regular interpreter can simply continue from the new next +opcode. For this reason it's necessary to put "jit_can_enter" and +"jit_can_leave" next to each other, control-flow-wise -- +i.e. "jit_can_enter" should be at the end of JUMP_ABSOLUTE and CONTINUE, +so that they are immediately followed by the global "jit_can_leave". + +Note that "jit_can_enter", in the regular interpreter, has another goal +too: it should quickly check if machine code was already emitted for the +next opcode, and if so, jump to it -- i.e. do a call to it. As above +the call to the machine code will make the current function execution +progress and when it returns we can go on interpreter it. + +PyPy contains some custom logic to virtualize the frame and the value +stack; in this new model it should go somewhere related to +"jit_can_enter". + +The "jit_can_enter" hint becomes nothing in the rainbow interpreter's +bytecode. Conversely, the "jit_can_leave" hint becomes nothing in +the regular interpreter, but an important bytecode in the rainbow +bytecode -- a global merge point. + +Very lazy code generation +++++++++++++++++++++++++++++ + +Now to the controversial part (if the above wasn't already). The idea +is for the JIT to be as lazy as possible producing machine code. The +simplest approach allows us to always maintain a single JITState, never +a chained list of pending-to-be-compiled JITStates. (Note that this is +not *necessary*; it's quite possible that it's better to combine +approaches and compile things a bit more eagerly along several paths. +I'm mostly decribing the other extreme here.) + +The basic idea is to stop compiling early, and wait before execution +actually followed one of the possible paths often enough before +continuing. "Early" means at some red splits and all promotions. The +picture is that the JIT should compile a single straight-code path +corresponding to maybe half an opcode or a few opcodes, and then wait; +then compile a bit more, and wait; and progress like this. In this +model we get the nice effect that in a Python-level loop, we would end +up compiling only the loop instead of the whole function that contains +it: indeed, the "jit_can_enter" profiling only triggers on the start of +the loop, and the stop-early logic means that the path that exits the +loop is cold and will not be compiled. + +Red splits and promotions +++++++++++++++++++++++++++++++++++++++++ + +We would identify two kinds of red splits: the ones that just correspond +to "simple if-then-else" patterns; and the "complicated" ones. We can +be more clever about simple if-then-else patterns, but for all other red +splits, we would just stop emitting machine code. The JIT puts in the +machine code a jump to a special "fall-back rainbow interpreter". This +interpreter is a variant that considers everything as green and just +interprets everything normally. The idea is that when execution reaches +the red split, in the middle of the rainbow bytecode of whatever +function of the Python interpreter, we only want to produce more machine +code for the hot path; so we have to do something to continue executing +when we don't want to generate more code immediately. + +The "something" in question, the fall-back rainbow interpreter, is quite +slow, but only runs until the end of the current opcode and can directly +perform all nested calls instead of interpreting them. When it reaches +the "jit_can_leave", it then returns; as described in the "hints" +section this should be a return from the initial call to the JIT or the +machine code -- a call which was in "jit_can_enter" in the regular +interpreter. So the control flow is now in the regular interpreter, +which can go on interpreting at its normal speed from there. + +All in all I guess that there is a chance that the fallback rainbow +interpreter is not too much of an overhead. The important point is that +whenever we use the fallback rainbow interpreter, we also update +counters, and when enough executions have been seen, we compile the hot +path (and only the hot path, unless we find out quickly that the other +path is hot enough too). So after the compilation converges overall, +the fallback rainbow interpreter is only ever executed on the cold +paths. + +As noted above, we can (later) be clever about simple if-then-else +patterns, and always immediately compile both branches. If we still +want a single JITState, we need to make sure that it's a good idea to +always merge the two states at the end of the two branches; a criteria +could be that an if-then-else is "simple enough" if the branches contain +no allocation (i.e. no potential new virtual stuff that could raise +DontMerge in the current rvalue.py). This should be good enough to +directly compile machine code like:: + + x = 5 + if condition: + x += 1 + do_more_stuff + +Promotions are similar to red splits -- go to the fall-back rainbow +interpreter, which update counters, and later resumes compilation for +the values that seem to be hot. For further improvements, this also +makes it easy to decide, looking at the counters, that a site is +"megamorphic", i.e. receives tons of different values with no clear +winner. For this case we can really compile a megamorphic path where +the promotion did not occur (i.e. the value stays as a red variable +after all). The megamorphic path is "hot", in a sense, so compiling for +it puts the fallback rainbow interpreter out of the hot path again. + +About calls: non-residual calls would always just return a single +JITState in this simplified model, so no need for the careful red/yellow +call logic (at least for now). Residual calls, like now, would be +followed by the equivalent of a promotion, checking if the residual call +caused an exception or forced devirtualization. + +About local merge points: in this model of a single JITState, I vaguely +suspect that it gives better results to have *less* local merge points, +e.g. only at the beginning of local loops. To be experimented with. It +might remove the need for the DontMerge exception and the need to +maintain (and linearly scan through) more than one state per green key. From antocuni at codespeak.net Sat Feb 9 12:00:26 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 9 Feb 2008 12:00:26 +0100 (CET) Subject: [pypy-svn] r51354 - in pypy/dist/pypy/translator/cli: . src test Message-ID: <20080209110026.4B05F1684C9@codespeak.net> Author: antocuni Date: Sat Feb 9 12:00:23 2008 New Revision: 51354 Modified: pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/ilgenerator.py pypy/dist/pypy/translator/cli/metavm.py pypy/dist/pypy/translator/cli/opcodes.py pypy/dist/pypy/translator/cli/src/pypylib.cs pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: add support to set static fields of cli classes Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Sat Feb 9 12:00:23 2008 @@ -32,6 +32,15 @@ else: return s_ImpossibleValue + def setattr(self, s_attr, s_value): + assert self.is_constant() + assert s_attr.is_constant + cliclass = self.const + attrname = s_attr.const + if attrname not in cliclass._static_fields: + return s_ImpossibleValue + # XXX: check types? + def simple_call(self, *s_args): assert self.is_constant() return SomeOOInstance(self.const._INSTANCE) @@ -102,9 +111,17 @@ assert attrname in self.cli_class._static_fields TYPE = self.cli_class._static_fields[attrname] c_class = hop.inputarg(hop.args_r[0], arg=0) - c_name = hop.inputconst(ootype.Void, hop.args_v[1].value) + c_name = hop.inputconst(ootype.Void, attrname) return hop.genop("cli_getstaticfield", [c_class, c_name], resulttype=hop.r_result.lowleveltype) + def rtype_setattr(self, hop): + attrname = hop.args_v[1].value + assert attrname in self.cli_class._static_fields + c_class = hop.inputarg(hop.args_r[0], arg=0) + c_name = hop.inputconst(ootype.Void, attrname) + v_value = hop.inputarg(hop.args_r[2], arg=2) + return hop.genop("cli_setstaticfield", [c_class, c_name, v_value], resulttype=hop.r_result.lowleveltype) + def rtype_simple_call(self, hop): # TODO: resolve constructor overloading INSTANCE = hop.args_r[0].cli_class._INSTANCE Modified: pypy/dist/pypy/translator/cli/ilgenerator.py ============================================================================== --- pypy/dist/pypy/translator/cli/ilgenerator.py (original) +++ pypy/dist/pypy/translator/cli/ilgenerator.py Sat Feb 9 12:00:23 2008 @@ -244,6 +244,9 @@ def load_static_field(self, cts_type, name): self.opcode('ldsfld', '%s %s' % (cts_type, name)) + def store_static_field(self, cts_type, name): + self.opcode('stsfld', '%s %s' % (cts_type, name)) + def emit(self, opcode, *args): self.opcode(opcode,*args) Modified: pypy/dist/pypy/translator/cli/metavm.py ============================================================================== --- pypy/dist/pypy/translator/cli/metavm.py (original) +++ pypy/dist/pypy/translator/cli/metavm.py Sat Feb 9 12:00:23 2008 @@ -224,6 +224,15 @@ desc = '%s::%s' % (cli_class._name, fldname) generator.ilasm.load_static_field(cts_type, desc) +class _SetStaticField(MicroInstruction): + def render(self, generator, op): + cli_class = op.args[0].value + fldname = op.args[1].value + TYPE = op.result.concretetype + cts_type = generator.cts.lltype_to_cts(TYPE) + desc = '%s::%s' % (cli_class._name, fldname) + generator.load(op.args[2]) + generator.ilasm.store_static_field(cts_type, desc) OOTYPE_TO_MNEMONIC = { ootype.Signed: 'i4', @@ -252,4 +261,5 @@ TypeOf = _TypeOf() EventHandler = _EventHandler() GetStaticField = _GetStaticField() +SetStaticField = _SetStaticField() CastPrimitive = _CastPrimitive() Modified: pypy/dist/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/cli/opcodes.py (original) +++ pypy/dist/pypy/translator/cli/opcodes.py Sat Feb 9 12:00:23 2008 @@ -1,7 +1,7 @@ from pypy.translator.cli.metavm import Call, CallMethod, \ IndirectCall, GetField, SetField, DownCast, NewCustomDict,\ MapException, Box, Unbox, NewArray, GetArrayElem, SetArrayElem,\ - TypeOf, CastPrimitive, EventHandler, GetStaticField + TypeOf, CastPrimitive, EventHandler, GetStaticField, SetStaticField from pypy.translator.oosupport.metavm import PushArg, PushAllArgs, StoreResult, InstructionList,\ New, RuntimeNew, CastTo, PushPrimitive, OOString, OOUnicode from pypy.translator.cli.cts import WEAKREF @@ -44,6 +44,7 @@ 'cli_arraylength': 'ldlen', 'cli_eventhandler': [EventHandler], 'cli_getstaticfield': [GetStaticField], + 'cli_setstaticfield': [SetStaticField], 'oois': 'ceq', 'oononnull': [PushAllArgs, 'ldnull', 'ceq']+Not, 'instanceof': [CastTo, 'ldnull', 'cgt.un'], @@ -106,7 +107,7 @@ # trick. 'cast_bool_to_int': [PushAllArgs, 'ldc.i4.0', 'ceq']+Not, 'cast_bool_to_uint': [PushAllArgs, 'ldc.i4.0', 'ceq']+Not, - 'cast_bool_to_float': [PushAllArgs, 'ldc.i4 0', 'ceq']+Not+['conv.r8'], + 'cast_bool_to_float': [PushAllArgs, 'ldc.i4.0', 'ceq']+Not+['conv.r8'], 'cast_char_to_int': DoNothing, 'cast_unichar_to_int': DoNothing, 'cast_int_to_char': DoNothing, Modified: pypy/dist/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/dist/pypy/translator/cli/src/pypylib.cs Sat Feb 9 12:00:23 2008 @@ -57,6 +57,12 @@ public delegate int DelegateType_int__int_int(int a, int b); public delegate int DelegateType_int__int_100(int a0, int a1, int a2, int a3, int a4, int a5, int a6, int a7, int a8, int a9, int a10, int a11, int a12, int a13, int a14, int a15, int a16, int a17, int a18, int a19, int a20, int a21, int a22, int a23, int a24, int a25, int a26, int a27, int a28, int a29, int a30, int a31, int a32, int a33, int a34, int a35, int a36, int a37, int a38, int a39, int a40, int a41, int a42, int a43, int a44, int a45, int a46, int a47, int a48, int a49, int a50, int a51, int a52, int a53, int a54, int a55, int a56, int a57, int a58, int a59, int a60, int a61, int a62, int a63, int a64, int a65, int a66, int a67, int a68, int a69, int a70, int a71, int a72, int a73, int a74, int a75, int a76, int a77, int a78, int a79, int a80, int a81, int a82, int a83, int a84, int a85, int a86, int a87, int a88, int a89, int a90, int a91, int a92, int a93, int a94, int a95, int a96, int a97, int a98, int a99); + public class Constants + { + public static object const1; + public static object const2; + } + public class Utils { public static DynamicMethod CreateDynamicMethod(string name, Type res, Type[] args) Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Sat Feb 9 12:00:23 2008 @@ -469,12 +469,13 @@ assert res == 42 def test_static_fields(self): - py.test.skip("does not work, and no need to implement") + Constants = CLR.pypy.runtime.Constants def fn(): - op = OpCodes.Add - return op.get_Name() + obj = System.Object() + Constants.const1 = obj + return Constants.const1 is obj res = self.interpret(fn, []) - assert self.ll_to_string(res) == 'add' + assert res def test_pypylib(self): def fn(): From antocuni at codespeak.net Sat Feb 9 15:15:19 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 9 Feb 2008 15:15:19 +0100 (CET) Subject: [pypy-svn] r51355 - in pypy/dist/pypy: jit/codegen/cli jit/codegen/cli/test translator/cli translator/cli/src Message-ID: <20080209141519.9244416850E@codespeak.net> Author: antocuni Date: Sat Feb 9 15:15:17 2008 New Revision: 51355 Modified: pypy/dist/pypy/jit/codegen/cli/operation.py pypy/dist/pypy/jit/codegen/cli/rgenop.py pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/src/pypylib.cs Log: - first tentative to add support for GenConsts that contain objects; each of those is stored in a static field, but for now only a fixed number of those is available; - improve the automatic generation of Operations from the descriptions in cli/opcodes.py - as a result, some more tests pass :-) Modified: pypy/dist/pypy/jit/codegen/cli/operation.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/operation.py (original) +++ pypy/dist/pypy/jit/codegen/cli/operation.py Sat Feb 9 15:15:17 2008 @@ -65,7 +65,37 @@ self.gv_res().store(self.il) +class Call(Operation): + + def __init__(self, il, sigtoken, gv_fnptr, args_gv): + from pypy.jit.codegen.cli.rgenop import token2clitype + self.il = il + self.sigtoken = sigtoken + self.gv_fnptr = gv_fnptr + self.args_gv = args_gv + self._restype = token2clitype(sigtoken[1]) + + def restype(self): + return self._restype + + def emit(self): + from pypy.jit.codegen.cli.rgenop import sigtoken2clitype + delegate_type = sigtoken2clitype(self.sigtoken) + meth_invoke = delegate_type.GetMethod('Invoke') + self.gv_fnptr.load(self.il) + self.il.Emit(OpCodes.Castclass, delegate_type) + for gv_arg in self.args_gv: + gv_arg.load(self.il) + self.il.EmitCall(OpCodes.Callvirt, meth_invoke, None) + self.storeResult() + + + def opcode2attrname(opcode): + if opcode == 'ldc.r8 0': + return 'Ldc_R8, 0' # XXX this is a hack + if opcode == 'ldc.i8 0': + return 'Ldc_I8, 0' # XXX this is a hack parts = map(str.capitalize, opcode.split('.')) return '_'.join(parts) @@ -97,9 +127,37 @@ elif value is cli_opcodes.DoNothing: out[opname] = SameAs else: - pass # XXX: handle remaining ops + renderCustomOp(opname, baseclass, value, out) return out +def renderCustomOp(opname, baseclass, steps, out): + assert steps + body = [] + for step in steps: + if step is cli_opcodes.PushAllArgs: + body.append('self.pushAllArgs()') + elif step is cli_opcodes.StoreResult: + body.append('self.storeResult()') + elif isinstance(step, str): + if 'call' in step: + return # XXX, fix this + attrname = opcode2attrname(step) + body.append('self.il.Emit(OpCodes.%s)' % attrname) + elif isinstance(step, cli_opcodes.MapException): + return # XXX, TODO + else: + return # ignore it for now + + if cli_opcodes.StoreResult not in steps: + body.append('self.storeResult()') + + emit = py.code.Source('\n'.join(body)) + emit = emit.putaround('def emit(self):') + source = emit.putaround('class %(opname)s (%(baseclass)s):' % locals()) + code = source.compile() + exec code in globals(), out + + UNARYOPS = fillops(cli_opcodes.unary_ops, "UnaryOp") BINARYOPS = fillops(cli_opcodes.binary_ops, "BinaryOp") Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/rgenop.py Sat Feb 9 15:15:17 2008 @@ -7,6 +7,7 @@ from pypy.translator.cli.dotnet import CLR, typeof, new_array, clidowncast System = CLR.System Utils = CLR.pypy.runtime.Utils +Constants = CLR.pypy.runtime.Constants OpCodes = System.Reflection.Emit.OpCodes def token2clitype(tok): @@ -96,30 +97,51 @@ def __repr__(self): return "const=%s" % self.value -SM_INT__INT = ootype.StaticMethod([ootype.Signed], ootype.Signed) -SM_INT__INT_INT = ootype.StaticMethod([ootype.Signed, ootype.Signed], ootype.Signed) -SM_INT__INT_100 = ootype.StaticMethod([ootype.Signed] * 100, ootype.Signed) +class BaseConst(GenConst): + def __init__(self, num): + self.num = num + self.fieldname = "const" + str(num) + + def getobj(self): + t = typeof(Constants) + return t.GetField(self.fieldname).GetValue(None) + + def setobj(self, obj): + t = typeof(Constants) + t.GetField(self.fieldname).SetValue(None, obj) -class ObjectConst(GenConst): + def load(self, il): + t = typeof(Constants) + field = t.GetField(self.fieldname) + il.Emit(OpCodes.Ldsfld, field) - def __init__(self, obj): - self.obj = obj +SM_INT__INT = ootype.StaticMethod([ootype.Signed], ootype.Signed) +SM_INT__INT_INT = ootype.StaticMethod([ootype.Signed, ootype.Signed], ootype.Signed) +SM_INT__INT_100 = ootype.StaticMethod([ootype.Signed] * 100, ootype.Signed) +class FunctionConst(BaseConst): + @specialize.arg(1) def revealconst(self, T): - # XXX: probably you can't mix StaticMethod and others OOTypes if T == SM_INT__INT: DelegateType = CLR.pypy.runtime.DelegateType_int__int - return clidowncast(DelegateType, self.obj) + return clidowncast(DelegateType, self.getobj()) elif T == SM_INT__INT_INT: DelegateType = CLR.pypy.runtime.DelegateType_int__int_int - return clidowncast(DelegateType, self.obj) + return clidowncast(DelegateType, self.getobj()) elif T == SM_INT__INT_100: DelegateType = CLR.pypy.runtime.DelegateType_int__int_100 - return clidowncast(DelegateType, self.obj) + return clidowncast(DelegateType, self.getobj()) else: - assert isinstance(T, ootype.OOType) - return ootype.oodowncast(T, self.obj) + assert False + +class ObjectConst(BaseConst): + + @specialize.arg(1) + def revealconst(self, T): + assert isinstance(T, ootype.OOType) + return ootype.oodowncast(T, self.obj) + class Label(GenLabel): def __init__(self, label, inputargs_gv): @@ -132,6 +154,13 @@ def __init__(self): self.meth = None self.il = None + self.constcount = 0 + + def newconst(self, cls): + assert self.constcount < 3 # the number of static fields declared in Constants + res = cls(self.constcount) + self.constcount += 1 + return res @specialize.genconst(1) def genconst(self, llvalue): @@ -139,7 +168,9 @@ if T is ootype.Signed: return IntConst(llvalue) elif isinstance(T, ootype.OOType): - return ObjectConst(llvalue) + const = self.newconst(ObjectConst) + const.setobj(llvalue) + return const else: assert False, "XXX not implemented" @@ -178,10 +209,17 @@ self.inputargs_gv = [] for i in range(len(args)): self.inputargs_gv.append(GenArgVar(i, args[i])) - self.gv_entrypoint = ObjectConst(None) # XXX? + self.gv_entrypoint = self.rgenop.newconst(FunctionConst) self.sigtoken = sigtoken self.isOpen = False - + + @specialize.arg(1) + def genop1(self, opname, gv_arg): + opcls = ops.getopclass1(opname) + op = opcls(self.il, gv_arg) + self.emit(op) + return op.gv_res() + @specialize.arg(1) def genop2(self, opname, gv_arg1, gv_arg2): opcls = ops.getopclass2(opname) @@ -189,6 +227,11 @@ self.emit(op) return op.gv_res() + def genop_call(self, sigtoken, gv_fnptr, args_gv): + op = ops.Call(self.il, sigtoken, gv_fnptr, args_gv) + self.emit(op) + return op.gv_res() + def emit(self, op): op.emit() @@ -212,7 +255,7 @@ def end(self): delegate_type = sigtoken2clitype(self.sigtoken) myfunc = self.meth.CreateDelegate(delegate_type) - self.gv_entrypoint.obj = myfunc + self.gv_entrypoint.setobj(myfunc) def enter_next_block(self, kinds, args_gv): for i in range(len(args_gv)): Modified: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Sat Feb 9 15:15:17 2008 @@ -10,10 +10,13 @@ 'test_adder', 'test_dummy', 'test_hide_and_reveal', - 'test_hide_and_reveal_p', + # 'test_hide_and_reveal_p', # think about this 'test_largedummy_direct', # _compile works if we set a higher maxstack 'test_branching', 'test_goto', + 'test_if', + # 'test_switch', # no promotion/flexswitch for now please :-) + 'test_fact', ] for p in prefixes: @@ -38,7 +41,7 @@ def cast(self, gv, nb_args): "NOT_RPYTHON" def fn(*args): - return gv.obj.Invoke(*args) + return gv.getobj().Invoke(*args) return fn def directtesthelper(self, FUNCTYPE, func): Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Sat Feb 9 15:15:17 2008 @@ -306,7 +306,7 @@ self._load_class() return getattr(self._PythonNet_class, attr) else: - raise AttributeError + raise AttributeError, attr def __call__(self, *args): self._load_class() Modified: pypy/dist/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/dist/pypy/translator/cli/src/pypylib.cs Sat Feb 9 15:15:17 2008 @@ -59,6 +59,7 @@ public class Constants { + public static object const0; public static object const1; public static object const2; } From fijal at codespeak.net Sat Feb 9 15:32:31 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 9 Feb 2008 15:32:31 +0100 (CET) Subject: [pypy-svn] r51356 - in pypy/dist/pypy/rlib: . test Message-ID: <20080209143231.E695A1684DC@codespeak.net> Author: fijal Date: Sat Feb 9 15:32:29 2008 New Revision: 51356 Added: pypy/dist/pypy/rlib/rbuffer.py (contents, props changed) pypy/dist/pypy/rlib/test/test_rbuffer.py (contents, props changed) Log: Add stub file for rbuffer protocol (the pure-rpython buffer, potentially used in many places (_rawffi, array, buffer...)) Added: pypy/dist/pypy/rlib/rbuffer.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rlib/rbuffer.py Sat Feb 9 15:32:29 2008 @@ -0,0 +1,21 @@ + +""" Implementation of pure-rpython low-level buffer, which allows +you to take address of underlaying memory. Will be used to implement +buffer protocol on app-level +""" + +from pypy.rpython.lltypesystem import lltype, rffi + +class RBuffer: + ll_buffer = lltype.nullptr(rffi.CCHARP.TO) + + def __init__(self, size): + self.ll_buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw') + self.size = size + + def address(self): + return rffi.cast(rffi.INT, self.ll_buffer) + + def free(self): + if self.ll_buffer: + lltype.free(self.ll_buffer, flavor='raw') Added: pypy/dist/pypy/rlib/test/test_rbuffer.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rlib/test/test_rbuffer.py Sat Feb 9 15:32:29 2008 @@ -0,0 +1,10 @@ + +import py +from pypy.rlib.rbuffer import RBuffer + +class TestRbuffer: + def test_creation(self): + buf = RBuffer(3) + assert buf.address() + buf.free() + From fijal at codespeak.net Sat Feb 9 15:41:13 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 9 Feb 2008 15:41:13 +0100 (CET) Subject: [pypy-svn] r51357 - in pypy/dist/pypy/rlib: . test Message-ID: <20080209144113.A109D16852E@codespeak.net> Author: fijal Date: Sat Feb 9 15:41:11 2008 New Revision: 51357 Modified: pypy/dist/pypy/rlib/rbuffer.py pypy/dist/pypy/rlib/test/test_rbuffer.py Log: setitem/getitem Modified: pypy/dist/pypy/rlib/rbuffer.py ============================================================================== --- pypy/dist/pypy/rlib/rbuffer.py (original) +++ pypy/dist/pypy/rlib/rbuffer.py Sat Feb 9 15:41:11 2008 @@ -5,17 +5,34 @@ """ from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rlib.rarithmetic import r_uint class RBuffer: ll_buffer = lltype.nullptr(rffi.CCHARP.TO) - def __init__(self, size): - self.ll_buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw') + def __init__(self, size, address=r_uint(0)): + if address == 0: + self.ll_buffer = lltype.malloc(rffi.CCHARP.TO, size, + zero=True, flavor='raw') + else: + self.ll_buffer = rffi.cast(rffi.CCHARP, address) self.size = size def address(self): - return rffi.cast(rffi.INT, self.ll_buffer) + return rffi.cast(rffi.UINT, self.ll_buffer) + + def getitem(self, item): + # XXX think how to avoid multiple layers of boundary checks + if item >= self.size or item < 0: + raise IndexError(item) + return self.ll_buffer[item] + + def setitem(self, item, value): + if item >= self.size or item < 0: + raise IndexError(item) + self.ll_buffer[item] = value def free(self): if self.ll_buffer: lltype.free(self.ll_buffer, flavor='raw') + self.ll_buffer = lltype.nullptr(rffi.CCHARP.TO) Modified: pypy/dist/pypy/rlib/test/test_rbuffer.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rbuffer.py (original) +++ pypy/dist/pypy/rlib/test/test_rbuffer.py Sat Feb 9 15:41:11 2008 @@ -7,4 +7,10 @@ buf = RBuffer(3) assert buf.address() buf.free() - + + def test_getsetitem(self): + buf = RBuffer(10) + assert buf.getitem(3) == '\x00' + buf.setitem(4, '\x01') + assert buf.getitem(4) == '\x01' + buf.free() From fijal at codespeak.net Sat Feb 9 15:42:33 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 9 Feb 2008 15:42:33 +0100 (CET) Subject: [pypy-svn] r51358 - pypy/dist/pypy/rlib/test Message-ID: <20080209144233.7E83F168542@codespeak.net> Author: fijal Date: Sat Feb 9 15:42:32 2008 New Revision: 51358 Modified: pypy/dist/pypy/rlib/test/test_rbuffer.py Log: fromaddress Modified: pypy/dist/pypy/rlib/test/test_rbuffer.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rbuffer.py (original) +++ pypy/dist/pypy/rlib/test/test_rbuffer.py Sat Feb 9 15:42:32 2008 @@ -14,3 +14,10 @@ buf.setitem(4, '\x01') assert buf.getitem(4) == '\x01' buf.free() + + def test_fromaddress(self): + buf = RBuffer(10) + buf.setitem(2, '\x03') + buf2 = RBuffer(7, buf.address() + 2) + assert buf2.getitem(0) == '\x03' + buf.free() From fijal at codespeak.net Sat Feb 9 17:26:34 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 9 Feb 2008 17:26:34 +0100 (CET) Subject: [pypy-svn] r51360 - in pypy/dist/pypy/rlib: . test Message-ID: <20080209162634.C15F416842C@codespeak.net> Author: fijal Date: Sat Feb 9 17:26:29 2008 New Revision: 51360 Modified: pypy/dist/pypy/rlib/rbuffer.py pypy/dist/pypy/rlib/test/test_rbuffer.py Log: getslice Modified: pypy/dist/pypy/rlib/rbuffer.py ============================================================================== --- pypy/dist/pypy/rlib/rbuffer.py (original) +++ pypy/dist/pypy/rlib/rbuffer.py Sat Feb 9 17:26:29 2008 @@ -27,6 +27,9 @@ raise IndexError(item) return self.ll_buffer[item] + def getslice(self, start, end): + return "".join([self.getitem(item) for item in range(start, end)]) + def setitem(self, item, value): if item >= self.size or item < 0: raise IndexError(item) Modified: pypy/dist/pypy/rlib/test/test_rbuffer.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_rbuffer.py (original) +++ pypy/dist/pypy/rlib/test/test_rbuffer.py Sat Feb 9 17:26:29 2008 @@ -21,3 +21,11 @@ buf2 = RBuffer(7, buf.address() + 2) assert buf2.getitem(0) == '\x03' buf.free() + + def test_getslice(self): + buf = RBuffer(10) + buf.setitem(0, '\x01') + buf.setitem(1, '\x02') + buf.setitem(2, '\x03') + assert buf.getslice(0, 3) == '\x01\x02\x03' + buf.free() From fijal at codespeak.net Sun Feb 10 12:30:14 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 10 Feb 2008 12:30:14 +0100 (CET) Subject: [pypy-svn] r51365 - pypy/extradoc/talk/rupy2008 Message-ID: <20080210113014.B58A91684F9@codespeak.net> Author: fijal Date: Sun Feb 10 12:30:10 2008 New Revision: 51365 Added: pypy/extradoc/talk/rupy2008/bio.txt (contents, props changed) Log: My bio Added: pypy/extradoc/talk/rupy2008/bio.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/rupy2008/bio.txt Sun Feb 10 12:30:10 2008 @@ -0,0 +1,5 @@ +Maciej Fijalkowski is a core pypy and py.test developer. He's working for +merlinux GmbH on pypy and py.test from the period of 6th EU +framework programme. He's giving talks at community conferences such as +EuroPython and Pycon. + From antocuni at codespeak.net Sun Feb 10 12:37:01 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 10 Feb 2008 12:37:01 +0100 (CET) Subject: [pypy-svn] r51366 - pypy/dist/pypy/jit/codegen/cli Message-ID: <20080210113701.EAFE41684D3@codespeak.net> Author: antocuni Date: Sun Feb 10 12:37:01 2008 New Revision: 51366 Added: pypy/dist/pypy/jit/codegen/cli/dumpgenerator.py (contents, props changed) Modified: pypy/dist/pypy/jit/codegen/cli/operation.py pypy/dist/pypy/jit/codegen/cli/rgenop.py Log: don't render operations on BranchBuilder on the fly, but save them to be rendered later; this is needed when calling pause(). Added: pypy/dist/pypy/jit/codegen/cli/dumpgenerator.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/jit/codegen/cli/dumpgenerator.py Sun Feb 10 12:37:01 2008 @@ -0,0 +1,33 @@ +class DumpGenerator: + def __init__(self, il, filename='dynamicmethod.il'): + self.il = il + self.out = file(filename, 'w') + self.localcount = 0 + self.labels = {} + + def _fmt(self, arg): + from System.Reflection import Emit + if isinstance(arg, Emit.LocalBuilder): + return 'v%d' % arg.LocalIndex + elif isinstance(arg, Emit.Label): + return 'label%d' % self.labels[arg] + return repr(arg) + + def Emit(self, opcode, *args): + arglist = ', '.join(map(self._fmt, args)) + self.out.write(' %s %s\n' % (opcode.Name, arglist)) + return self.il.Emit(opcode, *args) + + def DeclareLocal(self, t): + return self.il.DeclareLocal(t) + + def DefineLabel(self): + lbl = self.il.DefineLabel() + count = len(self.labels) + self.labels[lbl] = count + return lbl + + def MarkLabel(self, lbl): + self.out.write('\n%s:\n' % self._fmt(lbl)) + return self.il.MarkLabel(lbl) + Modified: pypy/dist/pypy/jit/codegen/cli/operation.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/operation.py (original) +++ pypy/dist/pypy/jit/codegen/cli/operation.py Sun Feb 10 12:37:01 2008 @@ -65,6 +65,51 @@ self.gv_res().store(self.il) +class FollowLink(Operation): + + def __init__(self, il, outputargs_gv, inputargs_gv, label): + self.il = il + self.outputargs_gv = outputargs_gv + self.inputargs_gv = inputargs_gv + self.label = label + + def restype(self): + return None + + def emit(self): + for i in range(len(self.outputargs_gv)): + self.outputargs_gv[i].load(self.il) + self.inputargs_gv[i].store(self.il) + self.il.Emit(OpCodes.Br, self.label) + +class BranchIf(Operation): + + def __init__(self, il, gv_cond, opcode, label): + self.il = il + self.gv_cond = gv_cond + self.opcode = opcode + self.label = label + + def restype(self): + return None + + def emit(self): + self.gv_cond.load(self.il) + self.il.Emit(self.opcode, self.label) + +class Return(Operation): + + def __init__(self, il, gv_x): + self.il = il + self.gv_x = gv_x + + def restype(self): + return None + + def emit(self): + self.gv_x.load(self.il) + self.il.Emit(OpCodes.Ret) + class Call(Operation): def __init__(self, il, sigtoken, gv_fnptr, args_gv): Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/rgenop.py Sun Feb 10 12:37:01 2008 @@ -4,12 +4,15 @@ from pypy.jit.codegen.model import AbstractRGenOp, GenBuilder, GenLabel from pypy.jit.codegen.model import GenVarOrConst, GenVar, GenConst, CodeGenSwitch from pypy.jit.codegen.cli import operation as ops +from pypy.jit.codegen.cli.dumpgenerator import DumpGenerator from pypy.translator.cli.dotnet import CLR, typeof, new_array, clidowncast System = CLR.System Utils = CLR.pypy.runtime.Utils Constants = CLR.pypy.runtime.Constants OpCodes = System.Reflection.Emit.OpCodes +DEBUG = False + def token2clitype(tok): if tok == '': return typeof(System.Int32) @@ -199,19 +202,24 @@ return builder, builder.gv_entrypoint, builder.inputargs_gv[:] - class Builder(GenBuilder): def __init__(self, rgenop, name, res, args, sigtoken): self.rgenop = rgenop self.meth = Utils.CreateDynamicMethod(name, res, args) self.il = self.meth.GetILGenerator() + if DEBUG: + self.il = DumpGenerator(self.il) self.inputargs_gv = [] for i in range(len(args)): self.inputargs_gv.append(GenArgVar(i, args[i])) self.gv_entrypoint = self.rgenop.newconst(FunctionConst) self.sigtoken = sigtoken self.isOpen = False + self.operations = [] + self.branches = [] + self.retlabel = self.il.DefineLabel() + self.gv_returnvar = None @specialize.arg(1) def genop1(self, opname, gv_arg): @@ -235,24 +243,35 @@ def emit(self, op): op.emit() + def appendbranch(self, branch): + self.branches.append(branch) + def start_writing(self): self.isOpen = True def finish_and_return(self, sigtoken, gv_returnvar): - gv_returnvar.load(self.il) - self.il.Emit(OpCodes.Ret) + self.il.Emit(OpCodes.Br, self.retlabel) + self.gv_returnvar = gv_returnvar self.isOpen = False def finish_and_goto(self, outputargs_gv, target): inputargs_gv = target.inputargs_gv assert len(inputargs_gv) == len(outputargs_gv) - for i in range(len(outputargs_gv)): - outputargs_gv[i].load(self.il) - inputargs_gv[i].store(self.il) - self.il.Emit(OpCodes.Br, target.label) + op = ops.FollowLink(self.il, outputargs_gv, inputargs_gv, target.label) + self.emit(op) self.isOpen = False def end(self): + # render all the pending branches + for branch in self.branches: + branch.replayops() + + # render the return block for last, else the verifier could complain + self.il.MarkLabel(self.retlabel) + op = ops.Return(self.il, self.gv_returnvar) + self.emit(op) + + # build the delegate delegate_type = sigtoken2clitype(self.sigtoken) myfunc = self.meth.CreateDelegate(delegate_type) self.gv_entrypoint.setobj(myfunc) @@ -268,9 +287,11 @@ def _jump_if(self, gv_condition, opcode): label = self.il.DefineLabel() - gv_condition.load(self.il) - self.il.Emit(opcode, label) - return BranchBuilder(self, label) + op = ops.BranchIf(self.il, gv_condition, opcode, label) + self.emit(op) + branch = BranchBuilder(self, label) + self.appendbranch(branch) + return branch def jump_if_false(self, gv_condition, args_for_jump_gv): return self._jump_if(gv_condition, OpCodes.Brfalse) @@ -284,12 +305,11 @@ self.parent = parent self.label = label self.il = parent.il + self.operations = [] self.isOpen = False def start_writing(self): - assert not self.parent.isOpen self.isOpen = True - self.il.MarkLabel(self.label) @specialize.arg(1) def genop2(self, opname, gv_arg1, gv_arg2): @@ -297,3 +317,22 @@ # feel like fixing now. Try to uncomment this and run # test_goto_compile to see why it fails return Builder.genop2(self, opname, gv_arg1, gv_arg2) + + def finish_and_return(self, sigtoken, gv_returnvar): + op = ops.Return(self.parent.il, gv_returnvar) + self.emit(op) + self.isOpen = False + + def emit(self, op): + self.operations.append(op) + + def appendbranch(self, branch): + self.parent.appendbranch(branch) + + def replayops(self): + assert not self.isOpen + assert not self.parent.isOpen + il = self.parent.il + il.MarkLabel(self.label) + for op in self.operations: + op.emit() From hpk at codespeak.net Sun Feb 10 12:38:03 2008 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sun, 10 Feb 2008 12:38:03 +0100 (CET) Subject: [pypy-svn] r51367 - pypy/extradoc/talk/rupy2008 Message-ID: <20080210113803.90A9A1684CC@codespeak.net> Author: hpk Date: Sun Feb 10 12:38:02 2008 New Revision: 51367 Modified: pypy/extradoc/talk/rupy2008/bio.txt Log: trying to help with english and content a bit. Modified: pypy/extradoc/talk/rupy2008/bio.txt ============================================================================== --- pypy/extradoc/talk/rupy2008/bio.txt (original) +++ pypy/extradoc/talk/rupy2008/bio.txt Sun Feb 10 12:38:02 2008 @@ -1,5 +1,7 @@ -Maciej Fijalkowski is a core pypy and py.test developer. He's working for -merlinux GmbH on pypy and py.test from the period of 6th EU -framework programme. He's giving talks at community conferences such as -EuroPython and Pycon. +Maciej Fijalkowski is a core developer of PyPy - +a Python Interpreter and Compiler and of py.test, +a popular testing tool. He has successfully completed a +Summer of Code project and was contracted by merlinux (germany) +for several tasks in the PyPy EU research project. He gave +talks at international European and US Python community conferences. From fijal at codespeak.net Sun Feb 10 12:43:19 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 10 Feb 2008 12:43:19 +0100 (CET) Subject: [pypy-svn] r51368 - pypy/extradoc/talk/sfi2008 Message-ID: <20080210114319.0FE171684F5@codespeak.net> Author: fijal Date: Sun Feb 10 12:43:18 2008 New Revision: 51368 Added: pypy/extradoc/talk/sfi2008/ Log: Directory for sfi 2008 (sfi.org.pl, everything in polish sorry) From xoraxax at codespeak.net Sun Feb 10 14:31:37 2008 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Sun, 10 Feb 2008 14:31:37 +0100 (CET) Subject: [pypy-svn] r51369 - pypy/dist/pypy/objspace/test Message-ID: <20080210133137.C415D168513@codespeak.net> Author: xoraxax Date: Sun Feb 10 14:31:35 2008 New Revision: 51369 Removed: pypy/dist/pypy/objspace/test/problem.py pypy/dist/pypy/objspace/test/problems.py Log: Remove problem{,s}.py, remaining artefact of the logic space and cslib removal. From arigo at codespeak.net Sun Feb 10 14:33:07 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 10 Feb 2008 14:33:07 +0100 (CET) Subject: [pypy-svn] r51370 - pypy/dist/pypy/doc/discussion Message-ID: <20080210133307.C1CC616850B@codespeak.net> Author: arigo Date: Sun Feb 10 14:33:06 2008 New Revision: 51370 Modified: pypy/dist/pypy/doc/discussion/jit-refactoring-plan.txt Log: Some more thoughts. Modified: pypy/dist/pypy/doc/discussion/jit-refactoring-plan.txt ============================================================================== --- pypy/dist/pypy/doc/discussion/jit-refactoring-plan.txt (original) +++ pypy/dist/pypy/doc/discussion/jit-refactoring-plan.txt Sun Feb 10 14:33:06 2008 @@ -181,3 +181,35 @@ e.g. only at the beginning of local loops. To be experimented with. It might remove the need for the DontMerge exception and the need to maintain (and linearly scan through) more than one state per green key. + +Random improvement ideas +++++++++++++++++++++++++++++++++ + +- in the global merge point "jit_can_leave", so far we'd + record one state snapshot for each opcode; instead, we can + use the idea implemented in the flow object space of only + recording the state at the beginning of an opcode that actually + causes machine code to be produced (or, more practically, to + throw away the latest recorded state if no new machine code + was generated in the meantime). + +- maybe it's a bit of a mess but the fallback rainbow interpreter + could also record profiling information about more than one + red split or promotion -- all the ones it finds alongs its + path. + +- I didn't think about the impact of this model on our compact + Path objects. As step one we can always make complete state + snapshot at each red split and promotion, and reintroduce + the compact Paths as step two. + +- compiling of more code: we could tweak the flexswitch + interface. For example, instead of "please add a new path", + it would make sense to have an API "please override the + switch completely so that it has this new set of paths". + +- we also need a "jit_can_enter" at the end of the stack + unroller corresponding to CONTINUE, for the case where the + "continue" statement was in a try:finally:. This is not + necessarily a problem, just a note that we have to allow + this hint to be in some subfunction, potentially. From arigo at codespeak.net Sun Feb 10 16:43:20 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 10 Feb 2008 16:43:20 +0100 (CET) Subject: [pypy-svn] r51371 - in pypy/dist/pypy: rpython/lltypesystem rpython/memory/gctransform translator/c/gcc translator/c/gcc/test translator/c/src Message-ID: <20080210154320.7142D168544@codespeak.net> Author: arigo Date: Sun Feb 10 16:43:17 2008 New Revision: 51371 Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/rpython/memory/gctransform/asmgcroot.py pypy/dist/pypy/translator/c/gcc/test/test_trackgcroot.py pypy/dist/pypy/translator/c/gcc/trackgcroot.py pypy/dist/pypy/translator/c/src/mem.h Log: Tweak the tables produced for the asmgcc root finder. This should reduce the size of the executable by quite a bit (still checking, might be about 0.5 MB). Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Sun Feb 10 16:43:17 2008 @@ -412,6 +412,7 @@ 'llvm_frameaddress': LLOp(sideeffects=False), 'llvm_gcmapstart': LLOp(sideeffects=False), 'llvm_gcmapend': LLOp(sideeffects=False), + 'llvm_gccallshapes': LLOp(sideeffects=False), 'llvm_store_gcroot': LLOp(), 'llvm_load_gcroot': LLOp(), Modified: pypy/dist/pypy/rpython/memory/gctransform/asmgcroot.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/asmgcroot.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/asmgcroot.py Sun Feb 10 16:43:17 2008 @@ -50,11 +50,12 @@ def _asm_callback(initialframedata): self.walk_stack_from(initialframedata) self._asm_callback = _asm_callback + self._shape_decompressor = ShapeDecompressor() def setup_root_walker(self): - # The gcmap table is a list of pairs of pointers: + # The gcmap table is a list of entries, two machine words each: # void *SafePointAddress; - # void *Shape; + # int Shape; # Here, i.e. when the program starts, we sort it # in-place on the SafePointAddress to allow for more # efficient searches. @@ -98,9 +99,9 @@ callback from the GC code for each GC root found in 'caller'. """ # - # The gcmap table is a list of pairs of pointers: + # The gcmap table is a list of entries, two machine words each: # void *SafePointAddress; - # void *Shape; + # int Shape; # # A "safe point" is the return address of a call. # The "shape" of a safe point is a list of integers @@ -148,20 +149,26 @@ gcmapend = llop.llvm_gcmapend(llmemory.Address) item = binary_search(gcmapstart, gcmapend, retaddr) if item.address[0] != retaddr: - # retaddr not found! - llop.debug_fatalerror(lltype.Void, "cannot find gc roots!") - return False + # 'retaddr' not exactly found. Check that 'item' the start of a + # compressed range containing 'retaddr'. + if retaddr > item.address[0] and item.signed[1] < 0: + pass # ok + else: + llop.debug_fatalerror(lltype.Void, "cannot find gc roots!") + return False # # found! Enumerate the GC roots in the caller frame # - shape = item.address[1] + shape = item.signed[1] + if shape < 0: + shape = ~ shape # can ignore this "range" marker here + self._shape_decompressor.setpos(shape) collect_stack_root = self.gcdata._gc_collect_stack_root gc = self.gc - LIVELOCS = 1 + CALLEE_SAVED_REGS + 1 # index of the first gc root loc - livecount = shape.signed[LIVELOCS-1] - while livecount > 0: - livecount -= 1 - location = shape.signed[LIVELOCS + livecount] + while True: + location = self._shape_decompressor.next() + if location == 0: + break addr = self.getlocation(callee, location) if addr.address[0] != llmemory.NULL: collect_stack_root(gc, addr) @@ -169,17 +176,18 @@ # track where the caller_frame saved the registers from its own # caller # - location = shape.signed[0] - caller.frame_address = self.getlocation(callee, location) - if not caller.frame_address: # marker that means "I'm the frame - return False # of the entry point, stop walking" - reg = 0 - while reg < CALLEE_SAVED_REGS: - location = shape.signed[1+reg] + reg = CALLEE_SAVED_REGS - 1 + while reg >= 0: + location = self._shape_decompressor.next() addr = self.getlocation(callee, location) caller.regs_stored_at[reg] = addr - reg += 1 - return True + reg -= 1 + + location = self._shape_decompressor.next() + caller.frame_address = self.getlocation(callee, location) + # we get a NULL marker to mean "I'm the frame + # of the entry point, stop walking" + return caller.frame_address != llmemory.NULL def getlocation(self, callee, location): """Get the location in the 'caller' frame of a variable, based @@ -241,6 +249,9 @@ This is an insertion sort, so it's slowish unless the array is mostly sorted already (which is what I expect, but XXX check this). """ + # XXX this should check that it's not changing the relative order + # of entry and the following entry in case it's a compressed "range" + # entry, i.e. "entry.signed[1] < 0". next = start while next < end: # assuming the interval from start (included) to next (excluded) @@ -259,6 +270,31 @@ # ____________________________________________________________ +class ShapeDecompressor: + _alloc_flavor_ = "raw" + + def setpos(self, pos): + gccallshapes = llop.llvm_gccallshapes(llmemory.Address) + self.addr = gccallshapes + pos + + def next(self): + value = 0 + addr = self.addr + while True: + b = ord(addr.char[0]) + addr += 1 + value += b + if b < 0x80: + break + value = (value - 0x80) << 7 + self.addr = addr + if value & 1: + value = ~ value + value = value >> 1 + return value + +# ____________________________________________________________ + # # The special pypy_asm_stackwalk(), implemented directly in # assembler, fills information about the current stack top in an Modified: pypy/dist/pypy/translator/c/gcc/test/test_trackgcroot.py ============================================================================== --- pypy/dist/pypy/translator/c/gcc/test/test_trackgcroot.py (original) +++ pypy/dist/pypy/translator/c/gcc/test/test_trackgcroot.py Sun Feb 10 16:43:17 2008 @@ -6,6 +6,8 @@ from pypy.translator.c.gcc.trackgcroot import LOC_EBP_BASED, LOC_ESP_BASED from pypy.translator.c.gcc.trackgcroot import GcRootTracker from pypy.translator.c.gcc.trackgcroot import FunctionGcRootTracker +from pypy.translator.c.gcc.trackgcroot import compress_callshape +from pypy.translator.c.gcc.trackgcroot import decompress_callshape from StringIO import StringIO this_dir = py.path.local(__file__).dirpath() @@ -36,6 +38,14 @@ LOC_EBP_BASED+24, LOC_EBP_BASED+28)) == expected +def test_compress_callshape(): + shape = (1, -3, 0x1234, -0x5678, 0x234567, + -0x765432, 0x61626364, -0x41424344) + bytes = list(compress_callshape(shape)) + print bytes + assert len(bytes) == 1+1+2+3+4+4+5+5+1 + assert decompress_callshape(bytes) == list(shape) + def test_find_functions(): source = """\ \t.p2align 4,,15 Modified: pypy/dist/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/dist/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/dist/pypy/translator/c/gcc/trackgcroot.py Sun Feb 10 16:43:17 2008 @@ -35,6 +35,8 @@ def dump(self, output): assert self.seen_main shapes = {} + shapelines = [] + shapeofs = 0 print >> output, """\t.text .globl pypy_asm_stackwalk .type pypy_asm_stackwalk, @function @@ -68,30 +70,26 @@ print >> output, '\t.align\t4' print >> output, '\t.globl\t__gcmapstart' print >> output, '__gcmapstart:' - for label, state in self.gcmaptable: - if state not in shapes: - lst = ['__gcmap_shape'] - for n in state: - if n < 0: - n = 'm%d' % (-n,) - lst.append(str(n)) - shapes[state] = '_'.join(lst) + for label, state, is_range in self.gcmaptable: + try: + n = shapes[state] + except KeyError: + n = shapes[state] = shapeofs + bytes = [str(b) for b in compress_callshape(state)] + shapelines.append('\t/*%d*/\t.byte\t%s\n' % ( + shapeofs, + ', '.join(bytes))) + shapeofs += len(bytes) + if is_range: + n = ~ n print >> output, '\t.long\t%s' % (label,) - print >> output, '\t.long\t%s' % (shapes[state],) + print >> output, '\t.long\t%d' % (n,) print >> output, '\t.globl\t__gcmapend' print >> output, '__gcmapend:' print >> output, '\t.section\t.rodata' - print >> output, '\t.align\t4' - keys = shapes.keys() - keys.sort() - FIXED = 1 + len(CALLEE_SAVE_REGISTERS) - for state in keys: - print >> output, '%s:' % (shapes[state],) - for i in range(FIXED): - print >> output, '\t.long\t%d' % (state[i],) - print >> output, '\t.long\t%d' % (len(state)-FIXED,) - for p in state[FIXED:]: - print >> output, '\t.long\t%d' % (p,) # gcroots + print >> output, '\t.globl\t__gccallshapes' + print >> output, '__gccallshapes:' + output.writelines(shapelines) def find_functions(self, iterlines): functionlines = [] @@ -115,10 +113,13 @@ yield False, functionlines def process(self, iterlines, newfile, entrypoint='main', filename='?'): + self.localgcmaptable = [] for in_function, lines in self.find_functions(iterlines): if in_function: lines = self.process_function(lines, entrypoint, filename) newfile.writelines(lines) + self.gcmaptable.extend(compress_gcmaptable(self.localgcmaptable)) + del self.localgcmaptable self.files_seen += 1 def process_function(self, lines, entrypoint, filename): @@ -131,7 +132,7 @@ if self.verbose > 1: for label, state in table: print >> sys.stderr, label, '\t', format_callshape(state) - self.gcmaptable.extend(table) + self.localgcmaptable.extend(table) self.seen_main |= tracker.is_main return tracker.lines @@ -897,6 +898,82 @@ ', '.join(result[1:5]), ', '.join(result[5:])) +# __________ table compression __________ + +def compress_gcmaptable(table): + # Compress ranges table[i:j] of entries with the same state + # into a single entry whose label is the start of the range. + # The last element in the table is never compressed in this + # way for debugging reasons, to avoid that a random address + # in memory gets mapped to the last element in the table + # just because it's the closest address. + # Also, compress_gcmaptable() should be called after each + # .s file processed -- otherwise the result depends on the + # linker not rearranging the .s files in memory, which looks + # fragile. + i = 0 + limit = len(table) - 1 # only process entries table[:limit] + while i < len(table): + label1, state = table[i] + is_range = False + j = i + 1 + while j < limit and table[j][1] == state: + is_range = True + j += 1 + # now all entries in table[i:j] have the same state + yield (label1, state, is_range) + i = j + +def compress_callshape(shape): + # For a single shape, this turns the list of integers into a list of + # bytes and reverses the order of the entries. The length is + # encoded by inserting a 0 marker after the gc roots coming from + # shape[5:] and before the 5 values coming from shape[4] to + # shape[0]. In practice it seems that shapes contain many integers + # whose value is up to a few thousands, which the algorithm below + # compresses down to 2 bytes. Very small values compress down to a + # single byte. + assert len(shape) >= 5 + shape = list(shape) + assert 0 not in shape[5:] + shape.insert(5, 0) + result = [] + for loc in shape: + if loc < 0: + loc = (-loc) * 2 - 1 + else: + loc = loc * 2 + flag = 0 + while loc >= 0x80: + result.append(int(loc & 0x7F) | flag) + flag = 0x80 + loc >>= 7 + result.append(int(loc) | flag) + result.reverse() + return result + +def decompress_callshape(bytes): + # For tests. This logic is copied in asmgcroot.py. + result = [] + n = 0 + while n < len(bytes): + value = 0 + while True: + b = bytes[n] + n += 1 + value += b + if b < 0x80: + break + value = (value - 0x80) << 7 + if value & 1: + value = ~ value + value = value >> 1 + result.append(value) + result.reverse() + assert result[5] == 0 + del result[5] + return result + if __name__ == '__main__': if sys.argv and sys.argv[1] == '-v': Modified: pypy/dist/pypy/translator/c/src/mem.h ============================================================================== --- pypy/dist/pypy/translator/c/src/mem.h (original) +++ pypy/dist/pypy/translator/c/src/mem.h Sun Feb 10 16:43:17 2008 @@ -10,7 +10,7 @@ extern char __gcmapstart; extern char __gcmapend; -extern char* __gcmap_frame_address(void); +extern char __gccallshapes; #define PYPY_GCROOT(p) asm ("/* GCROOT %0 */" : "=g" (p) : "0" (p) : "memory") #define pypy_asm_gcroot(p) ({void*_r; \ @@ -19,10 +19,7 @@ #define OP_LLVM_GCMAPSTART(r) r = &__gcmapstart #define OP_LLVM_GCMAPEND(r) r = &__gcmapend -#define OP_LLVM_FRAMEADDRESS(r) asm ("pypygetframeaddress %0" : "=r" (r)) -/* NB. we cannot use __builtin_frame_address(0) - apparently, gcc thinks - it can return %ebp even if -fomit-frame-pointer is specified, which - doesn't work. So we need a bit of help from trackgcroot.py... */ +#define OP_LLVM_GCCALLSHAPES(r) r = &__gccallshapes #define RAW_MALLOC_ZERO_FILLED 0 From cfbolz at codespeak.net Sun Feb 10 17:01:31 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 10 Feb 2008 17:01:31 +0100 (CET) Subject: [pypy-svn] r51372 - pypy/extradoc/talk/s3-2008 Message-ID: <20080210160131.2EDA7168563@codespeak.net> Author: cfbolz Date: Sun Feb 10 17:01:30 2008 New Revision: 51372 Added: pypy/extradoc/talk/s3-2008/ pypy/extradoc/talk/s3-2008/outline.txt (contents, props changed) Log: outline for a potential s3 paper Added: pypy/extradoc/talk/s3-2008/outline.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/s3-2008/outline.txt Sun Feb 10 17:01:30 2008 @@ -0,0 +1,59 @@ +=========================================================== +PyPy's Python Interpreter Architecture and its Applications +=========================================================== + +Introduction +============ + + - some kind of intro about Python + - object-oriented imperative language + - rich library of builtin-types + - complex dispatching rules for operations + + - and PyPy: + - Python implementation in RPython + - translate RPython to other languages + - try to reach extremely good compatibility + - also try to get a simple and flexible interpreter architecture + + +The Object Space Architecture +============================= + + - PyPy Python-interpreter architecture + - interpreter split into two parts: + - normal bytecode interpreter + - responsible for control flow + - treats all objects as black boxes + - only way to gain information: typed unwraps, particularly is_true + - object space + - is responsible for object implementation and operation semantics + - knows nothing about execution + +Applications +============ + + - flow object space + - goal: derive control flow of functions + - abstract interpretation + - flow object space is a new value domain: Variables and Constants + + - compile-time meta-programming, comprehensive changes + - thunk object space + - reflective object space + +Prior Work +========== + + - interpreter architecture + - compare to CPython, Jython and IronPython? + - parallels to the typical eval/apply model of Scheme interpreters + + - flow graph construction + - usually through parsing + + - lazy computation + - XXX find some papers + + - reflective object space + - reflective towers From cfbolz at codespeak.net Sun Feb 10 17:47:25 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 10 Feb 2008 17:47:25 +0100 (CET) Subject: [pypy-svn] r51373 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter/test Message-ID: <20080210164725.DE68C168548@codespeak.net> Author: cfbolz Date: Sun Feb 10 17:47:23 2008 New Revision: 51373 Added: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vdict.py - copied, changed from r51125, pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_vdict.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vlist.py - copied, changed from r51125, pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_vlist.py Removed: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_vdict.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_vlist.py Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py Log: some reshuffling of tests Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Sun Feb 10 17:47:23 2008 @@ -456,7 +456,12 @@ else: self.register_redvar(result, self.redvar_position(arg)) return - XXX + if "deepfreeze" in hints: + if self.varcolor(result) == "red": + self.register_redvar(result, self.redvar_position(arg)) + else: + self.register_greenvar(result, self.green_position(arg)) + return def args_of_call(self, args, colored_as): result = [] @@ -472,10 +477,10 @@ pass def serialize_op_direct_call(self, op): + kind, exc = self.guess_call_kind(op) targets = dict(self.graphs_from(op)) assert len(targets) == 1 targetgraph, = targets.values() - kind, exc = self.guess_call_kind(op) if kind == "red" or kind == "gray": graphindex = self.graph_position(targetgraph) args = targetgraph.getargs() @@ -642,6 +647,7 @@ self.register_redvar(op.result) def serialize_op_getinteriorfield(self, op): + color = self.opcolor(op) structvar = op.args[0] PTRTYPE = structvar.concretetype # no virtualizable access read here @@ -657,11 +663,14 @@ indexes = [] for arg in indices_v: indexes.append(self.serialize_oparg("red", arg)) - self.emit("red_getinteriorfield", structindex, interiordescindex, - deepfrozen) + self.emit("%s_getinteriorfield" % color, structindex, + interiordescindex, deepfrozen) self.emit(len(indexes)) self.emit(*indexes) - self.register_redvar(op.result) + if color == "red": + self.register_redvar(op.result) + else: + self.register_greenvar(op.result) def serialize_op_setinteriorfield(self, op): structvar = op.args[0] @@ -688,10 +697,10 @@ interiordescindex, indices_v = self.interiordesc( op, PTRTYPE, len(op.args) - 1) assert interiordescindex != -1 - structindex = self.serialize_oparg(color, structvar) + structindex = self.serialize_oparg("red", structvar) indexes = [] for arg in indices_v: - indexes.append(self.serialize_oparg(color, arg)) + indexes.append(self.serialize_oparg("red", arg)) self.emit("%s_getinteriorarraysize" % color, structindex, interiordescindex) self.emit(len(indexes)) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Sun Feb 10 17:47:23 2008 @@ -163,6 +163,7 @@ self.frame.local_boxes.append(box) def green_result(self, gv): + assert gv.is_const self.frame.local_green.append(gv) def newjitstate(self, newjitstate): @@ -403,6 +404,16 @@ structbox, indexboxes) self.red_result(resultbox) + def opimpl_green_getinteriorfield(self): + structbox = self.get_redarg() + interiordesc = self.frame.bytecode.interiordescs[self.load_2byte()] + deepfrozen = self.load_bool() + indexboxes = self.get_red_varargs() + resultbox = interiordesc.gengetinteriorfield(self.jitstate, deepfrozen, + structbox, indexboxes) + assert resultbox.is_constant() + self.green_result(resultbox.getgenvar(self.jitstate)) + def opimpl_red_setinteriorfield(self): destbox = self.get_redarg() interiordesc = self.frame.bytecode.interiordescs[self.load_2byte()] @@ -419,12 +430,15 @@ self.red_result(resultbox) def opimpl_green_getinteriorarraysize(self): - arraygenconst = self.get_greenarg() + # XXX make a green version that does not use the constant folding of + # the red one + arraybox = self.get_redarg() interiordesc = self.frame.bytecode.interiordescs[self.load_2byte()] - indexgenconsts = self.get_green_varargs() + indexboxes = self.get_red_varargs() resultbox = interiordesc.gengetinteriorarraysize( self.jitstate, arraybox, indexboxes) - self.red_result(resultbox) + assert resultbox.is_constant() + self.green_result(resultbox.getgenvar(self.jitstate)) # ____________________________________________________________ # construction-time interface Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sun Feb 10 17:47:23 2008 @@ -18,6 +18,8 @@ from pypy.rlib.objectmodel import keepalive_until_here from pypy import conftest +P_NOVIRTUAL = HintAnnotatorPolicy(novirtualcontainer=True) + def getargtypes(annotator, values): return [annotation(annotator, x) for x in values] @@ -47,7 +49,8 @@ del cls._cache del cls._cache_order - def serialize(self, func, values, backendoptimize=False): + def serialize(self, func, values, policy=None, + inline=None, backendoptimize=False): key = func, backendoptimize try: cache, argtypes = self._cache[key] @@ -68,13 +71,18 @@ rtyper = t.buildrtyper(type_system = self.type_system) rtyper.specialize() self.rtyper = rtyper + if inline: + from pypy.translator.backendopt.inline import auto_inlining + auto_inlining(t, threshold=inline) if backendoptimize: from pypy.translator.backendopt.all import backend_optimizations backend_optimizations(t) graph1 = graphof(t, func) # build hint annotator types - hannotator = HintAnnotator(base_translator=t, policy=P_OOPSPEC_NOVIRTUAL) + if policy is None: + policy = P_OOPSPEC_NOVIRTUAL + hannotator = HintAnnotator(base_translator=t, policy=policy) hs = hannotator.build_types(graph1, [SomeLLAbstractConstant(v.concretetype, {OriginFlags(): True}) for v in graph1.getargs()]) @@ -117,7 +125,8 @@ assert len(ll_function.convert_arguments) == len(values) values = [decoder(value) for decoder, value in zip( ll_function.convert_arguments, values)] - writer, jitcode, argcolors = self.serialize(ll_function, values) + writer, jitcode, argcolors = self.serialize(ll_function, values, + **kwds) rgenop = writer.RGenOp() sigtoken = rgenop.sigToken(self.RESIDUAL_FUNCTYPE) builder, gv_generated, inputargs_gv = rgenop.newgraph(sigtoken, "generated") @@ -697,8 +706,7 @@ res = self.interpret(ll_function, [], []) assert res.x == 123 - def test_plus_minus_all_inlined(self): - py.test.skip("arrays and structs are not working") + def test_plus_minus(self): def ll_plus_minus(s, x, y): acc = x n = len(s) @@ -903,5 +911,169 @@ assert res == True self.check_insns({'setfield': 2, 'getfield': 1}) + def test_deepfrozen_interior(self): + T = lltype.Struct('T', ('x', lltype.Signed)) + A = lltype.Array(T) + S = lltype.GcStruct('S', ('a', A)) + s = lltype.malloc(S, 3, zero=True) + s.a[2].x = 42 + def f(n): + s1 = hint(s, variable=True) + s1 = hint(s1, deepfreeze=True) + return s1.a[n].x + + # malloc-remove the interior ptr + res = self.interpret(f, [2], [0], backendoptimize=True) + assert res == 42 + self.check_insns({}) + + def test_compile_time_const_tuple(self): + py.test.skip("no clue what's wrong") + d = {(4, 5): 42, (6, 7): 12} + def f(a, b): + d1 = hint(d, deepfreeze=True) + return d1[a, b] + + # malloc-remove the interior ptr + res = self.interpret(f, [4, 5], [0, 1], + backendoptimize=True) + assert res == 42 + self.check_insns({}) + + def test_residual_red_call(self): + py.test.skip("needs promote") + def g(x): + return x+1 + + def f(x): + return 2*g(x) + + res = self.interpret(f, [20], [], policy=StopAtXPolicy(g)) + assert res == 42 + self.check_insns(int_add=0) + + def test_residual_red_call_with_exc(self): + py.test.skip("needs promote") + def h(x): + if x > 0: + return x+1 + else: + raise ValueError + + def g(x): + return 2*h(x) + + def f(x): + try: + return g(x) + except ValueError: + return 7 + + stop_at_h = StopAtXPolicy(h) + res = self.interpret(f, [20], [], policy=stop_at_h) + assert res == 42 + self.check_insns(int_add=0) + + res = self.interpret(f, [-20], [], policy=stop_at_h) + assert res == 7 + self.check_insns(int_add=0) + + def test_red_call_ignored_result(self): + def g(n): + return n * 7 + def f(n, m): + g(n) # ignore the result + return m + + res = self.interpret(f, [4, 212], [], policy=P_NOVIRTUAL) + assert res == 212 + + def test_simple_meth(self): + py.test.skip("needs promote") + class Base(object): + def m(self): + raise NotImplementedError + pass # for inspect.getsource() bugs + + class Concrete(Base): + def m(self): + return 42 + pass # for inspect.getsource() bugs + + def f(flag): + if flag: + o = Base() + else: + o = Concrete() + return o.m() + + res = self.interpret(f, [0], [0], policy=P_NOVIRTUAL) + assert res == 42 + self.check_insns({}) + + res = self.interpret(f, [0], [], policy=P_NOVIRTUAL) + assert res == 42 + self.check_insns(indirect_call=0) + + def test_simple_red_meth(self): + py.test.skip("needs promote") + class Base(object): + def m(self, n): + raise NotImplementedError + pass # for inspect.getsource() bugs + + class Concrete(Base): + def m(self, n): + return 21*n + pass # for inspect.getsource() bugs + + def f(flag, x): + if flag: + o = Base() + else: + o = Concrete() + return o.m(x) + + res = self.interpret(f, [0, 2], [0], policy=P_NOVIRTUAL) + assert res == 42 + self.check_insns({'int_mul': 1}) + + def test_simple_red_meth_vars_around(self): + py.test.skip("needs promote") + class Base(object): + def m(self, n): + raise NotImplementedError + pass # for inspect.getsource() bugs + + class Concrete(Base): + def m(self, n): + return 21*n + pass # for inspect.getsource() bugs + + def f(flag, x, y, z): + if flag: + o = Base() + else: + o = Concrete() + return (o.m(x)+y)-z + + res = self.interpret(f, [0, 2, 7, 5], [0], policy=P_NOVIRTUAL) + assert res == 44 + self.check_insns({'int_mul': 1, 'int_add': 1, 'int_sub': 1}) + + def test_green_red_mismatch_in_call(self): + #py.test.skip("WIP") + def add(a,b, u): + return a+b + + def f(x, y, u): + r = add(x+1,y+1, u) + z = x+y + z = hint(z, concrete=True) + r # this checks that 'r' is green + return hint(z, variable=True) + + res = self.interpret(f, [4, 5, 0], [], policy=P_NOVIRTUAL) + assert res == 20 + class TestLLType(SimpleTests): type_system = "lltype" Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py Sun Feb 10 17:47:23 2008 @@ -468,165 +468,6 @@ assert res == f(True, -1000) self.check_insns({'int_ge': 2, 'int_add': 1}) - def test_simple_meth(self): - class Base(object): - def m(self): - raise NotImplementedError - pass # for inspect.getsource() bugs - - class Concrete(Base): - def m(self): - return 42 - pass # for inspect.getsource() bugs - - def f(flag): - if flag: - o = Base() - else: - o = Concrete() - return o.m() - - res = self.timeshift(f, [0], [0], policy=P_NOVIRTUAL) - assert res == 42 - self.check_insns({}) - - res = self.timeshift(f, [0], [], policy=P_NOVIRTUAL) - assert res == 42 - self.check_insns(indirect_call=0) - - def test_simple_red_meth(self): - class Base(object): - def m(self, n): - raise NotImplementedError - pass # for inspect.getsource() bugs - - class Concrete(Base): - def m(self, n): - return 21*n - pass # for inspect.getsource() bugs - - def f(flag, x): - if flag: - o = Base() - else: - o = Concrete() - return o.m(x) - - res = self.timeshift(f, [0, 2], [0], policy=P_NOVIRTUAL) - assert res == 42 - self.check_insns({'int_mul': 1}) - - def test_simple_red_meth_vars_around(self): - class Base(object): - def m(self, n): - raise NotImplementedError - pass # for inspect.getsource() bugs - - class Concrete(Base): - def m(self, n): - return 21*n - pass # for inspect.getsource() bugs - - def f(flag, x, y, z): - if flag: - o = Base() - else: - o = Concrete() - return (o.m(x)+y)-z - - res = self.timeshift(f, [0, 2, 7, 5], [0], policy=P_NOVIRTUAL) - assert res == 44 - self.check_insns({'int_mul': 1, 'int_add': 1, 'int_sub': 1}) - - def test_deepfrozen_interior(self): - T = lltype.Struct('T', ('x', lltype.Signed)) - A = lltype.Array(T) - S = lltype.GcStruct('S', ('a', A)) - s = lltype.malloc(S, 3, zero=True) - s.a[2].x = 42 - def f(n): - s1 = hint(s, variable=True) - s1 = hint(s1, deepfreeze=True) - return s1.a[n].x - - # malloc-remove the interior ptr - res = self.timeshift(f, [2], [0], policy=P_NOVIRTUAL, - backendoptimize=True) - assert res == 42 - self.check_insns({}) - - def test_compile_time_const_tuple(self): - d = {(4, 5): 42, (6, 7): 12} - def f(a, b): - d1 = hint(d, deepfreeze=True) - return d1[a, b] - - # malloc-remove the interior ptr - res = self.timeshift(f, [4, 5], [0, 1], policy=P_NOVIRTUAL, - backendoptimize=True) - assert res == 42 - self.check_insns({}) - - def test_green_red_mismatch_in_call(self): - #py.test.skip("WIP") - def add(a,b, u): - return a+b - - def f(x, y, u): - r = add(x+1,y+1, u) - z = x+y - z = hint(z, concrete=True) + r # this checks that 'r' is green - return hint(z, variable=True) - - res = self.timeshift(f, [4, 5, 0], [], policy=P_NOVIRTUAL) - assert res == 20 - - def test_residual_red_call(self): - def g(x): - return x+1 - - def f(x): - return 2*g(x) - - res = self.timeshift(f, [20], [], policy=StopAtXPolicy(g)) - assert res == 42 - self.check_insns(int_add=0) - - def test_residual_red_call_with_exc(self): - def h(x): - if x > 0: - return x+1 - else: - raise ValueError - - def g(x): - return 2*h(x) - - def f(x): - try: - return g(x) - except ValueError: - return 7 - - stop_at_h = StopAtXPolicy(h) - res = self.timeshift(f, [20], [], policy=stop_at_h) - assert res == 42 - self.check_insns(int_add=0) - - res = self.timeshift(f, [-20], [], policy=stop_at_h) - assert res == 7 - self.check_insns(int_add=0) - - def test_red_call_ignored_result(self): - def g(n): - return n * 7 - def f(n, m): - g(n) # ignore the result - return m - - res = self.timeshift(f, [4, 212], [], policy=P_NOVIRTUAL) - assert res == 212 - def test_green_char_at_merge(self): def f(c, x): c = chr(c) From arigo at codespeak.net Sun Feb 10 18:44:06 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 10 Feb 2008 18:44:06 +0100 (CET) Subject: [pypy-svn] r51374 - in pypy/dist/pypy: rpython/lltypesystem rpython/memory/gctransform translator/c/gcc translator/c/gcc/test translator/c/src Message-ID: <20080210174406.54621168509@codespeak.net> Author: arigo Date: Sun Feb 10 18:44:01 2008 New Revision: 51374 Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/rpython/memory/gctransform/asmgcroot.py pypy/dist/pypy/translator/c/gcc/test/test_trackgcroot.py pypy/dist/pypy/translator/c/gcc/trackgcroot.py pypy/dist/pypy/translator/c/src/mem.h Log: Revert r51371 for now. Either it causes a rare "cannot find gc roots!" condition, or it's just because the working copy I compiled this pypy-c-asmgcc-faassen from was not completely up-to-date. We'll see. Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Sun Feb 10 18:44:01 2008 @@ -412,7 +412,6 @@ 'llvm_frameaddress': LLOp(sideeffects=False), 'llvm_gcmapstart': LLOp(sideeffects=False), 'llvm_gcmapend': LLOp(sideeffects=False), - 'llvm_gccallshapes': LLOp(sideeffects=False), 'llvm_store_gcroot': LLOp(), 'llvm_load_gcroot': LLOp(), Modified: pypy/dist/pypy/rpython/memory/gctransform/asmgcroot.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/asmgcroot.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/asmgcroot.py Sun Feb 10 18:44:01 2008 @@ -50,12 +50,11 @@ def _asm_callback(initialframedata): self.walk_stack_from(initialframedata) self._asm_callback = _asm_callback - self._shape_decompressor = ShapeDecompressor() def setup_root_walker(self): - # The gcmap table is a list of entries, two machine words each: + # The gcmap table is a list of pairs of pointers: # void *SafePointAddress; - # int Shape; + # void *Shape; # Here, i.e. when the program starts, we sort it # in-place on the SafePointAddress to allow for more # efficient searches. @@ -99,9 +98,9 @@ callback from the GC code for each GC root found in 'caller'. """ # - # The gcmap table is a list of entries, two machine words each: + # The gcmap table is a list of pairs of pointers: # void *SafePointAddress; - # int Shape; + # void *Shape; # # A "safe point" is the return address of a call. # The "shape" of a safe point is a list of integers @@ -149,26 +148,20 @@ gcmapend = llop.llvm_gcmapend(llmemory.Address) item = binary_search(gcmapstart, gcmapend, retaddr) if item.address[0] != retaddr: - # 'retaddr' not exactly found. Check that 'item' the start of a - # compressed range containing 'retaddr'. - if retaddr > item.address[0] and item.signed[1] < 0: - pass # ok - else: - llop.debug_fatalerror(lltype.Void, "cannot find gc roots!") - return False + # retaddr not found! + llop.debug_fatalerror(lltype.Void, "cannot find gc roots!") + return False # # found! Enumerate the GC roots in the caller frame # - shape = item.signed[1] - if shape < 0: - shape = ~ shape # can ignore this "range" marker here - self._shape_decompressor.setpos(shape) + shape = item.address[1] collect_stack_root = self.gcdata._gc_collect_stack_root gc = self.gc - while True: - location = self._shape_decompressor.next() - if location == 0: - break + LIVELOCS = 1 + CALLEE_SAVED_REGS + 1 # index of the first gc root loc + livecount = shape.signed[LIVELOCS-1] + while livecount > 0: + livecount -= 1 + location = shape.signed[LIVELOCS + livecount] addr = self.getlocation(callee, location) if addr.address[0] != llmemory.NULL: collect_stack_root(gc, addr) @@ -176,18 +169,17 @@ # track where the caller_frame saved the registers from its own # caller # - reg = CALLEE_SAVED_REGS - 1 - while reg >= 0: - location = self._shape_decompressor.next() + location = shape.signed[0] + caller.frame_address = self.getlocation(callee, location) + if not caller.frame_address: # marker that means "I'm the frame + return False # of the entry point, stop walking" + reg = 0 + while reg < CALLEE_SAVED_REGS: + location = shape.signed[1+reg] addr = self.getlocation(callee, location) caller.regs_stored_at[reg] = addr - reg -= 1 - - location = self._shape_decompressor.next() - caller.frame_address = self.getlocation(callee, location) - # we get a NULL marker to mean "I'm the frame - # of the entry point, stop walking" - return caller.frame_address != llmemory.NULL + reg += 1 + return True def getlocation(self, callee, location): """Get the location in the 'caller' frame of a variable, based @@ -249,9 +241,6 @@ This is an insertion sort, so it's slowish unless the array is mostly sorted already (which is what I expect, but XXX check this). """ - # XXX this should check that it's not changing the relative order - # of entry and the following entry in case it's a compressed "range" - # entry, i.e. "entry.signed[1] < 0". next = start while next < end: # assuming the interval from start (included) to next (excluded) @@ -270,31 +259,6 @@ # ____________________________________________________________ -class ShapeDecompressor: - _alloc_flavor_ = "raw" - - def setpos(self, pos): - gccallshapes = llop.llvm_gccallshapes(llmemory.Address) - self.addr = gccallshapes + pos - - def next(self): - value = 0 - addr = self.addr - while True: - b = ord(addr.char[0]) - addr += 1 - value += b - if b < 0x80: - break - value = (value - 0x80) << 7 - self.addr = addr - if value & 1: - value = ~ value - value = value >> 1 - return value - -# ____________________________________________________________ - # # The special pypy_asm_stackwalk(), implemented directly in # assembler, fills information about the current stack top in an Modified: pypy/dist/pypy/translator/c/gcc/test/test_trackgcroot.py ============================================================================== --- pypy/dist/pypy/translator/c/gcc/test/test_trackgcroot.py (original) +++ pypy/dist/pypy/translator/c/gcc/test/test_trackgcroot.py Sun Feb 10 18:44:01 2008 @@ -6,8 +6,6 @@ from pypy.translator.c.gcc.trackgcroot import LOC_EBP_BASED, LOC_ESP_BASED from pypy.translator.c.gcc.trackgcroot import GcRootTracker from pypy.translator.c.gcc.trackgcroot import FunctionGcRootTracker -from pypy.translator.c.gcc.trackgcroot import compress_callshape -from pypy.translator.c.gcc.trackgcroot import decompress_callshape from StringIO import StringIO this_dir = py.path.local(__file__).dirpath() @@ -38,14 +36,6 @@ LOC_EBP_BASED+24, LOC_EBP_BASED+28)) == expected -def test_compress_callshape(): - shape = (1, -3, 0x1234, -0x5678, 0x234567, - -0x765432, 0x61626364, -0x41424344) - bytes = list(compress_callshape(shape)) - print bytes - assert len(bytes) == 1+1+2+3+4+4+5+5+1 - assert decompress_callshape(bytes) == list(shape) - def test_find_functions(): source = """\ \t.p2align 4,,15 Modified: pypy/dist/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/dist/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/dist/pypy/translator/c/gcc/trackgcroot.py Sun Feb 10 18:44:01 2008 @@ -35,8 +35,6 @@ def dump(self, output): assert self.seen_main shapes = {} - shapelines = [] - shapeofs = 0 print >> output, """\t.text .globl pypy_asm_stackwalk .type pypy_asm_stackwalk, @function @@ -70,26 +68,30 @@ print >> output, '\t.align\t4' print >> output, '\t.globl\t__gcmapstart' print >> output, '__gcmapstart:' - for label, state, is_range in self.gcmaptable: - try: - n = shapes[state] - except KeyError: - n = shapes[state] = shapeofs - bytes = [str(b) for b in compress_callshape(state)] - shapelines.append('\t/*%d*/\t.byte\t%s\n' % ( - shapeofs, - ', '.join(bytes))) - shapeofs += len(bytes) - if is_range: - n = ~ n + for label, state in self.gcmaptable: + if state not in shapes: + lst = ['__gcmap_shape'] + for n in state: + if n < 0: + n = 'm%d' % (-n,) + lst.append(str(n)) + shapes[state] = '_'.join(lst) print >> output, '\t.long\t%s' % (label,) - print >> output, '\t.long\t%d' % (n,) + print >> output, '\t.long\t%s' % (shapes[state],) print >> output, '\t.globl\t__gcmapend' print >> output, '__gcmapend:' print >> output, '\t.section\t.rodata' - print >> output, '\t.globl\t__gccallshapes' - print >> output, '__gccallshapes:' - output.writelines(shapelines) + print >> output, '\t.align\t4' + keys = shapes.keys() + keys.sort() + FIXED = 1 + len(CALLEE_SAVE_REGISTERS) + for state in keys: + print >> output, '%s:' % (shapes[state],) + for i in range(FIXED): + print >> output, '\t.long\t%d' % (state[i],) + print >> output, '\t.long\t%d' % (len(state)-FIXED,) + for p in state[FIXED:]: + print >> output, '\t.long\t%d' % (p,) # gcroots def find_functions(self, iterlines): functionlines = [] @@ -113,13 +115,10 @@ yield False, functionlines def process(self, iterlines, newfile, entrypoint='main', filename='?'): - self.localgcmaptable = [] for in_function, lines in self.find_functions(iterlines): if in_function: lines = self.process_function(lines, entrypoint, filename) newfile.writelines(lines) - self.gcmaptable.extend(compress_gcmaptable(self.localgcmaptable)) - del self.localgcmaptable self.files_seen += 1 def process_function(self, lines, entrypoint, filename): @@ -132,7 +131,7 @@ if self.verbose > 1: for label, state in table: print >> sys.stderr, label, '\t', format_callshape(state) - self.localgcmaptable.extend(table) + self.gcmaptable.extend(table) self.seen_main |= tracker.is_main return tracker.lines @@ -898,82 +897,6 @@ ', '.join(result[1:5]), ', '.join(result[5:])) -# __________ table compression __________ - -def compress_gcmaptable(table): - # Compress ranges table[i:j] of entries with the same state - # into a single entry whose label is the start of the range. - # The last element in the table is never compressed in this - # way for debugging reasons, to avoid that a random address - # in memory gets mapped to the last element in the table - # just because it's the closest address. - # Also, compress_gcmaptable() should be called after each - # .s file processed -- otherwise the result depends on the - # linker not rearranging the .s files in memory, which looks - # fragile. - i = 0 - limit = len(table) - 1 # only process entries table[:limit] - while i < len(table): - label1, state = table[i] - is_range = False - j = i + 1 - while j < limit and table[j][1] == state: - is_range = True - j += 1 - # now all entries in table[i:j] have the same state - yield (label1, state, is_range) - i = j - -def compress_callshape(shape): - # For a single shape, this turns the list of integers into a list of - # bytes and reverses the order of the entries. The length is - # encoded by inserting a 0 marker after the gc roots coming from - # shape[5:] and before the 5 values coming from shape[4] to - # shape[0]. In practice it seems that shapes contain many integers - # whose value is up to a few thousands, which the algorithm below - # compresses down to 2 bytes. Very small values compress down to a - # single byte. - assert len(shape) >= 5 - shape = list(shape) - assert 0 not in shape[5:] - shape.insert(5, 0) - result = [] - for loc in shape: - if loc < 0: - loc = (-loc) * 2 - 1 - else: - loc = loc * 2 - flag = 0 - while loc >= 0x80: - result.append(int(loc & 0x7F) | flag) - flag = 0x80 - loc >>= 7 - result.append(int(loc) | flag) - result.reverse() - return result - -def decompress_callshape(bytes): - # For tests. This logic is copied in asmgcroot.py. - result = [] - n = 0 - while n < len(bytes): - value = 0 - while True: - b = bytes[n] - n += 1 - value += b - if b < 0x80: - break - value = (value - 0x80) << 7 - if value & 1: - value = ~ value - value = value >> 1 - result.append(value) - result.reverse() - assert result[5] == 0 - del result[5] - return result - if __name__ == '__main__': if sys.argv and sys.argv[1] == '-v': Modified: pypy/dist/pypy/translator/c/src/mem.h ============================================================================== --- pypy/dist/pypy/translator/c/src/mem.h (original) +++ pypy/dist/pypy/translator/c/src/mem.h Sun Feb 10 18:44:01 2008 @@ -10,7 +10,7 @@ extern char __gcmapstart; extern char __gcmapend; -extern char __gccallshapes; +extern char* __gcmap_frame_address(void); #define PYPY_GCROOT(p) asm ("/* GCROOT %0 */" : "=g" (p) : "0" (p) : "memory") #define pypy_asm_gcroot(p) ({void*_r; \ @@ -19,7 +19,10 @@ #define OP_LLVM_GCMAPSTART(r) r = &__gcmapstart #define OP_LLVM_GCMAPEND(r) r = &__gcmapend -#define OP_LLVM_GCCALLSHAPES(r) r = &__gccallshapes +#define OP_LLVM_FRAMEADDRESS(r) asm ("pypygetframeaddress %0" : "=r" (r)) +/* NB. we cannot use __builtin_frame_address(0) - apparently, gcc thinks + it can return %ebp even if -fomit-frame-pointer is specified, which + doesn't work. So we need a bit of help from trackgcroot.py... */ #define RAW_MALLOC_ZERO_FILLED 0 From fijal at codespeak.net Sun Feb 10 18:51:22 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 10 Feb 2008 18:51:22 +0100 (CET) Subject: [pypy-svn] r51375 - in pypy/dist/pypy/objspace/std: . test Message-ID: <20080210175122.4049416850B@codespeak.net> Author: fijal Date: Sun Feb 10 18:51:21 2008 New Revision: 51375 Modified: pypy/dist/pypy/objspace/std/objspace.py pypy/dist/pypy/objspace/std/test/test_stdobjspace.py Log: space.sliceindices (needs more testing, coming in a moment) Modified: pypy/dist/pypy/objspace/std/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/objspace.py (original) +++ pypy/dist/pypy/objspace/std/objspace.py Sun Feb 10 18:51:21 2008 @@ -598,6 +598,13 @@ len(t), expected_length) return t + def sliceindices(self, w_slice, w_length): + if isinstance(w_slice, W_SliceObject): + a, b, c = w_slice.indices3(self, self.int_w(w_length)) + return self.newtuple([a, b, c]) + w_indices = self.getattr(w_slice, self.wrap('indices')) + return self.call_function(w_indices, w_length) + def is_(self, w_one, w_two): # XXX a bit of hacking to gain more speed if w_one is w_two: Modified: pypy/dist/pypy/objspace/std/test/test_stdobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_stdobjspace.py (original) +++ pypy/dist/pypy/objspace/std/test/test_stdobjspace.py Sun Feb 10 18:51:21 2008 @@ -1,7 +1,5 @@ from pypy.interpreter.error import OperationError - - class TestW_StdObjSpace: def test_wrap_wrap(self): @@ -33,3 +31,11 @@ assert ('pop', True) in res assert ('reverse', True) in res assert ('popitem', True) not in res + + def test_sliceindices(self): + space = self.space + w = space.wrap + w_slice = space.newslice(w(1), w(2), w(1)) + assert space.unpacktuple(space.sliceindices(w_slice, w(3))) == [1,2,1] + + From antocuni at codespeak.net Mon Feb 11 10:36:08 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 11 Feb 2008 10:36:08 +0100 (CET) Subject: [pypy-svn] r51377 - in pypy/dist/pypy/jit/codegen/cli: . test Message-ID: <20080211093608.BBC061683F6@codespeak.net> Author: antocuni Date: Mon Feb 11 10:36:06 2008 New Revision: 51377 Modified: pypy/dist/pypy/jit/codegen/cli/operation.py pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Log: implement emit() for unaryops; test_calling_pause_* now pass Modified: pypy/dist/pypy/jit/codegen/cli/operation.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/operation.py (original) +++ pypy/dist/pypy/jit/codegen/cli/operation.py Mon Feb 11 10:36:06 2008 @@ -39,6 +39,12 @@ def pushAllArgs(self): self.gv_x.load(self.il) + def emit(self): + self.pushAllArgs() + self.il.Emit(self.getOpCode()) + self.storeResult() + + class BinaryOp(Operation): def __init__(self, il, gv_x, gv_y): self.il = il Modified: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Mon Feb 11 10:36:06 2008 @@ -17,6 +17,7 @@ 'test_if', # 'test_switch', # no promotion/flexswitch for now please :-) 'test_fact', + 'test_calling_pause', ] for p in prefixes: From antocuni at codespeak.net Mon Feb 11 10:38:51 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 11 Feb 2008 10:38:51 +0100 (CET) Subject: [pypy-svn] r51378 - pypy/dist/pypy/jit/codegen/cli/test Message-ID: <20080211093851.684891683FA@codespeak.net> Author: antocuni Date: Mon Feb 11 10:38:50 2008 New Revision: 51378 Modified: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Log: these tests pass out of the box :-) Modified: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Mon Feb 11 10:38:50 2008 @@ -18,6 +18,8 @@ # 'test_switch', # no promotion/flexswitch for now please :-) 'test_fact', 'test_calling_pause', + 'test_longwinded_and', + 'test_condition_result_cross_link_direct', ] for p in prefixes: From antocuni at codespeak.net Mon Feb 11 11:57:25 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 11 Feb 2008 11:57:25 +0100 (CET) Subject: [pypy-svn] r51379 - in pypy/dist/pypy/jit/codegen/cli: . test Message-ID: <20080211105725.0C9191683E8@codespeak.net> Author: antocuni Date: Mon Feb 11 11:57:24 2008 New Revision: 51379 Modified: pypy/dist/pypy/jit/codegen/cli/operation.py pypy/dist/pypy/jit/codegen/cli/rgenop.py pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Log: fix restype() for cast operations; test_multiple_cmps pass Modified: pypy/dist/pypy/jit/codegen/cli/operation.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/operation.py (original) +++ pypy/dist/pypy/jit/codegen/cli/operation.py Mon Feb 11 11:57:24 2008 @@ -44,6 +44,8 @@ self.il.Emit(self.getOpCode()) self.storeResult() + def getOpCode(self): + raise NotImplementedError class BinaryOp(Operation): def __init__(self, il, gv_x, gv_y): @@ -157,11 +159,15 @@ return True return False -def fillops(ops, baseclass): - # monkey-patch boolean operations - def restype(self): - return typeof(System.Boolean) +def restype_bool(self): return typeof(System.Boolean) +def restype_int(self): return typeof(System.Int32) +def restype_uint(self): return typeof(System.Int32) +def restype_float(self): return typeof(System.Double) +def restype_char(self): return typeof(System.Char) +def restype_unichar(self): return typeof(System.Char) +def restype_longlong(self): return typeof(System.Int64) +def fillops(ops, baseclass): out = {} for opname, value in ops.iteritems(): if isinstance(value, str): @@ -173,12 +179,19 @@ """ % locals()) code = source.compile() exec code in globals(), out - if is_comparison(opname): - out[opname].restype = restype elif value is cli_opcodes.DoNothing: out[opname] = SameAs else: renderCustomOp(opname, baseclass, value, out) + + # fix the restype for comparison ops and casts + if is_comparison(opname): + out[opname].restype = restype_bool + elif opname != 'cast_primitive' and opname.startswith('cast_'): + _, _, _, to = opname.split('_') + funcname = 'restype_%s' % to + out[opname].restype = globals()[funcname] + return out def renderCustomOp(opname, baseclass, steps, out): Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/rgenop.py Mon Feb 11 11:57:24 2008 @@ -11,6 +11,7 @@ Constants = CLR.pypy.runtime.Constants OpCodes = System.Reflection.Emit.OpCodes +DUMP_IL = False DEBUG = False def token2clitype(tok): @@ -208,7 +209,7 @@ self.rgenop = rgenop self.meth = Utils.CreateDynamicMethod(name, res, args) self.il = self.meth.GetILGenerator() - if DEBUG: + if DUMP_IL: self.il = DumpGenerator(self.il) self.inputargs_gv = [] for i in range(len(args)): @@ -226,14 +227,36 @@ opcls = ops.getopclass1(opname) op = opcls(self.il, gv_arg) self.emit(op) - return op.gv_res() + gv_res = op.gv_res() + if DEBUG: + self.il.EmitWriteLine(opname) + self.writeline_gv(gv_arg) + self.writeline_gv(gv_res) + self.il.EmitWriteLine('') + return gv_res + @specialize.arg(1) def genop2(self, opname, gv_arg1, gv_arg2): opcls = ops.getopclass2(opname) op = opcls(self.il, gv_arg1, gv_arg2) self.emit(op) - return op.gv_res() + gv_res = op.gv_res() + if DEBUG: + self.il.EmitWriteLine(opname) + self.writeline_gv(gv_arg1) + self.writeline_gv(gv_arg2) + self.writeline_gv(gv_res) + self.il.EmitWriteLine('') + return gv_res + + def writeline_gv(self, gv): + if isinstance(gv, GenLocalVar): + self.il.EmitWriteLine(gv.v) + elif isinstance(gv, IntConst): + self.il.EmitWriteLine('%s' % gv.value) + else: + assert False def genop_call(self, sigtoken, gv_fnptr, args_gv): op = ops.Call(self.il, sigtoken, gv_fnptr, args_gv) Modified: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Mon Feb 11 11:57:24 2008 @@ -20,6 +20,7 @@ 'test_calling_pause', 'test_longwinded_and', 'test_condition_result_cross_link_direct', + 'test_multiple_cmps', ] for p in prefixes: From arigo at codespeak.net Mon Feb 11 12:50:25 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 11 Feb 2008 12:50:25 +0100 (CET) Subject: [pypy-svn] r51380 - in pypy/dist/pypy: rpython/lltypesystem rpython/memory/gctransform translator/c/gcc translator/c/gcc/test translator/c/src Message-ID: <20080211115025.7F5221683EA@codespeak.net> Author: arigo Date: Mon Feb 11 12:50:21 2008 New Revision: 51380 Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/rpython/memory/gctransform/asmgcroot.py pypy/dist/pypy/translator/c/gcc/test/test_trackgcroot.py pypy/dist/pypy/translator/c/gcc/trackgcroot.py pypy/dist/pypy/translator/c/src/mem.h Log: Re-check-in r51371 with a fix: compress_gcmaptable() should be called after each function processed -- otherwise the result depends on the linker not rearranging the functions in memory, which is fragile (and wrong e.g. with "make profopt"). Reduces the size of a faassen build by 400 KB (only 4% but well). Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Mon Feb 11 12:50:21 2008 @@ -412,6 +412,7 @@ 'llvm_frameaddress': LLOp(sideeffects=False), 'llvm_gcmapstart': LLOp(sideeffects=False), 'llvm_gcmapend': LLOp(sideeffects=False), + 'llvm_gccallshapes': LLOp(sideeffects=False), 'llvm_store_gcroot': LLOp(), 'llvm_load_gcroot': LLOp(), Modified: pypy/dist/pypy/rpython/memory/gctransform/asmgcroot.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/asmgcroot.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/asmgcroot.py Mon Feb 11 12:50:21 2008 @@ -50,11 +50,12 @@ def _asm_callback(initialframedata): self.walk_stack_from(initialframedata) self._asm_callback = _asm_callback + self._shape_decompressor = ShapeDecompressor() def setup_root_walker(self): - # The gcmap table is a list of pairs of pointers: + # The gcmap table is a list of entries, two machine words each: # void *SafePointAddress; - # void *Shape; + # int Shape; # Here, i.e. when the program starts, we sort it # in-place on the SafePointAddress to allow for more # efficient searches. @@ -98,9 +99,9 @@ callback from the GC code for each GC root found in 'caller'. """ # - # The gcmap table is a list of pairs of pointers: + # The gcmap table is a list of entries, two machine words each: # void *SafePointAddress; - # void *Shape; + # int Shape; # # A "safe point" is the return address of a call. # The "shape" of a safe point is a list of integers @@ -148,20 +149,26 @@ gcmapend = llop.llvm_gcmapend(llmemory.Address) item = binary_search(gcmapstart, gcmapend, retaddr) if item.address[0] != retaddr: - # retaddr not found! - llop.debug_fatalerror(lltype.Void, "cannot find gc roots!") - return False + # 'retaddr' not exactly found. Check that 'item' the start of a + # compressed range containing 'retaddr'. + if retaddr > item.address[0] and item.signed[1] < 0: + pass # ok + else: + llop.debug_fatalerror(lltype.Void, "cannot find gc roots!") + return False # # found! Enumerate the GC roots in the caller frame # - shape = item.address[1] + shape = item.signed[1] + if shape < 0: + shape = ~ shape # can ignore this "range" marker here + self._shape_decompressor.setpos(shape) collect_stack_root = self.gcdata._gc_collect_stack_root gc = self.gc - LIVELOCS = 1 + CALLEE_SAVED_REGS + 1 # index of the first gc root loc - livecount = shape.signed[LIVELOCS-1] - while livecount > 0: - livecount -= 1 - location = shape.signed[LIVELOCS + livecount] + while True: + location = self._shape_decompressor.next() + if location == 0: + break addr = self.getlocation(callee, location) if addr.address[0] != llmemory.NULL: collect_stack_root(gc, addr) @@ -169,17 +176,18 @@ # track where the caller_frame saved the registers from its own # caller # - location = shape.signed[0] - caller.frame_address = self.getlocation(callee, location) - if not caller.frame_address: # marker that means "I'm the frame - return False # of the entry point, stop walking" - reg = 0 - while reg < CALLEE_SAVED_REGS: - location = shape.signed[1+reg] + reg = CALLEE_SAVED_REGS - 1 + while reg >= 0: + location = self._shape_decompressor.next() addr = self.getlocation(callee, location) caller.regs_stored_at[reg] = addr - reg += 1 - return True + reg -= 1 + + location = self._shape_decompressor.next() + caller.frame_address = self.getlocation(callee, location) + # we get a NULL marker to mean "I'm the frame + # of the entry point, stop walking" + return caller.frame_address != llmemory.NULL def getlocation(self, callee, location): """Get the location in the 'caller' frame of a variable, based @@ -241,6 +249,9 @@ This is an insertion sort, so it's slowish unless the array is mostly sorted already (which is what I expect, but XXX check this). """ + # XXX this should check that it's not changing the relative order + # of entry and the following entry in case it's a compressed "range" + # entry, i.e. "entry.signed[1] < 0". next = start while next < end: # assuming the interval from start (included) to next (excluded) @@ -259,6 +270,31 @@ # ____________________________________________________________ +class ShapeDecompressor: + _alloc_flavor_ = "raw" + + def setpos(self, pos): + gccallshapes = llop.llvm_gccallshapes(llmemory.Address) + self.addr = gccallshapes + pos + + def next(self): + value = 0 + addr = self.addr + while True: + b = ord(addr.char[0]) + addr += 1 + value += b + if b < 0x80: + break + value = (value - 0x80) << 7 + self.addr = addr + if value & 1: + value = ~ value + value = value >> 1 + return value + +# ____________________________________________________________ + # # The special pypy_asm_stackwalk(), implemented directly in # assembler, fills information about the current stack top in an Modified: pypy/dist/pypy/translator/c/gcc/test/test_trackgcroot.py ============================================================================== --- pypy/dist/pypy/translator/c/gcc/test/test_trackgcroot.py (original) +++ pypy/dist/pypy/translator/c/gcc/test/test_trackgcroot.py Mon Feb 11 12:50:21 2008 @@ -6,6 +6,8 @@ from pypy.translator.c.gcc.trackgcroot import LOC_EBP_BASED, LOC_ESP_BASED from pypy.translator.c.gcc.trackgcroot import GcRootTracker from pypy.translator.c.gcc.trackgcroot import FunctionGcRootTracker +from pypy.translator.c.gcc.trackgcroot import compress_callshape +from pypy.translator.c.gcc.trackgcroot import decompress_callshape from StringIO import StringIO this_dir = py.path.local(__file__).dirpath() @@ -36,6 +38,14 @@ LOC_EBP_BASED+24, LOC_EBP_BASED+28)) == expected +def test_compress_callshape(): + shape = (1, -3, 0x1234, -0x5678, 0x234567, + -0x765432, 0x61626364, -0x41424344) + bytes = list(compress_callshape(shape)) + print bytes + assert len(bytes) == 1+1+2+3+4+4+5+5+1 + assert decompress_callshape(bytes) == list(shape) + def test_find_functions(): source = """\ \t.p2align 4,,15 Modified: pypy/dist/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/dist/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/dist/pypy/translator/c/gcc/trackgcroot.py Mon Feb 11 12:50:21 2008 @@ -35,6 +35,8 @@ def dump(self, output): assert self.seen_main shapes = {} + shapelines = [] + shapeofs = 0 print >> output, """\t.text .globl pypy_asm_stackwalk .type pypy_asm_stackwalk, @function @@ -68,30 +70,26 @@ print >> output, '\t.align\t4' print >> output, '\t.globl\t__gcmapstart' print >> output, '__gcmapstart:' - for label, state in self.gcmaptable: - if state not in shapes: - lst = ['__gcmap_shape'] - for n in state: - if n < 0: - n = 'm%d' % (-n,) - lst.append(str(n)) - shapes[state] = '_'.join(lst) + for label, state, is_range in self.gcmaptable: + try: + n = shapes[state] + except KeyError: + n = shapes[state] = shapeofs + bytes = [str(b) for b in compress_callshape(state)] + shapelines.append('\t/*%d*/\t.byte\t%s\n' % ( + shapeofs, + ', '.join(bytes))) + shapeofs += len(bytes) + if is_range: + n = ~ n print >> output, '\t.long\t%s' % (label,) - print >> output, '\t.long\t%s' % (shapes[state],) + print >> output, '\t.long\t%d' % (n,) print >> output, '\t.globl\t__gcmapend' print >> output, '__gcmapend:' print >> output, '\t.section\t.rodata' - print >> output, '\t.align\t4' - keys = shapes.keys() - keys.sort() - FIXED = 1 + len(CALLEE_SAVE_REGISTERS) - for state in keys: - print >> output, '%s:' % (shapes[state],) - for i in range(FIXED): - print >> output, '\t.long\t%d' % (state[i],) - print >> output, '\t.long\t%d' % (len(state)-FIXED,) - for p in state[FIXED:]: - print >> output, '\t.long\t%d' % (p,) # gcroots + print >> output, '\t.globl\t__gccallshapes' + print >> output, '__gccallshapes:' + output.writelines(shapelines) def find_functions(self, iterlines): functionlines = [] @@ -131,7 +129,7 @@ if self.verbose > 1: for label, state in table: print >> sys.stderr, label, '\t', format_callshape(state) - self.gcmaptable.extend(table) + self.gcmaptable.extend(compress_gcmaptable(table)) self.seen_main |= tracker.is_main return tracker.lines @@ -897,6 +895,82 @@ ', '.join(result[1:5]), ', '.join(result[5:])) +# __________ table compression __________ + +def compress_gcmaptable(table): + # Compress ranges table[i:j] of entries with the same state + # into a single entry whose label is the start of the range. + # The last element in the table is never compressed in this + # way for debugging reasons, to avoid that a random address + # in memory gets mapped to the last element in the table + # just because it's the closest address. + # To be on the safe side, compress_gcmaptable() should be called + # after each function processed -- otherwise the result depends on + # the linker not rearranging the functions in memory, which is + # fragile (and wrong e.g. with "make profopt"). + i = 0 + limit = len(table) - 1 # only process entries table[:limit] + while i < len(table): + label1, state = table[i] + is_range = False + j = i + 1 + while j < limit and table[j][1] == state: + is_range = True + j += 1 + # now all entries in table[i:j] have the same state + yield (label1, state, is_range) + i = j + +def compress_callshape(shape): + # For a single shape, this turns the list of integers into a list of + # bytes and reverses the order of the entries. The length is + # encoded by inserting a 0 marker after the gc roots coming from + # shape[5:] and before the 5 values coming from shape[4] to + # shape[0]. In practice it seems that shapes contain many integers + # whose value is up to a few thousands, which the algorithm below + # compresses down to 2 bytes. Very small values compress down to a + # single byte. + assert len(shape) >= 5 + shape = list(shape) + assert 0 not in shape[5:] + shape.insert(5, 0) + result = [] + for loc in shape: + if loc < 0: + loc = (-loc) * 2 - 1 + else: + loc = loc * 2 + flag = 0 + while loc >= 0x80: + result.append(int(loc & 0x7F) | flag) + flag = 0x80 + loc >>= 7 + result.append(int(loc) | flag) + result.reverse() + return result + +def decompress_callshape(bytes): + # For tests. This logic is copied in asmgcroot.py. + result = [] + n = 0 + while n < len(bytes): + value = 0 + while True: + b = bytes[n] + n += 1 + value += b + if b < 0x80: + break + value = (value - 0x80) << 7 + if value & 1: + value = ~ value + value = value >> 1 + result.append(value) + result.reverse() + assert result[5] == 0 + del result[5] + return result + if __name__ == '__main__': if sys.argv and sys.argv[1] == '-v': Modified: pypy/dist/pypy/translator/c/src/mem.h ============================================================================== --- pypy/dist/pypy/translator/c/src/mem.h (original) +++ pypy/dist/pypy/translator/c/src/mem.h Mon Feb 11 12:50:21 2008 @@ -10,7 +10,7 @@ extern char __gcmapstart; extern char __gcmapend; -extern char* __gcmap_frame_address(void); +extern char __gccallshapes; #define PYPY_GCROOT(p) asm ("/* GCROOT %0 */" : "=g" (p) : "0" (p) : "memory") #define pypy_asm_gcroot(p) ({void*_r; \ @@ -19,10 +19,7 @@ #define OP_LLVM_GCMAPSTART(r) r = &__gcmapstart #define OP_LLVM_GCMAPEND(r) r = &__gcmapend -#define OP_LLVM_FRAMEADDRESS(r) asm ("pypygetframeaddress %0" : "=r" (r)) -/* NB. we cannot use __builtin_frame_address(0) - apparently, gcc thinks - it can return %ebp even if -fomit-frame-pointer is specified, which - doesn't work. So we need a bit of help from trackgcroot.py... */ +#define OP_LLVM_GCCALLSHAPES(r) r = &__gccallshapes #define RAW_MALLOC_ZERO_FILLED 0 From cfbolz at codespeak.net Mon Feb 11 14:15:50 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 11 Feb 2008 14:15:50 +0100 (CET) Subject: [pypy-svn] r51382 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter Message-ID: <20080211131550.C21F51683EE@codespeak.net> Author: cfbolz Date: Mon Feb 11 14:15:48 2008 New Revision: 51382 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vlist.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/oop.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/vdict.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/vlist.py Log: some support for virtual lists Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 11 14:15:48 2008 @@ -4,6 +4,7 @@ from pypy.jit.hintannotator.model import originalconcretetype from pypy.jit.hintannotator import model as hintmodel from pypy.jit.timeshifter import rtimeshift, rvalue, rcontainer, exception +from pypy.jit.timeshifter import oop from pypy.jit.timeshifter.greenkey import KeyDesc from pypy.jit.rainbow.interpreter import JitCode, JitInterpreter from pypy.translator.backendopt.removenoops import remove_same_as @@ -14,10 +15,10 @@ class BytecodeWriter(object): def __init__(self, t, hannotator, RGenOp): self.translator = t - self.annotator = t.annotator + self.rtyper = hannotator.base_translator.rtyper self.hannotator = hannotator etrafo = hannotator.exceptiontransformer - type_system = hannotator.base_translator.rtyper.type_system.name + type_system = self.rtyper.type_system.name self.exceptiondesc = exception.ExceptionDesc( RGenOp, etrafo, type_system, False) self.interpreter = JitInterpreter(self.exceptiondesc) @@ -44,6 +45,7 @@ self.fielddescs = [] self.arrayfielddescs = [] self.interiordescs = [] + self.oopspecdescs = [] self.called_bytecodes = [] self.num_mergepoints = 0 self.graph_color = self.graph_calling_color(graph) @@ -71,6 +73,8 @@ self.arrayfielddesc_positions = {} # mapping (TYPE, path) to index self.interiordesc_positions = {} + # mapping (fnobj, can_raise) to index + self.oopspecdesc_positions = {} # mapping graphs to index self.graph_positions = {} # mapping fnobjs to index @@ -91,6 +95,7 @@ self.fielddescs, self.arrayfielddescs, self.interiordescs, + self.oopspecdescs, self.called_bytecodes, self.num_mergepoints, self.graph_color, @@ -352,6 +357,17 @@ self.arrayfielddesc_positions[TYPE] = result return result + def oopspecdesc_position(self, fnobj, canraise): + key = fnobj, canraise + if key in self.oopspecdesc_positions: + return self.oopspecdesc_positions[key] + oopspecdesc = oop.OopSpecDesc(self.RGenOp, self.rtyper, + fnobj, canraise) + result = len(self.oopspecdescs) + self.oopspecdescs.append(oopspecdesc) + self.oopspecdesc_positions[key] = result + return result + def graph_position(self, graph): if graph in self.graph_positions: return self.graph_positions[graph] @@ -478,6 +494,45 @@ def serialize_op_direct_call(self, op): kind, exc = self.guess_call_kind(op) + print op, kind, exc + if kind == "oopspec": + from pypy.jit.timeshifter.oop import Index + fnobj = op.args[0].value._obj + oopspecdescindex = self.oopspecdesc_position(fnobj, exc) + oopspecdesc = self.oopspecdescs[oopspecdescindex] + opargs = op.args[1:] + args_v = [] + args = [] + for obj in oopspecdesc.argtuple: + if isinstance(obj, Index): + v = opargs[obj.n] + else: + v = flowmodel.Constant(obj, lltype.typeOf(obj)) + args_v.append(v) + args.append(self.serialize_oparg("red", v)) + + ll_handler = oopspecdesc.ll_handler + couldfold = oopspecdesc.couldfold + missing_args = ((ll_handler.func_code.co_argcount - 3) - + len(oopspecdesc.argtuple)) + assert missing_args >= 0 + if missing_args > 0: + assert (ll_handler.func_defaults[-missing_args:] == + (None,) * missing_args) + + if oopspecdesc.is_method: + hs_self = self.hannotator.binding( + opargs[oopspecdesc.argtuple[0].n]) + deepfrozen = hs_self.deepfrozen + else: + deepfrozen = False + + self.emit("red_oopspec_call_%s" % (len(args), )) + self.emit(oopspecdescindex) + self.emit(deepfrozen) + self.emit(*args) + self.register_redvar(op.result) + return targets = dict(self.graphs_from(op)) assert len(targets) == 1 targetgraph, = targets.values() Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Mon Feb 11 14:15:48 2008 @@ -19,8 +19,8 @@ def __init__(self, name, code, constants, typekinds, redboxclasses, keydescs, structtypedescs, fielddescs, arrayfielddescs, - interiordescs, called_bytecodes, num_mergepoints, graph_color, - nonrainbow_functions, is_portal): + interiordescs, oopspecdescs, called_bytecodes, num_mergepoints, + graph_color, nonrainbow_functions, is_portal): self.name = name self.code = code self.constants = constants @@ -31,6 +31,7 @@ self.fielddescs = fielddescs self.arrayfielddescs = arrayfielddescs self.interiordescs = interiordescs + self.oopspecdescs = oopspecdescs self.called_bytecodes = called_bytecodes self.num_mergepoints = num_mergepoints self.graph_color = graph_color @@ -166,6 +167,10 @@ assert gv.is_const self.frame.local_green.append(gv) + def green_result_from_red(self, box): + assert box.is_constant() + self.green_result(box.getgenvar(self.jitstate)) + def newjitstate(self, newjitstate): self.jitstate = newjitstate self.queue = None @@ -319,14 +324,43 @@ # XXX all this jitstate.greens business is a bit messy self.green_result(self.jitstate.greens[0]) + def opimpl_red_oopspec_call_1(self): + oopspecindex = self.load_2byte() + deepfrozen = self.load_bool() + arg1 = self.get_redarg() + oopspec = self.frame.bytecode.oopspecdescs[oopspecindex] + result = oopspec.ll_handler(self.jitstate, oopspec, deepfrozen, arg1) + self.red_result(result) + + def opimpl_red_oopspec_call_2(self): + oopspecindex = self.load_2byte() + deepfrozen = self.load_bool() + arg1 = self.get_redarg() + arg2 = self.get_redarg() + oopspec = self.frame.bytecode.oopspecdescs[oopspecindex] + result = oopspec.ll_handler(self.jitstate, oopspec, deepfrozen, arg1, arg2) + self.red_result(result) + + def opimpl_red_oopspec_call_3(self): + oopspecindex = self.load_2byte() + deepfrozen = self.load_bool() + arg1 = self.get_redarg() + arg2 = self.get_redarg() + arg3 = self.get_redarg() + oopspec = self.frame.bytecode.oopspecdescs[oopspecindex] + result = oopspec.ll_handler(self.jitstate, oopspec, deepfrozen, arg1, arg2, arg3) + self.red_result(result) + # exceptions def opimpl_read_exctype(self): - XXX + box = rtimeshift.getexctypebox(self.jitstate) + self.red_result(box) def opimpl_read_excvalue(self): - XXX + box = rtimeshift.getexcvaluebox(self.jitstate) + self.red_result(box) def opimpl_write_exctype(self): typebox = self.get_redarg() @@ -411,8 +445,7 @@ indexboxes = self.get_red_varargs() resultbox = interiordesc.gengetinteriorfield(self.jitstate, deepfrozen, structbox, indexboxes) - assert resultbox.is_constant() - self.green_result(resultbox.getgenvar(self.jitstate)) + self.green_result_from_red(resultbox) def opimpl_red_setinteriorfield(self): destbox = self.get_redarg() @@ -437,8 +470,8 @@ indexboxes = self.get_red_varargs() resultbox = interiordesc.gengetinteriorarraysize( self.jitstate, arraybox, indexboxes) - assert resultbox.is_constant() - self.green_result(resultbox.getgenvar(self.jitstate)) + self.green_result_from_red(resultbox) + # ____________________________________________________________ # construction-time interface Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vlist.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vlist.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vlist.py Mon Feb 11 14:15:48 2008 @@ -4,7 +4,6 @@ from pypy.rlib.jit import hint P_OOPSPEC = HintAnnotatorPolicy(novirtualcontainer=True, oopspec=True) -py.test.skip("implement me") class TestVList(AbstractInterpretationTest): @@ -66,6 +65,7 @@ self.check_insns({'int_is_true': 1}) def test_force(self): + py.test.skip("implement me") def ll_function(n): lst = [] lst.append(n) @@ -134,6 +134,7 @@ self.check_insns({}) def test_frozen_list(self): + py.test.skip("implement me") lst = [5, 7, 9] def ll_function(x): mylist = hint(lst, deepfreeze=True) @@ -146,6 +147,7 @@ self.check_insns({}) def test_frozen_list_indexerror(self): + py.test.skip("implement me") lst = [5, 7, 9] def ll_function(x): mylist = hint(lst, deepfreeze=True) @@ -161,6 +163,7 @@ self.check_insns({}) def test_bogus_index_while_compiling(self): + py.test.skip("implement me") class Y: pass Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/oop.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/oop.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/oop.py Mon Feb 11 14:15:48 2008 @@ -20,7 +20,7 @@ do_call = None - def __init__(self, hrtyper, fnobj, can_raise): + def __init__(self, RGenOp, rtyper, fnobj, can_raise): ll_func = fnobj._callable FUNCTYPE = lltype.typeOf(fnobj) nb_args = len(FUNCTYPE.ARGS) @@ -34,8 +34,8 @@ if args.strip() == ',': args = '()' argnames = ll_func.func_code.co_varnames[:nb_args] - d = dict(zip(argnames, [Index(n) for n in range(nb_args)])) - self.argtuple = eval(args, d) + argname2index = dict(zip(argnames, [Index(n) for n in range(nb_args)])) + self.argtuple = eval(args, argname2index) # end of rather XXX'edly hackish parsing OOPARGTYPES = [] @@ -54,7 +54,6 @@ if ARGTYPE is not lltype.Void: self.residualargsources.append(arg_llsig_to_oopsig[i]) - RGenOp = hrtyper.RGenOp self.args_gv = [None] * nb_args fnptr = fnobj._as_ptr() self.gv_fnptr = RGenOp.constPrebuiltGlobal(fnptr) @@ -86,7 +85,7 @@ vmodule = __import__('pypy.jit.timeshifter.v%s' % (typename,), None, None, [method]) - self.typedesc = vmodule.TypeDesc(hrtyper, SELFTYPE) + self.typedesc = vmodule.TypeDesc(RGenOp, rtyper, SELFTYPE) self.ll_handler = getattr(vmodule, method) self.couldfold = getattr(self.ll_handler, 'couldfold', False) Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/vdict.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/vdict.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/vdict.py Mon Feb 11 14:15:48 2008 @@ -269,7 +269,7 @@ def oop_newdict(jitstate, oopspecdesc): return oopspecdesc.typedesc.factory() -def oop_dict_setitem(jitstate, oopspecdesc, selfbox, keybox, valuebox): +def oop_dict_setitem(jitstate, oopspecdesc, deepfrozen, selfbox, keybox, valuebox): content = selfbox.content if isinstance(content, AbstractVirtualDict) and keybox.is_constant(): content.setitem(keybox, valuebox) Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/vlist.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/vlist.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/vlist.py Mon Feb 11 14:15:48 2008 @@ -27,9 +27,7 @@ class ListTypeDesc(object): __metaclass__ = cachedtype - def __init__(self, hrtyper, LIST): - RGenOp = hrtyper.RGenOp - rtyper = hrtyper.rtyper + def __init__(self, RGenOp, rtyper, LIST): self.LIST = LIST self.LISTPTR = lltype.Ptr(LIST) self.ptrkind = RGenOp.kindToken(self.LISTPTR) @@ -269,13 +267,13 @@ content.reshape(jitstate, shapemask, memo) -def oop_newlist(jitstate, oopspecdesc, lengthbox, itembox=None): +def oop_newlist(jitstate, oopspecdesc, deepfrozen, lengthbox, itembox=None): if lengthbox.is_constant(): length = rvalue.ll_getvalue(lengthbox, lltype.Signed) return oopspecdesc.typedesc.factory(length, itembox) return oopspecdesc.residual_call(jitstate, [lengthbox, itembox]) -def oop_list_copy(jitstate, oopspecdesc, selfbox): +def oop_list_copy(jitstate, oopspecdesc, deepfrozen, selfbox): content = selfbox.content if isinstance(content, VirtualList): copybox = oopspecdesc.typedesc.factory(0, None) @@ -304,14 +302,14 @@ deepfrozen=deepfrozen) oop_list_nonzero.couldfold = True -def oop_list_append(jitstate, oopspecdesc, selfbox, itembox): +def oop_list_append(jitstate, oopspecdesc, deepfrozen, selfbox, itembox): content = selfbox.content if isinstance(content, VirtualList): content.item_boxes.append(itembox) else: oopspecdesc.residual_call(jitstate, [selfbox, itembox]) -def oop_list_insert(jitstate, oopspecdesc, selfbox, indexbox, itembox): +def oop_list_insert(jitstate, oopspecdesc, deepfrozen, selfbox, indexbox, itembox): content = selfbox.content if isinstance(content, VirtualList) and indexbox.is_constant(): index = rvalue.ll_getvalue(indexbox, lltype.Signed) @@ -321,7 +319,7 @@ else: oopspecdesc.residual_call(jitstate, [selfbox, indexbox, itembox]) -def oop_list_concat(jitstate, oopspecdesc, selfbox, otherbox): +def oop_list_concat(jitstate, oopspecdesc, deepfrozen, selfbox, otherbox): content = selfbox.content if isinstance(content, VirtualList): assert isinstance(otherbox, rvalue.PtrRedBox) @@ -335,7 +333,7 @@ return newbox return oopspecdesc.residual_call(jitstate, [selfbox, otherbox]) -def oop_list_pop(jitstate, oopspecdesc, selfbox, indexbox=None): +def oop_list_pop(jitstate, oopspecdesc, deepfrozen, selfbox, indexbox=None): content = selfbox.content if indexbox is None: if isinstance(content, VirtualList): @@ -355,7 +353,7 @@ return oopspecdesc.residual_exception(jitstate, IndexError) return oopspecdesc.residual_call(jitstate, [selfbox, indexbox]) -def oop_list_reverse(jitstate, oopspecdesc, selfbox): +def oop_list_reverse(jitstate, oopspecdesc, deepfrozen, selfbox): content = selfbox.content if isinstance(content, VirtualList): content.item_boxes.reverse() @@ -375,7 +373,7 @@ deepfrozen=deepfrozen) oop_list_getitem.couldfold = True -def oop_list_setitem(jitstate, oopspecdesc, selfbox, indexbox, itembox): +def oop_list_setitem(jitstate, oopspecdesc, deepfrozen, selfbox, indexbox, itembox): content = selfbox.content if isinstance(content, VirtualList) and indexbox.is_constant(): index = rvalue.ll_getvalue(indexbox, lltype.Signed) @@ -386,7 +384,7 @@ else: oopspecdesc.residual_call(jitstate, [selfbox, indexbox, itembox]) -def oop_list_delitem(jitstate, oopspecdesc, selfbox, indexbox): +def oop_list_delitem(jitstate, oopspecdesc, deepfrozen, selfbox, indexbox): content = selfbox.content if isinstance(content, VirtualList) and indexbox.is_constant(): index = rvalue.ll_getvalue(indexbox, lltype.Signed) From cfbolz at codespeak.net Mon Feb 11 14:21:18 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 11 Feb 2008 14:21:18 +0100 (CET) Subject: [pypy-svn] r51383 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter Message-ID: <20080211132118.F2B591683EE@codespeak.net> Author: cfbolz Date: Mon Feb 11 14:21:18 2008 New Revision: 51383 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vdict.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/vdict.py Log: make all vdict tests pass Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Mon Feb 11 14:21:18 2008 @@ -324,6 +324,13 @@ # XXX all this jitstate.greens business is a bit messy self.green_result(self.jitstate.greens[0]) + def opimpl_red_oopspec_call_0(self): + oopspecindex = self.load_2byte() + deepfrozen = self.load_bool() + oopspec = self.frame.bytecode.oopspecdescs[oopspecindex] + result = oopspec.ll_handler(self.jitstate, oopspec, deepfrozen) + self.red_result(result) + def opimpl_red_oopspec_call_1(self): oopspecindex = self.load_2byte() deepfrozen = self.load_bool() Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vdict.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vdict.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vdict.py Mon Feb 11 14:21:18 2008 @@ -5,7 +5,6 @@ P_OOPSPEC = HintAnnotatorPolicy(novirtualcontainer = True, oopspec = True) -py.test.skip("implement me") class TestVDict(AbstractInterpretationTest): Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/vdict.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/vdict.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/vdict.py Mon Feb 11 14:21:18 2008 @@ -95,9 +95,7 @@ class DictTypeDesc(object): __metaclass__ = cachedtype - def __init__(self, hrtyper, DICT): - RGenOp = hrtyper.RGenOp - rtyper = hrtyper.rtyper + def __init__(self, RGenOp, rtyper, DICT): bk = rtyper.annotator.bookkeeper self.DICT = DICT self.DICTPTR = lltype.Ptr(DICT) @@ -266,7 +264,7 @@ raise NotImplementedError -def oop_newdict(jitstate, oopspecdesc): +def oop_newdict(jitstate, oopspecdesc, deepfrozen): return oopspecdesc.typedesc.factory() def oop_dict_setitem(jitstate, oopspecdesc, deepfrozen, selfbox, keybox, valuebox): From antocuni at codespeak.net Mon Feb 11 14:25:20 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 11 Feb 2008 14:25:20 +0100 (CET) Subject: [pypy-svn] r51384 - in pypy/dist/pypy/jit/codegen/cli: . test Message-ID: <20080211132520.EBEEE168049@codespeak.net> Author: antocuni Date: Mon Feb 11 14:25:18 2008 New Revision: 51384 Modified: pypy/dist/pypy/jit/codegen/cli/operation.py pypy/dist/pypy/jit/codegen/cli/rgenop.py pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Log: delay the rendering of the return blocks until the end() of the code generation; this is needed to make the CLI verifier happy; two more tests pass Modified: pypy/dist/pypy/jit/codegen/cli/operation.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/operation.py (original) +++ pypy/dist/pypy/jit/codegen/cli/operation.py Mon Feb 11 14:25:18 2008 @@ -90,7 +90,7 @@ self.inputargs_gv[i].store(self.il) self.il.Emit(OpCodes.Br, self.label) -class BranchIf(Operation): +class Branch(Operation): def __init__(self, il, gv_cond, opcode, label): self.il = il @@ -102,7 +102,8 @@ return None def emit(self): - self.gv_cond.load(self.il) + if self.gv_cond is not None: + self.gv_cond.load(self.il) self.il.Emit(self.opcode, self.label) class Return(Operation): Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/rgenop.py Mon Feb 11 14:25:18 2008 @@ -219,8 +219,7 @@ self.isOpen = False self.operations = [] self.branches = [] - self.retlabel = self.il.DefineLabel() - self.gv_returnvar = None + self.returnblocks = [] @specialize.arg(1) def genop1(self, opname, gv_arg): @@ -269,12 +268,17 @@ def appendbranch(self, branch): self.branches.append(branch) + def appendreturn(self, retlabel, gv_returnvar): + self.returnblocks.append((retlabel, gv_returnvar)) + def start_writing(self): self.isOpen = True def finish_and_return(self, sigtoken, gv_returnvar): - self.il.Emit(OpCodes.Br, self.retlabel) - self.gv_returnvar = gv_returnvar + retlabel = self.il.DefineLabel() + op = ops.Branch(self.il, None, OpCodes.Br, retlabel) + self.emit(op) + self.appendreturn(retlabel, gv_returnvar) self.isOpen = False def finish_and_goto(self, outputargs_gv, target): @@ -289,10 +293,11 @@ for branch in self.branches: branch.replayops() - # render the return block for last, else the verifier could complain - self.il.MarkLabel(self.retlabel) - op = ops.Return(self.il, self.gv_returnvar) - self.emit(op) + # render the return blocks for last, else the verifier could complain + for retlabel, gv_returnvar in self.returnblocks: + self.il.MarkLabel(retlabel) + op = ops.Return(self.il, gv_returnvar) + self.emit(op) # build the delegate delegate_type = sigtoken2clitype(self.sigtoken) @@ -310,7 +315,7 @@ def _jump_if(self, gv_condition, opcode): label = self.il.DefineLabel() - op = ops.BranchIf(self.il, gv_condition, opcode, label) + op = ops.Branch(self.il, gv_condition, opcode, label) self.emit(op) branch = BranchBuilder(self, label) self.appendbranch(branch) @@ -341,17 +346,15 @@ # test_goto_compile to see why it fails return Builder.genop2(self, opname, gv_arg1, gv_arg2) - def finish_and_return(self, sigtoken, gv_returnvar): - op = ops.Return(self.parent.il, gv_returnvar) - self.emit(op) - self.isOpen = False - def emit(self, op): self.operations.append(op) def appendbranch(self, branch): self.parent.appendbranch(branch) + def appendreturn(self, retlabel, gv_returnvar): + self.parent.appendreturn(retlabel, gv_returnvar) + def replayops(self): assert not self.isOpen assert not self.parent.isOpen Modified: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Mon Feb 11 14:25:18 2008 @@ -21,6 +21,8 @@ 'test_longwinded_and', 'test_condition_result_cross_link_direct', 'test_multiple_cmps', + 'test_flipped_cmp_with_immediate', + 'test_tight_loop', ] for p in prefixes: From fijal at codespeak.net Mon Feb 11 15:46:51 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 11 Feb 2008 15:46:51 +0100 (CET) Subject: [pypy-svn] r51385 - in pypy/dist/pypy/objspace/std: . test Message-ID: <20080211144651.753151683F2@codespeak.net> Author: fijal Date: Mon Feb 11 15:46:49 2008 New Revision: 51385 Modified: pypy/dist/pypy/objspace/std/objspace.py pypy/dist/pypy/objspace/std/test/test_stdobjspace.py Log: sliceindices shall return unpacked tuple. the test and fix for the fallback version. Modified: pypy/dist/pypy/objspace/std/objspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/objspace.py (original) +++ pypy/dist/pypy/objspace/std/objspace.py Mon Feb 11 15:46:49 2008 @@ -601,9 +601,14 @@ def sliceindices(self, w_slice, w_length): if isinstance(w_slice, W_SliceObject): a, b, c = w_slice.indices3(self, self.int_w(w_length)) - return self.newtuple([a, b, c]) + return (a, b, c) w_indices = self.getattr(w_slice, self.wrap('indices')) - return self.call_function(w_indices, w_length) + w_tup = self.call_function(w_indices, w_length) + l_w = self.unpackiterable(w_tup) + if not len(l_w) == 3: + raise OperationError(self.w_ValueError, + self.wrap("Expected tuple of length 3")) + return self.int_w(l_w[0]), self.int_w(l_w[1]), self.int_w(l_w[2]) def is_(self, w_one, w_two): # XXX a bit of hacking to gain more speed Modified: pypy/dist/pypy/objspace/std/test/test_stdobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/std/test/test_stdobjspace.py (original) +++ pypy/dist/pypy/objspace/std/test/test_stdobjspace.py Mon Feb 11 15:46:49 2008 @@ -1,4 +1,5 @@ from pypy.interpreter.error import OperationError +from pypy.interpreter.gateway import app2interp class TestW_StdObjSpace: @@ -34,8 +35,14 @@ def test_sliceindices(self): space = self.space + w_obj = space.appexec([], """(): + class Stuff(object): + def indices(self, l): + return 1,2,3 + return Stuff() + """) w = space.wrap w_slice = space.newslice(w(1), w(2), w(1)) - assert space.unpacktuple(space.sliceindices(w_slice, w(3))) == [1,2,1] - + assert space.sliceindices(w_slice, w(3)) == (1,2,1) + assert space.sliceindices(w_obj, w(3)) == (1,2,3) From fijal at codespeak.net Mon Feb 11 16:12:40 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 11 Feb 2008 16:12:40 +0100 (CET) Subject: [pypy-svn] r51386 - pypy/dist/pypy/module/__builtin__ Message-ID: <20080211151240.318351683E0@codespeak.net> Author: fijal Date: Mon Feb 11 16:12:39 2008 New Revision: 51386 Added: pypy/dist/pypy/module/__builtin__/interp_buffer.py (contents, props changed) Log: Write down enough buffer interface to past test suite. In short this means that our testing suite sucks. Not turned on Added: pypy/dist/pypy/module/__builtin__/interp_buffer.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/__builtin__/interp_buffer.py Mon Feb 11 16:12:39 2008 @@ -0,0 +1,71 @@ + +""" Implementation of buffer protocol, interp level +""" + +from pypy.rlib.rbuffer import RBuffer +from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable,\ + Arguments +from pypy.interpreter.typedef import TypeDef, GetSetProperty,\ + interp_attrproperty +from pypy.interpreter.gateway import interp2app +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.objspace.std.sliceobject import W_SliceObject + +class W_Buffer(Wrappable): + def __init__(self, space, w_arg): + if space.is_true(space.isinstance(w_arg, space.w_unicode)): + arg = space.unicode_w(w_arg) + size = len(arg) * rffi.sizeof(lltype.UniChar) + self._buffer = RBuffer(size) + # XXX memory copy interface, redesign and share + UNICODE_ARRAY_P = rffi.CArrayPtr(lltype.UniChar) + ll_buffer = rffi.cast(UNICODE_ARRAY_P, self._buffer.ll_buffer) + for i in range(len(arg)): + ll_buffer[i] = arg[i] + else: + size = space.int_w(space.len(w_arg)) + self._buffer = RBuffer(size) + for i in range(size): + self._buffer.setitem(i, rffi.cast(lltype.Char, space.int_w( + space.getitem(w_arg, space.wrap(i))))) + self.size = size + + def getitem(self, space, w_item): + if space.is_true(space.isinstance(w_item, space.w_slice)): + start, stop, step = space.sliceindices(w_item, + space.wrap(self.size)) + # XXX a bit of code duplication from listobject + + if (step < 0 and stop >= start) or (step > 0 and start >= stop): + slicelength = 0 + elif step < 0: + slicelength = (stop - start + 1) / step + 1 + else: + slicelength = (stop - start - 1) / step + 1 + res = ['\x00'] * slicelength + for i in range(slicelength): + res[i] = self._buffer.getitem(start) + start += step + return space.wrap("".join(res)) + return space.wrap(self._buffer.getitem(space.int_w(w_item))) + getitem.unwrap_spec = ['self', ObjSpace, W_Root] + + def len(self, space): + return space.wrap(self._buffer.size) + len.unwrap_spec = ['self', ObjSpace] + + def delete(self): + # XXX when exactly??? + self._buffer.free() + +def descr_new_buffer(space, w_type, w_arg): + return space.wrap(W_Buffer(space, w_arg)) +descr_new_buffer.unwrap_spec = [ObjSpace, W_Root, W_Root] + +W_Buffer.typedef = TypeDef( + 'buffer', + __new__ = interp2app(descr_new_buffer), + __del__ = interp2app(W_Buffer.delete), + __len__ = interp2app(W_Buffer.len), + __getitem__ = interp2app(W_Buffer.getitem), +) From antocuni at codespeak.net Mon Feb 11 16:32:40 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 11 Feb 2008 16:32:40 +0100 (CET) Subject: [pypy-svn] r51387 - in pypy/dist/pypy/jit/codegen/cli: . test Message-ID: <20080211153240.543F11683ED@codespeak.net> Author: antocuni Date: Mon Feb 11 16:32:36 2008 New Revision: 51387 Modified: pypy/dist/pypy/jit/codegen/cli/dumpgenerator.py pypy/dist/pypy/jit/codegen/cli/operation.py pypy/dist/pypy/jit/codegen/cli/rgenop.py pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Log: labels must be marked just before the operations they refer to; one more test passes Modified: pypy/dist/pypy/jit/codegen/cli/dumpgenerator.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/dumpgenerator.py (original) +++ pypy/dist/pypy/jit/codegen/cli/dumpgenerator.py Mon Feb 11 16:32:36 2008 @@ -2,15 +2,17 @@ def __init__(self, il, filename='dynamicmethod.il'): self.il = il self.out = file(filename, 'w') - self.localcount = 0 + self.locals = {} self.labels = {} def _fmt(self, arg): - from System.Reflection import Emit + from System.Reflection import Emit, FieldInfo if isinstance(arg, Emit.LocalBuilder): return 'v%d' % arg.LocalIndex elif isinstance(arg, Emit.Label): return 'label%d' % self.labels[arg] + elif isinstance(arg, FieldInfo): + return '%s %s::%s' % (arg.FieldType.FullName, arg.DeclaringType.FullName, arg.Name) return repr(arg) def Emit(self, opcode, *args): @@ -18,8 +20,22 @@ self.out.write(' %s %s\n' % (opcode.Name, arglist)) return self.il.Emit(opcode, *args) + def EmitCall(self, opcode, meth, vartypes): + assert vartypes is None + rettype = meth.ReturnType.FullName + clsname = meth.DeclaringType.FullName + params = meth.GetParameters() + types = [p.ParameterType.FullName for p in params] + arglist = ', '.join(types) + desc = '%s %s::%s(%s)' % (rettype, clsname, meth.Name, arglist) + self.out.write(' %s %s\n' % (opcode.Name, desc)) + return self.il.EmitCall(opcode, meth, vartypes) + def DeclareLocal(self, t): - return self.il.DeclareLocal(t) + v = self.il.DeclareLocal(t) + vname = self._fmt(v) + self.locals[vname] = t + return v def DefineLabel(self): lbl = self.il.DefineLabel() @@ -31,3 +47,8 @@ self.out.write('\n%s:\n' % self._fmt(lbl)) return self.il.MarkLabel(lbl) + def __del__(self): + decls = ['%s %s' % (t.FullName, name) for name, t in self.locals.iteritems()] + self.out.write('.locals init (') + self.out.write(', '.join(decls)) + self.out.write(')\n') Modified: pypy/dist/pypy/jit/codegen/cli/operation.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/operation.py (original) +++ pypy/dist/pypy/jit/codegen/cli/operation.py Mon Feb 11 16:32:36 2008 @@ -72,6 +72,17 @@ self.gv_x.load(self.il) self.gv_res().store(self.il) +class MarkLabel(Operation): + + def __init__(self, il, label): + self.il = il + self.label = label + + def restype(self): + return None + + def emit(self): + self.il.MarkLabel(self.label) class FollowLink(Operation): Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/rgenop.py Mon Feb 11 16:32:36 2008 @@ -310,7 +310,7 @@ op.emit() args_gv[i] = op.gv_res() label = self.il.DefineLabel() - self.il.MarkLabel(label) + self.emit(ops.MarkLabel(self.il, label)) return Label(label, args_gv) def _jump_if(self, gv_condition, opcode): Modified: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Mon Feb 11 16:32:36 2008 @@ -23,6 +23,7 @@ 'test_multiple_cmps', 'test_flipped_cmp_with_immediate', 'test_tight_loop', + 'test_jump_to_block_with_many_vars', ] for p in prefixes: From antocuni at codespeak.net Mon Feb 11 16:37:19 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 11 Feb 2008 16:37:19 +0100 (CET) Subject: [pypy-svn] r51388 - in pypy/dist/pypy/jit/codegen/cli: . test Message-ID: <20080211153719.612991683F2@codespeak.net> Author: antocuni Date: Mon Feb 11 16:37:17 2008 New Revision: 51388 Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Log: implement op_same_as (not very smart for now, it just renames everything) Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/rgenop.py Mon Feb 11 16:37:17 2008 @@ -261,6 +261,11 @@ op = ops.Call(self.il, sigtoken, gv_fnptr, args_gv) self.emit(op) return op.gv_res() + + def genop_same_as(self, kindtoken, gv_x): + op = ops.SameAs(self.il, gv_x) + self.emit(op) + return op.gv_res() def emit(self, op): op.emit() Modified: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Mon Feb 11 16:37:17 2008 @@ -24,6 +24,7 @@ 'test_flipped_cmp_with_immediate', 'test_tight_loop', 'test_jump_to_block_with_many_vars', + 'test_same_as', ] for p in prefixes: From antocuni at codespeak.net Mon Feb 11 16:42:53 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 11 Feb 2008 16:42:53 +0100 (CET) Subject: [pypy-svn] r51389 - in pypy/dist/pypy/jit/codegen: cli/test test Message-ID: <20080211154253.787F41683ED@codespeak.net> Author: antocuni Date: Mon Feb 11 16:42:52 2008 New Revision: 51389 Modified: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py pypy/dist/pypy/jit/codegen/test/rgenop_tests.py Log: this test passes out of the box, but we need to rename the function from "f" to "fn" else gencli thinks it's a char and can't find the appropriate overload for CreateDynamicMethod; it's a gencli bug, but it's much easier to insert the workaround here. Modified: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Mon Feb 11 16:42:52 2008 @@ -25,6 +25,7 @@ 'test_tight_loop', 'test_jump_to_block_with_many_vars', 'test_same_as', + 'test_pause_and_resume', ] for p in prefixes: Modified: pypy/dist/pypy/jit/codegen/test/rgenop_tests.py ============================================================================== --- pypy/dist/pypy/jit/codegen/test/rgenop_tests.py (original) +++ pypy/dist/pypy/jit/codegen/test/rgenop_tests.py Mon Feb 11 16:42:52 2008 @@ -549,7 +549,7 @@ # return w signed_kind = rgenop.kindToken(lltype.Signed) sigtoken = rgenop.sigToken(T.FUNC) - builder, gv_callable, [gv_x] = rgenop.newgraph(sigtoken, "f") + builder, gv_callable, [gv_x] = rgenop.newgraph(sigtoken, "fn") builder.start_writing() gv_one = rgenop.genconst(1) @@ -1322,7 +1322,6 @@ gvs.append(builder2.genop_call(sigtoken, gv_verysmall_callable, [gv_x2])) - while len(gvs) > 1: gvs.append(builder2.genop2("int_add", gvs.pop(), gvs.pop())) From antocuni at codespeak.net Mon Feb 11 17:05:39 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 11 Feb 2008 17:05:39 +0100 (CET) Subject: [pypy-svn] r51390 - in pypy/dist/pypy: jit/codegen/cli jit/codegen/cli/test translator/cli/src Message-ID: <20080211160539.AB5AB1683FC@codespeak.net> Author: antocuni Date: Mon Feb 11 17:05:38 2008 New Revision: 51390 Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py pypy/dist/pypy/translator/cli/src/pypylib.cs Log: more passing tests Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/rgenop.py Mon Feb 11 17:05:38 2008 @@ -22,9 +22,11 @@ def sigtoken2clitype(tok): if tok == ([''], ''): - return typeof(CLR.pypy.runtime.DelegateType_int__int) + return typeof(CLR.pypy.runtime.DelegateType_int__int_1) elif tok == (['', ''], ''): - return typeof(CLR.pypy.runtime.DelegateType_int__int_int) + return typeof(CLR.pypy.runtime.DelegateType_int__int_2) + elif tok == ([''] * 3, ''): + return typeof(CLR.pypy.runtime.DelegateType_int__int_3) elif tok == ([''] * 100, ''): return typeof(CLR.pypy.runtime.DelegateType_int__int_100) else: @@ -89,8 +91,12 @@ @specialize.arg(1) def revealconst(self, T): - assert T is ootype.Signed - return self.value + if T is ootype.Signed: + return self.value + elif T is ootype.Bool: + return bool(self.value) + else: + assert False def getCliType(self): return typeof(System.Int32) @@ -120,18 +126,22 @@ il.Emit(OpCodes.Ldsfld, field) -SM_INT__INT = ootype.StaticMethod([ootype.Signed], ootype.Signed) -SM_INT__INT_INT = ootype.StaticMethod([ootype.Signed, ootype.Signed], ootype.Signed) +SM_INT__INT_1 = ootype.StaticMethod([ootype.Signed], ootype.Signed) +SM_INT__INT_2 = ootype.StaticMethod([ootype.Signed] * 2, ootype.Signed) +SM_INT__INT_3 = ootype.StaticMethod([ootype.Signed] * 3, ootype.Signed) SM_INT__INT_100 = ootype.StaticMethod([ootype.Signed] * 100, ootype.Signed) class FunctionConst(BaseConst): @specialize.arg(1) def revealconst(self, T): - if T == SM_INT__INT: - DelegateType = CLR.pypy.runtime.DelegateType_int__int + if T == SM_INT__INT_1: + DelegateType = CLR.pypy.runtime.DelegateType_int__int_1 return clidowncast(DelegateType, self.getobj()) - elif T == SM_INT__INT_INT: - DelegateType = CLR.pypy.runtime.DelegateType_int__int_int + elif T == SM_INT__INT_2: + DelegateType = CLR.pypy.runtime.DelegateType_int__int_2 + return clidowncast(DelegateType, self.getobj()) + elif T == SM_INT__INT_3: + DelegateType = CLR.pypy.runtime.DelegateType_int__int_3 return clidowncast(DelegateType, self.getobj()) elif T == SM_INT__INT_100: DelegateType = CLR.pypy.runtime.DelegateType_int__int_100 @@ -171,6 +181,8 @@ T = ootype.typeOf(llvalue) if T is ootype.Signed: return IntConst(llvalue) + elif T is ootype.Bool: + return IntConst(int(llvalue)) elif isinstance(T, ootype.OOType): const = self.newconst(ObjectConst) const.setobj(llvalue) @@ -360,6 +372,9 @@ def appendreturn(self, retlabel, gv_returnvar): self.parent.appendreturn(retlabel, gv_returnvar) + def end(self): + self.parent.end() + def replayops(self): assert not self.isOpen assert not self.parent.isOpen Modified: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Mon Feb 11 17:05:38 2008 @@ -15,7 +15,8 @@ 'test_branching', 'test_goto', 'test_if', - # 'test_switch', # no promotion/flexswitch for now please :-) + # 'test_switch', # no promotion/flexswitch for now please :-) + # 'test_defaultonly_switch', # the same 'test_fact', 'test_calling_pause', 'test_longwinded_and', @@ -26,6 +27,21 @@ 'test_jump_to_block_with_many_vars', 'test_same_as', 'test_pause_and_resume', + 'test_like_residual_red_call_with_exc', + 'test_call_functions_with_different_signatures', + 'test_bool_not_direct', + # 'test_read_frame_var', # not for now + # 'test_write_frame_place', + # 'test_write_lots_of_frame_places_direct', + # 'test_read_frame_place_direct', + # 'test_read_frame_place_compile' + # 'test_frame_vars_like_the_frontend_direct', + 'test_unaliasing_variables_direct', + # 'test_from_random_direct', # mono crashes + 'test_from_random_2_direct', + # 'test_from_random_3_direct', # we need yet another delegate type + 'test_from_random_4_direct', + # 'test_from_random_5_direct', # we need yet another delegate type ] for p in prefixes: Modified: pypy/dist/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/dist/pypy/translator/cli/src/pypylib.cs Mon Feb 11 17:05:38 2008 @@ -53,8 +53,9 @@ namespace pypy.runtime { - public delegate int DelegateType_int__int(int a); - public delegate int DelegateType_int__int_int(int a, int b); + public delegate int DelegateType_int__int_1(int a); + public delegate int DelegateType_int__int_2(int a, int b); + public delegate int DelegateType_int__int_3(int a, int b, int c); public delegate int DelegateType_int__int_100(int a0, int a1, int a2, int a3, int a4, int a5, int a6, int a7, int a8, int a9, int a10, int a11, int a12, int a13, int a14, int a15, int a16, int a17, int a18, int a19, int a20, int a21, int a22, int a23, int a24, int a25, int a26, int a27, int a28, int a29, int a30, int a31, int a32, int a33, int a34, int a35, int a36, int a37, int a38, int a39, int a40, int a41, int a42, int a43, int a44, int a45, int a46, int a47, int a48, int a49, int a50, int a51, int a52, int a53, int a54, int a55, int a56, int a57, int a58, int a59, int a60, int a61, int a62, int a63, int a64, int a65, int a66, int a67, int a68, int a69, int a70, int a71, int a72, int a73, int a74, int a75, int a76, int a77, int a78, int a79, int a80, int a81, int a82, int a83, int a84, int a85, int a86, int a87, int a88, int a89, int a90, int a91, int a92, int a93, int a94, int a95, int a96, int a97, int a98, int a99); public class Constants From cfbolz at codespeak.net Mon Feb 11 20:48:35 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 11 Feb 2008 20:48:35 +0100 (CET) Subject: [pypy-svn] r51395 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080211194835.0007D16804D@codespeak.net> Author: cfbolz Date: Mon Feb 11 20:48:33 2008 New Revision: 51395 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vlist.py Log: three more virtual list tests pass Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 11 20:48:33 2008 @@ -1,4 +1,5 @@ from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.objectmodel import we_are_translated from pypy.objspace.flow import model as flowmodel from pypy.rpython.lltypesystem import lltype from pypy.jit.hintannotator.model import originalconcretetype @@ -10,8 +11,6 @@ from pypy.translator.backendopt.removenoops import remove_same_as - - class BytecodeWriter(object): def __init__(self, t, hannotator, RGenOp): self.translator = t @@ -295,9 +294,9 @@ def redvar_position(self, arg): return self.redvar_positions[arg] - def register_greenvar(self, arg, where=-1): + def register_greenvar(self, arg, where=None): assert isinstance(arg, flowmodel.Variable) - if where == -1: + if where is None: where = self.free_green[self.current_block] self.free_green[self.current_block] += 1 self.greenvar_positions[arg] = where @@ -382,27 +381,41 @@ self.graph_positions[graph] = index return index - def nonrainbow_position(self, fnptr): + def nonrainbow_position(self, fnptr, *voidargs): fn = fnptr._obj - if fn in self.nonrainbow_positions: - return self.nonrainbow_positions[fn] + key = fn, voidargs + if key in self.nonrainbow_positions: + return self.nonrainbow_positions[key] FUNCTYPE = lltype.typeOf(fn) - argiter = unrolling_iterable(enumerate(FUNCTYPE.ARGS)) + argiter = unrolling_iterable(FUNCTYPE.ARGS) numargs = len(FUNCTYPE.ARGS) - def call_normal_function(interpreter, greenargs, redargs): - assert len(redargs) == 0 - assert len(greenargs) == numargs + def call_normal_function(interpreter, greenargs): + assert len(greenargs) + len(voidargs) == numargs args = () - for i, ARG in argiter: - genconst = greenargs[i] - arg = genconst.revealconst(ARG) - args += (arg, ) + j = 0 + k = 0 + for ARG in argiter: + if ARG == lltype.Void: + # XXX terrible hack + arg = voidargs[k] + if not we_are_translated(): + arg._TYPE = lltype.Void + args += (arg, ) + k += 1 + else: + genconst = greenargs[j] + arg = genconst.revealconst(ARG) + args += (arg, ) + j += 1 rgenop = interpreter.jitstate.curbuilder.rgenop - result = rgenop.genconst(fnptr(*args)) + try: + result = rgenop.genconst(fnptr(*args)) + except Exception, e: + XXX # need to create a default result and set exception interpreter.green_result(result) result = len(self.nonrainbow_functions) self.nonrainbow_functions.append(call_normal_function) - self.nonrainbow_positions[fn] = result + self.nonrainbow_positions[key] = result return result def interiordesc(self, op, PTRTYPE, nb_offsets): @@ -533,6 +546,20 @@ self.emit(*args) self.register_redvar(op.result) return + elif kind == "green": + voidargs = [const.value for const in op.args[1:] + if const.concretetype == lltype.Void] + pos = self.nonrainbow_position(op.args[0].value, *voidargs) + emitted_args = [] + for v in op.args[1:]: + if v.concretetype != lltype.Void: + emitted_args.append(self.serialize_oparg("green", v)) + self.emit("green_direct_call") + self.emit(len(emitted_args)) + self.emit(*emitted_args) + self.emit(pos) + self.register_greenvar(op.result) + return targets = dict(self.graphs_from(op)) assert len(targets) == 1 targetgraph, = targets.values() @@ -546,14 +573,6 @@ if kind == "red": self.register_redvar(op.result) self.emit("red_after_direct_call") - elif kind == "green": - pos = self.nonrainbow_position(op.args[0].value) - args = targetgraph.getargs() - emitted_args = self.args_of_call(op.args[1:], args) - self.emit("green_direct_call") - self.emit(*emitted_args) - self.emit(pos) - self.register_greenvar(op.result) elif kind == "yellow": graphindex = self.graph_position(targetgraph) args = targetgraph.getargs() Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Mon Feb 11 20:48:33 2008 @@ -299,10 +299,9 @@ def opimpl_green_direct_call(self): greenargs = self.get_green_varargs() - redargs = self.get_red_varargs() index = self.load_2byte() function = self.frame.bytecode.nonrainbow_functions[index] - function(self, greenargs, redargs) + function(self, greenargs) def opimpl_yellow_direct_call(self): greenargs = self.get_green_varargs() Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 11 20:48:33 2008 @@ -95,6 +95,7 @@ self.graph = graph2 writer = BytecodeWriter(t, hannotator, self.RGenOp) jitcode = writer.make_bytecode(graph2) + rtyper.specialize_more_blocks() argcolors = [] # make residual functype Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Mon Feb 11 20:48:33 2008 @@ -246,7 +246,7 @@ writer, jitcode = self.serialize(ll_function, [int]) assert jitcode.code == assemble(writer.interpreter, - "green_direct_call", 1, 0, 0, 0, + "green_direct_call", 1, 0, 0, "make_redbox", 1, 0, "make_new_redvars", 1, 0, "make_new_greenvars", 0, Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vlist.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vlist.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vlist.py Mon Feb 11 20:48:33 2008 @@ -65,7 +65,6 @@ self.check_insns({'int_is_true': 1}) def test_force(self): - py.test.skip("implement me") def ll_function(n): lst = [] lst.append(n) @@ -134,7 +133,6 @@ self.check_insns({}) def test_frozen_list(self): - py.test.skip("implement me") lst = [5, 7, 9] def ll_function(x): mylist = hint(lst, deepfreeze=True) From simonb at codespeak.net Mon Feb 11 23:42:17 2008 From: simonb at codespeak.net (simonb at codespeak.net) Date: Mon, 11 Feb 2008 23:42:17 +0100 (CET) Subject: [pypy-svn] r51400 - pypy/dist/pypy/translator/tool Message-ID: <20080211224217.BE2491683EC@codespeak.net> Author: simonb Date: Mon Feb 11 23:42:15 2008 New Revision: 51400 Modified: pypy/dist/pypy/translator/tool/cbuild.py Log: distutils expects a list Modified: pypy/dist/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/dist/pypy/translator/tool/cbuild.py (original) +++ pypy/dist/pypy/translator/tool/cbuild.py Mon Feb 11 23:42:15 2008 @@ -273,7 +273,7 @@ include_dirs=include_dirs, library_dirs=library_dirs, extra_compile_args=extra_compile_args, - libraries=libraries,) + libraries=list(libraries),) ], 'script_name': 'setup.py', 'script_args': ['-q', 'build_ext', '--inplace', '--force'], From simonb at codespeak.net Mon Feb 11 23:46:52 2008 From: simonb at codespeak.net (simonb at codespeak.net) Date: Mon, 11 Feb 2008 23:46:52 +0100 (CET) Subject: [pypy-svn] r51401 - pypy/dist/pypy/rpython/tool Message-ID: <20080211224652.CBCBD1683EC@codespeak.net> Author: simonb Date: Mon Feb 11 23:46:52 2008 New Revision: 51401 Modified: pypy/dist/pypy/rpython/tool/genrffi.py Log: update to use new rffi interface Modified: pypy/dist/pypy/rpython/tool/genrffi.py ============================================================================== --- pypy/dist/pypy/rpython/tool/genrffi.py (original) +++ pypy/dist/pypy/rpython/tool/genrffi.py Mon Feb 11 23:46:52 2008 @@ -4,6 +4,7 @@ from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem import lltype from pypy.rlib.unroll import unrolling_iterable +from pypy.translator.tool.cbuild import ExternalCompilationInfo import py @@ -44,10 +45,17 @@ self.ns = {} else: self.ns = ns - self.includes = includes - self.libraries = libraries self.include_dirs = include_dirs + CConfig = type('CConfig', (object,), {}) + CConfig._compilation_info_ = ExternalCompilationInfo( + includes = list(includes), + include_dirs = list(include_dirs), + libraries = list(libraries), + ) + + self.CConfig = CConfig + def proc_struct(self, tp): name = tp.__name__ struct = self.ns.get(name) @@ -88,8 +96,7 @@ ll_item = rffi.llexternal( name, arg_tps, self.proc_tp(func.restype), - includes=self.includes, libraries=self.libraries, - include_dirs=self.include_dirs) + compilation_info=self.CConfig._compilation_info_) self.ns[name] = ll_item return ll_item From fijal at codespeak.net Tue Feb 12 10:14:09 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 12 Feb 2008 10:14:09 +0100 (CET) Subject: [pypy-svn] r51403 - pypy/dist/pypy/objspace/std Message-ID: <20080212091409.F15BC168404@codespeak.net> Author: fijal Date: Tue Feb 12 10:14:07 2008 New Revision: 51403 Modified: pypy/dist/pypy/objspace/std/stringobject.py Log: Use a proper exception signature. The thing is that such errors are extremely hard to debug due to strange errors in normalize_exception. Modified: pypy/dist/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/dist/pypy/objspace/std/stringobject.py (original) +++ pypy/dist/pypy/objspace/std/stringobject.py Tue Feb 12 10:14:07 2008 @@ -44,8 +44,12 @@ if ord(s[i]) > 127: raise OperationError( space.w_UnicodeDecodeError, - space.wrap(("'ascii' codec can't decode byte %s in position %s:" - " ordinal not in range(128)") % (hex(ord(s[i])), i))) + space.newtuple([ + space.wrap('ascii'), + space.wrap(s), + space.wrap(i), + space.wrap(i+1), + space.wrap("ordinal not in range(128)")])) def unicode_w__String(space, w_self): # XXX should this use the default encoding? From fijal at codespeak.net Tue Feb 12 11:52:48 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 12 Feb 2008 11:52:48 +0100 (CET) Subject: [pypy-svn] r51404 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080212105248.9F9F31683F3@codespeak.net> Author: fijal Date: Tue Feb 12 11:52:46 2008 New Revision: 51404 Modified: pypy/dist/pypy/lib/app_test/ctypes/test_unicode.py Log: These tests are passing nowadays Modified: pypy/dist/pypy/lib/app_test/ctypes/test_unicode.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_unicode.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_unicode.py Tue Feb 12 11:52:46 2008 @@ -1,7 +1,6 @@ # coding: latin-1 import ctypes import py -py.test.skip("Unsupported") try: ctypes.c_wchar @@ -29,7 +28,7 @@ assert wcslen(u"ab\u2070") == 3 # string args are converted assert wcslen("abc") == 3 - raises(ctypes.ArgumentError, wcslen, "ab?") + py.test.raises(ctypes.ArgumentError, wcslen, "ab?") def test_ascii_replace(self): ctypes.set_conversion_mode("ascii", "replace") @@ -68,12 +67,12 @@ assert buf[:] == u"ab\0\0\0\0" class TestString(TestUnicode): - def setup_method(self): + def setup_method(self, method): self.prev_conv_mode = ctypes.set_conversion_mode("ascii", "strict") func.argtypes = [ctypes.c_char_p] func.restype = ctypes.c_char_p - def teardown_method(self): + def teardown_method(self, method): ctypes.set_conversion_mode(*self.prev_conv_mode) func.argtypes = None func.restype = ctypes.c_int From fijal at codespeak.net Tue Feb 12 11:53:03 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 12 Feb 2008 11:53:03 +0100 (CET) Subject: [pypy-svn] r51405 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080212105303.E74011683F5@codespeak.net> Author: fijal Date: Tue Feb 12 11:53:02 2008 New Revision: 51405 Modified: pypy/dist/pypy/lib/_ctypes/builtin.py pypy/dist/pypy/lib/_ctypes/function.py pypy/dist/pypy/lib/_ctypes/primitive.py Log: unicode support. Modified: pypy/dist/pypy/lib/_ctypes/builtin.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/builtin.py (original) +++ pypy/dist/pypy/lib/_ctypes/builtin.py Tue Feb 12 11:53:02 2008 @@ -1,6 +1,10 @@ import _rawffi, sys +class ConvMode: + encoding = 'ascii' + errors = 'strict' + _memmove_addr = ('memmove', 'libc.so.6') _memset_addr = ('memset', 'libc.so.6') @@ -10,8 +14,11 @@ obj = ctypes.c_char_p._CData_input(addr)[0] return _rawffi.charp2rawstring(obj, lgt) -def set_conversion_mode(one, two): - pass +def set_conversion_mode(encoding, errors): + old_cm = ConvMode.encoding, ConvMode.errors + ConvMode.errors = errors + ConvMode.encoding = encoding + return old_cm def _wstring_at_addr(addr, lgt): import ctypes Modified: pypy/dist/pypy/lib/_ctypes/function.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/function.py (original) +++ pypy/dist/pypy/lib/_ctypes/function.py Tue Feb 12 11:53:02 2008 @@ -112,5 +112,5 @@ try: return [argtype._CData_input(arg) for argtype, arg in zip(argtypes, args)] - except TypeError, e: - raise ArgumentError(e.args[0]) + except (UnicodeError, TypeError), e: + raise ArgumentError(str(e)) Modified: pypy/dist/pypy/lib/_ctypes/primitive.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/primitive.py (original) +++ pypy/dist/pypy/lib/_ctypes/primitive.py Tue Feb 12 11:53:02 2008 @@ -3,6 +3,7 @@ SIMPLE_TYPE_CHARS = "cbBhHiIlLdfuzZqQPXOv" from _ctypes.basics import _CData, _CDataMeta, cdata_from_address +from _ctypes.builtin import ConvMode class NULL(object): pass @@ -68,7 +69,10 @@ return _rawffi.charp2string(addr) def _setvalue(self, value): - if isinstance(value, str): + if isinstance(value, basestring): + if isinstance(value, unicode): + value = value.encode(ConvMode.encoding, + ConvMode.errors) array = _rawffi.Array('c')(len(value)+1, value) value = array.buffer # XXX free 'array' later @@ -87,7 +91,10 @@ return _wstring_at_addr(addr, -1) def _setvalue(self, value): - if isinstance(value, str): + if isinstance(value, basestring): + if isinstance(value, str): + value = value.decode(ConvMode.encoding, + ConvMode.errors) array = _rawffi.Array('u')(len(value)+1, value) value = array.buffer # XXX free 'array' later @@ -142,6 +149,27 @@ return SimpleType.from_param(self, value) result.from_param = classmethod(from_param) + elif tp == 'u': + def _setvalue(self, val): + if isinstance(val, str): + val = val.decode(ConvMode.encoding, ConvMode.errors) + # possible if we use 'ignore' + if val: + self._buffer[0] = val + def _getvalue(self): + return self._buffer[0] + result.value = property(_getvalue, _setvalue) + + elif tp == 'c': + def _setvalue(self, val): + if isinstance(val, unicode): + val = val.encode(ConvMode.encoding, ConvMode.errors) + if val: + self._buffer[0] = val + def _getvalue(self): + return self._buffer[0] + result.value = property(_getvalue, _setvalue) + return result from_address = cdata_from_address From fijal at codespeak.net Tue Feb 12 11:56:28 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 12 Feb 2008 11:56:28 +0100 (CET) Subject: [pypy-svn] r51406 - pypy/dist/pypy/doc/discussion Message-ID: <20080212105628.8ADEE16840B@codespeak.net> Author: fijal Date: Tue Feb 12 11:56:27 2008 New Revision: 51406 Modified: pypy/dist/pypy/doc/discussion/ctypes_todo.txt Log: Tihs is done by now Modified: pypy/dist/pypy/doc/discussion/ctypes_todo.txt ============================================================================== --- pypy/dist/pypy/doc/discussion/ctypes_todo.txt (original) +++ pypy/dist/pypy/doc/discussion/ctypes_todo.txt Tue Feb 12 11:56:27 2008 @@ -19,12 +19,8 @@ - not all combinations of arrays/structures inlined in other arrays/structures work. - - unicode is not implemented (all the conversion functions and friends) - - bitfields are not implemented - - * keepalive is not there (everything is kept alive forever basically) * as all stuff is applevel, we cannot have it really fast right now. From fijal at codespeak.net Tue Feb 12 14:52:18 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 12 Feb 2008 14:52:18 +0100 (CET) Subject: [pypy-svn] r51410 - in pypy/dist/pypy/module/_rawffi: . test Message-ID: <20080212135218.03785168054@codespeak.net> Author: fijal Date: Tue Feb 12 14:52:15 2008 New Revision: 51410 Added: pypy/dist/pypy/module/_rawffi/test/test_tracker.py (contents, props changed) pypy/dist/pypy/module/_rawffi/tracker.py (contents, props changed) Modified: pypy/dist/pypy/module/_rawffi/__init__.py pypy/dist/pypy/module/_rawffi/interp_rawffi.py Log: Some support for object allocation tracing. Modified: pypy/dist/pypy/module/_rawffi/__init__.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/__init__.py (original) +++ pypy/dist/pypy/module/_rawffi/__init__.py Tue Feb 12 14:52:15 2008 @@ -5,6 +5,7 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.module._rawffi.interp_rawffi import W_CDLL from pypy.rpython.lltypesystem import lltype, rffi +from pypy.module._rawffi.tracker import Tracker class Module(MixedModule): applevelname = '_rawffi' @@ -21,6 +22,7 @@ 'charp2string' : 'interp_rawffi.charp2string', 'charp2rawstring' : 'interp_rawffi.charp2rawstring', 'CallbackPtr' : 'callback.W_CallbackPtr', + '_num_of_allocated_objects' : 'tracker.num_of_allocated_objects', } appleveldefs = { Modified: pypy/dist/pypy/module/_rawffi/interp_rawffi.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/interp_rawffi.py (original) +++ pypy/dist/pypy/module/_rawffi/interp_rawffi.py Tue Feb 12 14:52:15 2008 @@ -11,6 +11,7 @@ from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.rarithmetic import intmask, r_uint, r_singlefloat +from pypy.module._rawffi.tracker import tracker def _signed_type_for(TYPE): sz = rffi.sizeof(TYPE) @@ -195,6 +196,9 @@ else: self.ll_buffer = lltype.malloc(rffi.VOIDP.TO, size, flavor='raw', zero=True) + if tracker.DO_TRACING: + ll_buf = rffi.cast(rffi.UINT, self.ll_buffer) + tracker.trace_allocation(ll_buf, self) def getbuffer(space, self): return space.wrap(rffi.cast(lltype.Unsigned, self.ll_buffer)) @@ -210,6 +214,9 @@ def free(self, space): if not self.ll_buffer: raise segfault_exception(space, "freeing NULL pointer") + if tracker.DO_TRACING: + ll_buf = rffi.cast(rffi.UINT, self.ll_buffer) + tracker.trace_free(ll_buf) lltype.free(self.ll_buffer, flavor='raw') self.ll_buffer = lltype.nullptr(rffi.VOIDP.TO) free.unwrap_spec = ['self', ObjSpace] Added: pypy/dist/pypy/module/_rawffi/test/test_tracker.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/_rawffi/test/test_tracker.py Tue Feb 12 14:52:15 2008 @@ -0,0 +1,29 @@ + +from pypy.conftest import gettestobjspace +from pypy.module._rawffi.tracker import Tracker + +class AppTestTracker: + def setup_class(cls): + Tracker.DO_TRACING = True + space = gettestobjspace(usemodules=('_rawffi','struct')) + cls.space = space + + def test_array(self): + import _rawffi + assert _rawffi._num_of_allocated_objects() == 0 + a = _rawffi.Array('c')(3) + assert _rawffi._num_of_allocated_objects() == 1 + a.free() + assert _rawffi._num_of_allocated_objects() == 0 + + def test_structure(self): + import _rawffi + assert _rawffi._num_of_allocated_objects() == 0 + s = _rawffi.Structure([('a', 'i'), ('b', 'i')])() + assert _rawffi._num_of_allocated_objects() == 1 + s.free() + assert _rawffi._num_of_allocated_objects() == 0 + + def teardown_class(cls): + Tracker.DO_TRACING = False + Added: pypy/dist/pypy/module/_rawffi/tracker.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/_rawffi/tracker.py Tue Feb 12 14:52:15 2008 @@ -0,0 +1,29 @@ + +""" The file that keeps track about freed/kept-alive objects allocated +by _rawffi. Used for debugging ctypes +""" +from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable, \ + Arguments + +class Tracker(object): + DO_TRACING = False + + def __init__(self): + self.alloced = {} + + def trace_allocation(self, address, obj): + self.alloced[address] = obj + + def trace_free(self, address): + del self.alloced[address] + +# single, global, static object to keep all tracker info +tracker = Tracker() + +def num_of_allocated_objects(space): + return space.wrap(len(tracker.alloced)) +num_of_allocated_objects.unwrap_spec = [ObjSpace] + +def print_alloced_objects(space): + xxx + # eventually inspect and print what's left from applevel From arigo at codespeak.net Tue Feb 12 14:57:07 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 12 Feb 2008 14:57:07 +0100 (CET) Subject: [pypy-svn] r51411 - in pypy/dist/pypy/translator/c: . src Message-ID: <20080212135707.361AE1683FA@codespeak.net> Author: arigo Date: Tue Feb 12 14:57:06 2008 New Revision: 51411 Added: pypy/dist/pypy/translator/c/src/linuxmemchk.c - copied, changed from r51353, psyco/dist/c/linuxmemchk.c Modified: pypy/dist/pypy/translator/c/genc.py pypy/dist/pypy/translator/c/src/standalone.h Log: Copy linuxmemchk.c from Psyco. To use it, run "make linuxmemchk" with the Makefile produced by genc. Modified: pypy/dist/pypy/translator/c/genc.py ============================================================================== --- pypy/dist/pypy/translator/c/genc.py (original) +++ pypy/dist/pypy/translator/c/genc.py Tue Feb 12 14:57:06 2008 @@ -848,8 +848,14 @@ \t$(MAKE) CFLAGS="-g -DRPY_ASSERT -DDO_LOG_EXC" debug_mem: +\t$(MAKE) CFLAGS="-g -DRPY_ASSERT -DTRIVIAL_MALLOC_DEBUG" + +no_obmalloc: \t$(MAKE) CFLAGS="-g -DRPY_ASSERT -DNO_OBMALLOC" +linuxmemchk: +\t$(MAKE) CFLAGS="-g -DRPY_ASSERT -DLINUXMEMCHK" + llsafer: \t$(MAKE) CFLAGS="-O2 -DRPY_LL_ASSERT" Modified: pypy/dist/pypy/translator/c/src/standalone.h ============================================================================== --- pypy/dist/pypy/translator/c/src/standalone.h (original) +++ pypy/dist/pypy/translator/c/src/standalone.h Tue Feb 12 14:57:06 2008 @@ -10,15 +10,26 @@ #define NO_OBMALLOC #endif #endif -#ifdef NO_OBMALLOC -void *PyObject_Malloc(size_t n) { return malloc(n); } -void *PyObject_Realloc(void *p, size_t n) { return realloc(p, n); } -void PyObject_Free(void *p) { if (p) { *((int*)p) = 0xDDDDDDDD; } free(p); } + +#if defined(TRIVIAL_MALLOC_DEBUG) + void *PyObject_Malloc(size_t n) { return malloc(n); } + void *PyObject_Realloc(void *p, size_t n) { return realloc(p, n); } + void PyObject_Free(void *p) { if (p) { *((int*)p) = 0xDDDDDDDD; } free(p); } + +#elif defined(LINUXMEMCHK) +# include "linuxmemchk.c" + +#elif defined(NO_OBMALLOC) + void *PyObject_Malloc(size_t n) { return malloc(n); } + void *PyObject_Realloc(void *p, size_t n) { return realloc(p, n); } + void PyObject_Free(void *p) { free(p); } + #else # ifndef WITH_PYMALLOC # define WITH_PYMALLOC # endif # include "obmalloc.c" + #endif #endif From fijal at codespeak.net Tue Feb 12 15:23:00 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 12 Feb 2008 15:23:00 +0100 (CET) Subject: [pypy-svn] r51412 - pypy/dist/pypy/module/_rawffi Message-ID: <20080212142300.55FB11683CE@codespeak.net> Author: fijal Date: Tue Feb 12 15:22:59 2008 New Revision: 51412 Modified: pypy/dist/pypy/module/_rawffi/interp_rawffi.py Log: For hashing, don't compare signed and unsigned Modified: pypy/dist/pypy/module/_rawffi/interp_rawffi.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/interp_rawffi.py (original) +++ pypy/dist/pypy/module/_rawffi/interp_rawffi.py Tue Feb 12 15:22:59 2008 @@ -197,7 +197,7 @@ self.ll_buffer = lltype.malloc(rffi.VOIDP.TO, size, flavor='raw', zero=True) if tracker.DO_TRACING: - ll_buf = rffi.cast(rffi.UINT, self.ll_buffer) + ll_buf = rffi.cast(rffi.INT, self.ll_buffer) tracker.trace_allocation(ll_buf, self) def getbuffer(space, self): @@ -215,7 +215,7 @@ if not self.ll_buffer: raise segfault_exception(space, "freeing NULL pointer") if tracker.DO_TRACING: - ll_buf = rffi.cast(rffi.UINT, self.ll_buffer) + ll_buf = rffi.cast(rffi.INT, self.ll_buffer) tracker.trace_free(ll_buf) lltype.free(self.ll_buffer, flavor='raw') self.ll_buffer = lltype.nullptr(rffi.VOIDP.TO) From cfbolz at codespeak.net Tue Feb 12 16:48:01 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 12 Feb 2008 16:48:01 +0100 (CET) Subject: [pypy-svn] r51413 - pypy/dist/pypy/lang/smalltalk Message-ID: <20080212154801.BC7851683E4@codespeak.net> Author: cfbolz Date: Tue Feb 12 16:48:00 2008 New Revision: 51413 Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py Log: tiny optimization to smalltalk vm Modified: pypy/dist/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/dist/pypy/lang/smalltalk/interpreter.py Tue Feb 12 16:48:00 2008 @@ -356,6 +356,9 @@ def longJumpIfFalse(self, interp): self.jumpConditional(interp.FALSE,self.longJumpPosition()) + # RPython trick: specialize the following function on its second argument + # this makes sure that the primitive call is a direct one + @objectmodel.specialize.arg(1) def callPrimitive(self, primitive, selector, argcount, interp): # WARNING: this is used for bytecodes for which it is safe to # directly call the primitive. In general, it is not safe: for @@ -364,12 +367,14 @@ # else that the user put in a class in an 'at:' method. # The rule of thumb is that primitives with only int and float # in their unwrap_spec are safe. - try: - # note that argcount does not include self - primitives.prim_table[primitive](interp, argcount) - # the primitive pushes the result (if any) onto the stack itself - except primitives.PrimitiveFailedError: - self._sendSelfSelector(selector, argcount, interp) + for i, func in primitives.unrolling_prim_table: + if i == primitive: + try: + func(interp, argcount) + return + except primitives.PrimitiveFailedError: + break + self._sendSelfSelector(selector, argcount, interp) def callPrimitive2(self, primitive1, primitive2, selector, argcount, interp): From arigo at codespeak.net Tue Feb 12 17:44:59 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 12 Feb 2008 17:44:59 +0100 (CET) Subject: [pypy-svn] r51414 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080212164459.C1F301680AB@codespeak.net> Author: arigo Date: Tue Feb 12 17:44:57 2008 New Revision: 51414 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: I love pyc files Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Tue Feb 12 17:44:57 2008 @@ -7,7 +7,6 @@ from pypy.jit.rainbow.codewriter import BytecodeWriter, label, tlabel, assemble from pypy.jit.codegen.llgraph.rgenop import RGenOp as LLRGenOp from pypy.jit.rainbow.test.test_serializegraph import AbstractSerializationTest -from pypy.jit.rainbow import bytecode from pypy.jit.timeshifter import rtimeshift, rvalue from pypy.rpython.lltypesystem import lltype, rstr from pypy.rpython.llinterp import LLInterpreter From cfbolz at codespeak.net Tue Feb 12 18:29:10 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 12 Feb 2008 18:29:10 +0100 (CET) Subject: [pypy-svn] r51417 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20080212172910.C318F1680B8@codespeak.net> Author: cfbolz Date: Tue Feb 12 18:29:08 2008 New Revision: 51417 Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py Log: add a comment Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Tue Feb 12 18:29:08 2008 @@ -311,6 +311,7 @@ # walk over list of objects with finalizers # if it is not copied, add it to the list of to-be-called finalizers # and copy it, to me make the finalizer runnable + # NOTE: the caller is calling scan_copied, so no need to do it here new_with_finalizer = self.AddressLinkedList() while self.objects_with_finalizers.non_empty(): obj = self.objects_with_finalizers.pop() From antocuni at codespeak.net Wed Feb 13 10:06:15 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 13 Feb 2008 10:06:15 +0100 (CET) Subject: [pypy-svn] r51428 - in pypy/dist/pypy: rpython/ootypesystem translator/cli translator/cli/src translator/cli/test Message-ID: <20080213090615.41FA21680AB@codespeak.net> Author: antocuni Date: Wed Feb 13 10:06:14 2008 New Revision: 51428 Modified: pypy/dist/pypy/rpython/ootypesystem/rootype.py pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/metavm.py pypy/dist/pypy/translator/cli/query.py pypy/dist/pypy/translator/cli/src/pypylib.cs pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: a much saner approach to handle dynamically generated delegates; they are converted to the equivalent StaticMethod, and called by indirect_call; this way, gencli automatically handles the generation of the corresponding delegate class. Hardcoded delegate classes are still used but only for testing, when we interprets things on pythonnet, so they have been moved to the pypy.test namespace. Modified: pypy/dist/pypy/rpython/ootypesystem/rootype.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rootype.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rootype.py Wed Feb 13 10:06:14 2008 @@ -105,6 +105,12 @@ def __init__(self, METHODTYPE): self.lowleveltype = METHODTYPE + def rtype_simple_call(self, hop): + vlist = hop.inputargs(*hop.args_r) + cgraphs = hop.inputconst(ootype.Void, None) + vlist.append(cgraphs) + return hop.genop("indirect_call", vlist, resulttype = hop.r_result.lowleveltype) + class __extend__(pairtype(OOInstanceRepr, OOBoundMethRepr)): Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Wed Feb 13 10:06:14 2008 @@ -2,7 +2,7 @@ from pypy.tool.pairtype import pair, pairtype from pypy.annotation.model import SomeObject, SomeInstance, SomeOOInstance, SomeInteger, s_None,\ - s_ImpossibleValue, lltype_to_annotation, annotation_to_lltype, SomeChar, SomeString, SomePBC + s_ImpossibleValue, lltype_to_annotation, annotation_to_lltype, SomeChar, SomeString, SomeOOStaticMeth from pypy.annotation.unaryop import immutablevalue from pypy.annotation.binaryop import _make_none_union from pypy.annotation import model as annmodel @@ -534,8 +534,13 @@ hop.genop('cli_setelem', [v_array, c_index, v_elem], ootype.Void) return v_array - -def typeof(cliClass): +def typeof(cliClass_or_type): + if isinstance(cliClass_or_type, ootype.StaticMethod): + FUNCTYPE = cliClass_or_type + cliClass = known_delegates[FUNCTYPE] + else: + assert isinstance(cliClass_or_type, CliClass) + cliClass = cliClass_or_type TYPE = cliClass._INSTANCE return PythonNet.System.Type.GetType(TYPE._assembly_qualified_name) @@ -571,25 +576,37 @@ return hop.genop('cli_eventhandler', [v_obj, c_methodname], hop.r_result.lowleveltype) -def clidowncast(cliClass, obj): +def clidowncast(obj, TYPE): return obj class Entry(ExtRegistryEntry): _about_ = clidowncast - def compute_result_annotation(self, s_type, s_value): - assert s_type.is_constant() - cliClass = s_type.const - TYPE = cliClass._INSTANCE - assert ootype.isSubclass(TYPE, s_value.ootype) - return SomeOOInstance(TYPE) + def compute_result_annotation(self, s_value, s_type): + if isinstance(s_type.const, ootype.OOType): + TYPE = s_type.const + else: + cliClass = s_type.const + TYPE = cliClass._INSTANCE + if isinstance(TYPE, ootype.StaticMethod): + assert ootype.isSubclass(s_value.ootype, CLR.System.Delegate._INSTANCE) + return SomeOOStaticMeth(TYPE) + else: + assert ootype.isSubclass(TYPE, s_value.ootype) + return SomeOOInstance(TYPE) def specialize_call(self, hop): - assert isinstance(hop.args_s[0].const, CliClass) - assert isinstance(hop.args_s[1], annmodel.SomeOOInstance) - v_inst = hop.inputarg(hop.args_r[1], arg=1) + assert isinstance(hop.args_s[0], annmodel.SomeOOInstance) + v_inst = hop.inputarg(hop.args_r[0], arg=0) return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype) from pypy.translator.cli.query import CliNamespace CLR = CliNamespace(None) CLR._buildtree() + +known_delegates = { + ootype.StaticMethod([ootype.Signed], ootype.Signed): CLR.pypy.test.DelegateType_int__int_1, + ootype.StaticMethod([ootype.Signed] * 2, ootype.Signed): CLR.pypy.test.DelegateType_int__int_2, + ootype.StaticMethod([ootype.Signed] * 3, ootype.Signed): CLR.pypy.test.DelegateType_int__int_3, + ootype.StaticMethod([ootype.Signed] * 100, ootype.Signed): CLR.pypy.test.DelegateType_int__int_100 + } Modified: pypy/dist/pypy/translator/cli/metavm.py ============================================================================== --- pypy/dist/pypy/translator/cli/metavm.py (original) +++ pypy/dist/pypy/translator/cli/metavm.py Wed Feb 13 10:06:14 2008 @@ -191,10 +191,14 @@ class _TypeOf(MicroInstruction): def render(self, generator, op): - v_type, = op.args - assert v_type.concretetype is ootype.Void - cliClass = v_type.value - fullname = cliClass._INSTANCE._name + c_type, = op.args + assert c_type.concretetype is ootype.Void + if isinstance(c_type.value, ootype.StaticMethod): + FUNC = c_type.value + fullname = generator.cts.lltype_to_cts(FUNC) + else: + cliClass = c_type.value + fullname = cliClass._INSTANCE._name generator.ilasm.opcode('ldtoken', fullname) generator.ilasm.call('class [mscorlib]System.Type class [mscorlib]System.Type::GetTypeFromHandle(valuetype [mscorlib]System.RuntimeTypeHandle)') Modified: pypy/dist/pypy/translator/cli/query.py ============================================================================== --- pypy/dist/pypy/translator/cli/query.py (original) +++ pypy/dist/pypy/translator/cli/query.py Wed Feb 13 10:06:14 2008 @@ -229,6 +229,7 @@ parent, name = fullname.rsplit('.', 1) parent = getattr_ex(self, parent) setattr(parent, name, placeholder) + self.System.Object # XXX hack def __getattribute__(self, attr): value = object.__getattribute__(self, attr) Modified: pypy/dist/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/dist/pypy/translator/cli/src/pypylib.cs Wed Feb 13 10:06:14 2008 @@ -49,15 +49,16 @@ return string.Format("ExceptionWrapper('{0}')", obj.GetType().FullName); } } -} -namespace pypy.runtime -{ public delegate int DelegateType_int__int_1(int a); public delegate int DelegateType_int__int_2(int a, int b); public delegate int DelegateType_int__int_3(int a, int b, int c); public delegate int DelegateType_int__int_100(int a0, int a1, int a2, int a3, int a4, int a5, int a6, int a7, int a8, int a9, int a10, int a11, int a12, int a13, int a14, int a15, int a16, int a17, int a18, int a19, int a20, int a21, int a22, int a23, int a24, int a25, int a26, int a27, int a28, int a29, int a30, int a31, int a32, int a33, int a34, int a35, int a36, int a37, int a38, int a39, int a40, int a41, int a42, int a43, int a44, int a45, int a46, int a47, int a48, int a49, int a50, int a51, int a52, int a53, int a54, int a55, int a56, int a57, int a58, int a59, int a60, int a61, int a62, int a63, int a64, int a65, int a66, int a67, int a68, int a69, int a70, int a71, int a72, int a73, int a74, int a75, int a76, int a77, int a78, int a79, int a80, int a81, int a82, int a83, int a84, int a85, int a86, int a87, int a88, int a89, int a90, int a91, int a92, int a93, int a94, int a95, int a96, int a97, int a98, int a99); +} + +namespace pypy.runtime +{ public class Constants { public static object const0; Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Wed Feb 13 10:06:14 2008 @@ -13,21 +13,8 @@ ArrayList = CLR.System.Collections.ArrayList OpCodes = System.Reflection.Emit.OpCodes DynamicMethod = System.Reflection.Emit.DynamicMethod -DelegateType = CLR.pypy.runtime.DelegateType_int__int_int Utils = CLR.pypy.runtime.Utils - -# RPython function, used by test_dynamic_method and test_call_delegate -def build_fn(): - tInt = typeof(System.Int32) - args = init_array(System.Type, tInt, tInt) - meth = Utils.CreateDynamicMethod("add", tInt, args) - il = meth.GetILGenerator() - il.Emit(OpCodes.Ldarg_0) - il.Emit(OpCodes.Ldarg_1) - il.Emit(OpCodes.Add) - il.Emit(OpCodes.Ret) - myfunc = meth.CreateDelegate(typeof(DelegateType)) - return clidowncast(DelegateType, myfunc) +FUNCTYPE = ootype.StaticMethod([ootype.Signed, ootype.Signed], ootype.Signed) class TestDotnetAnnotation(object): @@ -371,19 +358,40 @@ assert res is True def test_typeof_pypylib(self): - DelegateType = CLR.pypy.runtime.DelegateType_int__int_int + DelegateType = CLR.pypy.test.DelegateType_int__int_2 def fn(): return typeof(DelegateType) is not None res = self.interpret(fn, []) assert res is True + def test_typeof_functype(self): + # this test is overridden in TestPythonnet + def fn(): + t = typeof(FUNCTYPE) + return t.get_Name() + res = self.interpret(fn, []) + assert res.startswith('StaticMethod__') + def test_clidowncast(self): def fn(): a = ArrayList() b = ArrayList() a.Add(b) c = a.get_Item(0) # type of c is Object - c = clidowncast(ArrayList, c) + c = clidowncast(c, ArrayList) + c.Add(None) + return c.get_Item(0) + res = self.interpret(fn, []) + assert res is None + + def test_clidowncast_lltype(self): + ARRAY_LIST = ArrayList._INSTANCE + def fn(): + a = ArrayList() + b = ArrayList() + a.Add(b) + c = a.get_Item(0) # type of c is Object + c = clidowncast(c, ARRAY_LIST) c.Add(None) return c.get_Item(0) res = self.interpret(fn, []) @@ -483,16 +491,21 @@ res = self.interpret(fn, []) assert self.ll_to_string(res) == '42' - def test_dynamic_method(self): - def fn(): - myfunc = build_fn() - return myfunc.Invoke(30, 12) - res = self.interpret(fn, []) - assert res == 42 - def test_call_delegate(self): + def build_fn(): + tInt = typeof(System.Int32) + args = init_array(System.Type, tInt, tInt) + meth = Utils.CreateDynamicMethod("add", tInt, args) + il = meth.GetILGenerator() + il.Emit(OpCodes.Ldarg_0) + il.Emit(OpCodes.Ldarg_1) + il.Emit(OpCodes.Add) + il.Emit(OpCodes.Ret) + myfunc = meth.CreateDelegate(typeof(FUNCTYPE)) + return myfunc + def fn(): - myfunc = build_fn() + myfunc = clidowncast(build_fn(), FUNCTYPE) return myfunc(30, 12) res = self.interpret(fn, []) assert res == 42 @@ -519,3 +532,9 @@ def test_whitout_box(self): pass # it makes sense only during translation + def test_typeof_functype(self): + def fn(): + t = typeof(FUNCTYPE) + return t.get_Name() + res = self.interpret(fn, []) + assert res == 'DelegateType_int__int_2' From antocuni at codespeak.net Wed Feb 13 10:16:45 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 13 Feb 2008 10:16:45 +0100 (CET) Subject: [pypy-svn] r51429 - in pypy/dist/pypy: jit/codegen/cli translator/cli translator/cli/test Message-ID: <20080213091645.4D79F1683BC@codespeak.net> Author: antocuni Date: Wed Feb 13 10:16:44 2008 New Revision: 51429 Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: make the cli codegen using the new and better interface to handle dynamic delegates Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/rgenop.py Wed Feb 13 10:16:44 2008 @@ -14,6 +14,11 @@ DUMP_IL = False DEBUG = False +SM_INT__INT_1 = ootype.StaticMethod([ootype.Signed], ootype.Signed) +SM_INT__INT_2 = ootype.StaticMethod([ootype.Signed] * 2, ootype.Signed) +SM_INT__INT_3 = ootype.StaticMethod([ootype.Signed] * 3, ootype.Signed) +SM_INT__INT_100 = ootype.StaticMethod([ootype.Signed] * 100, ootype.Signed) + def token2clitype(tok): if tok == '': return typeof(System.Int32) @@ -22,13 +27,13 @@ def sigtoken2clitype(tok): if tok == ([''], ''): - return typeof(CLR.pypy.runtime.DelegateType_int__int_1) + return typeof(SM_INT__INT_1) elif tok == (['', ''], ''): - return typeof(CLR.pypy.runtime.DelegateType_int__int_2) + return typeof(SM_INT__INT_2) elif tok == ([''] * 3, ''): - return typeof(CLR.pypy.runtime.DelegateType_int__int_3) + return typeof(SM_INT__INT_3) elif tok == ([''] * 100, ''): - return typeof(CLR.pypy.runtime.DelegateType_int__int_100) + return typeof(SM_INT__INT_100) else: assert False @@ -126,28 +131,11 @@ il.Emit(OpCodes.Ldsfld, field) -SM_INT__INT_1 = ootype.StaticMethod([ootype.Signed], ootype.Signed) -SM_INT__INT_2 = ootype.StaticMethod([ootype.Signed] * 2, ootype.Signed) -SM_INT__INT_3 = ootype.StaticMethod([ootype.Signed] * 3, ootype.Signed) -SM_INT__INT_100 = ootype.StaticMethod([ootype.Signed] * 100, ootype.Signed) class FunctionConst(BaseConst): @specialize.arg(1) def revealconst(self, T): - if T == SM_INT__INT_1: - DelegateType = CLR.pypy.runtime.DelegateType_int__int_1 - return clidowncast(DelegateType, self.getobj()) - elif T == SM_INT__INT_2: - DelegateType = CLR.pypy.runtime.DelegateType_int__int_2 - return clidowncast(DelegateType, self.getobj()) - elif T == SM_INT__INT_3: - DelegateType = CLR.pypy.runtime.DelegateType_int__int_3 - return clidowncast(DelegateType, self.getobj()) - elif T == SM_INT__INT_100: - DelegateType = CLR.pypy.runtime.DelegateType_int__int_100 - return clidowncast(DelegateType, self.getobj()) - else: - assert False + return clidowncast(self.getobj(), T) class ObjectConst(BaseConst): @@ -382,3 +370,6 @@ il.MarkLabel(self.label) for op in self.operations: op.emit() + +global_rgenop = RCliGenOp() +RCliGenOp.constPrebuiltGlobal = global_rgenop.genconst Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Wed Feb 13 10:16:44 2008 @@ -589,7 +589,7 @@ cliClass = s_type.const TYPE = cliClass._INSTANCE if isinstance(TYPE, ootype.StaticMethod): - assert ootype.isSubclass(s_value.ootype, CLR.System.Delegate._INSTANCE) + assert ootype.isSubclass(s_value.ootype, CLR.System.Object._INSTANCE) return SomeOOStaticMeth(TYPE) else: assert ootype.isSubclass(TYPE, s_value.ootype) Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Wed Feb 13 10:16:44 2008 @@ -274,6 +274,7 @@ assert self.interpret(fn, []) == 42+43 def test_array_setitem_None(self): + py.test.skip('Mono bug :-(') def fn(): x = init_array(System.Object, box(42), box(43)) x[0] = None From arigo at codespeak.net Wed Feb 13 12:18:04 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 13 Feb 2008 12:18:04 +0100 (CET) Subject: [pypy-svn] r51430 - pypy/dist/pypy/doc/discussion Message-ID: <20080213111804.1ABA21680AB@codespeak.net> Author: arigo Date: Wed Feb 13 12:18:02 2008 New Revision: 51430 Added: pypy/dist/pypy/doc/discussion/finalizer-order.txt (contents, props changed) Log: Finalizer ordering draft. Added: pypy/dist/pypy/doc/discussion/finalizer-order.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/discussion/finalizer-order.txt Wed Feb 13 12:18:02 2008 @@ -0,0 +1,152 @@ +Ordering finalizers in the SemiSpace GC +======================================= + +Goal +---- + +After a collection, the SemiSpace GC should call the finalizers on +*some* of the objects that have one and that have become unreachable. +Basically, if there is a reference chain from an object a to an object b +then it should not call the finalizer for b immediately, but just keep b +alive and try again to call its finalizer after the next collection. + +This basic idea fails when there are cycles. It's not a good idea to +keep the objects alive forever or to never call any of the finalizers. +The model we came up with is that in this case, we could just call the +finalizer of one of the objects in the cycle -- but only, of course, if +there are no other objects outside the cycle that has a finalizer and a +reference to the cycle. + +More precisely, given the graph of references between objects:: + + for each strongly connected component C of the graph: + if C has at least one object with a finalizer: + if there is no object outside C which has a finalizer and + indirectly references the objects in C: + mark one of the objects of C that has a finalizer + copy C and all objects it references to the new space + + for each marked object: + detach the finalizer (so that it's not called more than once) + call the finalizer + +Algorithm +--------- + +During deal_with_objects_with_finalizers(), each object x can be in 4 +possible states:: + + state[x] == 0: unreachable + state[x] == 1: (temporary state, see below) + state[x] == 2: reachable from any finalizer + state[x] == 3: alive + +Initially, objects are in state 0 or 3 depending on whether they have +been copied or not by the regular sweep done just before. The invariant +is that if there is a reference from x to y, then state[y] >= state[x]. + +The state 2 is used for objects that are reachable from a finalizer but +that may be in the same strongly connected component than the finalizer. +The state of these objects goes to 3 when we prove that they can be +reached from a finalizer which is definitely not in the same strongly +connected component. Finalizers on objects with state 3 must not be +called. + +Let closure(x) be the list of objects reachable from x, including x +itself. Pseudo-code (high-level) to get the list of marked objects:: + + marked = [] + for x in objects_with_finalizers: + if state[x] != 0: + continue + marked.append(x) + for y in closure(x): + if state[y] == 0: + state[y] = 2 + elif state[y] == 2: + state[y] = 3 + for x in marked: + assert state[x] >= 2 + if state[x] != 2: + marked.remove(x) + +This does the right thing independently on the order in which the +objects_with_finalizers are enumerated. First assume that [x1, .., xn] +are all in the same unreachable strongly connected component; no object +with finalizer references this strongly connected component from +outside. Then: + +* when x1 is processed, state[x1] == .. == state[xn] == 0 independently + of whatever else we did before. So x1 gets marked and we set + state[x1] = .. = state[xn] = 2. + +* when x2, ... xn are processed, their state is != 0 so we do nothing. + +* in the final loop, only x1 is marked and state[x1] == 2 so it stays + marked. + +Now, let's assume that x1 and x2 are not in the same strongly connected +component and there is a reference path from x1 to x2. Then: + +* if x1 is enumerated before x2, then x2 is in closure(x1) and so its + state gets at least >= 2 when we process x1. When we process x2 later + we just skip it ("continue" line) and so it doesn't get marked. + +* if x2 is enumerated before x1, then when we process x2 we mark it and + set its state to >= 2 (before x2 is in closure(x2)), and then when we + process x1 we set state[x2] == 3. So in the final loop x2 gets + removed from the "marked" list. + +I think that it proves that the algorithm is doing what we want. + +The next step is to remove the use of closure() in the algorithm in such +a way that the new algorithm has a reasonable performance -- linear in +the number of objects whose state it manipulates:: + + marked = [] + for x in objects_with_finalizers: + if state[x] != 0: + continue + marked.append(x) + recursing on the objects y starting from x: + if state[y] == 0: + state[y] = 1 + follow y's children recursively + elif state[y] == 2: + state[y] = 3 + follow y's children recursively + else: + don't need to recurse inside y + recursing on the objects y starting from x: + if state[y] == 1: + state[y] = 2 + follow y's children recursively + else: + don't need to recurse inside y + for x in marked: + assert state[x] >= 2 + if state[x] != 2: + marked.remove(x) + +In this algorithm we follow the children of each object at most 3 times, +when the state of the object changes from 0 to 1 to 2 to 3. In a visit +that doesn't change the state of an object, we don't follow its children +recursively. + +In practice we can encode the 4 states with a single extra bit in the +header: + + ===== ============= ======== ==================== + state is_forwarded? bit set? bit set in the copy? + ===== ============= ======== ==================== + 0 no no n/a + 1 no yes n/a + 2 yes yes yes + 3 yes whatever no + ===== ============= ======== ==================== + +So the loop above that does the transition from state 1 to state 2 is +really just a copy(x) followed by scan_copied(). We must also clear the +bit in the copy at the end, to clean up before the next collection +(which means recursively bumping the state from 2 to 3 in the final +loop). From arigo at codespeak.net Wed Feb 13 13:25:40 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 13 Feb 2008 13:25:40 +0100 (CET) Subject: [pypy-svn] r51431 - in pypy/dist/pypy/translator/c: . src Message-ID: <20080213122540.039A41683CC@codespeak.net> Author: arigo Date: Wed Feb 13 13:25:39 2008 New Revision: 51431 Modified: pypy/dist/pypy/translator/c/gc.py pypy/dist/pypy/translator/c/src/mem.h Log: Attempt to avoid a stack overflow situation with Boehm, by replacing recursive calls to finalizers with calls from a flat loop. Modified: pypy/dist/pypy/translator/c/gc.py ============================================================================== --- pypy/dist/pypy/translator/c/gc.py (original) +++ pypy/dist/pypy/translator/c/gc.py Wed Feb 13 13:25:39 2008 @@ -199,7 +199,7 @@ pass # yield 'assert(GC_all_interior_pointers == 0);' else: yield 'GC_all_interior_pointers = 0;' - yield 'GC_init();' + yield 'boehm_gc_startup_code();' def get_real_weakref_type(self): return boehm.WEAKLINK Modified: pypy/dist/pypy/translator/c/src/mem.h ============================================================================== --- pypy/dist/pypy/translator/c/src/mem.h (original) +++ pypy/dist/pypy/translator/c/src/mem.h Wed Feb 13 13:25:39 2008 @@ -128,6 +128,29 @@ else \ GC_GENERAL_REGISTER_DISAPPEARING_LINK(link, obj) +void boehm_gc_startup_code(void); + +#ifndef PYPY_NOT_MAIN_FILE +static void boehm_gc_finalizer_notifier(void) +{ + static int recursing = 0; + if (recursing) + return; /* GC_invoke_finalizers() will be done by the + boehm_gc_finalizer_notifier() that is + currently in the C stack, when we return there */ + recursing = 1; + while (GC_should_invoke_finalizers()) + GC_invoke_finalizers(); + recursing = 0; +} +void boehm_gc_startup_code(void) +{ + GC_init(); + GC_finalizer_notifier = &boehm_gc_finalizer_notifier; + GC_finalize_on_demand = 1; +} +#endif /* PYPY_NOT_MAIN_FILE */ + #endif /* USING_BOEHM_GC */ /************************************************************/ From arigo at codespeak.net Wed Feb 13 13:35:21 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 13 Feb 2008 13:35:21 +0100 (CET) Subject: [pypy-svn] r51432 - pypy/dist/pypy/translator/c Message-ID: <20080213123521.06C5C1683D1@codespeak.net> Author: arigo Date: Wed Feb 13 13:35:20 2008 New Revision: 51432 Modified: pypy/dist/pypy/translator/c/gc.py Log: I *think* that the explicit call to GC_invoke_finalizers() here was not needed. Now it goes in the way of the recursion protection introduced in r51431. Modified: pypy/dist/pypy/translator/c/gc.py ============================================================================== --- pypy/dist/pypy/translator/c/gc.py (original) +++ pypy/dist/pypy/translator/c/gc.py Wed Feb 13 13:35:20 2008 @@ -208,7 +208,7 @@ return boehm.convert_weakref_to(ptarget) def OP_GC__COLLECT(self, funcgen, op): - return 'GC_gcollect(); GC_invoke_finalizers();' + return 'GC_gcollect();' def OP_GC_SET_MAX_HEAP_SIZE(self, funcgen, op): nbytes = funcgen.expr(op.args[0]) From fijal at codespeak.net Wed Feb 13 13:39:03 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 13 Feb 2008 13:39:03 +0100 (CET) Subject: [pypy-svn] r51433 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080213123903.3EA9516805F@codespeak.net> Author: fijal Date: Wed Feb 13 13:39:02 2008 New Revision: 51433 Modified: pypy/dist/pypy/lib/app_test/ctypes/test_numbers.py Log: Check reference leaks. Modified: pypy/dist/pypy/lib/app_test/ctypes/test_numbers.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_numbers.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_numbers.py Wed Feb 13 13:39:02 2008 @@ -41,6 +41,25 @@ ################################################################ class TestNumber: + def setup_class(cls): + try: + import _rawffi + except ImportError: + pass + else: + cls.old_num = _rawffi._num_of_allocated_objects() + + def teardown_class(cls): + try: + import _rawffi + except ImportError: + pass + else: + import gc + gc.collect() + # there is one reference coming from the byref() above + assert _rawffi._num_of_allocated_objects() <= cls.old_num + def test_default_init(self): # default values are set to zero for t in signed_types + unsigned_types + float_types: @@ -175,7 +194,7 @@ ## def test_perf(self): ## check_perf() -from ctypes import _SimpleCData -class c_int_S(_SimpleCData): - _type_ = "i" - __slots__ = [] +#from ctypes import _SimpleCData +#class c_int_S(_SimpleCData): +# _type_ = "i" +# __slots__ = [] From fijal at codespeak.net Wed Feb 13 13:39:40 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 13 Feb 2008 13:39:40 +0100 (CET) Subject: [pypy-svn] r51434 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080213123940.2A08C1683CC@codespeak.net> Author: fijal Date: Wed Feb 13 13:39:39 2008 New Revision: 51434 Modified: pypy/dist/pypy/lib/_ctypes/pointer.py pypy/dist/pypy/lib/_ctypes/primitive.py Log: __del__ for primitive values, pointers keep alive stuff they reference. Modified: pypy/dist/pypy/lib/_ctypes/pointer.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/pointer.py (original) +++ pypy/dist/pypy/lib/_ctypes/pointer.py Wed Feb 13 13:39:39 2008 @@ -55,6 +55,7 @@ ffiarray = _rawffi.Array('P') def __init__(self, value=None): self._buffer = ffiarray(1) + self._objects = [value] # keepalive value if value is not None: self.contents = value self._ffiarray = ffiarray Modified: pypy/dist/pypy/lib/_ctypes/primitive.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/primitive.py (original) +++ pypy/dist/pypy/lib/_ctypes/primitive.py Wed Feb 13 13:39:39 2008 @@ -194,8 +194,10 @@ class _SimpleCData(_CData): __metaclass__ = SimpleType _type_ = 'i' + _needs_free = False def __init__(self, value=DEFAULT_VALUE): + self._needs_free = True self._buffer = self._ffiarray(1) if value is not DEFAULT_VALUE: self.value = value @@ -216,3 +218,9 @@ def __nonzero__(self): return self._buffer[0] not in (0, '\x00') + + def __del__(self): + if self._needs_free: + self._needs_free = False + self._buffer.free() + self._buffer = None From arigo at codespeak.net Wed Feb 13 14:00:55 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 13 Feb 2008 14:00:55 +0100 (CET) Subject: [pypy-svn] r51435 - pypy/dist/pypy/translator/c/test Message-ID: <20080213130055.063771683BE@codespeak.net> Author: arigo Date: Wed Feb 13 14:00:53 2008 New Revision: 51435 Modified: pypy/dist/pypy/translator/c/test/test_boehm.py Log: A test for r51431. Before that revision, this test causes a C stack overflow on my Linux laptop. Modified: pypy/dist/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_boehm.py (original) +++ pypy/dist/pypy/translator/c/test/test_boehm.py Wed Feb 13 14:00:53 2008 @@ -332,4 +332,36 @@ c_fn = self.getcompiled(fn, [int]) c_fn(100) + def test_nested_finalizers(self): + from pypy.rlib import rgc + class State: + pass + state = State() + def g(): + n = state.counter + if n > 0: + for i in range(5): + state.a = A(n) + state.a = None + rgc.collect() + return n + fun = g + for i in range(200): + def fun(next=fun): + return next() + 1 # prevents tail-call optimization + + class A: + def __init__(self, level): + self.level = level + def __del__(self): + if state.counter == self.level: + state.counter -= 1 + fun() + def fn(n): + state.counter = n + fun() + return state.counter + c_fn = self.getcompiled(fn, [int]) + res = c_fn(10000) + assert res == 0 From fijal at codespeak.net Wed Feb 13 14:21:05 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 13 Feb 2008 14:21:05 +0100 (CET) Subject: [pypy-svn] r51436 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080213132105.9104F1683EE@codespeak.net> Author: fijal Date: Wed Feb 13 14:21:05 2008 New Revision: 51436 Added: pypy/dist/pypy/lib/app_test/ctypes/support.py (contents, props changed) Modified: pypy/dist/pypy/lib/app_test/ctypes/test_functions.py pypy/dist/pypy/lib/app_test/ctypes/test_numbers.py pypy/dist/pypy/lib/app_test/ctypes/test_pointers.py pypy/dist/pypy/lib/app_test/ctypes/test_strings.py pypy/dist/pypy/lib/app_test/ctypes/test_structures.py Log: Introduce some keepalive checkers Added: pypy/dist/pypy/lib/app_test/ctypes/support.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lib/app_test/ctypes/support.py Wed Feb 13 14:21:05 2008 @@ -0,0 +1,20 @@ + +class BaseCTypesTestChecker: + def setup_class(cls): + try: + import _rawffi + except ImportError: + pass + else: + cls.old_num = _rawffi._num_of_allocated_objects() + + def teardown_class(cls): + try: + import _rawffi + except ImportError: + pass + else: + import gc + gc.collect() + # there is one reference coming from the byref() above + assert _rawffi._num_of_allocated_objects() <= cls.old_num Modified: pypy/dist/pypy/lib/app_test/ctypes/test_functions.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_functions.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_functions.py Wed Feb 13 14:21:05 2008 @@ -8,6 +8,7 @@ from ctypes import * import sys import py +from support import BaseCTypesTestChecker try: WINFUNCTYPE @@ -28,7 +29,7 @@ class RECT(Structure): _fields_ = [("left", c_int), ("top", c_int), ("right", c_int), ("bottom", c_int)] -class TestFunctions: +class TestFunctions(BaseCTypesTestChecker): def test_mro(self): # in Python 2.3, this raises TypeError: MRO conflict among bases classes, Modified: pypy/dist/pypy/lib/app_test/ctypes/test_numbers.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_numbers.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_numbers.py Wed Feb 13 14:21:05 2008 @@ -1,5 +1,6 @@ import py from ctypes import * +from support import BaseCTypesTestChecker import sys, struct def valid_ranges(*types): @@ -40,25 +41,7 @@ ################################################################ -class TestNumber: - def setup_class(cls): - try: - import _rawffi - except ImportError: - pass - else: - cls.old_num = _rawffi._num_of_allocated_objects() - - def teardown_class(cls): - try: - import _rawffi - except ImportError: - pass - else: - import gc - gc.collect() - # there is one reference coming from the byref() above - assert _rawffi._num_of_allocated_objects() <= cls.old_num +class TestNumber(BaseCTypesTestChecker): def test_default_init(self): # default values are set to zero Modified: pypy/dist/pypy/lib/app_test/ctypes/test_pointers.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_pointers.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_pointers.py Wed Feb 13 14:21:05 2008 @@ -1,5 +1,6 @@ import py from ctypes import * +from support import BaseCTypesTestChecker ctype_types = [c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float] @@ -11,7 +12,6 @@ mod._ctypes_test = str(conftest.sofile) class TestPointers: - def test_pointer_crash(self): class A(POINTER(c_ulong)): Modified: pypy/dist/pypy/lib/app_test/ctypes/test_strings.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_strings.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_strings.py Wed Feb 13 14:21:05 2008 @@ -1,8 +1,9 @@ import py from ctypes import * +from support import BaseCTypesTestChecker -class TestStringArray: +class TestStringArray(BaseCTypesTestChecker): def test_one(self): BUF = c_char * 4 @@ -53,7 +54,7 @@ except NameError: pass else: - class TestWString: + class TestWString(BaseCTypesTestChecker): def test(self): BUF = c_wchar * 4 Modified: pypy/dist/pypy/lib/app_test/ctypes/test_structures.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_structures.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_structures.py Wed Feb 13 14:21:05 2008 @@ -1,9 +1,10 @@ from ctypes import * from struct import calcsize +from support import BaseCTypesTestChecker import py -class TestSubclasses: +class TestSubclasses(BaseCTypesTestChecker): def test_subclass(self): class X(Structure): _fields_ = [("a", c_int)] @@ -42,7 +43,7 @@ assert Y._fields_ == [("b", c_int)] assert Z._fields_ == [("a", c_int)] -class TestStructure: +class TestStructure(BaseCTypesTestChecker): formats = {"c": c_char, "b": c_byte, "B": c_ubyte, @@ -348,7 +349,7 @@ assert "from_address" in dir(type(Structure)) assert "in_dll" in dir(type(Structure)) -class TestPointerMember: +class TestPointerMember:#(BaseCTypesTestChecker): def test_1(self): # a Structure with a POINTER field @@ -389,7 +390,7 @@ s.p = None assert s.x == 12345678 -class TestRecursiveStructure: +class TestRecursiveStructure(BaseCTypesTestChecker): def test_contains_itself(self): class Recursive(Structure): pass @@ -421,7 +422,7 @@ raise AssertionError, "AttributeError not raised" -class TestPatologicalCases: +class TestPatologicalCases(BaseCTypesTestChecker): def test_structure_overloading_getattr(self): class X(Structure): _fields_ = [('x', c_int)] From fijal at codespeak.net Wed Feb 13 14:21:34 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 13 Feb 2008 14:21:34 +0100 (CET) Subject: [pypy-svn] r51437 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080213132134.9DDC01683EE@codespeak.net> Author: fijal Date: Wed Feb 13 14:21:34 2008 New Revision: 51437 Modified: pypy/dist/pypy/lib/_ctypes/array.py pypy/dist/pypy/lib/_ctypes/basics.py pypy/dist/pypy/lib/_ctypes/pointer.py pypy/dist/pypy/lib/_ctypes/structure.py Log: Introduce some more dels Modified: pypy/dist/pypy/lib/_ctypes/array.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/array.py (original) +++ pypy/dist/pypy/lib/_ctypes/array.py Wed Feb 13 14:21:34 2008 @@ -105,9 +105,12 @@ class Array(_CData): __metaclass__ = ArrayMeta _ffiletter = 'P' + _needs_free = False def __init__(self, *args): self._buffer = self._ffiarray(self._length_) + self._needs_free = True + self._objects = [] for i, arg in enumerate(args): self[i] = arg @@ -133,6 +136,7 @@ if isinstance(index, slice): self._slice_setitem(index, value) return + self._objects.append(value) # keepalive value value = self._type_._CData_input(value) index = self._fix_index(index) if not isinstance(self._type_._ffishape, tuple): @@ -156,6 +160,12 @@ def _get_buffer_for_param(self): return self._buffer.byptr() + def __del__(self): + if self._needs_free: + self._buffer.free() + self._buffer = None + self._needs_free = False + ARRAY_CACHE = {} def create_array_type(base, length): Modified: pypy/dist/pypy/lib/_ctypes/basics.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/basics.py (original) +++ pypy/dist/pypy/lib/_ctypes/basics.py Wed Feb 13 14:21:34 2008 @@ -99,6 +99,7 @@ # fix the address, in case it's unsigned address = address & (sys.maxint * 2 + 1) instance = self.__new__(self) + instance._objects = [] lgt = getattr(self, '_length_', 1) instance._buffer = self._ffiarray.fromaddress(address, lgt) return instance Modified: pypy/dist/pypy/lib/_ctypes/pointer.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/pointer.py (original) +++ pypy/dist/pypy/lib/_ctypes/pointer.py Wed Feb 13 14:21:34 2008 @@ -55,6 +55,7 @@ ffiarray = _rawffi.Array('P') def __init__(self, value=None): self._buffer = ffiarray(1) + self._needs_free = True self._objects = [value] # keepalive value if value is not None: self.contents = value @@ -66,6 +67,7 @@ class _Pointer(_CData): __metaclass__ = PointerType + _needs_free = False def getcontents(self): addr = self._buffer[0] @@ -78,6 +80,7 @@ raise TypeError("expected %s instead of %s" % ( self._type_.__name__, type(value).__name__)) value = value._buffer + self._objects = [value] self._buffer[0] = value _get_slice_params = array_get_slice_params @@ -96,11 +99,18 @@ return self._type_._CData_output(self._subarray(index)) def __setitem__(self, index, value): + self._objects = [value] self._subarray(index)[0] = self._type_._CData_input(value)[0] def __nonzero__(self): return self._buffer[0] != 0 + def __del__(self): + if self._needs_free: + self._buffer.free() + self._needs_free = False + self._buffer = None + contents = property(getcontents, setcontents) Modified: pypy/dist/pypy/lib/_ctypes/structure.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/structure.py (original) +++ pypy/dist/pypy/lib/_ctypes/structure.py Wed Feb 13 14:21:34 2008 @@ -107,6 +107,7 @@ if not hasattr(self, '_ffistruct'): raise TypeError("Cannot instantiate structure, has no _fields_") self.__dict__['_buffer'] = self._ffistruct() + self.__dict__['_needs_free'] = True if len(args) > len(self._names): raise TypeError("too many arguments") for name, arg in zip(self._names, args): @@ -146,6 +147,7 @@ class Structure(_CData): __metaclass__ = StructureMeta _ffiletter = 'P' + _needs_free = False def _subarray(self, fieldtype, name): """Return a _rawffi array of length 1 whose address is the same as @@ -173,3 +175,9 @@ def _get_buffer_for_param(self): return self._buffer.byptr() + + def __del__(self): + if self._needs_free: + self._buffer.free() + self.__dict__['_buffer'] = None + self.__dict__['_needs_free'] = False From arigo at codespeak.net Wed Feb 13 15:41:07 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 13 Feb 2008 15:41:07 +0100 (CET) Subject: [pypy-svn] r51439 - in pypy/dist/pypy/rpython/memory: . gc gctransform test Message-ID: <20080213144107.AA2381683DC@codespeak.net> Author: arigo Date: Wed Feb 13 15:41:06 2008 New Revision: 51439 Modified: pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/gc/marksweep.py pypy/dist/pypy/rpython/memory/gc/semispace.py pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gcwrapper.py pypy/dist/pypy/rpython/memory/support.py pypy/dist/pypy/rpython/memory/test/test_support.py Log: Rename AddressLinkedList to AddressStack everywhere, and give it a complete stack behavior (the previous class would ignore append(NULL)). While I'm at it, here is an AddressDeque too. Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Wed Feb 13 15:41:06 2008 @@ -3,6 +3,7 @@ GCFLAG_IMMORTAL from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage from pypy.rpython.lltypesystem import lltype, llmemory, llarena +from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE from pypy.rlib.objectmodel import free_non_gc_object from pypy.rlib.debug import ll_assert from pypy.rpython.lltypesystem.lloperation import llop @@ -30,13 +31,13 @@ needs_write_barrier = True prebuilt_gc_objects_are_static_roots = False - def __init__(self, AddressLinkedList, + def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, nursery_size=128, min_nursery_size=128, auto_nursery_size=False, space_size=4096, max_space_size=sys.maxint//2+1): - SemiSpaceGC.__init__(self, AddressLinkedList, + SemiSpaceGC.__init__(self, chunk_size = chunk_size, space_size = space_size, max_space_size = max_space_size) assert min_nursery_size <= nursery_size <= space_size // 2 @@ -55,7 +56,7 @@ # of such objects is abused for this linked list; it needs to be # reset to its correct value when GCFLAG_NO_YOUNG_PTRS is set # again at the start of a collection. - self.young_objects_with_weakrefs = self.AddressLinkedList() + self.young_objects_with_weakrefs = self.AddressStack() self.set_nursery_size(self.initial_nursery_size) # the GC is fully setup now. The rest can make use of it. if self.auto_nursery_size: Modified: pypy/dist/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/dist/pypy/rpython/memory/gc/marksweep.py Wed Feb 13 15:41:06 2008 @@ -1,7 +1,8 @@ from pypy.rpython.lltypesystem.llmemory import raw_malloc, raw_free from pypy.rpython.lltypesystem.llmemory import raw_memcopy, raw_memclear from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage -from pypy.rpython.memory.support import get_address_linked_list +from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE +from pypy.rpython.memory.support import get_address_stack from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rlib.objectmodel import free_non_gc_object @@ -38,12 +39,12 @@ POOLNODE.become(lltype.Struct('gc_pool_node', ('linkedlist', HDRPTR), ('nextnode', POOLNODEPTR))) - def __init__(self, AddressLinkedList, start_heap_size=4096): + def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, start_heap_size=4096): self.heap_usage = 0 # at the end of the latest collection self.bytes_malloced = 0 # since the latest collection self.bytes_malloced_threshold = start_heap_size self.total_collection_time = 0.0 - self.AddressLinkedList = AddressLinkedList + self.AddressStack = get_address_stack(chunk_size) self.malloced_objects = lltype.nullptr(self.HDR) self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR) # these are usually only the small bits of memory that make a @@ -226,7 +227,7 @@ ## size_gc_header) # push the roots on the mark stack - objects = self.AddressLinkedList() # mark stack + objects = self.AddressStack() # mark stack self._mark_stack = objects self.root_walker.walk_roots( MarkSweepGC._mark_root, # stack roots @@ -477,7 +478,9 @@ self.trace(obj, self._add_reachable, objects) def _add_reachable(pointer, objects): - objects.append(pointer.address[0]) + obj = pointer.address[0] + if obj: + objects.append(obj) _add_reachable = staticmethod(_add_reachable) def statistics(self, index): @@ -558,7 +561,7 @@ curpool = self.x_swap_pool(lltype.nullptr(X_POOL)) size_gc_header = self.gcheaderbuilder.size_gc_header - oldobjects = self.AddressLinkedList() + oldobjects = self.AddressStack() # if no pool specified, use the current pool as the 'source' pool oldpool = clonedata.pool or curpool oldpool = lltype.cast_opaque_ptr(self.POOLPTR, oldpool) @@ -576,7 +579,7 @@ # a stack of addresses of places that still points to old objects # and that must possibly be fixed to point to a new copy - stack = self.AddressLinkedList() + stack = self.AddressStack() stack.append(llmemory.cast_ptr_to_adr(clonedata) + llmemory.offsetof(X_CLONE, 'gcobjectptr')) while stack.non_empty(): @@ -679,9 +682,11 @@ self.trace(obj, self._add_reachable_and_rename, objects) def _add_reachable_and_rename(self, pointer, objects): - if pointer.address[0] == self.x_become_target_addr: - pointer.address[0] = self.x_become_source_addr - objects.append(pointer.address[0]) + obj = pointer.address[0] + if obj: + if obj == self.x_become_target_addr: + obj = pointer.address[0] = self.x_become_source_addr + objects.append(obj) def x_become(self, target_addr, source_addr): # 1. mark from the roots, and also the objects that objects-with-del @@ -699,7 +704,7 @@ ## size_gc_header) # push the roots on the mark stack - objects = self.AddressLinkedList() # mark stack + objects = self.AddressStack() # mark stack self._mark_stack = objects # the last sweep did not clear the mark bit of static roots, # since they are not in the malloced_objects list @@ -843,8 +848,8 @@ _alloc_flavor_ = "raw" COLLECT_EVERY = 2000 - def __init__(self, AddressLinkedList, start_heap_size=4096): - MarkSweepGC.__init__(self, AddressLinkedList, start_heap_size) + def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, start_heap_size=4096): + MarkSweepGC.__init__(self, chunk_size, start_heap_size) self.count_mallocs = 0 def write_malloc_statistics(self, typeid, size, result, varsize): @@ -1021,7 +1026,7 @@ ## size_gc_header) # push the roots on the mark stack - objects = self.AddressLinkedList() # mark stack + objects = self.AddressStack() # mark stack self._mark_stack = objects self.root_walker.walk_roots( MarkSweepGC._mark_root, # stack roots Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Wed Feb 13 15:41:06 2008 @@ -1,7 +1,8 @@ from pypy.rpython.lltypesystem.llmemory import raw_malloc, raw_free from pypy.rpython.lltypesystem.llmemory import raw_memcopy, raw_memclear from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage -from pypy.rpython.memory.support import get_address_linked_list +from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE +from pypy.rpython.memory.support import get_address_stack from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rlib.objectmodel import free_non_gc_object @@ -27,13 +28,13 @@ HDR = lltype.Struct('header', ('forw', llmemory.Address), ('tid', lltype.Signed)) - def __init__(self, AddressLinkedList, space_size=4096, + def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096, max_space_size=sys.maxint//2+1): MovingGCBase.__init__(self) self.space_size = space_size self.max_space_size = max_space_size self.gcheaderbuilder = GCHeaderBuilder(self.HDR) - self.AddressLinkedList = AddressLinkedList + self.AddressStack = get_address_stack(chunk_size) def setup(self): self.tospace = llarena.arena_malloc(self.space_size, True) @@ -42,9 +43,9 @@ self.fromspace = llarena.arena_malloc(self.space_size, True) ll_assert(bool(self.fromspace), "couldn't allocate fromspace") self.free = self.tospace - self.objects_with_finalizers = self.AddressLinkedList() - self.run_finalizers = self.AddressLinkedList() - self.objects_with_weakrefs = self.AddressLinkedList() + self.objects_with_finalizers = self.AddressStack() + self.run_finalizers = self.AddressStack() + self.objects_with_weakrefs = self.AddressStack() self.finalizer_lock_count = 0 self.red_zone = 0 @@ -312,7 +313,7 @@ # if it is not copied, add it to the list of to-be-called finalizers # and copy it, to me make the finalizer runnable # NOTE: the caller is calling scan_copied, so no need to do it here - new_with_finalizer = self.AddressLinkedList() + new_with_finalizer = self.AddressStack() while self.objects_with_finalizers.non_empty(): obj = self.objects_with_finalizers.pop() if self.is_forwarded(obj): @@ -326,7 +327,7 @@ # walk over list of objects that contain weakrefs # if the object it references survives then update the weakref # otherwise invalidate the weakref - new_with_weakref = self.AddressLinkedList() + new_with_weakref = self.AddressStack() while self.objects_with_weakrefs.non_empty(): obj = self.objects_with_weakrefs.pop() if not self.is_forwarded(obj): @@ -348,7 +349,7 @@ def update_run_finalizers(self): # we are in an inner collection, caused by a finalizer # the run_finalizers objects need to be copied - new_run_finalizer = self.AddressLinkedList() + new_run_finalizer = self.AddressStack() while self.run_finalizers.non_empty(): obj = self.run_finalizers.pop() new_run_finalizer.append(self.copy(obj)) Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Wed Feb 13 15:41:06 2008 @@ -97,10 +97,8 @@ root_stack_depth = 163840 def __init__(self, translator): - from pypy.rpython.memory.support import get_address_linked_list from pypy.rpython.memory.gc.base import choose_gc_from_config super(FrameworkGCTransformer, self).__init__(translator, inline=True) - AddressLinkedList = get_address_linked_list() if hasattr(self, 'GC_PARAMS'): # for tests: the GC choice can be specified as class attributes from pypy.rpython.memory.gc.marksweep import MarkSweepGC @@ -131,7 +129,7 @@ self.gcdata = gcdata self.malloc_fnptr_cache = {} - gcdata.gc = GCClass(AddressLinkedList, **GC_PARAMS) + gcdata.gc = GCClass(**GC_PARAMS) root_walker = self.build_root_walker() gcdata.set_query_functions(gcdata.gc) gcdata.gc.set_root_walker(root_walker) Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Wed Feb 13 15:41:06 2008 @@ -1,7 +1,6 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llheap from pypy.rpython import llinterp from pypy.rpython.annlowlevel import llhelper -from pypy.rpython.memory.support import get_address_linked_list from pypy.rpython.memory import gctypelayout from pypy.objspace.flow.model import Constant @@ -9,8 +8,7 @@ class GCManagedHeap(object): def __init__(self, llinterp, flowgraphs, gc_class, GC_PARAMS={}): - self.AddressLinkedList = get_address_linked_list(10) - self.gc = gc_class(self.AddressLinkedList, **GC_PARAMS) + self.gc = gc_class(chunk_size = 10, **GC_PARAMS) self.gc.set_root_walker(LLInterpRootWalker(self)) self.llinterp = llinterp self.prepare_graphs(flowgraphs) Modified: pypy/dist/pypy/rpython/memory/support.py ============================================================================== --- pypy/dist/pypy/rpython/memory/support.py (original) +++ pypy/dist/pypy/rpython/memory/support.py Wed Feb 13 15:41:06 2008 @@ -1,14 +1,19 @@ from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rlib.objectmodel import free_non_gc_object +from pypy.rlib.objectmodel import free_non_gc_object, we_are_translated +from pypy.rlib.debug import ll_assert DEFAULT_CHUNK_SIZE = 1019 -def get_address_linked_list(chunk_size=DEFAULT_CHUNK_SIZE): + +def get_chunk_manager(chunk_size=DEFAULT_CHUNK_SIZE, cache={}): + try: + return cache[chunk_size] + except KeyError: + pass CHUNK = lltype.ForwardReference() - CHUNK.become(lltype.Struct('AddressLinkedListChunk', - ('previous', lltype.Ptr(CHUNK)), - ('length', lltype.Signed), + CHUNK.become(lltype.Struct('AddressChunk', + ('next', lltype.Ptr(CHUNK)), ('items', lltype.FixedSizeArray( llmemory.Address, chunk_size)))) null_chunk = lltype.nullptr(CHUNK) @@ -24,67 +29,150 @@ return lltype.malloc(CHUNK, flavor="raw") result = self.free_list - self.free_list = result.previous + self.free_list = result.next return result def put(self, chunk): - chunk.previous = self.free_list - self.free_list = chunk + if we_are_translated(): + chunk.next = self.free_list + self.free_list = chunk + else: + # Don't cache the old chunks but free them immediately. + # Helps debugging, and avoids that old chunks full of + # addresses left behind by a test end up in genc... + lltype.free(chunk, flavor="raw") unused_chunks = FreeList() + cache[chunk_size] = unused_chunks, null_chunk + return unused_chunks, null_chunk + + +def get_address_stack(chunk_size=DEFAULT_CHUNK_SIZE, cache={}): + try: + return cache[chunk_size] + except KeyError: + pass + + unused_chunks, null_chunk = get_chunk_manager(chunk_size) - class AddressLinkedList(object): + class AddressStack(object): _alloc_flavor_ = "raw" def __init__(self): self.chunk = unused_chunks.get() - self.chunk.previous = null_chunk - self.chunk.length = 0 + self.chunk.next = null_chunk + self.used_in_last_chunk = 0 + # invariant: self.used_in_last_chunk == 0 if and only if + # the AddressStack is empty def enlarge(self): new = unused_chunks.get() - new.previous = self.chunk - new.length = 0 + new.next = self.chunk self.chunk = new - return new + self.used_in_last_chunk = 0 enlarge._dont_inline_ = True def shrink(self): old = self.chunk - self.chunk = old.previous + self.chunk = old.next unused_chunks.put(old) - return self.chunk + self.used_in_last_chunk = chunk_size shrink._dont_inline_ = True def append(self, addr): - if addr == llmemory.NULL: - return - chunk = self.chunk - if chunk.length == chunk_size: - chunk = self.enlarge() - used_chunks = chunk.length - chunk.length = used_chunks + 1 - chunk.items[used_chunks] = addr + used = self.used_in_last_chunk + if used == chunk_size: + self.enlarge() + used = 0 + self.chunk.items[used] = addr + self.used_in_last_chunk = used + 1 # always > 0 here def non_empty(self): - chunk = self.chunk - return chunk.length != 0 or bool(chunk.previous) + return self.used_in_last_chunk != 0 def pop(self): - chunk = self.chunk - if chunk.length == 0: - chunk = self.shrink() - used_chunks = self.chunk.length - 1 - result = chunk.items[used_chunks] - chunk.length = used_chunks + used = self.used_in_last_chunk - 1 + ll_assert(used >= 0, "pop on empty AddressStack") + result = self.chunk.items[used] + self.used_in_last_chunk = used + if used == 0 and self.chunk.next: + self.shrink() return result def delete(self): cur = self.chunk while cur: - prev = cur.previous + next = cur.next + unused_chunks.put(cur) + cur = next + free_non_gc_object(self) + + cache[chunk_size] = AddressStack + return AddressStack + + +def get_address_deque(chunk_size=DEFAULT_CHUNK_SIZE, cache={}): + try: + return cache[chunk_size] + except KeyError: + pass + + unused_chunks, null_chunk = get_chunk_manager(chunk_size) + + class AddressDeque(object): + _alloc_flavor_ = "raw" + + def __init__(self): + chunk = unused_chunks.get() + chunk.next = null_chunk + self.oldest_chunk = self.newest_chunk = chunk + self.index_in_oldest = 0 + self.index_in_newest = 0 + + def enlarge(self): + new = unused_chunks.get() + new.next = null_chunk + self.newest_chunk.next = new + self.newest_chunk = new + self.index_in_newest = 0 + enlarge._dont_inline_ = True + + def shrink(self): + old = self.oldest_chunk + self.oldest_chunk = old.next + unused_chunks.put(old) + self.index_in_oldest = 0 + shrink._dont_inline_ = True + + def append(self, addr): + index = self.index_in_newest + if index == chunk_size: + self.enlarge() + index = 0 + self.newest_chunk.items[index] = addr + self.index_in_newest = index + 1 + + def non_empty(self): + return (self.oldest_chunk != self.newest_chunk + or self.index_in_oldest < self.index_in_newest) + + def popleft(self): + ll_assert(self.non_empty(), "pop on empty AddressDeque") + index = self.index_in_oldest + if index == chunk_size: + self.shrink() + index = 0 + result = self.oldest_chunk.items[index] + self.index_in_oldest = index + 1 + return result + + def delete(self): + cur = self.oldest_chunk + while cur: + next = cur.next unused_chunks.put(cur) - cur = prev + cur = next free_non_gc_object(self) - return AddressLinkedList + cache[chunk_size] = AddressDeque + return AddressDeque Modified: pypy/dist/pypy/rpython/memory/test/test_support.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_support.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_support.py Wed Feb 13 15:41:06 2008 @@ -1,17 +1,18 @@ from pypy.rlib.objectmodel import free_non_gc_object -from pypy.rpython.memory.support import get_address_linked_list +from pypy.rpython.memory.support import get_address_stack +from pypy.rpython.memory.support import get_address_deque from pypy.rpython.test.test_llinterp import interpret from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem.llmemory import raw_malloc, raw_free, NULL -class TestAddressLinkedList(object): +class TestAddressStack(object): def test_simple_access(self): - AddressLinkedList = get_address_linked_list() + AddressStack = get_address_stack() addr0 = raw_malloc(llmemory.sizeof(lltype.Signed)) addr1 = raw_malloc(llmemory.sizeof(lltype.Signed)) addr2 = raw_malloc(llmemory.sizeof(lltype.Signed)) - ll = AddressLinkedList() + ll = AddressStack() ll.append(addr0) ll.append(addr1) ll.append(addr2) @@ -27,20 +28,23 @@ assert not ll.non_empty() ll.append(addr0) ll.delete() - ll = AddressLinkedList() + ll = AddressStack() ll.append(addr0) ll.append(addr1) ll.append(addr2) + ll.append(NULL) + a = ll.pop() + assert a == NULL ll.delete() raw_free(addr2) raw_free(addr1) raw_free(addr0) def test_big_access(self): - AddressLinkedList = get_address_linked_list() + AddressStack = get_address_stack() addrs = [raw_malloc(llmemory.sizeof(lltype.Signed)) for i in range(3000)] - ll = AddressLinkedList() + ll = AddressStack() for i in range(3000): print i ll.append(addrs[i]) @@ -57,12 +61,32 @@ for addr in addrs: raw_free(addr) -def test_linked_list_annotate(): - AddressLinkedList = get_address_linked_list(60) + +class TestAddressDeque: + def test_big_access(self): + import random + AddressDeque = get_address_deque(10) + deque = AddressDeque() + expected = [] + for i in range(3000): + assert deque.non_empty() == (len(expected) > 0) + r = random.random() + if r < 0.51 and expected: + x = deque.popleft() + y = expected.pop(0) + assert x == y + else: + x = raw_malloc(llmemory.sizeof(lltype.Signed)) + deque.append(x) + expected.append(x) + + +def test_stack_annotate(): + AddressStack = get_address_stack(60) INT_SIZE = llmemory.sizeof(lltype.Signed) def f(): addr = raw_malloc(INT_SIZE*100) - ll = AddressLinkedList() + ll = AddressStack() ll.append(addr) ll.append(addr + INT_SIZE*1) ll.append(addr + INT_SIZE*2) @@ -86,7 +110,7 @@ a = ll.pop() res = res and (a - INT_SIZE*i == addr) ll.delete() - ll = AddressLinkedList() + ll = AddressStack() ll.append(addr) ll.append(addr + INT_SIZE*1) ll.append(addr + INT_SIZE*2) @@ -95,6 +119,6 @@ return res assert f() - AddressLinkedList = get_address_linked_list() + AddressStack = get_address_stack() res = interpret(f, [], malloc_check=False) assert res From arigo at codespeak.net Wed Feb 13 17:04:38 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 13 Feb 2008 17:04:38 +0100 (CET) Subject: [pypy-svn] r51440 - pypy/dist/pypy/rpython/lltypesystem Message-ID: <20080213160438.7D6711683EE@codespeak.net> Author: arigo Date: Wed Feb 13 17:04:34 2008 New Revision: 51440 Modified: pypy/dist/pypy/rpython/lltypesystem/lltype.py Log: Fix comment. Modified: pypy/dist/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lltype.py Wed Feb 13 17:04:34 2008 @@ -1513,8 +1513,8 @@ assert not '__dict__' in dir(_struct) -class _subarray(_parentable): # only for cast_subarray_pointer() - # and cast_structfield_pointer() +class _subarray(_parentable): # only for direct_fieldptr() + # and direct_arrayitems() _kind = "subarray" _cache = weakref.WeakKeyDictionary() # parentarray -> {subarrays} From arigo at codespeak.net Wed Feb 13 17:05:32 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 13 Feb 2008 17:05:32 +0100 (CET) Subject: [pypy-svn] r51441 - in pypy/dist/pypy/rpython/lltypesystem: . test Message-ID: <20080213160532.3C4E11683F7@codespeak.net> Author: arigo Date: Wed Feb 13 17:05:30 2008 New Revision: 51441 Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py Log: It was possible already to take the fakeaddress just past the end of an array of primitives; this allows the same for arrays of structures. Needed to llinterp the GCs handling dictionaries. Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py Wed Feb 13 17:05:30 2008 @@ -60,7 +60,18 @@ if isinstance(index, str): assert index.startswith('item') # itemN => N index = int(index[4:]) - return parent.getitem(index + self.repeat)._as_ptr() + index += self.repeat + if index == parent.getlength(): + # for references exactly to the end of the array + try: + endmarker = _end_markers[parent] + except KeyError: + endmarker = _endmarker_struct(A, parent=parent, + parentindex=index) + _end_markers[parent] = endmarker + return endmarker._as_ptr() + else: + return parent.getitem(index)._as_ptr() elif (isinstance(A, lltype.FixedSizeArray) and array_item_type_match(A.OF, self.TYPE)): # for array of primitives or pointers @@ -99,6 +110,24 @@ srcadr += ItemOffset(self.TYPE) dstadr += ItemOffset(self.TYPE) +_end_markers = weakref.WeakKeyDictionary() # -> _endmarker +class _endmarker_struct(lltype._struct): + __slots__ = () + def __new__(self, *args, **kwds): + return object.__new__(self) + def __init__(self, *args, **kwds): + lltype._struct.__init__(self, *args, **kwds) + self._storage = False + def __getattr__(self, name): + raise AttributeError("cannot access fields in the endmarker " + "structure at the end of the array") + def __setattr__(self, name, value): + if name.startswith('_'): + object.__setattr__(self, name, value) # '_xxx' attributes + elif self._storage is False: + raise AttributeError("cannot access fields in the endmarker " + "structure at the end of the array") + class FieldOffset(AddressOffset): Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_llmemory.py Wed Feb 13 17:05:30 2008 @@ -39,7 +39,35 @@ assert b.signed[0] == 123 b.signed[0] = 14 assert x[3] == 14 - + +def test_array_endaddress(): + A = lltype.GcArray(lltype.Signed) + x = lltype.malloc(A, 5) + x[4] = 321 + a = fakeaddress(x) + b = a + ArrayItemsOffset(A) + b += ItemOffset(lltype.Signed)*5 + assert b == a + ArrayItemsOffset(A) + ItemOffset(lltype.Signed)*5 + py.test.raises(IndexError, "b.signed[0]") + b -= ItemOffset(lltype.Signed) + assert b.signed[0] == 321 + +def test_structarray_endaddress(): + S = lltype.Struct('S', ('foo', lltype.Signed)) + A = lltype.GcArray(S) + x = lltype.malloc(A, 5) + x[4].foo = 321 + a = fakeaddress(x) + b = a + ArrayItemsOffset(A) + b += ItemOffset(S)*5 + assert b == a + ArrayItemsOffset(A) + ItemOffset(S)*5 + p = cast_adr_to_ptr(b, lltype.Ptr(S)) + py.test.raises(AttributeError, "p.foo") + py.test.raises(AttributeError, "p.foo = 55") + b -= ItemOffset(S) + p = cast_adr_to_ptr(b, lltype.Ptr(S)) + assert p.foo == 321 + def test_dont_mix_offsets_and_ints(): o = AddressOffset() py.test.raises(TypeError, "1 + o") From arigo at codespeak.net Wed Feb 13 17:06:30 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 13 Feb 2008 17:06:30 +0100 (CET) Subject: [pypy-svn] r51442 - pypy/dist/pypy/rpython/memory Message-ID: <20080213160630.A6B071683F2@codespeak.net> Author: arigo Date: Wed Feb 13 17:06:29 2008 New Revision: 51442 Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py Log: Fix for gcwrapper support for dictionaries (we see _interior_ptr objects here). Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Wed Feb 13 17:06:29 2008 @@ -177,10 +177,10 @@ return constants def reccollect(constants, llvalue): - T = lltype.typeOf(llvalue) - if isinstance(T, lltype.Ptr) and llvalue and llvalue._obj not in constants: + if (isinstance(llvalue, lltype._abstract_ptr) + and llvalue._obj is not None and llvalue._obj not in constants): + TYPE = llvalue._T constants[llvalue._obj] = True - TYPE = T.TO if isinstance(TYPE, lltype.Struct): for name in TYPE._names: reccollect(constants, getattr(llvalue, name)) From arigo at codespeak.net Wed Feb 13 17:07:55 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 13 Feb 2008 17:07:55 +0100 (CET) Subject: [pypy-svn] r51443 - pypy/dist/pypy/rpython/memory/test Message-ID: <20080213160755.EE5461683EE@codespeak.net> Author: arigo Date: Wed Feb 13 17:07:55 2008 New Revision: 51443 Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py Log: A test for the finalizer order. Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Wed Feb 13 17:07:55 2008 @@ -387,10 +387,105 @@ class TestSemiSpaceGC(GCTest): from pypy.rpython.memory.gc.semispace import SemiSpaceGC as GCClass + def test_finalizer_order(self): + py.test.skip("in-progress") + import random + from pypy.tool.algo import graphlib + + examples = [] + letters = 'abcdefghijklmnopqrstuvwxyz' + for i in range(20): + input = [] + edges = {} + for c in letters: + edges[c] = [] + # make up a random graph + for c in letters: + for j in range(random.randrange(0, 4)): + d = random.choice(letters) + edges[c].append(graphlib.Edge(c, d)) + input.append((c, d)) + # find the expected order in which destructors should be called + components = list(graphlib.strong_components(edges, edges)) + head = {} + for component in components: + c = component.keys()[0] + for d in component: + assert d not in head + head[d] = c + assert len(head) == len(letters) + strict = [] + for c, d in input: + if head[c] != head[d]: + strict.append((c, d)) + examples.append((input, components, strict)) + + class State: + pass + state = State() + class A: + def __init__(self, key): + self.key = key + self.refs = [] + def __del__(self): + assert state.age[self.key] == -1 + state.age[self.key] = state.time + + def build_example(input): + state.time = 0 + state.age = {} + vertices = {} + for c in letters: + vertices[c] = A(c) + state.age[c] = -1 + for c, d in input: + vertices[c].refs.append(d) + + def f(): + i = 0 + while i < len(examples): + input, components, strict = examples[i] + build_example(input) + while state.time < len(letters): + llop.gc__collect(lltype.Void) + state.time += 1 + # check that all instances have been finalized + if -1 in state.age.values(): + return i * 10 + 1 + # check that if a -> b and a and b are not in the same + # strong component, then a is finalized strictly before b + for c, d in strict: + if state.age[c] >= state.age[d]: + return i * 10 + 2 + # check that two instances in the same strong component + # are never finalized during the same collection + for component in components: + seen = {} + for c in component: + age = state.age[c] + if age in seen: + return i * 10 + 3 + seen[age] = True + i += 1 + return 0 + + res = self.interpret(f, []) + if res != 0: + import pprint + pprint.pprint(examples[res / 10]) + if res % 10 == 1: + py.test.fail("some instances have not been finalized at all") + if res % 10 == 2: + py.test.fail("the strict order is not respected") + if res % 10 == 3: + py.test.fail("two instances from the same component " + "have been finalized together") + assert 0 + class TestGrowingSemiSpaceGC(TestSemiSpaceGC): GC_PARAMS = {'space_size': 64} -class TestGenerationalGC(GCTest): +class TestGenerationalGC(TestSemiSpaceGC): from pypy.rpython.memory.gc.generation import GenerationGC as GCClass def test_coalloc(self): From arigo at codespeak.net Wed Feb 13 17:58:22 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 13 Feb 2008 17:58:22 +0100 (CET) Subject: [pypy-svn] r51444 - pypy/dist/pypy/rpython/memory/test Message-ID: <20080213165822.AD5AC1683FE@codespeak.net> Author: arigo Date: Wed Feb 13 17:58:20 2008 New Revision: 51444 Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py Log: We may not need all collections. This should speed up the test a little. Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Wed Feb 13 17:58:20 2008 @@ -430,6 +430,7 @@ def __del__(self): assert state.age[self.key] == -1 state.age[self.key] = state.time + state.progress = True def build_example(input): state.time = 0 @@ -447,7 +448,10 @@ input, components, strict = examples[i] build_example(input) while state.time < len(letters): + state.progress = False llop.gc__collect(lltype.Void) + if not state.progress: + break state.time += 1 # check that all instances have been finalized if -1 in state.age.values(): From fijal at codespeak.net Wed Feb 13 18:13:06 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 13 Feb 2008 18:13:06 +0100 (CET) Subject: [pypy-svn] r51447 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080213171306.9D77C1683CE@codespeak.net> Author: fijal Date: Wed Feb 13 18:13:06 2008 New Revision: 51447 Added: pypy/dist/pypy/lib/_ctypes/keepalive.txt (contents, props changed) Log: Draft for document describing keepalive behavior of ctypes. Added: pypy/dist/pypy/lib/_ctypes/keepalive.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/lib/_ctypes/keepalive.txt Wed Feb 13 18:13:06 2008 @@ -0,0 +1,16 @@ + +Few rules about ctypes keepalives: + +* Ownership: the memory is freed when owner of this memory is deleted. + Owner is the one who allocated the memory. So if we have: + + a = c_int(3) + b = c_int.from_address(a._buffer.buffer) # pypy notion of address access + + only a frees memory. + +* _objects: each ctypes object has (eventually) _objects dictionary in which + it simply keeps objects to be kept alive. + +* there are (some) rules about when to put stuff in it's objects when + accessing the fields etc. need to list them here. \ No newline at end of file From antocuni at codespeak.net Wed Feb 13 18:46:14 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 13 Feb 2008 18:46:14 +0100 (CET) Subject: [pypy-svn] r51448 - in pypy/dist/pypy/translator/cli: . test Message-ID: <20080213174614.7AC23168401@codespeak.net> Author: antocuni Date: Wed Feb 13 18:46:13 2008 New Revision: 51448 Modified: pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/metavm.py pypy/dist/pypy/translator/cli/opcodes.py pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: fieldinfo_for_const is a new function for getting the FieldInfo instance corresponding to the static field of the Constants clas where the constant resides. Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Wed Feb 13 18:46:13 2008 @@ -600,6 +600,22 @@ v_inst = hop.inputarg(hop.args_r[0], arg=0) return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype) + +def fieldinfo_for_const(const): + assert False, 'It works only after translation' + +class Entry(ExtRegistryEntry): + _about_ = fieldinfo_for_const + + def compute_result_annotation(self, s_const): + assert s_const.is_constant() + return SomeOOInstance(CLR.System.Reflection.FieldInfo._INSTANCE) + + def specialize_call(self, hop): + llvalue = hop.args_v[0].value + c_llvalue = hop.inputconst(ootype.Void, llvalue) + return hop.genop('cli_fieldinfo_for_const', [c_llvalue], resulttype = hop.r_result.lowleveltype) + from pypy.translator.cli.query import CliNamespace CLR = CliNamespace(None) CLR._buildtree() Modified: pypy/dist/pypy/translator/cli/metavm.py ============================================================================== --- pypy/dist/pypy/translator/cli/metavm.py (original) +++ pypy/dist/pypy/translator/cli/metavm.py Wed Feb 13 18:46:13 2008 @@ -238,6 +238,18 @@ generator.load(op.args[2]) generator.ilasm.store_static_field(cts_type, desc) +class _FieldInfoForConst(MicroInstruction): + def render(self, generator, op): + from pypy.translator.cli.constant import CONST_CLASS + llvalue = op.args[0].value + constgen = generator.db.constant_generator + const = constgen.record_const(llvalue) + generator.ilasm.opcode('ldtoken', CONST_CLASS) + generator.ilasm.call('class [mscorlib]System.Type class [mscorlib]System.Type::GetTypeFromHandle(valuetype [mscorlib]System.RuntimeTypeHandle)') + generator.ilasm.opcode('ldstr', '"%s"' % const.name) + generator.ilasm.call_method('class [mscorlib]System.Reflection.FieldInfo class [mscorlib]System.Type::GetField(string)', virtual=True) + + OOTYPE_TO_MNEMONIC = { ootype.Signed: 'i4', ootype.SignedLongLong: 'i8', @@ -266,4 +278,5 @@ EventHandler = _EventHandler() GetStaticField = _GetStaticField() SetStaticField = _SetStaticField() +FieldInfoForConst = _FieldInfoForConst() CastPrimitive = _CastPrimitive() Modified: pypy/dist/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/cli/opcodes.py (original) +++ pypy/dist/pypy/translator/cli/opcodes.py Wed Feb 13 18:46:13 2008 @@ -1,7 +1,8 @@ from pypy.translator.cli.metavm import Call, CallMethod, \ IndirectCall, GetField, SetField, DownCast, NewCustomDict,\ MapException, Box, Unbox, NewArray, GetArrayElem, SetArrayElem,\ - TypeOf, CastPrimitive, EventHandler, GetStaticField, SetStaticField + TypeOf, CastPrimitive, EventHandler, GetStaticField, SetStaticField,\ + FieldInfoForConst from pypy.translator.oosupport.metavm import PushArg, PushAllArgs, StoreResult, InstructionList,\ New, RuntimeNew, CastTo, PushPrimitive, OOString, OOUnicode from pypy.translator.cli.cts import WEAKREF @@ -45,6 +46,7 @@ 'cli_eventhandler': [EventHandler], 'cli_getstaticfield': [GetStaticField], 'cli_setstaticfield': [SetStaticField], + 'cli_fieldinfo_for_const': [FieldInfoForConst], 'oois': 'ceq', 'oononnull': [PushAllArgs, 'ldnull', 'ceq']+Not, 'instanceof': [CastTo, 'ldnull', 'cgt.un'], Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Wed Feb 13 18:46:13 2008 @@ -7,7 +7,8 @@ from pypy.translator.cli.test.runtest import CliTest from pypy.translator.cli.dotnet import SomeCliClass, SomeCliStaticMethod,\ NativeInstance, CLR, box, unbox, OverloadingResolver, NativeException,\ - native_exc, new_array, init_array, typeof, eventhandler, clidowncast + native_exc, new_array, init_array, typeof, eventhandler, clidowncast,\ + fieldinfo_for_const System = CLR.System ArrayList = CLR.System.Collections.ArrayList @@ -521,6 +522,21 @@ return f self.interpret(fn, []) + def test_fieldinfo_for_const(self): + A = ootype.Instance('A', ootype.ROOT, {'xx': ootype.Signed}) + const = ootype.new(A) + const.xx = 42 + def fn(): + fieldinfo = fieldinfo_for_const(const) + obj = fieldinfo.GetValue(None) + # get the 'xx' field by using reflection + t = obj.GetType() + x_info = t.GetField('xx') + x_value = x_info.GetValue(obj) + return unbox(x_value, ootype.Signed) + res = self.interpret(fn, []) + assert res == 42 + class TestPythonnet(TestDotnetRtyping): # don't interpreter functions but execute them directly through pythonnet @@ -539,3 +555,6 @@ return t.get_Name() res = self.interpret(fn, []) assert res == 'DelegateType_int__int_2' + + def test_fieldinfo_for_const(self): + pass # it makes sense only during translation From cfbolz at codespeak.net Wed Feb 13 20:33:56 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 13 Feb 2008 20:33:56 +0100 (CET) Subject: [pypy-svn] r51449 - pypy/dist/pypy/doc Message-ID: <20080213193356.F1730168407@codespeak.net> Author: cfbolz Date: Wed Feb 13 20:33:55 2008 New Revision: 51449 Modified: pypy/dist/pypy/doc/faq.txt Log: add faq item about mandelbrot fractals Modified: pypy/dist/pypy/doc/faq.txt ============================================================================== --- pypy/dist/pypy/doc/faq.txt (original) +++ pypy/dist/pypy/doc/faq.txt Wed Feb 13 20:33:55 2008 @@ -377,3 +377,9 @@ .. _`getting-started`: getting-started.html .. include:: _ref.txt + +---------------------------------------------------------- +Why does PyPy draw a Mandelbrot fractal while translating? +---------------------------------------------------------- + +Because it's fun. From pedronis at codespeak.net Wed Feb 13 21:45:04 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 13 Feb 2008 21:45:04 +0100 (CET) Subject: [pypy-svn] r51451 - in pypy/dist/pypy: config doc/config module/_ffi Message-ID: <20080213204504.74EFB1683FE@codespeak.net> Author: pedronis Date: Wed Feb 13 21:45:02 2008 New Revision: 51451 Removed: pypy/dist/pypy/doc/config/objspace.usemodules._ffi.txt pypy/dist/pypy/module/_ffi/ Modified: pypy/dist/pypy/config/pypyoption.py Log: _ffi is not being maintained atm Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Wed Feb 13 21:45:02 2008 @@ -35,7 +35,7 @@ del working_modules["termios"] -module_dependencies = {'_ffi': [("objspace.usemodules.struct", True)], +module_dependencies = { '_rawffi': [("objspace.usemodules.struct", True)], } if os.name == "posix": From antocuni at codespeak.net Thu Feb 14 09:51:46 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 14 Feb 2008 09:51:46 +0100 (CET) Subject: [pypy-svn] r51476 - pypy/dist/pypy/translator/oosupport Message-ID: <20080214085146.67D13168412@codespeak.net> Author: antocuni Date: Thu Feb 14 09:51:46 2008 New Revision: 51476 Modified: pypy/dist/pypy/translator/oosupport/constant.py Log: s/NotImplemented{Error,Exception}/NotImplemented Modified: pypy/dist/pypy/translator/oosupport/constant.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/constant.py (original) +++ pypy/dist/pypy/translator/oosupport/constant.py Thu Feb 14 09:51:46 2008 @@ -98,7 +98,7 @@ Loads the constant onto the stack. Can be invoked at any time. """ - raise NotImplementedError + raise NotImplemented def _store_constant(self, gen, const): """ @@ -107,7 +107,7 @@ stores the constant from the stack """ - raise NotImplementedError + raise NotImplemented # _________________________________________________________________ # Optional Constant Operations @@ -303,27 +303,27 @@ """ Invoked with the assembler and sorted list of constants before anything else. Expected to return a generator that will be passed around after that (the parameter named 'gen'). """ - raise NotImplementedError + raise NotImplemented def _declare_const(self, gen, const): """ Invoked once for each constant before any steps are created. """ - raise NotImplementedError + raise NotImplemented def _declare_step(self, gen, stepnum): """ Invoked to begin step #stepnum. stepnum starts with 0 (!) and proceeds monotonically. If _declare_step() is invoked, there will always be a corresponding call to _close_step(). """ - raise NotImplementedError + raise NotImplemented def _close_step(self, gen, stepnum): """ Invoked to end step #stepnum. Never invoked without a corresponding call from _declare_step() first. """ - raise NotImplementedError + raise NotImplemented def _end_gen_constants(self, gen, numsteps): """ Invoked as the very last thing. numsteps is the total number of steps that were created. """ - raise NotImplementedError + raise NotImplemented # ______________________________________________________________________ # Constant base class @@ -395,7 +395,7 @@ and any classes that are used are loaded. Called when the constant object is created. """ - raise NotImplementedException + raise NotImplemented def create_pointer(self, gen): """ @@ -413,7 +413,7 @@ the pointer from the stack in the process; otherwise, a pop is automatically inserted afterwards. """ - raise NotImplementedException + raise NotImplemented # ____________________________________________________________ # Internal helpers @@ -685,7 +685,7 @@ self.delegate_type = self.db.record_delegate(self.value._TYPE) def initialize_data(self, constgen, gen): - raise NotImplementedError + raise NotImplemented # ______________________________________________________________________ # Weak Reference constants From antocuni at codespeak.net Thu Feb 14 09:57:41 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 14 Feb 2008 09:57:41 +0100 (CET) Subject: [pypy-svn] r51477 - pypy/dist/pypy/translator/oosupport Message-ID: <20080214085741.7C6E9168410@codespeak.net> Author: antocuni Date: Thu Feb 14 09:57:41 2008 New Revision: 51477 Modified: pypy/dist/pypy/translator/oosupport/constant.py Log: (cfbolz) s/NotImplemented/NotImplementedError Modified: pypy/dist/pypy/translator/oosupport/constant.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/constant.py (original) +++ pypy/dist/pypy/translator/oosupport/constant.py Thu Feb 14 09:57:41 2008 @@ -98,7 +98,7 @@ Loads the constant onto the stack. Can be invoked at any time. """ - raise NotImplemented + raise NotImplementedError def _store_constant(self, gen, const): """ @@ -107,7 +107,7 @@ stores the constant from the stack """ - raise NotImplemented + raise NotImplementedError # _________________________________________________________________ # Optional Constant Operations @@ -303,27 +303,27 @@ """ Invoked with the assembler and sorted list of constants before anything else. Expected to return a generator that will be passed around after that (the parameter named 'gen'). """ - raise NotImplemented + raise NotImplementedError def _declare_const(self, gen, const): """ Invoked once for each constant before any steps are created. """ - raise NotImplemented + raise NotImplementedError def _declare_step(self, gen, stepnum): """ Invoked to begin step #stepnum. stepnum starts with 0 (!) and proceeds monotonically. If _declare_step() is invoked, there will always be a corresponding call to _close_step(). """ - raise NotImplemented + raise NotImplementedError def _close_step(self, gen, stepnum): """ Invoked to end step #stepnum. Never invoked without a corresponding call from _declare_step() first. """ - raise NotImplemented + raise NotImplementedError def _end_gen_constants(self, gen, numsteps): """ Invoked as the very last thing. numsteps is the total number of steps that were created. """ - raise NotImplemented + raise NotImplementedError # ______________________________________________________________________ # Constant base class @@ -395,7 +395,7 @@ and any classes that are used are loaded. Called when the constant object is created. """ - raise NotImplemented + raise NotImplementedError def create_pointer(self, gen): """ @@ -413,7 +413,7 @@ the pointer from the stack in the process; otherwise, a pop is automatically inserted afterwards. """ - raise NotImplemented + raise NotImplementedError # ____________________________________________________________ # Internal helpers @@ -685,7 +685,7 @@ self.delegate_type = self.db.record_delegate(self.value._TYPE) def initialize_data(self, constgen, gen): - raise NotImplemented + raise NotImplementedError # ______________________________________________________________________ # Weak Reference constants From antocuni at codespeak.net Thu Feb 14 10:03:41 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 14 Feb 2008 10:03:41 +0100 (CET) Subject: [pypy-svn] r51478 - in pypy/dist/pypy/translator/cli: . test Message-ID: <20080214090341.AA5A2168410@codespeak.net> Author: antocuni Date: Thu Feb 14 10:03:39 2008 New Revision: 51478 Modified: pypy/dist/pypy/translator/cli/constant.py pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: make the result of fieldinfo_for_const working also as a pbc Modified: pypy/dist/pypy/translator/cli/constant.py ============================================================================== --- pypy/dist/pypy/translator/cli/constant.py (original) +++ pypy/dist/pypy/translator/cli/constant.py Thu Feb 14 10:03:39 2008 @@ -28,7 +28,7 @@ from pypy.translator.oosupport.constant import \ push_constant, WeakRefConst, StaticMethodConst, CustomDictConst, \ ListConst, ClassConst, InstanceConst, RecordConst, DictConst, \ - BaseConstantGenerator + BaseConstantGenerator, AbstractConst from pypy.translator.cli.ilgenerator import CLIBaseGenerator from pypy.rpython.ootypesystem import ootype from pypy.translator.cli.comparer import EqualityComparer @@ -84,6 +84,14 @@ type = self.cts.lltype_to_cts(EXPECTED_TYPE) gen.ilasm.opcode('castclass', type) + def _create_complex_const(self, value): + from pypy.translator.cli.dotnet import _fieldinfo + if isinstance(value, _fieldinfo): + uniq = self.db.unique() + return CLIFieldInfoConst(self.db, value.llvalue, uniq) + else: + return BaseConstantGenerator._create_complex_const(self, value) + class FieldConstGenerator(CLIBaseConstGenerator): pass @@ -408,3 +416,26 @@ gen.ilasm.call_method('void %s::ll_set(object)' % self.get_type(), True) return True + +class CLIFieldInfoConst(AbstractConst): + def __init__(self, db, llvalue, count): + AbstractConst.__init__(self, db, llvalue, count) + self.name = 'FieldInfo__%d' % count + + def create_pointer(self, generator): + constgen = generator.db.constant_generator + const = constgen.record_const(self.value) + generator.ilasm.opcode('ldtoken', CONST_CLASS) + generator.ilasm.call('class [mscorlib]System.Type class [mscorlib]System.Type::GetTypeFromHandle(valuetype [mscorlib]System.RuntimeTypeHandle)') + generator.ilasm.opcode('ldstr', '"%s"' % const.name) + generator.ilasm.call_method('class [mscorlib]System.Reflection.FieldInfo class [mscorlib]System.Type::GetField(string)', virtual=True) + + def get_type(self): + from pypy.translator.cli.cts import CliClassType + return CliClassType('mscorlib', 'System.Reflection.FieldInfo') + + def initialize_data(self, constgen, gen): + pass + + def record_dependencies(self): + self.db.constant_generator.record_const(self.value) Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Thu Feb 14 10:03:39 2008 @@ -600,9 +600,13 @@ v_inst = hop.inputarg(hop.args_r[0], arg=0) return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype) +class _fieldinfo(object): + def __init__(self, llvalue): + self._TYPE = CLR.System.Reflection.FieldInfo._INSTANCE + self.llvalue = llvalue def fieldinfo_for_const(const): - assert False, 'It works only after translation' + return _fieldinfo(const) class Entry(ExtRegistryEntry): _about_ = fieldinfo_for_const @@ -616,6 +620,13 @@ c_llvalue = hop.inputconst(ootype.Void, llvalue) return hop.genop('cli_fieldinfo_for_const', [c_llvalue], resulttype = hop.r_result.lowleveltype) + +class Entry(ExtRegistryEntry): + _type_ = _fieldinfo + + def compute_annotation(self): + return SomeOOInstance(CLR.System.Reflection.FieldInfo._INSTANCE) + from pypy.translator.cli.query import CliNamespace CLR = CliNamespace(None) CLR._buildtree() Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Thu Feb 14 10:03:39 2008 @@ -537,6 +537,20 @@ res = self.interpret(fn, []) assert res == 42 + def test_fieldinfo_for_const_pbc(self): + A = ootype.Instance('A', ootype.ROOT, {'xx': ootype.Signed}) + const = ootype.new(A) + fieldinfo = fieldinfo_for_const(const) + def fn(): + const.xx = 42 + obj = fieldinfo.GetValue(None) + # get the 'xx' field by using reflection + t = obj.GetType() + x_info = t.GetField('xx') + x_value = x_info.GetValue(obj) + return unbox(x_value, ootype.Signed) + res = self.interpret(fn, []) + assert res == 42 class TestPythonnet(TestDotnetRtyping): # don't interpreter functions but execute them directly through pythonnet @@ -558,3 +572,6 @@ def test_fieldinfo_for_const(self): pass # it makes sense only during translation + + def test_fieldinfo_for_const_pbc(self): + pass # it makes sense only during translation From arigo at codespeak.net Thu Feb 14 10:07:50 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 14 Feb 2008 10:07:50 +0100 (CET) Subject: [pypy-svn] r51479 - pypy/dist/pypy/config Message-ID: <20080214090750.CA847168410@codespeak.net> Author: arigo Date: Thu Feb 14 10:07:50 2008 New Revision: 51479 Modified: pypy/dist/pypy/config/pypyoption.py Log: Document and make this module dependency more precise. Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Thu Feb 14 10:07:50 2008 @@ -35,7 +35,10 @@ del working_modules["termios"] -module_dependencies = { +module_dependencies = {} +module_suggests = { # the reason you want _rawffi is for ctypes, which + # itself needs the interp-level struct module + # because 'P' is missing from the app-level one '_rawffi': [("objspace.usemodules.struct", True)], } if os.name == "posix": @@ -85,6 +88,7 @@ default=modname in default_modules, cmdline="--withmod-%s" % (modname, ), requires=module_dependencies.get(modname, []), + suggests=module_suggests.get(modname, []), negation=modname not in essential_modules) for modname in all_modules]), From arigo at codespeak.net Thu Feb 14 10:56:11 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 14 Feb 2008 10:56:11 +0100 (CET) Subject: [pypy-svn] r51480 - pypy/branch/finalizer-order Message-ID: <20080214095611.3708F168416@codespeak.net> Author: arigo Date: Thu Feb 14 10:56:09 2008 New Revision: 51480 Added: pypy/branch/finalizer-order/ - copied from r51479, pypy/dist/ Log: (cfbolz, arigo) A branch to implement doc/discussion/finalizer-order.txt. From arigo at codespeak.net Thu Feb 14 13:54:48 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 14 Feb 2008 13:54:48 +0100 (CET) Subject: [pypy-svn] r51484 - in pypy/branch/finalizer-order/pypy: rpython/memory/gc rpython/memory/test translator/c/test Message-ID: <20080214125448.79C4616841E@codespeak.net> Author: arigo Date: Thu Feb 14 13:54:46 2008 New Revision: 51484 Added: pypy/branch/finalizer-order/pypy/rpython/memory/test/snippet.py Modified: pypy/branch/finalizer-order/pypy/rpython/memory/gc/generation.py pypy/branch/finalizer-order/pypy/rpython/memory/gc/semispace.py pypy/branch/finalizer-order/pypy/rpython/memory/test/test_gc.py pypy/branch/finalizer-order/pypy/translator/c/test/test_newgc.py Log: (cfbolz, arigo) The straightforward implementation of pypy/doc/discussion/finalizer-order.txt. Tests pass. Modified: pypy/branch/finalizer-order/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/finalizer-order/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/finalizer-order/pypy/rpython/memory/gc/generation.py Thu Feb 14 13:54:46 2008 @@ -1,6 +1,5 @@ import sys -from pypy.rpython.memory.gc.semispace import SemiSpaceGC, GCFLAGSHIFT, \ - GCFLAG_IMMORTAL +from pypy.rpython.memory.gc.semispace import SemiSpaceGC, GCFLAG_IMMORTAL from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE @@ -12,11 +11,11 @@ # in the nursery. It is initially set on all prebuilt and old objects, # and gets cleared by the write_barrier() when we write in them a # pointer to a young object. -GCFLAG_NO_YOUNG_PTRS = 1 << (GCFLAGSHIFT+1) +GCFLAG_NO_YOUNG_PTRS = SemiSpaceGC.first_unused_gcflag << 1 # The following flag is set for static roots which are not on the list # of static roots yet, but will appear with write barrier -GCFLAG_NO_HEAP_PTRS = 1 << (GCFLAGSHIFT+2) +GCFLAG_NO_HEAP_PTRS = SemiSpaceGC.first_unused_gcflag << 2 DEBUG_PRINT = False @@ -30,6 +29,7 @@ inline_simple_malloc_varsize = True needs_write_barrier = True prebuilt_gc_objects_are_static_roots = False + first_unused_gcflag = SemiSpaceGC.first_unused_gcflag << 3 def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, nursery_size=128, Modified: pypy/branch/finalizer-order/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/finalizer-order/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/finalizer-order/pypy/rpython/memory/gc/semispace.py Thu Feb 14 13:54:46 2008 @@ -2,7 +2,7 @@ from pypy.rpython.lltypesystem.llmemory import raw_memcopy, raw_memclear from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE -from pypy.rpython.memory.support import get_address_stack +from pypy.rpython.memory.support import get_address_stack, get_address_deque from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rlib.objectmodel import free_non_gc_object @@ -14,8 +14,9 @@ import sys, os TYPEID_MASK = 0xffff -GCFLAGSHIFT = 16 -GCFLAG_IMMORTAL = 1 << GCFLAGSHIFT +first_gcflag = 1 << 16 +GCFLAG_IMMORTAL = first_gcflag +GCFLAG_FINALIZATION_ORDERING = first_gcflag << 1 memoryError = MemoryError() @@ -24,6 +25,7 @@ inline_simple_malloc = True inline_simple_malloc_varsize = True needs_zero_gc_pointers = False + first_unused_gcflag = first_gcflag << 2 HDR = lltype.Struct('header', ('forw', llmemory.Address), ('tid', lltype.Signed)) @@ -35,6 +37,7 @@ self.max_space_size = max_space_size self.gcheaderbuilder = GCHeaderBuilder(self.HDR) self.AddressStack = get_address_stack(chunk_size) + self.AddressDeque = get_address_deque(chunk_size) def setup(self): self.tospace = llarena.arena_malloc(self.space_size, True) @@ -43,8 +46,8 @@ self.fromspace = llarena.arena_malloc(self.space_size, True) ll_assert(bool(self.fromspace), "couldn't allocate fromspace") self.free = self.tospace - self.objects_with_finalizers = self.AddressStack() - self.run_finalizers = self.AddressStack() + self.objects_with_finalizers = self.AddressDeque() + self.run_finalizers = self.AddressDeque() self.objects_with_weakrefs = self.AddressStack() self.finalizer_lock_count = 0 self.red_zone = 0 @@ -189,12 +192,11 @@ self.top_of_space = tospace + self.space_size scan = self.free = tospace self.collect_roots() - scan = self.scan_copied(scan) if self.run_finalizers.non_empty(): self.update_run_finalizers() - if self.objects_with_finalizers.non_empty(): - self.deal_with_objects_with_finalizers() scan = self.scan_copied(scan) + if self.objects_with_finalizers.non_empty(): + scan = self.deal_with_objects_with_finalizers(scan) if self.objects_with_weakrefs.non_empty(): self.invalidate_weakrefs() self.notify_objects_just_moved() @@ -308,20 +310,105 @@ else: hdr.forw = NULL - def deal_with_objects_with_finalizers(self): + def deal_with_objects_with_finalizers(self, scan): # walk over list of objects with finalizers # if it is not copied, add it to the list of to-be-called finalizers # and copy it, to me make the finalizer runnable - # NOTE: the caller is calling scan_copied, so no need to do it here - new_with_finalizer = self.AddressStack() + # We try to run the finalizers in a "reasonable" order, like + # CPython does. The details of this algorithm are in + # pypy/doc/discussion/finalizer-order.txt. + new_with_finalizer = self.AddressDeque() + marked = self.AddressDeque() + pending = self.AddressStack() while self.objects_with_finalizers.non_empty(): - obj = self.objects_with_finalizers.pop() - if self.is_forwarded(obj): - new_with_finalizer.append(self.get_forwarding_address(obj)) + x = self.objects_with_finalizers.popleft() + ll_assert(self._finalization_state(x) != 1, + "bad finalization state 1") + if self.is_forwarded(x): + new_with_finalizer.append(self.get_forwarding_address(x)) + continue + marked.append(x) + pending.append(x) + while pending.non_empty(): + y = pending.pop() + state = self._finalization_state(y) + if state == 0: + self._bump_finalization_state_from_0_to_1(y) + elif state == 2: + self._bump_finalization_state_from_2_to_3(y) + else: + continue # don't need to recurse inside y + self.trace(y, self._append_if_nonnull, pending) + scan = self._recursively_bump_finalization_state_from_1_to_2( + x, scan) + + while marked.non_empty(): + x = marked.popleft() + state = self._finalization_state(x) + ll_assert(state >= 2, "unexpected finalization state < 2") + newx = self.get_forwarding_address(x) + if state == 2: + self.run_finalizers.append(newx) + # we must also fix the state from 2 to 3 here, otherwise + # we leave the GCFLAG_FINALIZATION_ORDERING bit behind + # which will confuse the next collection + pending.append(x) + while pending.non_empty(): + y = pending.pop() + state = self._finalization_state(y) + if state == 2: + self._bump_finalization_state_from_2_to_3(y) + self.trace(y, self._append_if_nonnull, pending) else: - self.run_finalizers.append(self.copy(obj)) + new_with_finalizer.append(newx) + + pending.delete() + marked.delete() self.objects_with_finalizers.delete() self.objects_with_finalizers = new_with_finalizer + return scan + + def _append_if_nonnull(pointer, stack): + if pointer.address[0] != NULL: + stack.append(pointer.address[0]) + _append_if_nonnull = staticmethod(_append_if_nonnull) + + def _finalization_state(self, obj): + if self.is_forwarded(obj): + newobj = self.get_forwarding_address(obj) + hdr = self.header(newobj) + if hdr.tid & GCFLAG_FINALIZATION_ORDERING: + return 2 + else: + return 3 + else: + hdr = self.header(obj) + if hdr.tid & GCFLAG_FINALIZATION_ORDERING: + return 1 + else: + return 0 + + def _bump_finalization_state_from_0_to_1(self, obj): + ll_assert(self._finalization_state(obj) == 0, + "unexpected finalization state != 0") + hdr = self.header(obj) + hdr.tid |= GCFLAG_FINALIZATION_ORDERING + + def _bump_finalization_state_from_2_to_3(self, obj): + ll_assert(self._finalization_state(obj) == 2, + "unexpected finalization state != 2") + newobj = self.get_forwarding_address(obj) + hdr = self.header(newobj) + hdr.tid &= ~GCFLAG_FINALIZATION_ORDERING + + def _recursively_bump_finalization_state_from_1_to_2(self, obj, scan): + # recursively convert objects from state 1 to state 2. + # Note that copy() copies all bits, including the + # GCFLAG_FINALIZATION_ORDERING. The mapping between + # state numbers and the presence of this bit was designed + # for the following to work :-) + self.copy(obj) + return self.scan_copied(scan) def invalidate_weakrefs(self): # walk over list of objects that contain weakrefs @@ -349,9 +436,9 @@ def update_run_finalizers(self): # we are in an inner collection, caused by a finalizer # the run_finalizers objects need to be copied - new_run_finalizer = self.AddressStack() + new_run_finalizer = self.AddressDeque() while self.run_finalizers.non_empty(): - obj = self.run_finalizers.pop() + obj = self.run_finalizers.popleft() new_run_finalizer.append(self.copy(obj)) self.run_finalizers.delete() self.run_finalizers = new_run_finalizer @@ -363,7 +450,7 @@ try: while self.run_finalizers.non_empty(): #print "finalizer" - obj = self.run_finalizers.pop() + obj = self.run_finalizers.popleft() finalizer = self.getfinalizer(self.get_type_id(obj)) finalizer(obj) finally: Added: pypy/branch/finalizer-order/pypy/rpython/memory/test/snippet.py ============================================================================== --- (empty file) +++ pypy/branch/finalizer-order/pypy/rpython/memory/test/snippet.py Thu Feb 14 13:54:46 2008 @@ -0,0 +1,121 @@ +from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem.lloperation import llop + +class SemiSpaceGCTests: + large_tests_ok = False + + def test_finalizer_order(self): + import random + from pypy.tool.algo import graphlib + + examples = [] + if self.large_tests_ok: + letters = 'abcdefghijklmnopqrstuvwxyz' + COUNT = 20 + else: + letters = 'abcdefghijklm' + COUNT = 2 + for i in range(COUNT): + input = [] + edges = {} + for c in letters: + edges[c] = [] + # make up a random graph + for c in letters: + for j in range(random.randrange(0, 4)): + d = random.choice(letters) + edges[c].append(graphlib.Edge(c, d)) + input.append((c, d)) + # find the expected order in which destructors should be called + components = list(graphlib.strong_components(edges, edges)) + head = {} + for component in components: + c = component.keys()[0] + for d in component: + assert d not in head + head[d] = c + assert len(head) == len(letters) + strict = [] + for c, d in input: + if head[c] != head[d]: + strict.append((c, d)) + examples.append((input, components, strict)) + + class State: + pass + state = State() + class A: + def __init__(self, key): + self.key = key + self.refs = [] + def __del__(self): + assert state.age[self.key] == -1 + state.age[self.key] = state.time + state.progress = True + + def build_example(input): + state.time = 0 + state.age = {} + vertices = {} + for c in letters: + vertices[c] = A(c) + state.age[c] = -1 + for c, d in input: + vertices[c].refs.append(vertices[d]) + + def f(): + i = 0 + while i < len(examples): + input, components, strict = examples[i] + build_example(input) + while state.time < len(letters): + state.progress = False + llop.gc__collect(lltype.Void) + if not state.progress: + break + state.time += 1 + # summarize the finalization order + lst = [] + for c in letters: + lst.append('%s:%d' % (c, state.age[c])) + summary = ', '.join(lst) + + # check that all instances have been finalized + if -1 in state.age.values(): + return error(i, summary, "not all instances finalized") + # check that if a -> b and a and b are not in the same + # strong component, then a is finalized strictly before b + for c, d in strict: + if state.age[c] >= state.age[d]: + return error(i, summary, + "%s should be finalized before %s" + % (c, d)) + # check that two instances in the same strong component + # are never finalized during the same collection + for component in components: + seen = {} + for c in component: + age = state.age[c] + if age in seen: + d = seen[age] + return error(i, summary, + "%s and %s should not be finalized" + " at the same time" % (c, d)) + seen[age] = c + i += 1 + return "ok" + + def error(i, summary, msg): + return '%d\n%s\n%s' % (i, summary, msg) + + res = self.run(f) + if res != "ok": + i, summary, msg = res.split('\n') + i = int(i) + import pprint + print 'Example:' + pprint.pprint(examples[i]) + print 'Finalization ages:' + print summary + print msg + py.test.fail(msg) Modified: pypy/branch/finalizer-order/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/finalizer-order/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/finalizer-order/pypy/rpython/memory/test/test_gc.py Thu Feb 14 13:54:46 2008 @@ -3,8 +3,10 @@ #from pypy.rpython.memory.support import INT_SIZE from pypy.rpython.memory import gcwrapper +from pypy.rpython.memory.test import snippet from pypy.rpython.test.test_llinterp import get_interpreter from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem.rstr import STR from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.objectmodel import compute_unique_id @@ -35,6 +37,12 @@ self.GC_PARAMS) return interp.eval_graph(graph, values) + def run(self, func): # for snippet.py + res = self.interpret(func, []) + if lltype.typeOf(res) == lltype.Ptr(STR): + res = ''.join(res.chars) + return res + def test_llinterp_lists(self): #curr = simulator.current_size def malloc_a_lot(): @@ -384,108 +392,9 @@ class TestMarkSweepGC(GCTest): from pypy.rpython.memory.gc.marksweep import MarkSweepGC as GCClass -class TestSemiSpaceGC(GCTest): +class TestSemiSpaceGC(GCTest, snippet.SemiSpaceGCTests): from pypy.rpython.memory.gc.semispace import SemiSpaceGC as GCClass - def test_finalizer_order(self): - py.test.skip("in-progress") - import random - from pypy.tool.algo import graphlib - - examples = [] - letters = 'abcdefghijklmnopqrstuvwxyz' - for i in range(20): - input = [] - edges = {} - for c in letters: - edges[c] = [] - # make up a random graph - for c in letters: - for j in range(random.randrange(0, 4)): - d = random.choice(letters) - edges[c].append(graphlib.Edge(c, d)) - input.append((c, d)) - # find the expected order in which destructors should be called - components = list(graphlib.strong_components(edges, edges)) - head = {} - for component in components: - c = component.keys()[0] - for d in component: - assert d not in head - head[d] = c - assert len(head) == len(letters) - strict = [] - for c, d in input: - if head[c] != head[d]: - strict.append((c, d)) - examples.append((input, components, strict)) - - class State: - pass - state = State() - class A: - def __init__(self, key): - self.key = key - self.refs = [] - def __del__(self): - assert state.age[self.key] == -1 - state.age[self.key] = state.time - state.progress = True - - def build_example(input): - state.time = 0 - state.age = {} - vertices = {} - for c in letters: - vertices[c] = A(c) - state.age[c] = -1 - for c, d in input: - vertices[c].refs.append(d) - - def f(): - i = 0 - while i < len(examples): - input, components, strict = examples[i] - build_example(input) - while state.time < len(letters): - state.progress = False - llop.gc__collect(lltype.Void) - if not state.progress: - break - state.time += 1 - # check that all instances have been finalized - if -1 in state.age.values(): - return i * 10 + 1 - # check that if a -> b and a and b are not in the same - # strong component, then a is finalized strictly before b - for c, d in strict: - if state.age[c] >= state.age[d]: - return i * 10 + 2 - # check that two instances in the same strong component - # are never finalized during the same collection - for component in components: - seen = {} - for c in component: - age = state.age[c] - if age in seen: - return i * 10 + 3 - seen[age] = True - i += 1 - return 0 - - res = self.interpret(f, []) - if res != 0: - import pprint - pprint.pprint(examples[res / 10]) - if res % 10 == 1: - py.test.fail("some instances have not been finalized at all") - if res % 10 == 2: - py.test.fail("the strict order is not respected") - if res % 10 == 3: - py.test.fail("two instances from the same component " - "have been finalized together") - assert 0 - class TestGrowingSemiSpaceGC(TestSemiSpaceGC): GC_PARAMS = {'space_size': 64} Modified: pypy/branch/finalizer-order/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/finalizer-order/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/finalizer-order/pypy/translator/c/test/test_newgc.py Thu Feb 14 13:54:46 2008 @@ -9,6 +9,7 @@ from pypy.translator.c import genc, gc from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rpython.memory.test import snippet from pypy import conftest def compile_func(fn, inputtypes, t=None, gcpolicy="ref"): @@ -281,6 +282,12 @@ gcpolicy = "marksweep" should_be_moving = False + # interface for snippet.py + large_tests_ok = True + def run(self, func): + fn = self.getcompiled(func) + return fn() + def test_empty_collect(self): def f(): llop.gc__collect(lltype.Void) @@ -836,7 +843,7 @@ def test_weakref(self): py.test.skip("fails for some reason I couldn't figure out yet :-(") -class TestSemiSpaceGC(TestUsingFramework): +class TestSemiSpaceGC(TestUsingFramework, snippet.SemiSpaceGCTests): gcpolicy = "semispace" should_be_moving = True From arigo at codespeak.net Thu Feb 14 14:13:15 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 14 Feb 2008 14:13:15 +0100 (CET) Subject: [pypy-svn] r51485 - in pypy/dist/pypy: rpython/memory/gc rpython/memory/test translator/c/test Message-ID: <20080214131315.6200B16840E@codespeak.net> Author: arigo Date: Thu Feb 14 14:13:14 2008 New Revision: 51485 Added: pypy/dist/pypy/rpython/memory/test/snippet.py - copied unchanged from r51484, pypy/branch/finalizer-order/pypy/rpython/memory/test/snippet.py Modified: pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/gc/semispace.py pypy/dist/pypy/rpython/memory/test/test_gc.py pypy/dist/pypy/translator/c/test/test_newgc.py Log: (cfbolz, arigo) (Merge of branch/finalizer-order/pypy) The straightforward implementation of pypy/doc/discussion/finalizer-order.txt. Tests pass. Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Thu Feb 14 14:13:14 2008 @@ -1,6 +1,5 @@ import sys -from pypy.rpython.memory.gc.semispace import SemiSpaceGC, GCFLAGSHIFT, \ - GCFLAG_IMMORTAL +from pypy.rpython.memory.gc.semispace import SemiSpaceGC, GCFLAG_IMMORTAL from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE @@ -12,11 +11,11 @@ # in the nursery. It is initially set on all prebuilt and old objects, # and gets cleared by the write_barrier() when we write in them a # pointer to a young object. -GCFLAG_NO_YOUNG_PTRS = 1 << (GCFLAGSHIFT+1) +GCFLAG_NO_YOUNG_PTRS = SemiSpaceGC.first_unused_gcflag << 1 # The following flag is set for static roots which are not on the list # of static roots yet, but will appear with write barrier -GCFLAG_NO_HEAP_PTRS = 1 << (GCFLAGSHIFT+2) +GCFLAG_NO_HEAP_PTRS = SemiSpaceGC.first_unused_gcflag << 2 DEBUG_PRINT = False @@ -30,6 +29,7 @@ inline_simple_malloc_varsize = True needs_write_barrier = True prebuilt_gc_objects_are_static_roots = False + first_unused_gcflag = SemiSpaceGC.first_unused_gcflag << 3 def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, nursery_size=128, Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Thu Feb 14 14:13:14 2008 @@ -2,7 +2,7 @@ from pypy.rpython.lltypesystem.llmemory import raw_memcopy, raw_memclear from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE -from pypy.rpython.memory.support import get_address_stack +from pypy.rpython.memory.support import get_address_stack, get_address_deque from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rlib.objectmodel import free_non_gc_object @@ -14,8 +14,9 @@ import sys, os TYPEID_MASK = 0xffff -GCFLAGSHIFT = 16 -GCFLAG_IMMORTAL = 1 << GCFLAGSHIFT +first_gcflag = 1 << 16 +GCFLAG_IMMORTAL = first_gcflag +GCFLAG_FINALIZATION_ORDERING = first_gcflag << 1 memoryError = MemoryError() @@ -24,6 +25,7 @@ inline_simple_malloc = True inline_simple_malloc_varsize = True needs_zero_gc_pointers = False + first_unused_gcflag = first_gcflag << 2 HDR = lltype.Struct('header', ('forw', llmemory.Address), ('tid', lltype.Signed)) @@ -35,6 +37,7 @@ self.max_space_size = max_space_size self.gcheaderbuilder = GCHeaderBuilder(self.HDR) self.AddressStack = get_address_stack(chunk_size) + self.AddressDeque = get_address_deque(chunk_size) def setup(self): self.tospace = llarena.arena_malloc(self.space_size, True) @@ -43,8 +46,8 @@ self.fromspace = llarena.arena_malloc(self.space_size, True) ll_assert(bool(self.fromspace), "couldn't allocate fromspace") self.free = self.tospace - self.objects_with_finalizers = self.AddressStack() - self.run_finalizers = self.AddressStack() + self.objects_with_finalizers = self.AddressDeque() + self.run_finalizers = self.AddressDeque() self.objects_with_weakrefs = self.AddressStack() self.finalizer_lock_count = 0 self.red_zone = 0 @@ -189,12 +192,11 @@ self.top_of_space = tospace + self.space_size scan = self.free = tospace self.collect_roots() - scan = self.scan_copied(scan) if self.run_finalizers.non_empty(): self.update_run_finalizers() - if self.objects_with_finalizers.non_empty(): - self.deal_with_objects_with_finalizers() scan = self.scan_copied(scan) + if self.objects_with_finalizers.non_empty(): + scan = self.deal_with_objects_with_finalizers(scan) if self.objects_with_weakrefs.non_empty(): self.invalidate_weakrefs() self.notify_objects_just_moved() @@ -308,20 +310,105 @@ else: hdr.forw = NULL - def deal_with_objects_with_finalizers(self): + def deal_with_objects_with_finalizers(self, scan): # walk over list of objects with finalizers # if it is not copied, add it to the list of to-be-called finalizers # and copy it, to me make the finalizer runnable - # NOTE: the caller is calling scan_copied, so no need to do it here - new_with_finalizer = self.AddressStack() + # We try to run the finalizers in a "reasonable" order, like + # CPython does. The details of this algorithm are in + # pypy/doc/discussion/finalizer-order.txt. + new_with_finalizer = self.AddressDeque() + marked = self.AddressDeque() + pending = self.AddressStack() while self.objects_with_finalizers.non_empty(): - obj = self.objects_with_finalizers.pop() - if self.is_forwarded(obj): - new_with_finalizer.append(self.get_forwarding_address(obj)) + x = self.objects_with_finalizers.popleft() + ll_assert(self._finalization_state(x) != 1, + "bad finalization state 1") + if self.is_forwarded(x): + new_with_finalizer.append(self.get_forwarding_address(x)) + continue + marked.append(x) + pending.append(x) + while pending.non_empty(): + y = pending.pop() + state = self._finalization_state(y) + if state == 0: + self._bump_finalization_state_from_0_to_1(y) + elif state == 2: + self._bump_finalization_state_from_2_to_3(y) + else: + continue # don't need to recurse inside y + self.trace(y, self._append_if_nonnull, pending) + scan = self._recursively_bump_finalization_state_from_1_to_2( + x, scan) + + while marked.non_empty(): + x = marked.popleft() + state = self._finalization_state(x) + ll_assert(state >= 2, "unexpected finalization state < 2") + newx = self.get_forwarding_address(x) + if state == 2: + self.run_finalizers.append(newx) + # we must also fix the state from 2 to 3 here, otherwise + # we leave the GCFLAG_FINALIZATION_ORDERING bit behind + # which will confuse the next collection + pending.append(x) + while pending.non_empty(): + y = pending.pop() + state = self._finalization_state(y) + if state == 2: + self._bump_finalization_state_from_2_to_3(y) + self.trace(y, self._append_if_nonnull, pending) else: - self.run_finalizers.append(self.copy(obj)) + new_with_finalizer.append(newx) + + pending.delete() + marked.delete() self.objects_with_finalizers.delete() self.objects_with_finalizers = new_with_finalizer + return scan + + def _append_if_nonnull(pointer, stack): + if pointer.address[0] != NULL: + stack.append(pointer.address[0]) + _append_if_nonnull = staticmethod(_append_if_nonnull) + + def _finalization_state(self, obj): + if self.is_forwarded(obj): + newobj = self.get_forwarding_address(obj) + hdr = self.header(newobj) + if hdr.tid & GCFLAG_FINALIZATION_ORDERING: + return 2 + else: + return 3 + else: + hdr = self.header(obj) + if hdr.tid & GCFLAG_FINALIZATION_ORDERING: + return 1 + else: + return 0 + + def _bump_finalization_state_from_0_to_1(self, obj): + ll_assert(self._finalization_state(obj) == 0, + "unexpected finalization state != 0") + hdr = self.header(obj) + hdr.tid |= GCFLAG_FINALIZATION_ORDERING + + def _bump_finalization_state_from_2_to_3(self, obj): + ll_assert(self._finalization_state(obj) == 2, + "unexpected finalization state != 2") + newobj = self.get_forwarding_address(obj) + hdr = self.header(newobj) + hdr.tid &= ~GCFLAG_FINALIZATION_ORDERING + + def _recursively_bump_finalization_state_from_1_to_2(self, obj, scan): + # recursively convert objects from state 1 to state 2. + # Note that copy() copies all bits, including the + # GCFLAG_FINALIZATION_ORDERING. The mapping between + # state numbers and the presence of this bit was designed + # for the following to work :-) + self.copy(obj) + return self.scan_copied(scan) def invalidate_weakrefs(self): # walk over list of objects that contain weakrefs @@ -349,9 +436,9 @@ def update_run_finalizers(self): # we are in an inner collection, caused by a finalizer # the run_finalizers objects need to be copied - new_run_finalizer = self.AddressStack() + new_run_finalizer = self.AddressDeque() while self.run_finalizers.non_empty(): - obj = self.run_finalizers.pop() + obj = self.run_finalizers.popleft() new_run_finalizer.append(self.copy(obj)) self.run_finalizers.delete() self.run_finalizers = new_run_finalizer @@ -363,7 +450,7 @@ try: while self.run_finalizers.non_empty(): #print "finalizer" - obj = self.run_finalizers.pop() + obj = self.run_finalizers.popleft() finalizer = self.getfinalizer(self.get_type_id(obj)) finalizer(obj) finally: Modified: pypy/dist/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_gc.py Thu Feb 14 14:13:14 2008 @@ -3,8 +3,10 @@ #from pypy.rpython.memory.support import INT_SIZE from pypy.rpython.memory import gcwrapper +from pypy.rpython.memory.test import snippet from pypy.rpython.test.test_llinterp import get_interpreter from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem.rstr import STR from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.objectmodel import compute_unique_id @@ -35,6 +37,12 @@ self.GC_PARAMS) return interp.eval_graph(graph, values) + def run(self, func): # for snippet.py + res = self.interpret(func, []) + if lltype.typeOf(res) == lltype.Ptr(STR): + res = ''.join(res.chars) + return res + def test_llinterp_lists(self): #curr = simulator.current_size def malloc_a_lot(): @@ -384,108 +392,9 @@ class TestMarkSweepGC(GCTest): from pypy.rpython.memory.gc.marksweep import MarkSweepGC as GCClass -class TestSemiSpaceGC(GCTest): +class TestSemiSpaceGC(GCTest, snippet.SemiSpaceGCTests): from pypy.rpython.memory.gc.semispace import SemiSpaceGC as GCClass - def test_finalizer_order(self): - py.test.skip("in-progress") - import random - from pypy.tool.algo import graphlib - - examples = [] - letters = 'abcdefghijklmnopqrstuvwxyz' - for i in range(20): - input = [] - edges = {} - for c in letters: - edges[c] = [] - # make up a random graph - for c in letters: - for j in range(random.randrange(0, 4)): - d = random.choice(letters) - edges[c].append(graphlib.Edge(c, d)) - input.append((c, d)) - # find the expected order in which destructors should be called - components = list(graphlib.strong_components(edges, edges)) - head = {} - for component in components: - c = component.keys()[0] - for d in component: - assert d not in head - head[d] = c - assert len(head) == len(letters) - strict = [] - for c, d in input: - if head[c] != head[d]: - strict.append((c, d)) - examples.append((input, components, strict)) - - class State: - pass - state = State() - class A: - def __init__(self, key): - self.key = key - self.refs = [] - def __del__(self): - assert state.age[self.key] == -1 - state.age[self.key] = state.time - state.progress = True - - def build_example(input): - state.time = 0 - state.age = {} - vertices = {} - for c in letters: - vertices[c] = A(c) - state.age[c] = -1 - for c, d in input: - vertices[c].refs.append(d) - - def f(): - i = 0 - while i < len(examples): - input, components, strict = examples[i] - build_example(input) - while state.time < len(letters): - state.progress = False - llop.gc__collect(lltype.Void) - if not state.progress: - break - state.time += 1 - # check that all instances have been finalized - if -1 in state.age.values(): - return i * 10 + 1 - # check that if a -> b and a and b are not in the same - # strong component, then a is finalized strictly before b - for c, d in strict: - if state.age[c] >= state.age[d]: - return i * 10 + 2 - # check that two instances in the same strong component - # are never finalized during the same collection - for component in components: - seen = {} - for c in component: - age = state.age[c] - if age in seen: - return i * 10 + 3 - seen[age] = True - i += 1 - return 0 - - res = self.interpret(f, []) - if res != 0: - import pprint - pprint.pprint(examples[res / 10]) - if res % 10 == 1: - py.test.fail("some instances have not been finalized at all") - if res % 10 == 2: - py.test.fail("the strict order is not respected") - if res % 10 == 3: - py.test.fail("two instances from the same component " - "have been finalized together") - assert 0 - class TestGrowingSemiSpaceGC(TestSemiSpaceGC): GC_PARAMS = {'space_size': 64} Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Thu Feb 14 14:13:14 2008 @@ -9,6 +9,7 @@ from pypy.translator.c import genc, gc from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rpython.memory.test import snippet from pypy import conftest def compile_func(fn, inputtypes, t=None, gcpolicy="ref"): @@ -281,6 +282,12 @@ gcpolicy = "marksweep" should_be_moving = False + # interface for snippet.py + large_tests_ok = True + def run(self, func): + fn = self.getcompiled(func) + return fn() + def test_empty_collect(self): def f(): llop.gc__collect(lltype.Void) @@ -836,7 +843,7 @@ def test_weakref(self): py.test.skip("fails for some reason I couldn't figure out yet :-(") -class TestSemiSpaceGC(TestUsingFramework): +class TestSemiSpaceGC(TestUsingFramework, snippet.SemiSpaceGCTests): gcpolicy = "semispace" should_be_moving = True From arigo at codespeak.net Thu Feb 14 14:13:29 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 14 Feb 2008 14:13:29 +0100 (CET) Subject: [pypy-svn] r51486 - pypy/branch/finalizer-order Message-ID: <20080214131329.8C23616840E@codespeak.net> Author: arigo Date: Thu Feb 14 14:13:29 2008 New Revision: 51486 Removed: pypy/branch/finalizer-order/ Log: Branch already merged. From antocuni at codespeak.net Thu Feb 14 16:02:11 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 14 Feb 2008 16:02:11 +0100 (CET) Subject: [pypy-svn] r51489 - in pypy/dist/pypy/translator/cli: . test Message-ID: <20080214150211.3BCDB168029@codespeak.net> Author: antocuni Date: Thu Feb 14 16:02:10 2008 New Revision: 51489 Modified: pypy/dist/pypy/translator/cli/query.py pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: a test to show that bound delegates actually work Modified: pypy/dist/pypy/translator/cli/query.py ============================================================================== --- pypy/dist/pypy/translator/cli/query.py (original) +++ pypy/dist/pypy/translator/cli/query.py Thu Feb 14 16:02:10 2008 @@ -160,8 +160,7 @@ TYPE._assembly_qualified_name = self.AssemblyQualifiedName Class = CliClass(TYPE, {}, {}) self._cliclass = Class - # we need to check also for System.Array to prevent a circular recursion - if self.FullName in ('System.Object', 'System.Array'): + if self.FullName == 'System.Object': TYPE._set_superclass(ootype.ROOT) else: BASETYPE = get_ootype(self.BaseType) Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Thu Feb 14 16:02:10 2008 @@ -512,6 +512,27 @@ res = self.interpret(fn, []) assert res == 42 + def test_bound_delegate(self): + def build_fn(): + tObjArray = System.Type.GetType("System.Object[]") + tInt = typeof(System.Int32) + args = init_array(System.Type, tObjArray, tInt, tInt) + meth = Utils.CreateDynamicMethod("add", tInt, args) + il = meth.GetILGenerator() + il.Emit(OpCodes.Ldarg_1) + il.Emit(OpCodes.Ldarg_2) + il.Emit(OpCodes.Add) + il.Emit(OpCodes.Ret) + array = new_array(System.Object, 0) + myfunc = meth.CreateDelegate(typeof(FUNCTYPE), array) + return myfunc + + def fn(): + myfunc = clidowncast(build_fn(), FUNCTYPE) + return myfunc(30, 12) + res = self.interpret(fn, []) + assert res == 42 + def test_valuetype_field(self): class Foo: def __init__(self, x): From antocuni at codespeak.net Thu Feb 14 16:18:34 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 14 Feb 2008 16:18:34 +0100 (CET) Subject: [pypy-svn] r51491 - in pypy/dist/pypy/jit/codegen/cli: . test Message-ID: <20080214151834.48D3A168030@codespeak.net> Author: antocuni Date: Thu Feb 14 16:18:33 2008 New Revision: 51491 Modified: pypy/dist/pypy/jit/codegen/cli/operation.py pypy/dist/pypy/jit/codegen/cli/rgenop.py pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Log: test_hide_and_reveal_p finally works :-) Modified: pypy/dist/pypy/jit/codegen/cli/operation.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/operation.py (original) +++ pypy/dist/pypy/jit/codegen/cli/operation.py Thu Feb 14 16:18:33 2008 @@ -17,7 +17,7 @@ if self._gv_res is None: restype = self.restype() if restype is not None: - loc = self.il.DeclareLocal(restype) + loc = self.builder.il.DeclareLocal(restype) self._gv_res = GenLocalVar(loc) return self._gv_res @@ -28,38 +28,38 @@ raise NotImplementedError def storeResult(self): - self.gv_res().store(self.il) + self.gv_res().store(self.builder) class UnaryOp(Operation): - def __init__(self, il, gv_x): - self.il = il + def __init__(self, builder, gv_x): + self.builder = builder self.gv_x = gv_x def pushAllArgs(self): - self.gv_x.load(self.il) + self.gv_x.load(self.builder) def emit(self): self.pushAllArgs() - self.il.Emit(self.getOpCode()) + self.builder.il.Emit(self.getOpCode()) self.storeResult() def getOpCode(self): raise NotImplementedError class BinaryOp(Operation): - def __init__(self, il, gv_x, gv_y): - self.il = il + def __init__(self, builder, gv_x, gv_y): + self.builder = builder self.gv_x = gv_x self.gv_y = gv_y def pushAllArgs(self): - self.gv_x.load(self.il) - self.gv_y.load(self.il) + self.gv_x.load(self.builder) + self.gv_y.load(self.builder) def emit(self): self.pushAllArgs() - self.il.Emit(self.getOpCode()) + self.builder.il.Emit(self.getOpCode()) self.storeResult() def getOpCode(self): @@ -69,25 +69,25 @@ class SameAs(UnaryOp): def emit(self): gv_res = self.gv_res() - self.gv_x.load(self.il) - self.gv_res().store(self.il) + self.gv_x.load(self.builder) + self.gv_res().store(self.builder) class MarkLabel(Operation): - def __init__(self, il, label): - self.il = il + def __init__(self, builder, label): + self.builder = builder self.label = label def restype(self): return None def emit(self): - self.il.MarkLabel(self.label) + self.builder.il.MarkLabel(self.label) class FollowLink(Operation): - def __init__(self, il, outputargs_gv, inputargs_gv, label): - self.il = il + def __init__(self, builder, outputargs_gv, inputargs_gv, label): + self.builder = builder self.outputargs_gv = outputargs_gv self.inputargs_gv = inputargs_gv self.label = label @@ -97,14 +97,14 @@ def emit(self): for i in range(len(self.outputargs_gv)): - self.outputargs_gv[i].load(self.il) - self.inputargs_gv[i].store(self.il) - self.il.Emit(OpCodes.Br, self.label) + self.outputargs_gv[i].load(self.builder) + self.inputargs_gv[i].store(self.builder) + self.builder.il.Emit(OpCodes.Br, self.label) class Branch(Operation): - def __init__(self, il, gv_cond, opcode, label): - self.il = il + def __init__(self, builder, gv_cond, opcode, label): + self.builder = builder self.gv_cond = gv_cond self.opcode = opcode self.label = label @@ -114,27 +114,27 @@ def emit(self): if self.gv_cond is not None: - self.gv_cond.load(self.il) - self.il.Emit(self.opcode, self.label) + self.gv_cond.load(self.builder) + self.builder.il.Emit(self.opcode, self.label) class Return(Operation): - def __init__(self, il, gv_x): - self.il = il + def __init__(self, builder, gv_x): + self.builder = builder self.gv_x = gv_x def restype(self): return None def emit(self): - self.gv_x.load(self.il) - self.il.Emit(OpCodes.Ret) + self.gv_x.load(self.builder) + self.builder.il.Emit(OpCodes.Ret) class Call(Operation): - def __init__(self, il, sigtoken, gv_fnptr, args_gv): + def __init__(self, builder, sigtoken, gv_fnptr, args_gv): from pypy.jit.codegen.cli.rgenop import token2clitype - self.il = il + self.builder = builder self.sigtoken = sigtoken self.gv_fnptr = gv_fnptr self.args_gv = args_gv @@ -147,11 +147,11 @@ from pypy.jit.codegen.cli.rgenop import sigtoken2clitype delegate_type = sigtoken2clitype(self.sigtoken) meth_invoke = delegate_type.GetMethod('Invoke') - self.gv_fnptr.load(self.il) - self.il.Emit(OpCodes.Castclass, delegate_type) + self.gv_fnptr.load(self.builder) + self.builder.il.Emit(OpCodes.Castclass, delegate_type) for gv_arg in self.args_gv: - gv_arg.load(self.il) - self.il.EmitCall(OpCodes.Callvirt, meth_invoke, None) + gv_arg.load(self.builder) + self.builder.il.EmitCall(OpCodes.Callvirt, meth_invoke, None) self.storeResult() @@ -218,7 +218,7 @@ if 'call' in step: return # XXX, fix this attrname = opcode2attrname(step) - body.append('self.il.Emit(OpCodes.%s)' % attrname) + body.append('self.builder.il.Emit(OpCodes.%s)' % attrname) elif isinstance(step, cli_opcodes.MapException): return # XXX, TODO else: Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/rgenop.py Thu Feb 14 16:18:33 2008 @@ -43,10 +43,10 @@ def getCliType(self): raise NotImplementedError - def load(self, il): + def load(self, builder): raise NotImplementedError - def store(self, il): + def store(self, builder): raise NotImplementedError class GenArgVar(GenVar): @@ -57,20 +57,20 @@ def getCliType(self): return self.cliType - def load(self, il): + def load(self, builder): if self.index == 0: - il.Emit(OpCodes.Ldarg_0) + builder.il.Emit(OpCodes.Ldarg_0) elif self.index == 1: - il.Emit(OpCodes.Ldarg_1) + builder.il.Emit(OpCodes.Ldarg_1) elif self.index == 2: - il.Emit(OpCodes.Ldarg_2) + builder.il.Emit(OpCodes.Ldarg_2) elif self.index == 3: - il.Emit(OpCodes.Ldarg_3) + builder.il.Emit(OpCodes.Ldarg_3) else: - il.Emit(OpCodes.Ldarg, self.index) + builder.il.Emit(OpCodes.Ldarg, self.index) - def store(self, il): - il.Emit(OpCodes.Starg, self.index) + def store(self, builder): + builder.il.Emit(OpCodes.Starg, self.index) def __repr__(self): return "GenArgVar(%d)" % self.index @@ -82,11 +82,11 @@ def getCliType(self): return self.v.get_LocalType() - def load(self, il): - il.Emit(OpCodes.Ldloc, self.v) + def load(self, builder): + builder.il.Emit(OpCodes.Ldloc, self.v) - def store(self, il): - il.Emit(OpCodes.Stloc, self.v) + def store(self, builder): + builder.il.Emit(OpCodes.Stloc, self.v) class IntConst(GenConst): @@ -106,13 +106,14 @@ def getCliType(self): return typeof(System.Int32) - def load(self, il): - il.Emit(OpCodes.Ldc_I4, self.value) + def load(self, builder): + builder.il.Emit(OpCodes.Ldc_I4, self.value) def __repr__(self): return "const=%s" % self.value -class BaseConst(GenConst): +class FunctionConst(GenConst): + def __init__(self, num): self.num = num self.fieldname = "const" + str(num) @@ -125,19 +126,33 @@ t = typeof(Constants) t.GetField(self.fieldname).SetValue(None, obj) - def load(self, il): + def load(self, builder): t = typeof(Constants) field = t.GetField(self.fieldname) - il.Emit(OpCodes.Ldsfld, field) - + builder.il.Emit(OpCodes.Ldsfld, field) -class FunctionConst(BaseConst): - @specialize.arg(1) def revealconst(self, T): return clidowncast(self.getobj(), T) -class ObjectConst(BaseConst): +class ObjectConst(GenConst): + + def __init__(self, obj): + self.obj = obj + + def load(self, builder): + # check whether there is already an index associated to this const + try: + index = builder.genconsts[self.obj] + except KeyError: + index = len(builder.genconsts) + builder.genconsts[self.obj] = index + + t = self.obj.GetType() + builder.il.Emit(OpCodes.Ldarg_0) + builder.il.Emit(OpCodes.Ldc_i4, index) + builder.il.Emit(OpCodes.Ldelem_ref) + builder.il.Emit(OpCodes.Castclass, t) @specialize.arg(1) def revealconst(self, T): @@ -172,9 +187,7 @@ elif T is ootype.Bool: return IntConst(int(llvalue)) elif isinstance(T, ootype.OOType): - const = self.newconst(ObjectConst) - const.setobj(llvalue) - return const + return ObjectConst(llvalue) else: assert False, "XXX not implemented" @@ -195,9 +208,10 @@ def newgraph(self, sigtoken, name): argtoks, restok = sigtoken - args = new_array(System.Type, len(argtoks)) + args = new_array(System.Type, len(argtoks)+1) + args[0] = System.Type.GetType("System.Object[]") for i in range(len(argtoks)): - args[i] = token2clitype(argtoks[i]) + args[i+1] = token2clitype(argtoks[i]) res = token2clitype(restok) builder = Builder(self, name, res, args, sigtoken) return builder, builder.gv_entrypoint, builder.inputargs_gv[:] @@ -212,7 +226,8 @@ if DUMP_IL: self.il = DumpGenerator(self.il) self.inputargs_gv = [] - for i in range(len(args)): + # we start from 1 because the 1st arg is an Object[] containing the genconsts + for i in range(1, len(args)): self.inputargs_gv.append(GenArgVar(i, args[i])) self.gv_entrypoint = self.rgenop.newconst(FunctionConst) self.sigtoken = sigtoken @@ -220,11 +235,12 @@ self.operations = [] self.branches = [] self.returnblocks = [] + self.genconsts = {} @specialize.arg(1) def genop1(self, opname, gv_arg): opcls = ops.getopclass1(opname) - op = opcls(self.il, gv_arg) + op = opcls(self, gv_arg) self.emit(op) gv_res = op.gv_res() if DEBUG: @@ -238,7 +254,7 @@ @specialize.arg(1) def genop2(self, opname, gv_arg1, gv_arg2): opcls = ops.getopclass2(opname) - op = opcls(self.il, gv_arg1, gv_arg2) + op = opcls(self, gv_arg1, gv_arg2) self.emit(op) gv_res = op.gv_res() if DEBUG: @@ -258,12 +274,12 @@ assert False def genop_call(self, sigtoken, gv_fnptr, args_gv): - op = ops.Call(self.il, sigtoken, gv_fnptr, args_gv) + op = ops.Call(self, sigtoken, gv_fnptr, args_gv) self.emit(op) return op.gv_res() def genop_same_as(self, kindtoken, gv_x): - op = ops.SameAs(self.il, gv_x) + op = ops.SameAs(self, gv_x) self.emit(op) return op.gv_res() @@ -281,7 +297,7 @@ def finish_and_return(self, sigtoken, gv_returnvar): retlabel = self.il.DefineLabel() - op = ops.Branch(self.il, None, OpCodes.Br, retlabel) + op = ops.Branch(self, None, OpCodes.Br, retlabel) self.emit(op) self.appendreturn(retlabel, gv_returnvar) self.isOpen = False @@ -289,7 +305,7 @@ def finish_and_goto(self, outputargs_gv, target): inputargs_gv = target.inputargs_gv assert len(inputargs_gv) == len(outputargs_gv) - op = ops.FollowLink(self.il, outputargs_gv, inputargs_gv, target.label) + op = ops.FollowLink(self, outputargs_gv, inputargs_gv, target.label) self.emit(op) self.isOpen = False @@ -301,26 +317,30 @@ # render the return blocks for last, else the verifier could complain for retlabel, gv_returnvar in self.returnblocks: self.il.MarkLabel(retlabel) - op = ops.Return(self.il, gv_returnvar) + op = ops.Return(self, gv_returnvar) self.emit(op) + # initialize the array of genconsts + consts = new_array(System.Object, len(self.genconsts)) + for obj, i in self.genconsts.iteritems(): + consts[i] = obj # build the delegate delegate_type = sigtoken2clitype(self.sigtoken) - myfunc = self.meth.CreateDelegate(delegate_type) + myfunc = self.meth.CreateDelegate(delegate_type, consts) self.gv_entrypoint.setobj(myfunc) def enter_next_block(self, kinds, args_gv): for i in range(len(args_gv)): - op = ops.SameAs(self.il, args_gv[i]) + op = ops.SameAs(self, args_gv[i]) op.emit() args_gv[i] = op.gv_res() label = self.il.DefineLabel() - self.emit(ops.MarkLabel(self.il, label)) + self.emit(ops.MarkLabel(self, label)) return Label(label, args_gv) def _jump_if(self, gv_condition, opcode): label = self.il.DefineLabel() - op = ops.Branch(self.il, gv_condition, opcode, label) + op = ops.Branch(self, gv_condition, opcode, label) self.emit(op) branch = BranchBuilder(self, label) self.appendbranch(branch) Modified: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Thu Feb 14 16:18:33 2008 @@ -10,7 +10,7 @@ 'test_adder', 'test_dummy', 'test_hide_and_reveal', - # 'test_hide_and_reveal_p', # think about this + 'test_hide_and_reveal_p', 'test_largedummy_direct', # _compile works if we set a higher maxstack 'test_branching', 'test_goto', From arigo at codespeak.net Thu Feb 14 17:06:51 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 14 Feb 2008 17:06:51 +0100 (CET) Subject: [pypy-svn] r51492 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20080214160651.ACBC0168425@codespeak.net> Author: arigo Date: Thu Feb 14 17:06:49 2008 New Revision: 51492 Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py Log: Add a DEBUG_PRINT flag to semispace.py. For now it's just a global flag to change, not a nice translation option... Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Thu Feb 14 17:06:49 2008 @@ -18,6 +18,7 @@ GCFLAG_IMMORTAL = first_gcflag GCFLAG_FINALIZATION_ORDERING = first_gcflag << 1 +DEBUG_PRINT = False memoryError = MemoryError() class SemiSpaceGC(MovingGCBase): @@ -26,6 +27,8 @@ inline_simple_malloc_varsize = True needs_zero_gc_pointers = False first_unused_gcflag = first_gcflag << 2 + total_collection_time = 0.0 + total_collection_count = 0 HDR = lltype.Struct('header', ('forw', llmemory.Address), ('tid', lltype.Signed)) @@ -40,6 +43,9 @@ self.AddressDeque = get_address_deque(chunk_size) def setup(self): + if DEBUG_PRINT: + import time + self.program_start_time = time.time() self.tospace = llarena.arena_malloc(self.space_size, True) ll_assert(bool(self.tospace), "couldn't allocate tospace") self.top_of_space = self.tospace + self.space_size @@ -184,6 +190,16 @@ # to by the gc transformer, and the default argument would crash def semispace_collect(self, size_changing=False): + if DEBUG_PRINT: + import time + llop.debug_print(lltype.Void) + llop.debug_print(lltype.Void, + ".----------- Full collection ------------------") + start_usage = self.free - self.tospace + llop.debug_print(lltype.Void, + "| used before collection: ", + start_usage, "bytes") + start_time = time.time() #llop.debug_print(lltype.Void, 'semispace_collect', int(size_changing)) tospace = self.fromspace fromspace = self.tospace @@ -205,6 +221,41 @@ self.record_red_zone() self.execute_finalizers() #llop.debug_print(lltype.Void, 'collected', self.space_size, size_changing, self.top_of_space - self.free) + if DEBUG_PRINT: + end_time = time.time() + elapsed_time = end_time - start_time + self.total_collection_time += elapsed_time + self.total_collection_count += 1 + total_program_time = end_time - self.program_start_time + end_usage = self.free - self.tospace + llop.debug_print(lltype.Void, + "| used after collection: ", + end_usage, "bytes") + llop.debug_print(lltype.Void, + "| freed: ", + start_usage - end_usage, "bytes") + llop.debug_print(lltype.Void, + "| size of each semispace: ", + self.space_size, "bytes") + llop.debug_print(lltype.Void, + "| fraction of semispace now used: ", + end_usage * 100.0 / self.space_size, "%") + ct = self.total_collection_time + cc = self.total_collection_count + llop.debug_print(lltype.Void, + "| number of semispace_collects: ", + cc) + llop.debug_print(lltype.Void, + "| i.e.: ", + cc / total_program_time, "per second") + llop.debug_print(lltype.Void, + "| total time in semispace_collect: ", + ct, "seconds") + llop.debug_print(lltype.Void, + "| i.e.: ", + ct * 100.0 / total_program_time, "%") + llop.debug_print(lltype.Void, + "`----------------------------------------------") def record_red_zone(self): # red zone: if the space is more than 80% full, the next collection From pypy-svn at codespeak.net Thu Feb 14 20:31:37 2008 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Thu, 14 Feb 2008 20:31:37 +0100 (CET) Subject: [pypy-svn] February 71% OFF Message-ID: <20080214093126.13376.qmail@c134-234.icpnet.pl> An HTML attachment was scrubbed... URL: From antocuni at codespeak.net Thu Feb 14 20:53:11 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 14 Feb 2008 20:53:11 +0100 (CET) Subject: [pypy-svn] r51493 - in pypy/dist/pypy/translator/cli: . test Message-ID: <20080214195311.9C8E9168458@codespeak.net> Author: antocuni Date: Thu Feb 14 20:53:08 2008 New Revision: 51493 Modified: pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: add support to box/unbox lowlevel instances and records; mostly useful to cast everything to System.Object and back Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Thu Feb 14 20:53:08 2008 @@ -386,6 +386,12 @@ else: return None + if isinstance(TYPE, ootype.OOType) and TYPE is not ootype.String: + try: + return ootype.enforce(TYPE, x) + except TypeError: + return None + # TODO: do the typechecking also in the other cases # this is a workaround against a pythonnet limitation: you can't @@ -409,7 +415,7 @@ hop.exception_cannot_occur() TYPE = v_obj.concretetype - if (TYPE is ootype.String or isinstance(TYPE, (ootype.Instance, ootype.BuiltinType, NativeInstance))): + if (TYPE is ootype.String or isinstance(TYPE, (ootype.OOType, NativeInstance))): return hop.genop('ooupcast', [v_obj], hop.r_result.lowleveltype) else: if TYPE not in BOXABLE_TYPES: @@ -430,6 +436,8 @@ # can_be_None == True because it can always return None, if it fails classdef = self.bookkeeper.getuniqueclassdef(TYPE) return SomeInstance(classdef, can_be_None=True) + elif isinstance(TYPE, ootype.OOType): + return SomeOOInstance(TYPE) else: assert TYPE in BOXABLE_TYPES return OverloadingResolver.lltype_to_annotation(TYPE) @@ -437,7 +445,7 @@ def specialize_call(self, hop): TYPE = hop.args_v[1].value v_obj = hop.inputarg(hop.args_r[0], arg=0) - if TYPE is ootype.String or isinstance(TYPE, (type, types.ClassType)): + if TYPE is ootype.String or isinstance(TYPE, (type, types.ClassType)) or isinstance(TYPE, ootype.OOType): return hop.genop('oodowncast', [v_obj], hop.r_result.lowleveltype) else: c_type = hop.inputconst(ootype.Void, TYPE) Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Thu Feb 14 20:53:08 2008 @@ -431,6 +431,37 @@ res = self.interpret(fn, []) assert res is None + def test_box_unbox_ooinstance(self): + A = ootype.Instance('A', ootype.ROOT, {'xx': ootype.Signed}) + def fn(flag): + a = ootype.new(A) + a.xx = 42 + b_obj = box(a) + a2 = unbox(b_obj, A) + return a2.xx + res = self.interpret(fn, [True]) + assert res == 42 + + def test_box_unbox_ooinstance_fail(self): + A = ootype.Instance('A', ootype.ROOT, {'xx': ootype.Signed}) + def fn(flag): + b_obj = System.Object() + a2 = unbox(b_obj, A) + return a2 + res = self.interpret(fn, [True]) + assert res is None + + def test_box_unbox_oorecord(self): + A = ootype.Record({'xx': ootype.Signed}) + def fn(flag): + a = ootype.new(A) + a.xx = 42 + b_obj = box(a) + a2 = unbox(b_obj, A) + return a2.xx + res = self.interpret(fn, [True]) + assert res == 42 + def test_instance_wrapping(self): class Foo: pass From pypy-svn at codespeak.net Fri Feb 15 08:53:42 2008 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Fri, 15 Feb 2008 08:53:42 +0100 (CET) Subject: [pypy-svn] SALE 79% OFF Message-ID: <20080215135314.63513.qmail@s3d7a331d.oct-net.ne.jp> An HTML attachment was scrubbed... URL: From fijal at codespeak.net Fri Feb 15 09:51:57 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 09:51:57 +0100 (CET) Subject: [pypy-svn] r51503 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080215085157.71DDD16846A@codespeak.net> Author: fijal Date: Fri Feb 15 09:51:54 2008 New Revision: 51503 Modified: pypy/dist/pypy/lib/_ctypes/array.py pypy/dist/pypy/lib/_ctypes/basics.py pypy/dist/pypy/lib/_ctypes/pointer.py pypy/dist/pypy/lib/_ctypes/primitive.py pypy/dist/pypy/lib/_ctypes/structure.py Log: * Remove buggy keepalives. * Rename __del__ to delete. Let's first figure out exact rules. Modified: pypy/dist/pypy/lib/_ctypes/array.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/array.py (original) +++ pypy/dist/pypy/lib/_ctypes/array.py Fri Feb 15 09:51:54 2008 @@ -110,7 +110,6 @@ def __init__(self, *args): self._buffer = self._ffiarray(self._length_) self._needs_free = True - self._objects = [] for i, arg in enumerate(args): self[i] = arg @@ -136,7 +135,6 @@ if isinstance(index, slice): self._slice_setitem(index, value) return - self._objects.append(value) # keepalive value value = self._type_._CData_input(value) index = self._fix_index(index) if not isinstance(self._type_._ffishape, tuple): @@ -160,7 +158,7 @@ def _get_buffer_for_param(self): return self._buffer.byptr() - def __del__(self): + def delete(self): if self._needs_free: self._buffer.free() self._buffer = None Modified: pypy/dist/pypy/lib/_ctypes/basics.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/basics.py (original) +++ pypy/dist/pypy/lib/_ctypes/basics.py Fri Feb 15 09:51:54 2008 @@ -99,7 +99,6 @@ # fix the address, in case it's unsigned address = address & (sys.maxint * 2 + 1) instance = self.__new__(self) - instance._objects = [] lgt = getattr(self, '_length_', 1) instance._buffer = self._ffiarray.fromaddress(address, lgt) return instance Modified: pypy/dist/pypy/lib/_ctypes/pointer.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/pointer.py (original) +++ pypy/dist/pypy/lib/_ctypes/pointer.py Fri Feb 15 09:51:54 2008 @@ -56,7 +56,6 @@ def __init__(self, value=None): self._buffer = ffiarray(1) self._needs_free = True - self._objects = [value] # keepalive value if value is not None: self.contents = value self._ffiarray = ffiarray @@ -80,7 +79,6 @@ raise TypeError("expected %s instead of %s" % ( self._type_.__name__, type(value).__name__)) value = value._buffer - self._objects = [value] self._buffer[0] = value _get_slice_params = array_get_slice_params @@ -99,13 +97,12 @@ return self._type_._CData_output(self._subarray(index)) def __setitem__(self, index, value): - self._objects = [value] self._subarray(index)[0] = self._type_._CData_input(value)[0] def __nonzero__(self): return self._buffer[0] != 0 - def __del__(self): + def delete(self): if self._needs_free: self._buffer.free() self._needs_free = False Modified: pypy/dist/pypy/lib/_ctypes/primitive.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/primitive.py (original) +++ pypy/dist/pypy/lib/_ctypes/primitive.py Fri Feb 15 09:51:54 2008 @@ -219,7 +219,7 @@ def __nonzero__(self): return self._buffer[0] not in (0, '\x00') - def __del__(self): + def delete(self): if self._needs_free: self._needs_free = False self._buffer.free() Modified: pypy/dist/pypy/lib/_ctypes/structure.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/structure.py (original) +++ pypy/dist/pypy/lib/_ctypes/structure.py Fri Feb 15 09:51:54 2008 @@ -176,7 +176,7 @@ def _get_buffer_for_param(self): return self._buffer.byptr() - def __del__(self): + def delete(self): if self._needs_free: self._buffer.free() self.__dict__['_buffer'] = None From pedronis at codespeak.net Fri Feb 15 10:03:36 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 15 Feb 2008 10:03:36 +0100 (CET) Subject: [pypy-svn] r51504 - pypy/dist/pypy/rlib Message-ID: <20080215090336.7835616846A@codespeak.net> Author: pedronis Date: Fri Feb 15 10:03:35 2008 New Revision: 51504 Modified: pypy/dist/pypy/rlib/libffi.py Log: userdata needs to be raw allocated too, it stored away by ffi in places a movable gc cannot see Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Fri Feb 15 10:03:35 2008 @@ -176,6 +176,17 @@ buf[0] = arg push_arg_as_ffiptr._annspecialcase_ = 'specialize:argtype(1)' + +# type defs for callback and closure userdata +USERDATA_P = lltype.Ptr(lltype.ForwardReference()) +CALLBACK_TP = lltype.Ptr(lltype.FuncType([rffi.VOIDPP, rffi.VOIDP, USERDATA_P], + lltype.Void)) +USERDATA_P.TO.become(lltype.Struct('userdata', + ('callback', CALLBACK_TP), + ('addarg', rffi.INT), + hints={'callback':True})) + + def ll_callback(ffi_cif, ll_res, ll_args, ll_userdata): """ Callback specification. ffi_cif - something ffi specific, don't care @@ -214,24 +225,18 @@ lltype.free(self.ll_argtypes, flavor='raw') self.ll_argtypes = lltype.nullptr(FFI_TYPE_PP.TO) -USERDATA_P = lltype.Ptr(lltype.GcForwardReference()) -CALLBACK_TP = lltype.Ptr(lltype.FuncType([rffi.VOIDPP, rffi.VOIDP, USERDATA_P], - lltype.Void)) -USERDATA_P.TO.become(lltype.GcStruct('userdata', - ('callback', CALLBACK_TP), - ('addarg', rffi.INT), - hints={'callback':True})) - # as long as CallbackFuncPtr is kept alive, the underlaying userdata # is kept alive as well class CallbackFuncPtr(AbstractFuncPtr): ll_closure = lltype.nullptr(FFI_CLOSUREP.TO) ll_userdata = lltype.nullptr(USERDATA_P.TO) + # additional_arg should really be a non-heap type like a integer, + # it cannot be any kind of movable gc reference def __init__(self, argtypes, restype, func, additional_arg=0): AbstractFuncPtr.__init__(self, "callback", argtypes, restype) self.ll_closure = lltype.malloc(FFI_CLOSUREP.TO, flavor='raw') - self.ll_userdata = lltype.malloc(USERDATA_P.TO) + self.ll_userdata = lltype.malloc(USERDATA_P.TO, flavor='raw') self.ll_userdata.callback = rffi.llhelper(CALLBACK_TP, func) self.ll_userdata.addarg = additional_arg res = c_ffi_prep_closure(self.ll_closure, self.ll_cif, @@ -245,8 +250,8 @@ if self.ll_closure: lltype.free(self.ll_closure, flavor='raw') self.ll_closure = lltype.nullptr(FFI_CLOSUREP.TO) - # note that ll_userdata is a GC object and therefore does not need to - # be explicitely freed + lltype.free(self.ll_userdata, flavor='raw') + self.ll_userdata = lltype.nullptr(USERDATA_P.TO) class RawFuncPtr(AbstractFuncPtr): From fijal at codespeak.net Fri Feb 15 10:11:55 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 10:11:55 +0100 (CET) Subject: [pypy-svn] r51505 - pypy/dist/pypy/rlib Message-ID: <20080215091155.45BB0168465@codespeak.net> Author: fijal Date: Fri Feb 15 10:11:54 2008 New Revision: 51505 Modified: pypy/dist/pypy/rlib/libffi.py Log: Super-paranoia as usual Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Fri Feb 15 10:11:54 2008 @@ -250,6 +250,7 @@ if self.ll_closure: lltype.free(self.ll_closure, flavor='raw') self.ll_closure = lltype.nullptr(FFI_CLOSUREP.TO) + if self.ll_userdata: lltype.free(self.ll_userdata, flavor='raw') self.ll_userdata = lltype.nullptr(USERDATA_P.TO) From fijal at codespeak.net Fri Feb 15 10:20:42 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 10:20:42 +0100 (CET) Subject: [pypy-svn] r51506 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080215092042.0A92A168465@codespeak.net> Author: fijal Date: Fri Feb 15 10:20:42 2008 New Revision: 51506 Modified: pypy/dist/pypy/lib/app_test/ctypes/support.py Log: Disable reference check. Modified: pypy/dist/pypy/lib/app_test/ctypes/support.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/support.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/support.py Fri Feb 15 10:20:42 2008 @@ -9,6 +9,7 @@ cls.old_num = _rawffi._num_of_allocated_objects() def teardown_class(cls): + return try: import _rawffi except ImportError: From fijal at codespeak.net Fri Feb 15 10:21:08 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 10:21:08 +0100 (CET) Subject: [pypy-svn] r51507 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080215092108.3B019168465@codespeak.net> Author: fijal Date: Fri Feb 15 10:21:07 2008 New Revision: 51507 Modified: pypy/dist/pypy/lib/_ctypes/array.py pypy/dist/pypy/lib/_ctypes/basics.py pypy/dist/pypy/lib/_ctypes/pointer.py pypy/dist/pypy/lib/_ctypes/structure.py Log: Introduce _base and _index, following b_base and b_index in ctypes. Modified: pypy/dist/pypy/lib/_ctypes/array.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/array.py (original) +++ pypy/dist/pypy/lib/_ctypes/array.py Fri Feb 15 10:21:07 2008 @@ -150,7 +150,7 @@ if isinstance(index, slice): return self._slice_getitem(index) index = self._fix_index(index) - return self._type_._CData_output(self._subarray(index)) + return self._type_._CData_output(self._subarray(index), self, index) def __len__(self): return self._length_ Modified: pypy/dist/pypy/lib/_ctypes/basics.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/basics.py (original) +++ pypy/dist/pypy/lib/_ctypes/basics.py Fri Feb 15 10:21:07 2008 @@ -17,16 +17,18 @@ else: return self.from_param(as_parameter) - def _CData_input(self, value): + def _CData_input(self, value, base=None, index=-1): """Used when data enters into ctypes from user code. 'value' is some user-specified Python object, which is converted into a _rawffi array of length 1 containing the same value according to the type 'self'. """ cobj = self.from_param(value) + cobj.__dict__['_base'] = base + cobj.__dict__['_index'] = index return cobj._get_buffer_for_param() - def _CData_output(self, resarray): + def _CData_output(self, resarray, base=None, index=-1): assert isinstance(resarray, _rawffi.ArrayInstance) """Used when data exits ctypes and goes into user code. 'resarray' is a _rawffi array of length 1 containing the value, @@ -34,6 +36,8 @@ """ res = self.__new__(self) res.__dict__['_buffer'] = resarray + res.__dict__['_base'] = base + res.__dict__['_index'] = index return res.__ctypes_from_outparam__() def __mul__(self, other): Modified: pypy/dist/pypy/lib/_ctypes/pointer.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/pointer.py (original) +++ pypy/dist/pypy/lib/_ctypes/pointer.py Fri Feb 15 10:21:07 2008 @@ -94,10 +94,11 @@ def __getitem__(self, index): if isinstance(index, slice): return self._slice_getitem(index) - return self._type_._CData_output(self._subarray(index)) + return self._type_._CData_output(self._subarray(index), self, index) def __setitem__(self, index, value): - self._subarray(index)[0] = self._type_._CData_input(value)[0] + self._subarray(index)[0] = self._type_._CData_input(value, self, + index)[0] def __nonzero__(self): return self._buffer[0] != 0 Modified: pypy/dist/pypy/lib/_ctypes/structure.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/structure.py (original) +++ pypy/dist/pypy/lib/_ctypes/structure.py Fri Feb 15 10:21:07 2008 @@ -138,10 +138,12 @@ def _alignmentofinstances(self): return self._ffistruct.alignment - def _CData_output(self, resarray): + def _CData_output(self, resarray, base=None, index=-1): res = self.__new__(self) ffistruct = self._ffistruct.fromaddress(resarray.buffer) res.__dict__['_buffer'] = ffistruct + res.__dict__['_base'] = base + res.__dict__['_index'] = index return res.__ctypes_from_outparam__() class Structure(_CData): @@ -171,7 +173,8 @@ fieldtype = self._fieldtypes[name].ctype except KeyError: return _CData.__getattribute__(self, name) - return fieldtype._CData_output(self._subarray(fieldtype, name)) + return fieldtype._CData_output(self._subarray(fieldtype, name), self, + getattr(self.__class__, name).offset) def _get_buffer_for_param(self): return self._buffer.byptr() From antocuni at codespeak.net Fri Feb 15 10:39:21 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 15 Feb 2008 10:39:21 +0100 (CET) Subject: [pypy-svn] r51511 - in pypy/dist/pypy/translator/cli: . test Message-ID: <20080215093921.9BF64168464@codespeak.net> Author: antocuni Date: Fri Feb 15 10:39:20 2008 New Revision: 51511 Modified: pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: use unbox() instead of clidowncast() to convert delegates to StaticMethods; now box/unbox are used to (almost) consistently convert from the OOType hierarchy to the CLI hierarchy and back. Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Fri Feb 15 10:39:20 2008 @@ -386,7 +386,7 @@ else: return None - if isinstance(TYPE, ootype.OOType) and TYPE is not ootype.String: + if isinstance(TYPE, ootype.OOType) and TYPE is not ootype.String and not isinstance(TYPE, ootype.StaticMethod): try: return ootype.enforce(TYPE, x) except TypeError: @@ -428,7 +428,7 @@ def compute_result_annotation(self, x_s, type_s): assert isinstance(x_s, SomeOOInstance) - assert x_s.ootype == CLR.System.Object._INSTANCE + assert isinstance(x_s.ootype, NativeInstance) assert type_s.is_constant() TYPE = type_s.const if isinstance(TYPE, (type, types.ClassType)): @@ -436,6 +436,8 @@ # can_be_None == True because it can always return None, if it fails classdef = self.bookkeeper.getuniqueclassdef(TYPE) return SomeInstance(classdef, can_be_None=True) + elif isinstance(TYPE, ootype.StaticMethod): + return SomeOOStaticMeth(TYPE) elif isinstance(TYPE, ootype.OOType): return SomeOOInstance(TYPE) else: @@ -596,12 +598,8 @@ else: cliClass = s_type.const TYPE = cliClass._INSTANCE - if isinstance(TYPE, ootype.StaticMethod): - assert ootype.isSubclass(s_value.ootype, CLR.System.Object._INSTANCE) - return SomeOOStaticMeth(TYPE) - else: - assert ootype.isSubclass(TYPE, s_value.ootype) - return SomeOOInstance(TYPE) + assert ootype.isSubclass(TYPE, s_value.ootype) + return SomeOOInstance(TYPE) def specialize_call(self, hop): assert isinstance(hop.args_s[0], annmodel.SomeOOInstance) Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Fri Feb 15 10:39:20 2008 @@ -538,7 +538,7 @@ return myfunc def fn(): - myfunc = clidowncast(build_fn(), FUNCTYPE) + myfunc = unbox(build_fn(), FUNCTYPE) return myfunc(30, 12) res = self.interpret(fn, []) assert res == 42 @@ -559,7 +559,7 @@ return myfunc def fn(): - myfunc = clidowncast(build_fn(), FUNCTYPE) + myfunc = unbox(build_fn(), FUNCTYPE) return myfunc(30, 12) res = self.interpret(fn, []) assert res == 42 From fijal at codespeak.net Fri Feb 15 10:43:29 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 10:43:29 +0100 (CET) Subject: [pypy-svn] r51513 - in pypy/dist/pypy/lib: _ctypes app_test/ctypes Message-ID: <20080215094329.A5C9F168471@codespeak.net> Author: fijal Date: Fri Feb 15 10:43:29 2008 New Revision: 51513 Added: pypy/dist/pypy/lib/app_test/ctypes/test_base.py (contents, props changed) Modified: pypy/dist/pypy/lib/_ctypes/array.py Log: Test and a fix. Modified: pypy/dist/pypy/lib/_ctypes/array.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/array.py (original) +++ pypy/dist/pypy/lib/_ctypes/array.py Fri Feb 15 10:43:29 2008 @@ -150,7 +150,8 @@ if isinstance(index, slice): return self._slice_getitem(index) index = self._fix_index(index) - return self._type_._CData_output(self._subarray(index), self, index) + return self._type_._CData_output(self._subarray(index), self, + self._ffiarray.gettypecode(index)[0]) def __len__(self): return self._length_ Added: pypy/dist/pypy/lib/app_test/ctypes/test_base.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lib/app_test/ctypes/test_base.py Fri Feb 15 10:43:29 2008 @@ -0,0 +1,23 @@ + +from ctypes import * + +class TestCTypesBase: + def test_pointer(self): + p = pointer(pointer(c_int(2))) + x = p[0] + assert x._base is p + + def test_structure(self): + class X(Structure): + _fields_ = [('x', POINTER(c_int)), + ('y', POINTER(c_int))] + + x = X() + assert x.y._base is x + assert x.y._index == sizeof(POINTER(c_int)) + + def test_array(self): + X = POINTER(c_int) * 24 + x = X() + assert x[16]._base is x + assert x[16]._index == 16 * sizeof(POINTER(c_int)) From fijal at codespeak.net Fri Feb 15 11:03:59 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 11:03:59 +0100 (CET) Subject: [pypy-svn] r51514 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080215100359.E7DBA16847A@codespeak.net> Author: fijal Date: Fri Feb 15 11:03:58 2008 New Revision: 51514 Modified: pypy/dist/pypy/lib/_ctypes/keepalive.txt Log: Document places that call KeepRef Modified: pypy/dist/pypy/lib/_ctypes/keepalive.txt ============================================================================== --- pypy/dist/pypy/lib/_ctypes/keepalive.txt (original) +++ pypy/dist/pypy/lib/_ctypes/keepalive.txt Fri Feb 15 11:03:58 2008 @@ -13,4 +13,24 @@ it simply keeps objects to be kept alive. * there are (some) rules about when to put stuff in it's objects when - accessing the fields etc. need to list them here. \ No newline at end of file + accessing the fields etc. need to list them here. + + * each object has _objects dictionary. In this dict stuff is stored per + key, which should be a tuple of indices of storages. + +places storing stuff in _objects: + +* array item assignement, if we share the storage + +* pointer item assignement, if we share the storage + +* getitem on CDLL stores function pointer + +* CFuncPtr.__new__ (assuming that it's a ctypes callable), stores itself + (XXX???) + +* set contents on pointer + +* set value on primitive or __init__ on primitive + + \ No newline at end of file From pedronis at codespeak.net Fri Feb 15 12:38:18 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 15 Feb 2008 12:38:18 +0100 (CET) Subject: [pypy-svn] r51516 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080215113818.6C59716847A@codespeak.net> Author: pedronis Date: Fri Feb 15 12:38:15 2008 New Revision: 51516 Modified: pypy/dist/pypy/lib/app_test/ctypes/_ctypes_test.c Log: don't depend on Python.h anymore, tested only on recent linux Modified: pypy/dist/pypy/lib/app_test/ctypes/_ctypes_test.c ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/_ctypes_test.c (original) +++ pypy/dist/pypy/lib/app_test/ctypes/_ctypes_test.c Fri Feb 15 12:38:15 2008 @@ -1,18 +1,3 @@ -/***************************************************************** - This file should be kept compatible with Python 2.3, see PEP 291. - *****************************************************************/ - - -#include - -/* - Backwards compatibility: - Python2.2 used LONG_LONG instead of PY_LONG_LONG -*/ -#if defined(HAVE_LONG_LONG) && !defined(PY_LONG_LONG) -#define PY_LONG_LONG LONG_LONG -#endif - #ifdef MS_WIN32 #include #endif @@ -23,6 +8,15 @@ #define EXPORT(x) x #endif +#include +#include +#include + +#define HAVE_LONG_LONG +#define LONG_LONG long long +#define HAVE_WCHAR_H + + /* some functions handy for testing */ EXPORT(char *)my_strtok(char *token, const char *delim) @@ -165,15 +159,15 @@ } #ifdef HAVE_LONG_LONG -EXPORT(PY_LONG_LONG) _testfunc_q_bhilfdq(signed char b, short h, int i, long l, float f, - double d, PY_LONG_LONG q) +EXPORT(LONG_LONG) _testfunc_q_bhilfdq(signed char b, short h, int i, long l, float f, + double d, LONG_LONG q) { - return (PY_LONG_LONG)(b + h + i + l + f + d + q); + return (LONG_LONG)(b + h + i + l + f + d + q); } -EXPORT(PY_LONG_LONG) _testfunc_q_bhilfd(signed char b, short h, int i, long l, float f, double d) +EXPORT(LONG_LONG) _testfunc_q_bhilfd(signed char b, short h, int i, long l, float f, double d) { - return (PY_LONG_LONG)(b + h + i + l + f + d); + return (LONG_LONG)(b + h + i + l + f + d); } EXPORT(int) _testfunc_callback_i_if(int value, int (*func)(int)) @@ -186,10 +180,10 @@ return sum; } -EXPORT(PY_LONG_LONG) _testfunc_callback_q_qf(PY_LONG_LONG value, - PY_LONG_LONG (*func)(PY_LONG_LONG)) +EXPORT(LONG_LONG) _testfunc_callback_q_qf(LONG_LONG value, + LONG_LONG (*func)(LONG_LONG)) { - PY_LONG_LONG sum = 0; + LONG_LONG sum = 0; while (value != 0) { sum += func(value); @@ -296,15 +290,15 @@ { } -EXPORT(PY_LONG_LONG) last_tf_arg_s; -EXPORT(unsigned PY_LONG_LONG) last_tf_arg_u; +EXPORT(LONG_LONG) last_tf_arg_s; +EXPORT(unsigned LONG_LONG) last_tf_arg_u; struct BITS { int A: 1, B:2, C:3, D:4, E: 5, F: 6, G: 7, H: 8, I: 9; short M: 1, N: 2, O: 3, P: 4, Q: 5, R: 6, S: 7; }; -DL_EXPORT(void) set_bitfields(struct BITS *bits, char name, int value) +EXPORT(void) set_bitfields(struct BITS *bits, char name, int value) { switch (name) { case 'A': bits->A = value; break; @@ -327,7 +321,7 @@ } } -DL_EXPORT(int) unpack_bitfields(struct BITS *bits, char name) +EXPORT(int) unpack_bitfields(struct BITS *bits, char name) { switch (name) { case 'A': return bits->A; @@ -351,8 +345,8 @@ return 0; } -#define S last_tf_arg_s = (PY_LONG_LONG)c -#define U last_tf_arg_u = (unsigned PY_LONG_LONG)c +#define S last_tf_arg_s = (LONG_LONG)c +#define U last_tf_arg_u = (unsigned LONG_LONG)c EXPORT(signed char) tf_b(signed char c) { S; return c/3; } EXPORT(unsigned char) tf_B(unsigned char c) { U; return c/3; } @@ -362,8 +356,8 @@ EXPORT(unsigned int) tf_I(unsigned int c) { U; return c/3; } EXPORT(long) tf_l(long c) { S; return c/3; } EXPORT(unsigned long) tf_L(unsigned long c) { U; return c/3; } -EXPORT(PY_LONG_LONG) tf_q(PY_LONG_LONG c) { S; return c/3; } -EXPORT(unsigned PY_LONG_LONG) tf_Q(unsigned PY_LONG_LONG c) { U; return c/3; } +EXPORT(LONG_LONG) tf_q(LONG_LONG c) { S; return c/3; } +EXPORT(unsigned LONG_LONG) tf_Q(unsigned LONG_LONG c) { U; return c/3; } EXPORT(float) tf_f(float c) { S; return c/3; } EXPORT(double) tf_d(double c) { S; return c/3; } @@ -376,8 +370,8 @@ EXPORT(unsigned int) __stdcall s_tf_I(unsigned int c) { U; return c/3; } EXPORT(long) __stdcall s_tf_l(long c) { S; return c/3; } EXPORT(unsigned long) __stdcall s_tf_L(unsigned long c) { U; return c/3; } -EXPORT(PY_LONG_LONG) __stdcall s_tf_q(PY_LONG_LONG c) { S; return c/3; } -EXPORT(unsigned PY_LONG_LONG) __stdcall s_tf_Q(unsigned PY_LONG_LONG c) { U; return c/3; } +EXPORT(LONG_LONG) __stdcall s_tf_q(LONG_LONG c) { S; return c/3; } +EXPORT(unsigned LONG_LONG) __stdcall s_tf_Q(unsigned LONG_LONG c) { U; return c/3; } EXPORT(float) __stdcall s_tf_f(float c) { S; return c/3; } EXPORT(double) __stdcall s_tf_d(double c) { S; return c/3; } #endif @@ -391,8 +385,8 @@ EXPORT(unsigned int) tf_bI(signed char x, unsigned int c) { U; return c/3; } EXPORT(long) tf_bl(signed char x, long c) { S; return c/3; } EXPORT(unsigned long) tf_bL(signed char x, unsigned long c) { U; return c/3; } -EXPORT(PY_LONG_LONG) tf_bq(signed char x, PY_LONG_LONG c) { S; return c/3; } -EXPORT(unsigned PY_LONG_LONG) tf_bQ(signed char x, unsigned PY_LONG_LONG c) { U; return c/3; } +EXPORT(LONG_LONG) tf_bq(signed char x, LONG_LONG c) { S; return c/3; } +EXPORT(unsigned LONG_LONG) tf_bQ(signed char x, unsigned LONG_LONG c) { U; return c/3; } EXPORT(float) tf_bf(signed char x, float c) { S; return c/3; } EXPORT(double) tf_bd(signed char x, double c) { S; return c/3; } EXPORT(void) tv_i(int c) { S; return; } @@ -406,8 +400,8 @@ EXPORT(unsigned int) __stdcall s_tf_bI(signed char x, unsigned int c) { U; return c/3; } EXPORT(long) __stdcall s_tf_bl(signed char x, long c) { S; return c/3; } EXPORT(unsigned long) __stdcall s_tf_bL(signed char x, unsigned long c) { U; return c/3; } -EXPORT(PY_LONG_LONG) __stdcall s_tf_bq(signed char x, PY_LONG_LONG c) { S; return c/3; } -EXPORT(unsigned PY_LONG_LONG) __stdcall s_tf_bQ(signed char x, unsigned PY_LONG_LONG c) { U; return c/3; } +EXPORT(LONG_LONG) __stdcall s_tf_bq(signed char x, LONG_LONG c) { S; return c/3; } +EXPORT(unsigned LONG_LONG) __stdcall s_tf_bQ(signed char x, unsigned LONG_LONG c) { U; return c/3; } EXPORT(float) __stdcall s_tf_bf(signed char x, float c) { S; return c/3; } EXPORT(double) __stdcall s_tf_bd(signed char x, double c) { S; return c/3; } EXPORT(void) __stdcall s_tv_i(int c) { S; return; } From fijal at codespeak.net Fri Feb 15 12:43:03 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 12:43:03 +0100 (CET) Subject: [pypy-svn] r51517 - in pypy/dist/pypy/lib: _ctypes app_test/ctypes Message-ID: <20080215114303.46D0F168477@codespeak.net> Author: fijal Date: Fri Feb 15 12:43:02 2008 New Revision: 51517 Added: pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py (contents, props changed) Modified: pypy/dist/pypy/lib/_ctypes/array.py pypy/dist/pypy/lib/_ctypes/basics.py pypy/dist/pypy/lib/_ctypes/pointer.py Log: One test and keepalive logic simple enough to pass this test. Modified: pypy/dist/pypy/lib/_ctypes/array.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/array.py (original) +++ pypy/dist/pypy/lib/_ctypes/array.py Fri Feb 15 12:43:02 2008 @@ -110,6 +110,7 @@ def __init__(self, *args): self._buffer = self._ffiarray(self._length_) self._needs_free = True + self._objects = {} for i, arg in enumerate(args): self[i] = arg @@ -135,8 +136,10 @@ if isinstance(index, slice): self._slice_setitem(index, value) return - value = self._type_._CData_input(value) index = self._fix_index(index) + if getattr(value, '_objects', None): + self._objects[str(index)] = value._objects + value = self._type_._CData_input(value) if not isinstance(self._type_._ffishape, tuple): self._buffer[index] = value[0] # something more sophisticated, cannot set field directly Modified: pypy/dist/pypy/lib/_ctypes/basics.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/basics.py (original) +++ pypy/dist/pypy/lib/_ctypes/basics.py Fri Feb 15 12:43:02 2008 @@ -57,6 +57,7 @@ """ The most basic object for all ctypes types """ __metaclass__ = _CDataMeta + _objects = None def __init__(self, *args, **kwds): raise TypeError("%s has no type" % (type(self),)) Modified: pypy/dist/pypy/lib/_ctypes/pointer.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/pointer.py (original) +++ pypy/dist/pypy/lib/_ctypes/pointer.py Fri Feb 15 12:43:02 2008 @@ -78,6 +78,7 @@ if not isinstance(value, self._type_): raise TypeError("expected %s instead of %s" % ( self._type_.__name__, type(value).__name__)) + self._objects = {'1':value} value = value._buffer self._buffer[0] = value Added: pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py Fri Feb 15 12:43:02 2008 @@ -0,0 +1,16 @@ + +from ctypes import * + +class TestKeepalive: + """ Tests whether various objects land in _objects + or not + """ + def test_array_of_pointers(self): + A = POINTER(c_int) * 24 + a = A() + l = c_long(2) + p = pointer(l) + a[3] = p + assert l._objects is None + assert p._objects == {'1':l} + assert a._objects == {'3':{'1':l}} From fijal at codespeak.net Fri Feb 15 12:44:33 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 12:44:33 +0100 (CET) Subject: [pypy-svn] r51518 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080215114433.10126168477@codespeak.net> Author: fijal Date: Fri Feb 15 12:44:31 2008 New Revision: 51518 Modified: pypy/dist/pypy/lib/app_test/ctypes/_ctypes_test.c Log: Missing includes. Modified: pypy/dist/pypy/lib/app_test/ctypes/_ctypes_test.c ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/_ctypes_test.c (original) +++ pypy/dist/pypy/lib/app_test/ctypes/_ctypes_test.c Fri Feb 15 12:44:31 2008 @@ -8,6 +8,8 @@ #define EXPORT(x) x #endif +#include +#include #include #include #include From arigo at codespeak.net Fri Feb 15 13:07:46 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 15 Feb 2008 13:07:46 +0100 (CET) Subject: [pypy-svn] r51519 - pypy/dist/pypy/translator/c/gcc/test Message-ID: <20080215120746.46076168477@codespeak.net> Author: arigo Date: Fri Feb 15 13:07:45 2008 New Revision: 51519 Modified: pypy/dist/pypy/translator/c/gcc/test/test_asmgcroot.py Log: Fix test (needs to support string results). Modified: pypy/dist/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/dist/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/dist/pypy/translator/c/gcc/test/test_asmgcroot.py Fri Feb 15 13:07:45 2008 @@ -19,7 +19,10 @@ except MemoryError: print 'Result: MemoryError' else: - print 'Result:', res + if isinstance(res, int): + print 'Result:', res + else: + print 'Result: "%s"' % (res,) return 0 from pypy.config.pypyoption import get_pypy_config config = get_pypy_config(translating=True) @@ -54,6 +57,8 @@ result = lines[-1][len('Result:'):].strip() if result == 'MemoryError': raise MemoryError("subprocess got an RPython MemoryError") + if result.startswith('"') and result.endswith('"'): + return result[1:-1] else: return int(result) return run From fijal at codespeak.net Fri Feb 15 13:26:24 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 13:26:24 +0100 (CET) Subject: [pypy-svn] r51520 - in pypy/dist/pypy/lib: _ctypes app_test/ctypes Message-ID: <20080215122624.DC09E168476@codespeak.net> Author: fijal Date: Fri Feb 15 13:26:24 2008 New Revision: 51520 Modified: pypy/dist/pypy/lib/_ctypes/keepalive.txt pypy/dist/pypy/lib/_ctypes/structure.py pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py Log: Keepalive for struct. Modified: pypy/dist/pypy/lib/_ctypes/keepalive.txt ============================================================================== --- pypy/dist/pypy/lib/_ctypes/keepalive.txt (original) +++ pypy/dist/pypy/lib/_ctypes/keepalive.txt Fri Feb 15 13:26:24 2008 @@ -32,5 +32,3 @@ * set contents on pointer * set value on primitive or __init__ on primitive - - \ No newline at end of file Modified: pypy/dist/pypy/lib/_ctypes/structure.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/structure.py (original) +++ pypy/dist/pypy/lib/_ctypes/structure.py Fri Feb 15 13:26:24 2008 @@ -117,6 +117,7 @@ self.__setattr__(name, arg) for name, arg in kwds.items(): self.__setattr__(name, arg) + self.__dict__['_objects'] = {} res.__init__ = __init__ @@ -163,6 +164,9 @@ fieldtype = self._fieldtypes[name].ctype except KeyError: raise AttributeError(name) + if getattr(value, '_objects', None): + self._objects[str(getattr(self.__class__, name).offset)] = \ + value._objects value = fieldtype._CData_input(value) self._buffer.__setattr__(name, value[0]) Modified: pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py Fri Feb 15 13:26:24 2008 @@ -6,6 +6,7 @@ or not """ def test_array_of_pointers(self): + # tests array item assignements & pointer.contents = ... A = POINTER(c_int) * 24 a = A() l = c_long(2) @@ -14,3 +15,15 @@ assert l._objects is None assert p._objects == {'1':l} assert a._objects == {'3':{'1':l}} + + def test_structure_with_pointers(self): + class X(Structure): + _fields_ = [('x', POINTER(c_int))] + + x = X() + u = c_int(3) + p = pointer(u) + x.x = p + assert x.x._objects is None + assert p._objects == {'1': u} + assert x._objects == {'0': p._objects} From fijal at codespeak.net Fri Feb 15 13:29:01 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 13:29:01 +0100 (CET) Subject: [pypy-svn] r51521 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080215122901.E194616847A@codespeak.net> Author: fijal Date: Fri Feb 15 13:29:01 2008 New Revision: 51521 Modified: pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py Log: another (passing) test. Modified: pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py Fri Feb 15 13:29:01 2008 @@ -27,3 +27,12 @@ assert x.x._objects is None assert p._objects == {'1': u} assert x._objects == {'0': p._objects} + + def test_pointer_setitem(self): + x = c_int(2) + y = c_int(3) + p = pointer(x) + assert p._objects == {'1':x} + p[0] = y + assert p._objects.keys() == ['1'] + assert p._objects['1'].value == 3 From fijal at codespeak.net Fri Feb 15 13:37:35 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 13:37:35 +0100 (CET) Subject: [pypy-svn] r51522 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080215123735.DBA0A16847B@codespeak.net> Author: fijal Date: Fri Feb 15 13:37:34 2008 New Revision: 51522 Modified: pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py Log: another test Modified: pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py Fri Feb 15 13:37:34 2008 @@ -36,3 +36,8 @@ p[0] = y assert p._objects.keys() == ['1'] assert p._objects['1'].value == 3 + + def test_char_p(self): + assert c_char_p("abc")._objects == "abc" + assert c_int(3)._objects is None + From fijal at codespeak.net Fri Feb 15 13:37:43 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 13:37:43 +0100 (CET) Subject: [pypy-svn] r51523 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080215123743.2B46E1684BF@codespeak.net> Author: fijal Date: Fri Feb 15 13:37:42 2008 New Revision: 51523 Modified: pypy/dist/pypy/lib/_ctypes/primitive.py pypy/dist/pypy/lib/_ctypes/structure.py Log: another keepalive Modified: pypy/dist/pypy/lib/_ctypes/primitive.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/primitive.py (original) +++ pypy/dist/pypy/lib/_ctypes/primitive.py Fri Feb 15 13:37:42 2008 @@ -73,6 +73,7 @@ if isinstance(value, unicode): value = value.encode(ConvMode.encoding, ConvMode.errors) + self._objects = value array = _rawffi.Array('c')(len(value)+1, value) value = array.buffer # XXX free 'array' later @@ -95,6 +96,7 @@ if isinstance(value, str): value = value.decode(ConvMode.encoding, ConvMode.errors) + self._objects = value array = _rawffi.Array('u')(len(value)+1, value) value = array.buffer # XXX free 'array' later Modified: pypy/dist/pypy/lib/_ctypes/structure.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/structure.py (original) +++ pypy/dist/pypy/lib/_ctypes/structure.py Fri Feb 15 13:37:42 2008 @@ -108,6 +108,7 @@ raise TypeError("Cannot instantiate structure, has no _fields_") self.__dict__['_buffer'] = self._ffistruct() self.__dict__['_needs_free'] = True + self.__dict__['_objects'] = {} if len(args) > len(self._names): raise TypeError("too many arguments") for name, arg in zip(self._names, args): @@ -117,7 +118,6 @@ self.__setattr__(name, arg) for name, arg in kwds.items(): self.__setattr__(name, arg) - self.__dict__['_objects'] = {} res.__init__ = __init__ @@ -165,8 +165,8 @@ except KeyError: raise AttributeError(name) if getattr(value, '_objects', None): - self._objects[str(getattr(self.__class__, name).offset)] = \ - value._objects + key = str(getattr(self.__class__, name).offset) + self.__dict__['_objects'][key] = value._objects value = fieldtype._CData_input(value) self._buffer.__setattr__(name, value[0]) From pedronis at codespeak.net Fri Feb 15 13:46:17 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 15 Feb 2008 13:46:17 +0100 (CET) Subject: [pypy-svn] r51524 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080215124617.3DB991684C0@codespeak.net> Author: pedronis Date: Fri Feb 15 13:46:14 2008 New Revision: 51524 Modified: pypy/dist/pypy/lib/app_test/ctypes/_ctypes_test.c Log: pedantic change of include order Modified: pypy/dist/pypy/lib/app_test/ctypes/_ctypes_test.c ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/_ctypes_test.c (original) +++ pypy/dist/pypy/lib/app_test/ctypes/_ctypes_test.c Fri Feb 15 13:46:14 2008 @@ -8,11 +8,11 @@ #define EXPORT(x) x #endif -#include -#include #include #include #include +#include +#include #define HAVE_LONG_LONG #define LONG_LONG long long From fijal at codespeak.net Fri Feb 15 14:28:21 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 14:28:21 +0100 (CET) Subject: [pypy-svn] r51526 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080215132821.28D041684C0@codespeak.net> Author: fijal Date: Fri Feb 15 14:28:20 2008 New Revision: 51526 Modified: pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py Log: another keepalive test Modified: pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py Fri Feb 15 14:28:20 2008 @@ -37,7 +37,12 @@ assert p._objects.keys() == ['1'] assert p._objects['1'].value == 3 - def test_char_p(self): + def test_primitive(self): assert c_char_p("abc")._objects == "abc" assert c_int(3)._objects is None - + + def test_cfunc(self): + def f(): + pass + cf = CFUNCTYPE(c_int, c_int)(f) + assert cf._objects == {'0':cf} From fijal at codespeak.net Fri Feb 15 14:28:29 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 14:28:29 +0100 (CET) Subject: [pypy-svn] r51527 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080215132829.B23EF1684C4@codespeak.net> Author: fijal Date: Fri Feb 15 14:28:29 2008 New Revision: 51527 Modified: pypy/dist/pypy/lib/_ctypes/function.py Log: keepalive for function pointer Modified: pypy/dist/pypy/lib/_ctypes/function.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/function.py (original) +++ pypy/dist/pypy/lib/_ctypes/function.py Fri Feb 15 14:28:29 2008 @@ -40,6 +40,7 @@ def __init__(self, argument=None): self.callable = None self.name = None + self._objects = {'0':self} if isinstance(argument, int): self._buffer = _rawffi.Array('P').fromaddress(argument, 1) # XXX finish this one, we need to be able to jump there somehow From pedronis at codespeak.net Fri Feb 15 14:31:07 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 15 Feb 2008 14:31:07 +0100 (CET) Subject: [pypy-svn] r51529 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080215133107.7B0C21684C6@codespeak.net> Author: pedronis Date: Fri Feb 15 14:31:06 2008 New Revision: 51529 Modified: pypy/dist/pypy/lib/_ctypes/basics.py pypy/dist/pypy/lib/_ctypes/pointer.py Log: in this case index and base where not really going anywhere Modified: pypy/dist/pypy/lib/_ctypes/basics.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/basics.py (original) +++ pypy/dist/pypy/lib/_ctypes/basics.py Fri Feb 15 14:31:06 2008 @@ -17,15 +17,13 @@ else: return self.from_param(as_parameter) - def _CData_input(self, value, base=None, index=-1): + def _CData_input(self, value): """Used when data enters into ctypes from user code. 'value' is some user-specified Python object, which is converted into a _rawffi array of length 1 containing the same value according to the type 'self'. """ cobj = self.from_param(value) - cobj.__dict__['_base'] = base - cobj.__dict__['_index'] = index return cobj._get_buffer_for_param() def _CData_output(self, resarray, base=None, index=-1): Modified: pypy/dist/pypy/lib/_ctypes/pointer.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/pointer.py (original) +++ pypy/dist/pypy/lib/_ctypes/pointer.py Fri Feb 15 14:31:06 2008 @@ -98,8 +98,7 @@ return self._type_._CData_output(self._subarray(index), self, index) def __setitem__(self, index, value): - self._subarray(index)[0] = self._type_._CData_input(value, self, - index)[0] + self._subarray(index)[0] = self._type_._CData_input(value)[0] def __nonzero__(self): return self._buffer[0] != 0 From fijal at codespeak.net Fri Feb 15 14:31:53 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 14:31:53 +0100 (CET) Subject: [pypy-svn] r51530 - in pypy/dist/pypy/lib: _ctypes app_test/ctypes Message-ID: <20080215133153.DF6CB1683F4@codespeak.net> Author: fijal Date: Fri Feb 15 14:31:53 2008 New Revision: 51530 Modified: pypy/dist/pypy/lib/_ctypes/array.py pypy/dist/pypy/lib/_ctypes/pointer.py pypy/dist/pypy/lib/_ctypes/primitive.py pypy/dist/pypy/lib/_ctypes/structure.py pypy/dist/pypy/lib/app_test/ctypes/support.py pypy/dist/pypy/lib/app_test/ctypes/test_functions.py Log: Reintroduce __del__'s. Modified: pypy/dist/pypy/lib/_ctypes/array.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/array.py (original) +++ pypy/dist/pypy/lib/_ctypes/array.py Fri Feb 15 14:31:53 2008 @@ -162,7 +162,7 @@ def _get_buffer_for_param(self): return self._buffer.byptr() - def delete(self): + def __del__(self): if self._needs_free: self._buffer.free() self._buffer = None Modified: pypy/dist/pypy/lib/_ctypes/pointer.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/pointer.py (original) +++ pypy/dist/pypy/lib/_ctypes/pointer.py Fri Feb 15 14:31:53 2008 @@ -103,7 +103,7 @@ def __nonzero__(self): return self._buffer[0] != 0 - def delete(self): + def __del__(self): if self._needs_free: self._buffer.free() self._needs_free = False Modified: pypy/dist/pypy/lib/_ctypes/primitive.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/primitive.py (original) +++ pypy/dist/pypy/lib/_ctypes/primitive.py Fri Feb 15 14:31:53 2008 @@ -221,7 +221,7 @@ def __nonzero__(self): return self._buffer[0] not in (0, '\x00') - def delete(self): + def __del__(self): if self._needs_free: self._needs_free = False self._buffer.free() Modified: pypy/dist/pypy/lib/_ctypes/structure.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/structure.py (original) +++ pypy/dist/pypy/lib/_ctypes/structure.py Fri Feb 15 14:31:53 2008 @@ -183,7 +183,7 @@ def _get_buffer_for_param(self): return self._buffer.byptr() - def delete(self): + def __del__(self): if self._needs_free: self._buffer.free() self.__dict__['_buffer'] = None Modified: pypy/dist/pypy/lib/app_test/ctypes/support.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/support.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/support.py Fri Feb 15 14:31:53 2008 @@ -9,7 +9,7 @@ cls.old_num = _rawffi._num_of_allocated_objects() def teardown_class(cls): - return + #return try: import _rawffi except ImportError: Modified: pypy/dist/pypy/lib/app_test/ctypes/test_functions.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_functions.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_functions.py Fri Feb 15 14:31:53 2008 @@ -29,7 +29,7 @@ class RECT(Structure): _fields_ = [("left", c_int), ("top", c_int), ("right", c_int), ("bottom", c_int)] -class TestFunctions(BaseCTypesTestChecker): +class TestFunctions:#(BaseCTypesTestChecker): def test_mro(self): # in Python 2.3, this raises TypeError: MRO conflict among bases classes, From fijal at codespeak.net Fri Feb 15 14:34:33 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 14:34:33 +0100 (CET) Subject: [pypy-svn] r51531 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080215133433.512FC1684C8@codespeak.net> Author: fijal Date: Fri Feb 15 14:34:32 2008 New Revision: 51531 Modified: pypy/dist/pypy/lib/_ctypes/structure.py Log: use direct access to offset Modified: pypy/dist/pypy/lib/_ctypes/structure.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/structure.py (original) +++ pypy/dist/pypy/lib/_ctypes/structure.py Fri Feb 15 14:34:32 2008 @@ -177,8 +177,9 @@ fieldtype = self._fieldtypes[name].ctype except KeyError: return _CData.__getattribute__(self, name) + offset = self.__class__._fieldtypes[name].offset return fieldtype._CData_output(self._subarray(fieldtype, name), self, - getattr(self.__class__, name).offset) + offset) def _get_buffer_for_param(self): return self._buffer.byptr() From fijal at codespeak.net Fri Feb 15 14:48:47 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 14:48:47 +0100 (CET) Subject: [pypy-svn] r51532 - in pypy/dist/pypy/module/_rawffi: . test Message-ID: <20080215134847.B1AE516803E@codespeak.net> Author: fijal Date: Fri Feb 15 14:48:46 2008 New Revision: 51532 Modified: pypy/dist/pypy/module/_rawffi/array.py pypy/dist/pypy/module/_rawffi/test/test__rawffi.py Log: introduce segfault_exception for array as well Modified: pypy/dist/pypy/module/_rawffi/array.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/array.py (original) +++ pypy/dist/pypy/module/_rawffi/array.py Fri Feb 15 14:48:46 2008 @@ -115,6 +115,8 @@ # XXX don't allow negative indexes, nor slices def setitem(self, space, num, w_value): + if not self.ll_buffer: + raise segfault_exception(space, "setting element of freed array") if num >= self.length or num < 0: raise OperationError(space.w_IndexError, space.w_None) unwrap_value(space, push_elem, self.ll_buffer, num, self.shape.itemtp, @@ -122,6 +124,8 @@ setitem.unwrap_spec = ['self', ObjSpace, int, W_Root] def getitem(self, space, num): + if not self.ll_buffer: + raise segfault_exception(space, "accessing elements of freed array") if num >= self.length or num < 0: raise OperationError(space.w_IndexError, space.w_None) return wrap_value(space, get_elem, self.ll_buffer, num, Modified: pypy/dist/pypy/module/_rawffi/test/test__rawffi.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/test/test__rawffi.py (original) +++ pypy/dist/pypy/module/_rawffi/test/test__rawffi.py Fri Feb 15 14:48:46 2008 @@ -587,3 +587,19 @@ raises(ValueError, lib.getprimitive, 'z', 'ddddddd') raises(ValueError, lib.getprimitive, 'zzz', 'static_int') + def test_segfault_exception(self): + import _rawffi + S = _rawffi.Structure([('x', 'i')]) + s = S() + s.x = 3 + s.free() + raises(_rawffi.SegfaultException, s.__getattr__, 'x') + raises(_rawffi.SegfaultException, s.__setattr__, 'x', 3) + A = _rawffi.Array('c') + a = A(13) + a.free() + raises(_rawffi.SegfaultException, a.__getitem__, 3) + raises(_rawffi.SegfaultException, a.__setitem__, 3, 3) + + + From fijal at codespeak.net Fri Feb 15 14:54:25 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 14:54:25 +0100 (CET) Subject: [pypy-svn] r51533 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080215135425.865DF168487@codespeak.net> Author: fijal Date: Fri Feb 15 14:54:24 2008 New Revision: 51533 Modified: pypy/dist/pypy/lib/_ctypes/array.py pypy/dist/pypy/lib/_ctypes/basics.py pypy/dist/pypy/lib/_ctypes/function.py pypy/dist/pypy/lib/_ctypes/pointer.py pypy/dist/pypy/lib/_ctypes/structure.py Log: provide some comment about computing keepalive key. Modified: pypy/dist/pypy/lib/_ctypes/array.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/array.py (original) +++ pypy/dist/pypy/lib/_ctypes/array.py Fri Feb 15 14:54:24 2008 @@ -1,7 +1,8 @@ import _rawffi -from _ctypes.basics import _CData, cdata_from_address, _CDataMeta, sizeof +from _ctypes.basics import _CData, cdata_from_address, _CDataMeta, sizeof,\ + keepalive_key def _create_unicode(buffer, maxlength): res = [] @@ -138,7 +139,7 @@ return index = self._fix_index(index) if getattr(value, '_objects', None): - self._objects[str(index)] = value._objects + self._objects[keepalive_key(index)] = value._objects value = self._type_._CData_input(value) if not isinstance(self._type_._ffishape, tuple): self._buffer[index] = value[0] Modified: pypy/dist/pypy/lib/_ctypes/basics.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/basics.py (original) +++ pypy/dist/pypy/lib/_ctypes/basics.py Fri Feb 15 14:54:24 2008 @@ -2,6 +2,8 @@ import _rawffi import sys +keepalive_key = str # XXX fix this when provided with test + class ArgumentError(Exception): pass Modified: pypy/dist/pypy/lib/_ctypes/function.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/function.py (original) +++ pypy/dist/pypy/lib/_ctypes/function.py Fri Feb 15 14:54:24 2008 @@ -1,6 +1,6 @@ import types -from _ctypes.basics import _CData, _CDataMeta, ArgumentError +from _ctypes.basics import _CData, _CDataMeta, ArgumentError, keepalive_key import _rawffi class CFuncPtrType(_CDataMeta): @@ -40,7 +40,7 @@ def __init__(self, argument=None): self.callable = None self.name = None - self._objects = {'0':self} + self._objects = {keepalive_key(0):self} if isinstance(argument, int): self._buffer = _rawffi.Array('P').fromaddress(argument, 1) # XXX finish this one, we need to be able to jump there somehow Modified: pypy/dist/pypy/lib/_ctypes/pointer.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/pointer.py (original) +++ pypy/dist/pypy/lib/_ctypes/pointer.py Fri Feb 15 14:54:24 2008 @@ -1,7 +1,7 @@ import _rawffi from _ctypes.basics import _CData, _CDataMeta, cdata_from_address -from _ctypes.basics import sizeof, byref +from _ctypes.basics import sizeof, byref, keepalive_key from _ctypes.array import Array, array_get_slice_params, array_slice_getitem,\ array_slice_setitem @@ -78,7 +78,7 @@ if not isinstance(value, self._type_): raise TypeError("expected %s instead of %s" % ( self._type_.__name__, type(value).__name__)) - self._objects = {'1':value} + self._objects = {keepalive_key(1):value} value = value._buffer self._buffer[0] = value Modified: pypy/dist/pypy/lib/_ctypes/structure.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/structure.py (original) +++ pypy/dist/pypy/lib/_ctypes/structure.py Fri Feb 15 14:54:24 2008 @@ -1,6 +1,6 @@ import _rawffi -from _ctypes.basics import _CData, _CDataMeta +from _ctypes.basics import _CData, _CDataMeta, keepalive_key import inspect def round_up(size, alignment): @@ -165,7 +165,7 @@ except KeyError: raise AttributeError(name) if getattr(value, '_objects', None): - key = str(getattr(self.__class__, name).offset) + key = keepalive_key(getattr(self.__class__, name).offset) self.__dict__['_objects'][key] = value._objects value = fieldtype._CData_input(value) self._buffer.__setattr__(name, value[0]) From fijal at codespeak.net Fri Feb 15 15:07:26 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 15:07:26 +0100 (CET) Subject: [pypy-svn] r51534 - in pypy/dist/pypy/lib: _ctypes app_test/ctypes Message-ID: <20080215140726.55AFC16847F@codespeak.net> Author: fijal Date: Fri Feb 15 15:07:25 2008 New Revision: 51534 Modified: pypy/dist/pypy/lib/_ctypes/pointer.py pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py Log: A test and a fix Modified: pypy/dist/pypy/lib/_ctypes/pointer.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/pointer.py (original) +++ pypy/dist/pypy/lib/_ctypes/pointer.py Fri Feb 15 15:07:25 2008 @@ -79,6 +79,8 @@ raise TypeError("expected %s instead of %s" % ( self._type_.__name__, type(value).__name__)) self._objects = {keepalive_key(1):value} + if getattr(value, '_objects', None): + self._objects[keepalive_key(0)] = value._objects value = value._buffer self._buffer[0] = value Modified: pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py Fri Feb 15 15:07:25 2008 @@ -41,6 +41,16 @@ assert c_char_p("abc")._objects == "abc" assert c_int(3)._objects is None + def test_pointer_to_pointer(self): + l = c_long(2) + assert l._objects is None + + p1 = pointer(l) + assert p1._objects == {'1':l} + + p2 = pointer(p1) + assert p2._objects == {'1':p1, '0':{'1':l}} + def test_cfunc(self): def f(): pass From pedronis at codespeak.net Fri Feb 15 15:34:59 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 15 Feb 2008 15:34:59 +0100 (CET) Subject: [pypy-svn] r51540 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080215143459.2314416847F@codespeak.net> Author: pedronis Date: Fri Feb 15 15:34:57 2008 New Revision: 51540 Modified: pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py Log: skipped exploding tests about ._objects and keepalive logic for nested complex objects Modified: pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py Fri Feb 15 15:34:57 2008 @@ -1,3 +1,4 @@ +import py from ctypes import * @@ -56,3 +57,21 @@ pass cf = CFUNCTYPE(c_int, c_int)(f) assert cf._objects == {'0':cf} + + def test_array_of_struct_with_pointer(self): + py.test.skip("explodes right now") + class S(Structure): + _fields_ = [('x', c_int)] + PS = POINTER(S) + + class Q(Structure): + _fields_ = [('p', PS)] + + A = Q*10 + a=A() + s=S() + s.x=3 + a[3].p = pointer(s) + + print a._objects + From antocuni at codespeak.net Fri Feb 15 17:07:17 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 15 Feb 2008 17:07:17 +0100 (CET) Subject: [pypy-svn] r51542 - in pypy/dist/pypy: jit/codegen/cli jit/codegen/cli/test translator/cli translator/cli/src Message-ID: <20080215160717.6F58C16844D@codespeak.net> Author: antocuni Date: Fri Feb 15 17:07:16 2008 New Revision: 51542 Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/src/pypylib.cs Log: make the cli jit backend using the new box/unbox interface to cast delegates to static methods. Moreover, FunctionConsts are now properly handled just like other ObjectConsts, instead of being stored into static fields of the old Constant class. Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/rgenop.py Fri Feb 15 17:07:16 2008 @@ -5,10 +5,10 @@ from pypy.jit.codegen.model import GenVarOrConst, GenVar, GenConst, CodeGenSwitch from pypy.jit.codegen.cli import operation as ops from pypy.jit.codegen.cli.dumpgenerator import DumpGenerator -from pypy.translator.cli.dotnet import CLR, typeof, new_array, clidowncast +from pypy.translator.cli.dotnet import CLR, typeof, new_array, box, unbox System = CLR.System Utils = CLR.pypy.runtime.Utils -Constants = CLR.pypy.runtime.Constants +DelegateHolder = CLR.pypy.runtime.DelegateHolder OpCodes = System.Reflection.Emit.OpCodes DUMP_IL = False @@ -112,53 +112,67 @@ def __repr__(self): return "const=%s" % self.value -class FunctionConst(GenConst): - - def __init__(self, num): - self.num = num - self.fieldname = "const" + str(num) +class BaseConst(GenConst): + + def _get_index(self, builder): + # check whether there is already an index associated to this const + try: + index = builder.genconsts[self] + except KeyError: + index = len(builder.genconsts) + builder.genconsts[self] = index + return index + + def _load_from_array(self, builder, index, clitype): + builder.il.Emit(OpCodes.Ldarg_0) + builder.il.Emit(OpCodes.Ldc_I4, index) + builder.il.Emit(OpCodes.Ldelem_Ref) + builder.il.Emit(OpCodes.Castclass, clitype) def getobj(self): - t = typeof(Constants) - return t.GetField(self.fieldname).GetValue(None) + raise NotImplementedError + +class ObjectConst(BaseConst): + + def __init__(self, obj): + self.obj = obj - def setobj(self, obj): - t = typeof(Constants) - t.GetField(self.fieldname).SetValue(None, obj) + def getobj(self): + return self.obj def load(self, builder): - t = typeof(Constants) - field = t.GetField(self.fieldname) - builder.il.Emit(OpCodes.Ldsfld, field) + index = self._get_index(builder) + t = self.obj.GetType() + self._load_from_array(builder, index, t) @specialize.arg(1) def revealconst(self, T): - return clidowncast(self.getobj(), T) + assert isinstance(T, ootype.OOType) + return unbox(self.obj, T) -class ObjectConst(GenConst): - def __init__(self, obj): - self.obj = obj +class FunctionConst(BaseConst): - def load(self, builder): - # check whether there is already an index associated to this const - try: - index = builder.genconsts[self.obj] - except KeyError: - index = len(builder.genconsts) - builder.genconsts[self.obj] = index + def __init__(self, delegatetype): + self.holder = DelegateHolder() + self.delegatetype = delegatetype - t = self.obj.GetType() - builder.il.Emit(OpCodes.Ldarg_0) - builder.il.Emit(OpCodes.Ldc_i4, index) - builder.il.Emit(OpCodes.Ldelem_ref) - builder.il.Emit(OpCodes.Castclass, t) + def getobj(self): + return self.holder + + def load(self, builder): + holdertype = box(self.holder).GetType() + funcfield = holdertype.GetField('func') + delegatetype = self.delegatetype + index = self._get_index(builder) + self._load_from_array(builder, index, holdertype) + builder.il.Emit(OpCodes.Ldfld, funcfield) + builder.il.Emit(OpCodes.Castclass, delegatetype) @specialize.arg(1) def revealconst(self, T): assert isinstance(T, ootype.OOType) - return ootype.oodowncast(T, self.obj) - + return unbox(self.holder.GetFunc(), T) class Label(GenLabel): def __init__(self, label, inputargs_gv): @@ -173,12 +187,6 @@ self.il = None self.constcount = 0 - def newconst(self, cls): - assert self.constcount < 3 # the number of static fields declared in Constants - res = cls(self.constcount) - self.constcount += 1 - return res - @specialize.genconst(1) def genconst(self, llvalue): T = ootype.typeOf(llvalue) @@ -187,7 +195,7 @@ elif T is ootype.Bool: return IntConst(int(llvalue)) elif isinstance(T, ootype.OOType): - return ObjectConst(llvalue) + return ObjectConst(box(llvalue)) else: assert False, "XXX not implemented" @@ -229,8 +237,8 @@ # we start from 1 because the 1st arg is an Object[] containing the genconsts for i in range(1, len(args)): self.inputargs_gv.append(GenArgVar(i, args[i])) - self.gv_entrypoint = self.rgenop.newconst(FunctionConst) - self.sigtoken = sigtoken + self.delegatetype = sigtoken2clitype(sigtoken) + self.gv_entrypoint = FunctionConst(self.delegatetype) self.isOpen = False self.operations = [] self.branches = [] @@ -250,7 +258,6 @@ self.il.EmitWriteLine('') return gv_res - @specialize.arg(1) def genop2(self, opname, gv_arg1, gv_arg2): opcls = ops.getopclass2(opname) @@ -322,12 +329,11 @@ # initialize the array of genconsts consts = new_array(System.Object, len(self.genconsts)) - for obj, i in self.genconsts.iteritems(): - consts[i] = obj + for gv_const, i in self.genconsts.iteritems(): + consts[i] = gv_const.getobj() # build the delegate - delegate_type = sigtoken2clitype(self.sigtoken) - myfunc = self.meth.CreateDelegate(delegate_type, consts) - self.gv_entrypoint.setobj(myfunc) + myfunc = self.meth.CreateDelegate(self.delegatetype, consts) + self.gv_entrypoint.holder.SetFunc(myfunc) def enter_next_block(self, kinds, args_gv): for i in range(len(args_gv)): @@ -360,6 +366,7 @@ self.il = parent.il self.operations = [] self.isOpen = False + self.genconsts = parent.genconsts def start_writing(self): self.isOpen = True Modified: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Fri Feb 15 17:07:16 2008 @@ -61,12 +61,12 @@ def getcompiled(self, fn, annotation, annotatorpolicy): return compile_function(fn, annotation, annotatorpolicy=annotatorpolicy, - nowrap=True) + nowrap=False) def cast(self, gv, nb_args): "NOT_RPYTHON" def fn(*args): - return gv.getobj().Invoke(*args) + return gv.getobj().func.Invoke(*args) return fn def directtesthelper(self, FUNCTYPE, func): Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Fri Feb 15 17:07:16 2008 @@ -445,7 +445,8 @@ return OverloadingResolver.lltype_to_annotation(TYPE) def specialize_call(self, hop): - TYPE = hop.args_v[1].value + assert hop.args_s[1].is_constant() + TYPE = hop.args_s[1].const v_obj = hop.inputarg(hop.args_r[0], arg=0) if TYPE is ootype.String or isinstance(TYPE, (type, types.ClassType)) or isinstance(TYPE, ootype.OOType): return hop.genop('oodowncast', [v_obj], hop.r_result.lowleveltype) Modified: pypy/dist/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/dist/pypy/translator/cli/src/pypylib.cs Fri Feb 15 17:07:16 2008 @@ -59,11 +59,21 @@ namespace pypy.runtime { - public class Constants + + public class DelegateHolder { - public static object const0; - public static object const1; - public static object const2; + public Delegate func; + + // we need getter and setter because we can't directly access fields from RPython + public void SetFunc(Delegate func) + { + this.func = func; + } + + public Delegate GetFunc() + { + return this.func; + } } public class Utils From fijal at codespeak.net Fri Feb 15 17:09:12 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 17:09:12 +0100 (CET) Subject: [pypy-svn] r51543 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080215160912.2666616844F@codespeak.net> Author: fijal Date: Fri Feb 15 17:09:11 2008 New Revision: 51543 Modified: pypy/dist/pypy/lib/_ctypes/primitive.py Log: Store this (is unnecessary as we copy stuff anyway, but makes it more compliant) Modified: pypy/dist/pypy/lib/_ctypes/primitive.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/primitive.py (original) +++ pypy/dist/pypy/lib/_ctypes/primitive.py Fri Feb 15 17:09:11 2008 @@ -118,6 +118,7 @@ if isinstance(value, str): array = _rawffi.Array('c')(len(value)+1, value) value = array.buffer + self._objects = value # XXX free 'array' later elif value is None: value = 0 From fijal at codespeak.net Fri Feb 15 20:08:14 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 15 Feb 2008 20:08:14 +0100 (CET) Subject: [pypy-svn] r51545 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080215190814.626C216844C@codespeak.net> Author: fijal Date: Fri Feb 15 20:08:12 2008 New Revision: 51545 Modified: pypy/dist/pypy/lib/_ctypes/basics.py Log: Add a comment. Modified: pypy/dist/pypy/lib/_ctypes/basics.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/basics.py (original) +++ pypy/dist/pypy/lib/_ctypes/basics.py Fri Feb 15 20:08:12 2008 @@ -26,6 +26,8 @@ type 'self'. """ cobj = self.from_param(value) + # XXX this function segfaults randomly, because + # cobj is considered to be an owner of that, fix return cobj._get_buffer_for_param() def _CData_output(self, resarray, base=None, index=-1): From cfbolz at codespeak.net Sat Feb 16 00:36:31 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 16 Feb 2008 00:36:31 +0100 (CET) Subject: [pypy-svn] r51547 - in pypy/dist/pypy/doc: . discussion Message-ID: <20080215233631.22801168431@codespeak.net> Author: cfbolz Date: Sat Feb 16 00:36:29 2008 New Revision: 51547 Modified: pypy/dist/pypy/doc/discussion/ctypes_todo.txt pypy/dist/pypy/doc/interpreter.txt Log: typos Modified: pypy/dist/pypy/doc/discussion/ctypes_todo.txt ============================================================================== --- pypy/dist/pypy/doc/discussion/ctypes_todo.txt (original) +++ pypy/dist/pypy/doc/discussion/ctypes_todo.txt Sat Feb 16 00:36:29 2008 @@ -5,8 +5,8 @@ - for unions and structs, late assignement of _fields_ is somewhat buggy. Tests about behavior of getattr working properly on instances - are missing or not full. Some tests are skipped because I didn't understand - details. + are missing or not comprehensive. Some tests are skipped because I didn't + understand the details. - restype being a function is not working. Modified: pypy/dist/pypy/doc/interpreter.txt ============================================================================== --- pypy/dist/pypy/doc/interpreter.txt (original) +++ pypy/dist/pypy/doc/interpreter.txt Sat Feb 16 00:36:29 2008 @@ -31,7 +31,7 @@ translated with the rest of PyPy. Code objects contain -contensed information about their respective functions, class and +condensed information about their respective functions, class and module body source codes. Interpreting such code objects means instantiating and initializing a `Frame class`_ and then calling its ``frame.eval()`` method. This main entry point @@ -100,7 +100,7 @@ It is for this reason that PyPy resorts to generate specialized frame classes and functions at `initialization time`_ in order to let the annotator only see rather static -program flows with homogenous name-value assignments on +program flows with homogeneous name-value assignments on function invocations. .. _`how-to guide for descriptors`: http://users.rcn.com/python/download/Descriptor.htm @@ -190,7 +190,7 @@ * ``co_lnotab`` a helper table to compute the line-numbers corresponding to bytecodes In PyPy, code objects also have the responsibility of creating their Frame_ objects -via the `'create_frame()`` method. With proper parser and compiler support this should +via the `'create_frame()`` method. With proper parser and compiler support this would allow to create custom Frame objects extending the execution of functions in various ways. The several Frame_ classes already utilize this flexibility in order to implement Generators and Nested Scopes. From fijal at codespeak.net Sat Feb 16 14:15:03 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 16 Feb 2008 14:15:03 +0100 (CET) Subject: [pypy-svn] r51554 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080216131503.1EE1F168456@codespeak.net> Author: fijal Date: Sat Feb 16 14:15:03 2008 New Revision: 51554 Modified: pypy/dist/pypy/lib/_ctypes/array.py Log: it seems to be the case in ctypes that from_param returns self if it's of the right type. Modified: pypy/dist/pypy/lib/_ctypes/array.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/array.py (original) +++ pypy/dist/pypy/lib/_ctypes/array.py Sat Feb 16 14:15:03 2008 @@ -72,6 +72,9 @@ def from_param(self, value): # check for iterable + # shortcut + if isinstance(value, self): + return value try: iter(value) except ValueError: From fijal at codespeak.net Sat Feb 16 14:23:18 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 16 Feb 2008 14:23:18 +0100 (CET) Subject: [pypy-svn] r51555 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080216132318.E351616845F@codespeak.net> Author: fijal Date: Sat Feb 16 14:23:18 2008 New Revision: 51555 Modified: pypy/dist/pypy/lib/_ctypes/array.py Log: it seems that this code is completely irrelevant. Modified: pypy/dist/pypy/lib/_ctypes/array.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/array.py (original) +++ pypy/dist/pypy/lib/_ctypes/array.py Sat Feb 16 14:23:18 2008 @@ -70,20 +70,6 @@ def _alignmentofinstances(self): return self._type_._alignmentofinstances() - def from_param(self, value): - # check for iterable - # shortcut - if isinstance(value, self): - return value - try: - iter(value) - except ValueError: - return _CDataMeta.from_param(self, value) - else: - if len(value) > self._length_: - raise ValueError("%s too long" % (value,)) - return self(*value) - def array_get_slice_params(self, index): if index.step is not None: raise TypeError("3 arg slices not supported (for no reason)") From fijal at codespeak.net Sat Feb 16 14:32:20 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 16 Feb 2008 14:32:20 +0100 (CET) Subject: [pypy-svn] r51556 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080216133220.E9D0016845E@codespeak.net> Author: fijal Date: Sat Feb 16 14:32:20 2008 New Revision: 51556 Modified: pypy/dist/pypy/lib/_ctypes/array.py pypy/dist/pypy/lib/_ctypes/basics.py pypy/dist/pypy/lib/_ctypes/builtin.py pypy/dist/pypy/lib/_ctypes/function.py pypy/dist/pypy/lib/_ctypes/pointer.py pypy/dist/pypy/lib/_ctypes/structure.py pypy/dist/pypy/lib/_ctypes/union.py Log: strange, dangerous checkin. Make _CData_input unsafe. Now it returns the buffer and object to keep it alive. I cannot think right now about better solution, at least fixes segfaults (but I don't like it) Modified: pypy/dist/pypy/lib/_ctypes/array.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/array.py (original) +++ pypy/dist/pypy/lib/_ctypes/array.py Sat Feb 16 14:32:20 2008 @@ -129,7 +129,7 @@ index = self._fix_index(index) if getattr(value, '_objects', None): self._objects[keepalive_key(index)] = value._objects - value = self._type_._CData_input(value) + cobj, value = self._type_._CData_input(value) if not isinstance(self._type_._ffishape, tuple): self._buffer[index] = value[0] # something more sophisticated, cannot set field directly Modified: pypy/dist/pypy/lib/_ctypes/basics.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/basics.py (original) +++ pypy/dist/pypy/lib/_ctypes/basics.py Sat Feb 16 14:32:20 2008 @@ -28,7 +28,7 @@ cobj = self.from_param(value) # XXX this function segfaults randomly, because # cobj is considered to be an owner of that, fix - return cobj._get_buffer_for_param() + return cobj, cobj._get_buffer_for_param() def _CData_output(self, resarray, base=None, index=-1): assert isinstance(resarray, _rawffi.ArrayInstance) Modified: pypy/dist/pypy/lib/_ctypes/builtin.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/builtin.py (original) +++ pypy/dist/pypy/lib/_ctypes/builtin.py Sat Feb 16 14:32:20 2008 @@ -11,7 +11,8 @@ def _string_at_addr(addr, lgt): # address here can be almost anything import ctypes - obj = ctypes.c_char_p._CData_input(addr)[0] + cobj, arg = ctypes.c_char_p._CData_input(addr) + obj = arg[0] return _rawffi.charp2rawstring(obj, lgt) def set_conversion_mode(encoding, errors): @@ -22,7 +23,8 @@ def _wstring_at_addr(addr, lgt): import ctypes - obj = ctypes.c_wchar_p._CData_input(addr)[0] + cobj, arg = ctypes.c_wchar_p._CData_input(addr) + obj = arg[0] # XXX purely applevel if lgt == -1: lgt = sys.maxint Modified: pypy/dist/pypy/lib/_ctypes/function.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/function.py (original) +++ pypy/dist/pypy/lib/_ctypes/function.py Sat Feb 16 14:32:20 2008 @@ -77,7 +77,8 @@ argtypes = argtypes[:] + self._guess_argtypes(args[cut:]) restype = self._restype_ funcptr = self._getfuncptr(argtypes, restype) - resarray = funcptr(*self._wrap_args(argtypes, args)) + args = self._wrap_args(argtypes, args) + resarray = funcptr(*[arg for obj, arg in args]) if restype is not None: return restype._CData_output(resarray) Modified: pypy/dist/pypy/lib/_ctypes/pointer.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/pointer.py (original) +++ pypy/dist/pypy/lib/_ctypes/pointer.py Sat Feb 16 14:32:20 2008 @@ -100,7 +100,8 @@ return self._type_._CData_output(self._subarray(index), self, index) def __setitem__(self, index, value): - self._subarray(index)[0] = self._type_._CData_input(value)[0] + cobj, arg = self._type_._CData_input(value) + self._subarray(index)[0] = arg[0] def __nonzero__(self): return self._buffer[0] != 0 Modified: pypy/dist/pypy/lib/_ctypes/structure.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/structure.py (original) +++ pypy/dist/pypy/lib/_ctypes/structure.py Sat Feb 16 14:32:20 2008 @@ -167,7 +167,7 @@ if getattr(value, '_objects', None): key = keepalive_key(getattr(self.__class__, name).offset) self.__dict__['_objects'][key] = value._objects - value = fieldtype._CData_input(value) + cobj, value = fieldtype._CData_input(value) self._buffer.__setattr__(name, value[0]) def __getattribute__(self, name): Modified: pypy/dist/pypy/lib/_ctypes/union.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/union.py (original) +++ pypy/dist/pypy/lib/_ctypes/union.py Sat Feb 16 14:32:20 2008 @@ -79,6 +79,6 @@ fieldtype = self._fieldtypes[name].ctype except KeyError: raise AttributeError(name) - value = fieldtype._CData_input(value) + cobj, value = fieldtype._CData_input(value) buf = self._ffiarrays[name].fromaddress(self._buffer.buffer, 1) buf[0] = value[0] From cfbolz at codespeak.net Mon Feb 18 12:20:11 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 18 Feb 2008 12:20:11 +0100 (CET) Subject: [pypy-svn] r51577 - pypy/branch/smalltalk-shadow-changes Message-ID: <20080218112011.B008A1684CB@codespeak.net> Author: cfbolz Date: Mon Feb 18 12:20:09 2008 New Revision: 51577 Added: pypy/branch/smalltalk-shadow-changes/ - copied from r51576, pypy/dist/ Log: make a branch for some smalltalk refactorings From fijal at codespeak.net Mon Feb 18 12:24:50 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 12:24:50 +0100 (CET) Subject: [pypy-svn] r51578 - pypy/dist/pypy/module/_rawffi Message-ID: <20080218112450.61B631684CB@codespeak.net> Author: fijal Date: Mon Feb 18 12:24:49 2008 New Revision: 51578 Modified: pypy/dist/pypy/module/_rawffi/tracker.py Log: enable tracing by default. this yields performance penalty, but it's just simple int counter. Modified: pypy/dist/pypy/module/_rawffi/tracker.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/tracker.py (original) +++ pypy/dist/pypy/module/_rawffi/tracker.py Mon Feb 18 12:24:49 2008 @@ -6,7 +6,7 @@ Arguments class Tracker(object): - DO_TRACING = False + DO_TRACING = True def __init__(self): self.alloced = {} From tverwaes at codespeak.net Mon Feb 18 12:33:05 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Mon, 18 Feb 2008 12:33:05 +0100 (CET) Subject: [pypy-svn] r51579 - in pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk: . test Message-ID: <20080218113305.B91C51684CF@codespeak.net> Author: tverwaes Date: Mon Feb 18 12:33:05 2008 New Revision: 51579 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/classtable.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/constants.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/objtable.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py Log: fiddling with shadows Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/classtable.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/classtable.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/classtable.py Mon Feb 18 12:33:05 2008 @@ -75,7 +75,7 @@ create_classtable() def copy_in_globals_classes_known_to_the_vm(): - for name in constants.classes_in_special_object_table: + for name in constants.classes_needed_boot_vm: name = 'w_' + name globals()[name] = classtable[name] Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/constants.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/constants.py Mon Feb 18 12:33:05 2008 @@ -13,7 +13,20 @@ CLASS_SUPERCLASS_INDEX = 0 CLASS_METHODDICT_INDEX = 1 CLASS_FORMAT_INDEX = 2 -CLASS_NAME_INDEX = 6 # in the mini.image, at least +CLASS_NAME_INDEX = 6 # in the mini.image, at least + +NEXT_LINK_INDEX = 0 # " + +PROCESS_SUSPENDED_CONTEXT_INDEX = 1 # " +PROCESS_PRIORITY_INDEX = 2 # " +PROCESS_MY_LIST_INDEX = 3 # " + +FIRST_LINK_INDEX = 0 # " +LAST_LINK_INDEX = 1 # " +EXCESS_SIGNALS_INDEX = 2 # " + +SCHEDULER_PROCESS_LISTS_INDEX = 0 # " +SCHEDULER_ACTIVE_PROCESS_INDEX = 1 # " METHODDICT_TALLY_INDEX = 0 METHODDICT_VALUES_INDEX = 1 @@ -99,23 +112,52 @@ SO_FINALIZATION_SEMPAHORE = 41 SO_LARGENEGATIVEINTEGER_CLASS = 42 +classes_needed_boot_vm = [ + "SmallInteger", + "String", + "Array", + "Float", + "MethodContext", + "BlockContext", + "CompiledMethod", + "Character", + "ByteArray", +] + # XXX more missing? classes_in_special_object_table = { - "SmallInteger": SO_SMALLINTEGER_CLASS, - "Array": SO_ARRAY_CLASS, - "String": SO_STRING_CLASS, - "Float": SO_FLOAT_CLASS, - "BlockContext": SO_BLOCKCONTEXT_CLASS, - "MethodContext": SO_METHODCONTEXT_CLASS, - "Character": SO_CHARACTER_CLASS, - "ByteArray": SO_BYTEARRAY_CLASS, - "CompiledMethod": SO_COMPILEDMETHOD_CLASS, + "Bitmap" : SO_BITMAP_CLASS, + "SmallInteger" : SO_SMALLINTEGER_CLASS, + "String" : SO_STRING_CLASS, + "Array" : SO_ARRAY_CLASS, + "Float" : SO_FLOAT_CLASS, + "MethodContext" : SO_METHODCONTEXT_CLASS, + "BlockContext" : SO_BLOCKCONTEXT_CLASS, + "Point" : SO_POINT_CLASS, + "LargePositiveInteger" : SO_LARGEPOSITIVEINTEGER_CLASS, + "Display" : SO_DISPLAY_CLASS, + "Message" : SO_MESSAGE_CLASS, + "CompiledMethod" : SO_COMPILEDMETHOD_CLASS, + "Semaphore" : SO_SEMAPHORE_CLASS, + "Character" : SO_CHARACTER_CLASS, + "ByteArray" : SO_BYTEARRAY_CLASS, + "Process" : SO_PROCESS_CLASS, + "PseudoContext" : SO_PSEUDOCONTEXT_CLASS, + "TranslatedMethod" : SO_TRANSLATEDMETHOD_CLASS, + # "LargeNegativeInteger" : SO_LARGENEGATIVEINTEGER_CLASS, # Not available in mini.image } +objects_needed_boot_vm = [ + "nil", + "true", + "false", +] + objects_in_special_object_table = { "nil": SO_NIL, "true": SO_TRUE, "false": SO_FALSE, + "schedulerassociationpointer" : SO_SCHEDULERASSOCIATIONPOINTER, } TAGGED_MAXINT = 2 ** (LONG_BIT - 2) - 1 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py Mon Feb 18 12:33:05 2008 @@ -29,6 +29,7 @@ def __init__(self): self.w_active_context = None + self.cnt = 0 def interpret(self): try: Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py Mon Feb 18 12:33:05 2008 @@ -125,7 +125,8 @@ return "<%s %s>" % (self.__class__.__name__, self) def __str__(self): - if isinstance(self, W_PointersObject) and self._shadow is not None: + from pypy.lang.smalltalk.shadow import ClassShadow + if isinstance(self, W_PointersObject) and isinstance(self._shadow,ClassShadow): return "%s class" % (self.as_class_get_shadow().name or '?',) else: return "a %s" % (self.shadow_of_my_class().name or '?',) @@ -180,15 +181,44 @@ return (W_AbstractObjectWithClassReference.invariant(self) and isinstance(self._vars, list)) - def as_class_get_shadow(self): - from pypy.lang.smalltalk.shadow import ClassShadow + def as_special_get_shadow(self, TheClass): shadow = self._shadow if shadow is None: - self._shadow = shadow = ClassShadow(self) - assert isinstance(shadow, ClassShadow) # for now, the only kind + self._shadow = shadow = TheClass(self) + assert isinstance(shadow, TheClass) + return shadow + + def as_class_get_shadow(self): + from pypy.lang.smalltalk.shadow import ClassShadow + shadow = self.as_special_get_shadow(ClassShadow) shadow.check_for_updates() return shadow + def as_semaphore_get_shadow(self): + from pypy.lang.smalltalk.shadow import SemaphoreShadow + from pypy.lang.smalltalk import classtable + assert self.getclass() == classtable.classtable["w_Semaphore"] + return self.as_special_get_shadow(SemaphoreShadow) + + def as_linkedlist_get_shadow(self): + from pypy.lang.smalltalk.shadow import LinkedListShadow + from pypy.lang.smalltalk import classtable + return self.as_special_get_shadow(LinkedListShadow) + + def as_process_get_shadow(self): + from pypy.lang.smalltalk.shadow import ProcessShadow + from pypy.lang.smalltalk import classtable + assert self.getclass() == classtable.classtable["w_Process"] + return self.as_special_get_shadow(ProcessShadow) + + def as_scheduler_get_shadow(self): + from pypy.lang.smalltalk.shadow import SchedulerShadow + return self.as_special_get_shadow(SchedulerShadow) + + def as_association_get_shadow(self): + from pypy.lang.smalltalk.shadow import AssociationShadow + return self.as_special_get_shadow(AssociationShadow) + def equals(self, other): if not isinstance(other, W_PointersObject): return False @@ -398,14 +428,23 @@ def at0(self, index0): from pypy.lang.smalltalk import utility + # XXX Not tested + index0 -= self.headersize() if index0 < self.getliteralsize(): self.literalat0(index0) else: index0 = index0 - self.getliteralsize() + if index0 > len(self.bytes): + print "ERROR" + print self.getliteralsize() + print len(self.bytes) + print self.bytes return utility.wrap_int(ord(self.bytes[index0])) def atput0(self, index0, w_value): from pypy.lang.smalltalk import utility + # XXX Not tested + index0 -= self.headersize() if index0 < self.getliteralsize(): self.literalatput0(index0, w_value) else: @@ -452,8 +491,19 @@ # Invalid! raise IndexError - def store(self, index, value): - raise NotImplementedError + def store(self, index, w_value): + # XXX Untested code... + + from pypy.lang.smalltalk import utility, objtable + if index == constants.CTXPART_SENDER_INDEX: + self.w_sender = w_value + elif index == constants.CTXPART_PC_INDEX: + self.pc = utility.unwrap_int(w_value) + elif index == constants.CTXPART_STACKP_INDEX: + self.stack = [objtable.w_nil] * utility.unwrap_int(w_value) + else: + # Invalid! + raise IndexError # ______________________________________________________________________ # Method that contains the bytecode for this method/block context @@ -594,6 +644,27 @@ else: return W_ContextPart.fetch(self, index) + def store(self, index, w_object): + if index == constants.MTHDCTX_METHOD: + self._w_method = w_object + elif index == constants.MTHDCTX_RECEIVER_MAP: # what is this thing? + pass + elif index == constants.MTHDCTX_RECEIVER: + self.w_receiver = w_object + elif index >= constants.MTHDCTX_TEMP_FRAME_START: + # First set of indices are temporary variables: + offset = index - constants.MTHDCTX_TEMP_FRAME_START + if offset < len(self.temps): + self.temps[offset] = w_object + + # After that comes the stack: + offset -= len(self.temps) + stack_index = len(self.stack) - offset - 1 + self.stack[stack_index] = w_object + else: + W_ContextPart.store(self, index, w_object) + + # Use black magic to create w_nil without running the constructor, # thus allowing it to be used even in the constructor of its own # class. Note that we patch its class in objtable. Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/objtable.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/objtable.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/objtable.py Mon Feb 18 12:33:05 2008 @@ -30,5 +30,5 @@ objtable = {} -for name in constants.objects_in_special_object_table: +for name in constants.objects_needed_boot_vm: objtable["w_" + name] = globals()["w_" + name] Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py Mon Feb 18 12:33:05 2008 @@ -15,7 +15,7 @@ raise PrimitiveFailedError() def assert_valid_index(n0, w_obj): - if not 0 <= n0 < w_obj.size(): + if not 0 <= n0 < w_obj.primsize(): raise PrimitiveFailedError() # return the index, since from here on the annotator knows that # n0 cannot be negative @@ -276,6 +276,10 @@ w_res = utility.wrap_float(math.exp(f)) return w_res +# ___________________________________________________________________________ +# Failure + +FAIL = 18 # ___________________________________________________________________________ # Subscript and Stream Primitives @@ -380,7 +384,8 @@ s_class = w_rcvr.shadow_of_my_class() assert_bounds(n0, 0, s_class.instsize()) # only pointers have non-0 size - assert isinstance(w_rcvr, model.W_PointersObject) + # XXX Now MethodContext is still own format, leave + #assert isinstance(w_rcvr, model.W_PointersObject) return w_rcvr.fetch(n0) @expose_primitive(INST_VAR_AT_PUT, unwrap_spec=[object, index1_0, object]) @@ -389,7 +394,8 @@ s_class = w_rcvr.shadow_of_my_class() assert_bounds(n0, 0, s_class.instsize()) # only pointers have non-0 size - assert isinstance(w_rcvr, model.W_PointersObject) + # XXX Now MethodContext is still own format, leave + #assert isinstance(w_rcvr, model.W_PointersObject) w_rcvr.store(n0, w_value) return w_value @@ -694,9 +700,13 @@ w_frame.w_sender = interp.w_active_context interp.w_active_context = w_frame + @expose_primitive(PRIMITIVE_SIGNAL, unwrap_spec=[object]) def func(interp, w_rcvr): - raise PrimitiveNotYetWrittenError() + if w_rcvr.getclass() != classtable.classtable['w_Semaphore']: + raise PrimitiveFailedError() + w_rcvr.as_semaphore_get_shadow().synchronous_signal(interp) + return w_rcvr @expose_primitive(PRIMITIVE_WAIT, unwrap_spec=[object]) def func(interp, w_rcvr): @@ -712,7 +722,9 @@ @expose_primitive(PRIMITIVE_FLUSH_CACHE, unwrap_spec=[object]) def func(interp, w_rcvr): - raise PrimitiveNotYetWrittenError() + # XXX we currently don't care about bad flushes :) XXX + # raise PrimitiveNotYetWrittenError() + return w_rcvr # ___________________________________________________________________________ # PrimitiveLoadInstVar Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py Mon Feb 18 12:33:05 2008 @@ -43,9 +43,10 @@ self.update_shadow() def update_shadow(self): - "Update the ClassShadow with data from the w_self class." from pypy.lang.smalltalk import objtable + "Update the ClassShadow with data from the w_self class." + w_self = self.w_self # read and painfully decode the format classformat = utility.unwrap_int( @@ -85,8 +86,18 @@ # read the name if w_self.size() > constants.CLASS_NAME_INDEX: w_name = w_self.fetch(constants.CLASS_NAME_INDEX) - if isinstance(w_name, model.W_BytesObject): - self.name = w_name.as_string() + + # XXX This is highly experimental XXX + # if the name-pos of class is not bytesobject, + # we are probably holding a metaclass instead of a class. + # metaclasses hold a pointer to the real class in the last + # slot. This is pos 6 in mini.image and higher in squeak3.9 + if not isinstance(w_name, model.W_BytesObject): + w_realclass = w_self.fetch(w_self.size() - 1) + if w_realclass.size() > constants.CLASS_NAME_INDEX: + w_name = w_realclass.fetch(constants.CLASS_NAME_INDEX) + if isinstance(w_name, model.W_BytesObject): + self.name = w_name.as_string() # read the methoddict w_methoddict = w_self.fetch(constants.CLASS_METHODDICT_INDEX) w_values = w_methoddict.fetch(constants.METHODDICT_VALUES_INDEX) @@ -121,8 +132,16 @@ return model.W_BlockContext(None, None, 0, 0) elif w_cls == classtable.w_MethodContext: # From slang: Contexts must only be created with newForMethod: - raise error.PrimitiveFailedError - + # raise error.PrimitiveFailedError + # XXX XXX XXX XXX + # The above text is bogous. This does not come from slang but a + # method implementation of ContextPart in Squeak3.9 which + # overwrites the default compiledmethod to give this error. + # The mini.image -however- doesn't overwrite this method, and as + # far as I was able to trace it back, it -does- call this method. + from pypy.rlib.objectmodel import instantiate + return instantiate(model.W_MethodContext) + if self.instance_kind == POINTERS: return model.W_PointersObject(w_cls, self.instance_size+extrasize) elif self.instance_kind == WORDS: @@ -194,3 +213,149 @@ assert isinstance(method, model.W_CompiledMethod) self.methoddict[selector] = method method.w_compiledin = self.w_self + +class LinkedListShadow(AbstractShadow): + def __init__(self, w_self): + self.w_self = w_self + + def firstlink(self): + return self.w_self.at0(constants.FIRST_LINK_INDEX) + + def store_firstlink(self, w_object): + return self.w_self.atput0(constants.FIRST_LINK_INDEX, w_object) + + def lastlink(self): + return self.w_self.at0(constants.LAST_LINK_INDEX) + + def store_lastlink(self, w_object): + return self.w_self.atput0(constants.LAST_LINK_INDEX, w_object) + + def is_empty_list(self): + from pypy.lang.smalltalk import objtable + return self.firstlink() == objtable.w_nil + + def add_last_link(self, w_object): + if self.is_empty_list(): + self.store_firstlink(w_object) + else: + self.lastlink().store_next(w_object) + # XXX Slang version stores list in process here... + self.store_lastlink(w_object) + + def remove_first_link_of_list(self): + from pypy.lang.smalltalk import objtable + first = self.firstlink() + last = self.lastlink() + if first == last: + self.store_firstlink(objtable.w_nil) + self.store_lastlink(objtable.w_nil) + else: + next = first.as_process_get_shadow().next() + self.store_firstlink(next) + first.as_process_get_shadow().store_next(objtable.w_nil) + return first + +class SemaphoreShadow(LinkedListShadow): + """A shadow for Smalltalk objects that are semaphores + """ + def __init__(self, w_self): + self.w_self = w_self + + def put_to_sleep(self, s_process): + priority = s_process.priority() + s_scheduler = self.scheduler() + w_process_lists = s_scheduler.process_lists() + w_process_list = w_process_lists.at0(priority) + w_process_list.as_linkedlist_get_shadow().add_last_link(s_process.w_self) + s_process.store_my_list(w_process_list) + + def transfer_to(self, s_process, interp): + from pypy.lang.smalltalk import objtable + s_scheduler = self.scheduler() + s_old_process = s_scheduler.active_process() + s_scheduler.store_active_process(s_process) + s_old_process.store_suspended_context(interp.w_active_context) + interp.w_active_context = s_process.suspended_context() + s_process.store_suspended_context(objtable.w_nil) + #reclaimableContextCount := 0 + + def scheduler(self): + from pypy.lang.smalltalk import objtable + w_association = objtable.objtable["w_schedulerassociationpointer"] + w_scheduler = w_association.as_association_get_shadow().value() + return w_scheduler.as_scheduler_get_shadow() + + def resume(self, w_process, interp): + s_process = w_process.as_process_get_shadow() + s_scheduler = self.scheduler() + s_active_process = s_scheduler.active_process() + active_priority = s_active_process.priority() + new_priority = s_process.priority() + if new_priority > active_priority: + self.put_to_sleep(s_active_process) + self.transfer_to(s_process, interp) + else: + self.put_to_sleep(s_process) + + def synchronous_signal(self, interp): + print "SYNCHRONOUS SIGNAL" + if self.is_empty_list(): + w_value = self.w_self.at0(constants.EXCESS_SIGNALS_INDEX) + w_value = utility.wrap_int(utility.unwrap_int(w_value) + 1) + self.w_self.atput0(constants.EXCESS_SIGNALS_INDEX, w_value) + else: + self.resume(self.remove_first_link_of_list(), interp) + +class LinkShadow(AbstractShadow): + def __init__(self, w_self): + self.w_self = self + + def next(self): + return self.w_self.at0(constants.NEXT_LINK_INDEX) + + def store_next(self, w_object): + self.w_self.atput0(constants.NEXT_LINK_INDEX, w_object) + +class ProcessShadow(LinkShadow): + """A shadow for Smalltalk objects that are processes + """ + def __init__(self, w_self): + self.w_self = w_self + + def priority(self): + return utility.unwrap_int(self.w_self.at0(constants.PROCESS_PRIORITY_INDEX)) + + def my_list(self): + return self.w_self.at0(constants.PROCESS_MY_LIST_INDEX) + + def store_my_list(self, w_object): + self.w_self.atput0(constants.PROCESS_MY_LIST_INDEX, w_object) + + def suspended_context(self): + return self.w_self.at0(constants.PROCESS_SUSPENDED_CONTEXT_INDEX) + + def store_suspended_context(self, w_object): + self.w_self.atput0(constants.PROCESS_SUSPENDED_CONTEXT_INDEX, w_object) + +class AssociationShadow(AbstractShadow): + def __init__(self, w_self): + self.w_self = w_self + + def key(self): + return self.w_self.at0(constants.ASSOCIATION_KEY_INDEX) + + def value(self): + return self.w_self.at0(constants.ASSOCIATION_VALUE_INDEX) + +class SchedulerShadow(AbstractShadow): + def __init__(self, w_self): + self.w_self = w_self + + def active_process(self): + return self.w_self.at0(constants.SCHEDULER_ACTIVE_PROCESS_INDEX).as_process_get_shadow() + + def store_active_process(self, w_object): + self.w_self.atput0(constants.SCHEDULER_ACTIVE_PROCESS_INDEX, w_object) + + def process_lists(self): + return self.w_self.at0(constants.SCHEDULER_PROCESS_LISTS_INDEX) Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py Mon Feb 18 12:33:05 2008 @@ -123,15 +123,25 @@ for chunk in self.chunks.itervalues(): chunk.g_object.init_w_object() + def class_indxs(self): + from pypy.lang.smalltalk import constants + return map(lambda key: (key, constants.classes_in_special_object_table[key]), + constants.classes_needed_boot_vm) + + def object_indxs(self): + from pypy.lang.smalltalk import constants + return map(lambda key: (key, constants.objects_in_special_object_table[key]), + constants.objects_needed_boot_vm) + def assign_prebuilt_constants(self): from pypy.lang.smalltalk import classtable, constants, objtable # assign w_objects for objects that are already in classtable - for name, so_index in constants.classes_in_special_object_table.items(): + for name, so_index in self.class_indxs(): # w_object = getattr(classtable, "w_" + name) w_object = classtable.classtable["w_" + name] self.special_object(so_index).w_object = w_object # assign w_objects for objects that are already in objtable - for name, so_index in constants.objects_in_special_object_table.items(): + for name, so_index in self.object_indxs(): # w_object = getattr(objtable, "w_" + name) w_object = objtable.objtable["w_" + name] self.special_object(so_index).w_object = w_object @@ -197,10 +207,18 @@ class SqueakImage(object): def from_reader(self, reader): + from pypy.lang.smalltalk import constants, classtable self.special_objects = [g_object.w_object for g_object in reader.chunks[reader.specialobjectspointer] .g_object.pointers] + # After loading we know of more classes than before. Copy back. + for name, so_index in constants.classes_in_special_object_table.items(): + classtable.classtable["w_" + name] = self.special(so_index) + + # After loading we know of more objects than before. Copy back. self.objects = [chunk.g_object.w_object for chunk in reader.chunklist] + for name, so_index in constants.objects_in_special_object_table.items(): + objtable.objtable["w_" + name] = self.special(so_index) def special(self, index): return self.special_objects[index] Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py Mon Feb 18 12:33:05 2008 @@ -227,7 +227,7 @@ assert w_false is objtable.w_false def test_compile_method(): - py.test.skip("Not working yet.") + # py.test.skip("Not working yet.") sourcecode = """fib ^self < 2 ifTrue: [ 1 ] From fijal at codespeak.net Mon Feb 18 13:20:58 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 13:20:58 +0100 (CET) Subject: [pypy-svn] r51580 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080218122058.7CB151684C3@codespeak.net> Author: fijal Date: Mon Feb 18 13:20:57 2008 New Revision: 51580 Modified: pypy/dist/pypy/lib/app_test/ctypes/test_base.py pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py Log: Fix those tests. Modified: pypy/dist/pypy/lib/app_test/ctypes/test_base.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_base.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_base.py Mon Feb 18 13:20:57 2008 @@ -1,6 +1,8 @@ from ctypes import * +# WhiteBoxTests + class TestCTypesBase: def test_pointer(self): p = pointer(pointer(c_int(2))) @@ -20,4 +22,4 @@ X = POINTER(c_int) * 24 x = X() assert x[16]._base is x - assert x[16]._index == 16 * sizeof(POINTER(c_int)) + assert x[16]._index == 16 Modified: pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py Mon Feb 18 13:20:57 2008 @@ -59,7 +59,6 @@ assert cf._objects == {'0':cf} def test_array_of_struct_with_pointer(self): - py.test.skip("explodes right now") class S(Structure): _fields_ = [('x', c_int)] PS = POINTER(S) @@ -73,5 +72,5 @@ s.x=3 a[3].p = pointer(s) - print a._objects + assert a._objects['0:3']['1'] is s From fijal at codespeak.net Mon Feb 18 13:22:40 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 13:22:40 +0100 (CET) Subject: [pypy-svn] r51581 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080218122240.236B31684C3@codespeak.net> Author: fijal Date: Mon Feb 18 13:22:39 2008 New Revision: 51581 Modified: pypy/dist/pypy/lib/_ctypes/union.py Log: Missing del Modified: pypy/dist/pypy/lib/_ctypes/union.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/union.py (original) +++ pypy/dist/pypy/lib/_ctypes/union.py Mon Feb 18 13:22:39 2008 @@ -26,6 +26,7 @@ # malloc size size = self.__class__._sizeofinstances() self.__dict__['_buffer'] = _rawffi.Array('c')(size) + self.__dict__['_needs_free'] = True res.__init__ = __init__ return res @@ -65,6 +66,7 @@ class Union(_CData): __metaclass__ = UnionMeta _ffiletter = 'P' + _needs_free = False def __getattr__(self, name): try: @@ -82,3 +84,9 @@ cobj, value = fieldtype._CData_input(value) buf = self._ffiarrays[name].fromaddress(self._buffer.buffer, 1) buf[0] = value[0] + + def __del__(self): + if self._needs_free: + self._buffer.free() + self._buffer = None + self._needs_free = False From fijal at codespeak.net Mon Feb 18 13:22:57 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 13:22:57 +0100 (CET) Subject: [pypy-svn] r51582 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080218122257.0E3301684C3@codespeak.net> Author: fijal Date: Mon Feb 18 13:22:56 2008 New Revision: 51582 Modified: pypy/dist/pypy/lib/_ctypes/array.py pypy/dist/pypy/lib/_ctypes/basics.py pypy/dist/pypy/lib/_ctypes/structure.py Log: First approach at reasonable keepalives. Modified: pypy/dist/pypy/lib/_ctypes/array.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/array.py (original) +++ pypy/dist/pypy/lib/_ctypes/array.py Mon Feb 18 13:22:56 2008 @@ -143,8 +143,7 @@ if isinstance(index, slice): return self._slice_getitem(index) index = self._fix_index(index) - return self._type_._CData_output(self._subarray(index), self, - self._ffiarray.gettypecode(index)[0]) + return self._type_._CData_output(self._subarray(index), self, index) def __len__(self): return self._length_ Modified: pypy/dist/pypy/lib/_ctypes/basics.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/basics.py (original) +++ pypy/dist/pypy/lib/_ctypes/basics.py Mon Feb 18 13:22:56 2008 @@ -4,6 +4,19 @@ keepalive_key = str # XXX fix this when provided with test +def store_reference(where, base_key, target): + #self.__dict__['_objects'][key] = value._objects + if '_objects' in where.__dict__: + # shortcut + where.__dict__['_objects'][str(base_key)] = target + return + key = [base_key] + while not '_objects' in where.__dict__: + key.append(where.__dict__['_index']) + where = where.__dict__['_base'] + real_key = ":".join([str(i) for i in key]) + where.__dict__['_objects'][real_key] = target + class ArgumentError(Exception): pass Modified: pypy/dist/pypy/lib/_ctypes/structure.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/structure.py (original) +++ pypy/dist/pypy/lib/_ctypes/structure.py Mon Feb 18 13:22:56 2008 @@ -1,6 +1,7 @@ import _rawffi -from _ctypes.basics import _CData, _CDataMeta, keepalive_key +from _ctypes.basics import _CData, _CDataMeta, keepalive_key,\ + store_reference import inspect def round_up(size, alignment): @@ -166,7 +167,7 @@ raise AttributeError(name) if getattr(value, '_objects', None): key = keepalive_key(getattr(self.__class__, name).offset) - self.__dict__['_objects'][key] = value._objects + store_reference(self, key, value._objects) cobj, value = fieldtype._CData_input(value) self._buffer.__setattr__(name, value[0]) From fijal at codespeak.net Mon Feb 18 15:54:43 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 15:54:43 +0100 (CET) Subject: [pypy-svn] r51583 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080218145443.949121684CB@codespeak.net> Author: fijal Date: Mon Feb 18 15:54:42 2008 New Revision: 51583 Modified: pypy/dist/pypy/lib/app_test/ctypes/support.py pypy/dist/pypy/lib/app_test/ctypes/test_array.py pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py pypy/dist/pypy/lib/app_test/ctypes/test_unions.py Log: More strict checks Modified: pypy/dist/pypy/lib/app_test/ctypes/support.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/support.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/support.py Mon Feb 18 15:54:42 2008 @@ -6,6 +6,8 @@ except ImportError: pass else: + import gc + gc.collect() cls.old_num = _rawffi._num_of_allocated_objects() def teardown_class(cls): @@ -17,5 +19,7 @@ else: import gc gc.collect() + gc.collect() + gc.collect() # there is one reference coming from the byref() above - assert _rawffi._num_of_allocated_objects() <= cls.old_num + assert _rawffi._num_of_allocated_objects() == cls.old_num Modified: pypy/dist/pypy/lib/app_test/ctypes/test_array.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_array.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_array.py Mon Feb 18 15:54:42 2008 @@ -1,13 +1,15 @@ import py from ctypes import * +from support import BaseCTypesTestChecker +import _rawffi formats = "bBhHiIlLqQfd" formats = c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, \ c_long, c_ulonglong, c_float, c_double -class TestArray: +class TestArray(BaseCTypesTestChecker): def test_simple(self): # create classes holding simple numeric types, and check # various properties. @@ -109,8 +111,14 @@ assert sz[:] == "foo" assert sz.value == "foo" -class TestSophisticatedThings: +class TestSophisticatedThings(BaseCTypesTestChecker): def test_array_of_structures(self): + import gc + gc.collect() + gc.collect() + gc.collect() + from _rawffi import _num_of_allocated_objects as _num + assert _num() == 3 class X(Structure): _fields_ = [('x', c_int), ('y', c_int)] @@ -120,4 +128,4 @@ x.y = 3 y[1] = x assert y[1].y == 3 - + Modified: pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py Mon Feb 18 15:54:42 2008 @@ -73,4 +73,4 @@ a[3].p = pointer(s) assert a._objects['0:3']['1'] is s - + Modified: pypy/dist/pypy/lib/app_test/ctypes/test_unions.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_unions.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_unions.py Mon Feb 18 15:54:42 2008 @@ -1,7 +1,8 @@ from ctypes import * +from support import BaseCTypesTestChecker -class TestUnion: +class TestUnion(BaseCTypesTestChecker): def test_getattr(self): class Stuff(Union): _fields_ = [('x', c_char), ('y', c_int)] From fijal at codespeak.net Mon Feb 18 17:00:09 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 17:00:09 +0100 (CET) Subject: [pypy-svn] r51589 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080218160009.9C39C1684CA@codespeak.net> Author: fijal Date: Mon Feb 18 17:00:06 2008 New Revision: 51589 Modified: pypy/dist/pypy/lib/app_test/ctypes/support.py Log: one line less :) Modified: pypy/dist/pypy/lib/app_test/ctypes/support.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/support.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/support.py Mon Feb 18 17:00:06 2008 @@ -18,8 +18,7 @@ pass else: import gc - gc.collect() - gc.collect() - gc.collect() + for _ in range(3): + gc.collect() # there is one reference coming from the byref() above assert _rawffi._num_of_allocated_objects() == cls.old_num From fijal at codespeak.net Mon Feb 18 17:01:39 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 17:01:39 +0100 (CET) Subject: [pypy-svn] r51590 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080218160139.A42031684C3@codespeak.net> Author: fijal Date: Mon Feb 18 17:01:39 2008 New Revision: 51590 Modified: pypy/dist/pypy/lib/_ctypes/array.py pypy/dist/pypy/lib/_ctypes/basics.py pypy/dist/pypy/lib/_ctypes/builtin.py pypy/dist/pypy/lib/_ctypes/function.py pypy/dist/pypy/lib/_ctypes/pointer.py pypy/dist/pypy/lib/_ctypes/structure.py pypy/dist/pypy/lib/_ctypes/union.py Log: Slightly more subtle approach to freeing stuff. Let's see how it works out. Also, this has quite a bit of indirection, I think we shall start from writing tests for what we know about this now... Modified: pypy/dist/pypy/lib/_ctypes/array.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/array.py (original) +++ pypy/dist/pypy/lib/_ctypes/array.py Mon Feb 18 17:01:39 2008 @@ -2,7 +2,7 @@ import _rawffi from _ctypes.basics import _CData, cdata_from_address, _CDataMeta, sizeof,\ - keepalive_key + keepalive_key, CArgObject def _create_unicode(buffer, maxlength): res = [] @@ -129,14 +129,14 @@ index = self._fix_index(index) if getattr(value, '_objects', None): self._objects[keepalive_key(index)] = value._objects - cobj, value = self._type_._CData_input(value) + cobj, arg = self._type_._CData_input(value) if not isinstance(self._type_._ffishape, tuple): - self._buffer[index] = value[0] + self._buffer[index] = arg._buffer[0] # something more sophisticated, cannot set field directly else: from ctypes import memmove dest = self._buffer.itemaddress(index) - source = value[0] + source = arg._buffer[0] memmove(dest, source, self._type_._ffishape[0]) def __getitem__(self, index): @@ -149,7 +149,7 @@ return self._length_ def _get_buffer_for_param(self): - return self._buffer.byptr() + return CArgObject(self._buffer.byptr()) def __del__(self): if self._needs_free: Modified: pypy/dist/pypy/lib/_ctypes/basics.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/basics.py (original) +++ pypy/dist/pypy/lib/_ctypes/basics.py Mon Feb 18 17:01:39 2008 @@ -39,11 +39,9 @@ type 'self'. """ cobj = self.from_param(value) - # XXX this function segfaults randomly, because - # cobj is considered to be an owner of that, fix return cobj, cobj._get_buffer_for_param() - def _CData_output(self, resarray, base=None, index=-1): + def _CData_output(self, resarray, base=None, index=-1, needs_free=False): assert isinstance(resarray, _rawffi.ArrayInstance) """Used when data exits ctypes and goes into user code. 'resarray' is a _rawffi array of length 1 containing the value, @@ -53,6 +51,7 @@ res.__dict__['_buffer'] = resarray res.__dict__['_base'] = base res.__dict__['_index'] = index + res.__dict__['_needs_free'] = needs_free return res.__ctypes_from_outparam__() def __mul__(self, other): @@ -68,6 +67,17 @@ val._buffer = buffer return val +class CArgObject(object): + """ simple wrapper around buffer, just for the case of freeing + it afterwards + """ + def __init__(self, buffer): + self._buffer = buffer + + def __del__(self): + self._buffer.free() + self._buffer = None + class _CData(object): """ The most basic object for all ctypes types """ @@ -81,17 +91,7 @@ return self def _get_buffer_for_param(self): - return self._buffer - -#class CArgObject(object): -# def __init__(self, letter, raw_value, _type): -# self.ffiletter = letter -# self._array = raw_value -# self._type = _type - -# def __repr__(self): -# return "" % (self.ffiletter, self._array[0]) - + return self def sizeof(tp): if not isinstance(tp, _CDataMeta): Modified: pypy/dist/pypy/lib/_ctypes/builtin.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/builtin.py (original) +++ pypy/dist/pypy/lib/_ctypes/builtin.py Mon Feb 18 17:01:39 2008 @@ -12,7 +12,7 @@ # address here can be almost anything import ctypes cobj, arg = ctypes.c_char_p._CData_input(addr) - obj = arg[0] + obj = arg._buffer[0] return _rawffi.charp2rawstring(obj, lgt) def set_conversion_mode(encoding, errors): @@ -24,7 +24,7 @@ def _wstring_at_addr(addr, lgt): import ctypes cobj, arg = ctypes.c_wchar_p._CData_input(addr) - obj = arg[0] + obj = arg._buffer[0] # XXX purely applevel if lgt == -1: lgt = sys.maxint Modified: pypy/dist/pypy/lib/_ctypes/function.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/function.py (original) +++ pypy/dist/pypy/lib/_ctypes/function.py Mon Feb 18 17:01:39 2008 @@ -78,9 +78,11 @@ restype = self._restype_ funcptr = self._getfuncptr(argtypes, restype) args = self._wrap_args(argtypes, args) - resarray = funcptr(*[arg for obj, arg in args]) + resarray = funcptr(*[arg._buffer for obj, arg in args]) if restype is not None: - return restype._CData_output(resarray) + return restype._CData_output(resarray, needs_free=True) + else: + resarray.free() def _getfuncptr(self, argtypes, restype): if restype is None: Modified: pypy/dist/pypy/lib/_ctypes/pointer.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/pointer.py (original) +++ pypy/dist/pypy/lib/_ctypes/pointer.py Mon Feb 18 17:01:39 2008 @@ -101,7 +101,7 @@ def __setitem__(self, index, value): cobj, arg = self._type_._CData_input(value) - self._subarray(index)[0] = arg[0] + self._subarray(index)[0] = arg._buffer[0] def __nonzero__(self): return self._buffer[0] != 0 Modified: pypy/dist/pypy/lib/_ctypes/structure.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/structure.py (original) +++ pypy/dist/pypy/lib/_ctypes/structure.py Mon Feb 18 17:01:39 2008 @@ -1,7 +1,7 @@ import _rawffi from _ctypes.basics import _CData, _CDataMeta, keepalive_key,\ - store_reference + store_reference, CArgObject import inspect def round_up(size, alignment): @@ -168,8 +168,8 @@ if getattr(value, '_objects', None): key = keepalive_key(getattr(self.__class__, name).offset) store_reference(self, key, value._objects) - cobj, value = fieldtype._CData_input(value) - self._buffer.__setattr__(name, value[0]) + cobj, arg = fieldtype._CData_input(value) + self._buffer.__setattr__(name, arg._buffer[0]) def __getattribute__(self, name): if name == '_fieldtypes': @@ -183,7 +183,7 @@ offset) def _get_buffer_for_param(self): - return self._buffer.byptr() + return CArgObject(self._buffer.byptr()) def __del__(self): if self._needs_free: Modified: pypy/dist/pypy/lib/_ctypes/union.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/union.py (original) +++ pypy/dist/pypy/lib/_ctypes/union.py Mon Feb 18 17:01:39 2008 @@ -81,9 +81,9 @@ fieldtype = self._fieldtypes[name].ctype except KeyError: raise AttributeError(name) - cobj, value = fieldtype._CData_input(value) + cobj, arg = fieldtype._CData_input(value) buf = self._ffiarrays[name].fromaddress(self._buffer.buffer, 1) - buf[0] = value[0] + buf[0] = arg._buffer[0] def __del__(self): if self._needs_free: From fijal at codespeak.net Mon Feb 18 17:05:07 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 17:05:07 +0100 (CET) Subject: [pypy-svn] r51591 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080218160507.BAC571684C3@codespeak.net> Author: fijal Date: Mon Feb 18 17:05:07 2008 New Revision: 51591 Modified: pypy/dist/pypy/lib/_ctypes/union.py Log: update union, so there are no complaints regarding missing attributes in __del__ Modified: pypy/dist/pypy/lib/_ctypes/union.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/union.py (original) +++ pypy/dist/pypy/lib/_ctypes/union.py Mon Feb 18 17:05:07 2008 @@ -88,5 +88,5 @@ def __del__(self): if self._needs_free: self._buffer.free() - self._buffer = None - self._needs_free = False + self.__dict__['_buffer'] = None + self.__dict__['_needs_free'] = False From fijal at codespeak.net Mon Feb 18 17:31:08 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 17:31:08 +0100 (CET) Subject: [pypy-svn] r51595 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080218163108.5EDB91684CF@codespeak.net> Author: fijal Date: Mon Feb 18 17:31:08 2008 New Revision: 51595 Modified: pypy/dist/pypy/lib/_ctypes/array.py pypy/dist/pypy/lib/_ctypes/basics.py pypy/dist/pypy/lib/_ctypes/builtin.py pypy/dist/pypy/lib/_ctypes/pointer.py pypy/dist/pypy/lib/_ctypes/structure.py pypy/dist/pypy/lib/_ctypes/union.py Log: Simplify stuff a bit by having a direct way of accessing 0th element of buffer as buffer is usually not needed. Quite a bit easier. Modified: pypy/dist/pypy/lib/_ctypes/array.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/array.py (original) +++ pypy/dist/pypy/lib/_ctypes/array.py Mon Feb 18 17:31:08 2008 @@ -129,15 +129,14 @@ index = self._fix_index(index) if getattr(value, '_objects', None): self._objects[keepalive_key(index)] = value._objects - cobj, arg = self._type_._CData_input(value) + arg = self._type_._CData_value(value) if not isinstance(self._type_._ffishape, tuple): - self._buffer[index] = arg._buffer[0] + self._buffer[index] = arg # something more sophisticated, cannot set field directly else: from ctypes import memmove dest = self._buffer.itemaddress(index) - source = arg._buffer[0] - memmove(dest, source, self._type_._ffishape[0]) + memmove(dest, arg, self._type_._ffishape[0]) def __getitem__(self, index): if isinstance(index, slice): Modified: pypy/dist/pypy/lib/_ctypes/basics.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/basics.py (original) +++ pypy/dist/pypy/lib/_ctypes/basics.py Mon Feb 18 17:31:08 2008 @@ -41,6 +41,12 @@ cobj = self.from_param(value) return cobj, cobj._get_buffer_for_param() + def _CData_value(self, value): + cobj = self.from_param(value) + # we don't care here if this stuff will live afterwards, as we're + # interested only in value anyway + return cobj._get_buffer_for_param()._buffer[0] + def _CData_output(self, resarray, base=None, index=-1, needs_free=False): assert isinstance(resarray, _rawffi.ArrayInstance) """Used when data exits ctypes and goes into user code. Modified: pypy/dist/pypy/lib/_ctypes/builtin.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/builtin.py (original) +++ pypy/dist/pypy/lib/_ctypes/builtin.py Mon Feb 18 17:31:08 2008 @@ -11,9 +11,8 @@ def _string_at_addr(addr, lgt): # address here can be almost anything import ctypes - cobj, arg = ctypes.c_char_p._CData_input(addr) - obj = arg._buffer[0] - return _rawffi.charp2rawstring(obj, lgt) + arg = ctypes.c_char_p._CData_value(addr) + return _rawffi.charp2rawstring(arg, lgt) def set_conversion_mode(encoding, errors): old_cm = ConvMode.encoding, ConvMode.errors @@ -23,12 +22,11 @@ def _wstring_at_addr(addr, lgt): import ctypes - cobj, arg = ctypes.c_wchar_p._CData_input(addr) - obj = arg._buffer[0] + arg = ctypes.c_wchar_p._CData_value(addr) # XXX purely applevel if lgt == -1: lgt = sys.maxint - a = _rawffi.Array('u').fromaddress(obj, lgt) + a = _rawffi.Array('u').fromaddress(arg, lgt) res = [] for i in xrange(lgt): if lgt == sys.maxint and a[i] == '\x00': Modified: pypy/dist/pypy/lib/_ctypes/pointer.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/pointer.py (original) +++ pypy/dist/pypy/lib/_ctypes/pointer.py Mon Feb 18 17:31:08 2008 @@ -100,8 +100,7 @@ return self._type_._CData_output(self._subarray(index), self, index) def __setitem__(self, index, value): - cobj, arg = self._type_._CData_input(value) - self._subarray(index)[0] = arg._buffer[0] + self._subarray(index)[0] = self._type_._CData_value(value) def __nonzero__(self): return self._buffer[0] != 0 Modified: pypy/dist/pypy/lib/_ctypes/structure.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/structure.py (original) +++ pypy/dist/pypy/lib/_ctypes/structure.py Mon Feb 18 17:31:08 2008 @@ -168,8 +168,7 @@ if getattr(value, '_objects', None): key = keepalive_key(getattr(self.__class__, name).offset) store_reference(self, key, value._objects) - cobj, arg = fieldtype._CData_input(value) - self._buffer.__setattr__(name, arg._buffer[0]) + self._buffer.__setattr__(name, fieldtype._CData_value(value)) def __getattribute__(self, name): if name == '_fieldtypes': Modified: pypy/dist/pypy/lib/_ctypes/union.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/union.py (original) +++ pypy/dist/pypy/lib/_ctypes/union.py Mon Feb 18 17:31:08 2008 @@ -81,9 +81,8 @@ fieldtype = self._fieldtypes[name].ctype except KeyError: raise AttributeError(name) - cobj, arg = fieldtype._CData_input(value) buf = self._ffiarrays[name].fromaddress(self._buffer.buffer, 1) - buf[0] = arg._buffer[0] + buf[0] = fieldtype._CData_value(value) def __del__(self): if self._needs_free: From fijal at codespeak.net Mon Feb 18 17:43:00 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 17:43:00 +0100 (CET) Subject: [pypy-svn] r51596 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080218164300.87AAD1684D2@codespeak.net> Author: fijal Date: Mon Feb 18 17:42:58 2008 New Revision: 51596 Modified: pypy/dist/pypy/lib/app_test/ctypes/support.py pypy/dist/pypy/lib/app_test/ctypes/test_anon.py pypy/dist/pypy/lib/app_test/ctypes/test_array_in_pointer.py Log: it seems that for generation gc the magic number is 4 here. two more tracing tests. Modified: pypy/dist/pypy/lib/app_test/ctypes/support.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/support.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/support.py Mon Feb 18 17:42:58 2008 @@ -18,7 +18,7 @@ pass else: import gc - for _ in range(3): + for _ in range(4): gc.collect() # there is one reference coming from the byref() above - assert _rawffi._num_of_allocated_objects() == cls.old_num + assert _rawffi._num_of_allocated_objects() <= cls.old_num Modified: pypy/dist/pypy/lib/app_test/ctypes/test_anon.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_anon.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_anon.py Mon Feb 18 17:42:58 2008 @@ -1,6 +1,7 @@ from ctypes import * +from support import BaseCTypesTestChecker -class TestAnon: +class TestAnon(BaseCTypesTestChecker): def test_anon(self): class ANON(Union): Modified: pypy/dist/pypy/lib/app_test/ctypes/test_array_in_pointer.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_array_in_pointer.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_array_in_pointer.py Mon Feb 18 17:42:58 2008 @@ -2,6 +2,7 @@ from binascii import hexlify import re import py +from support import BaseCTypesTestChecker def dump(obj): # helper function to dump memory contents in hex, with a hyphen @@ -23,8 +24,9 @@ class Container(Structure): _fields_ = [("pvalues", POINTER(Value))] -class TestOne: +class TestOne(BaseCTypesTestChecker): def test(self): + from _rawffi import _num_of_allocated_objects as _num # create an array of 4 values val_array = (Value * 4)() From fijal at codespeak.net Mon Feb 18 17:49:29 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 17:49:29 +0100 (CET) Subject: [pypy-svn] r51597 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080218164929.93E06168449@codespeak.net> Author: fijal Date: Mon Feb 18 17:49:28 2008 New Revision: 51597 Modified: pypy/dist/pypy/lib/_ctypes/pointer.py Log: missing free. Modified: pypy/dist/pypy/lib/_ctypes/pointer.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/pointer.py (original) +++ pypy/dist/pypy/lib/_ctypes/pointer.py Mon Feb 18 17:49:28 2008 @@ -122,6 +122,7 @@ ptr = tp.__new__(tp) ptr._buffer = tp._ffiarray(1) ptr._buffer[0] = obj._buffer + ptr._needs_free = True return ptr if isinstance(obj, (int, long)): result = tp() @@ -132,4 +133,5 @@ % (type(obj),)) result = tp() result._buffer[0] = obj._buffer[0] + result._needs_free = True return result From fijal at codespeak.net Mon Feb 18 18:10:57 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 18:10:57 +0100 (CET) Subject: [pypy-svn] r51598 - in pypy/dist/pypy/module/_rawffi: . test Message-ID: <20080218171057.A57CE1684D8@codespeak.net> Author: fijal Date: Mon Feb 18 18:10:57 2008 New Revision: 51598 Modified: pypy/dist/pypy/module/_rawffi/callback.py pypy/dist/pypy/module/_rawffi/test/test__rawffi.py pypy/dist/pypy/module/_rawffi/test/test_tracker.py Log: Tracing for callbacks. Modified: pypy/dist/pypy/module/_rawffi/callback.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/callback.py (original) +++ pypy/dist/pypy/module/_rawffi/callback.py Mon Feb 18 18:10:57 2008 @@ -9,6 +9,7 @@ from pypy.module._rawffi.interp_rawffi import W_DataInstance, _get_type_,\ wrap_value, unwrap_value, unwrap_truncate_int, letter2tp from pypy.rlib.libffi import USERDATA_P, CallbackFuncPtr +from pypy.module._rawffi.tracker import tracker def callback(ll_args, ll_res, ll_userdata): userdata = rffi.cast(USERDATA_P, ll_userdata) @@ -52,9 +53,16 @@ self.ll_callback = CallbackFuncPtr(ffiargs, ffiresult, callback, number) self.ll_buffer = rffi.cast(rffi.VOIDP, self.ll_callback.ll_closure) - - #def free(self): - # del self.global_counter.CallbackPtr_by_number[self.number] + if tracker.DO_TRACING: + addr = rffi.cast(rffi.INT, self.ll_callback.ll_closure) + tracker.trace_allocation(addr, self) + + def free(self): + if tracker.DO_TRACING: + addr = rffi.cast(rffi.INT, self.ll_callback.ll_closure) + tracker.trace_free(addr) + del self.global_counter.CallbackPtr_by_number[self.number] + free.unwrap_spec = ['self'] def descr_new_callbackptr(space, w_type, w_callable, w_args, w_result): return W_CallbackPtr(space, w_callable, w_args, w_result) @@ -64,4 +72,5 @@ __new__ = interp2app(descr_new_callbackptr), byptr = interp2app(W_CallbackPtr.byptr), buffer = GetSetProperty(W_CallbackPtr.getbuffer), + free = interp2app(W_CallbackPtr.free), ) Modified: pypy/dist/pypy/module/_rawffi/test/test__rawffi.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/test/test__rawffi.py (original) +++ pypy/dist/pypy/module/_rawffi/test/test__rawffi.py Mon Feb 18 18:10:57 2008 @@ -600,6 +600,3 @@ a.free() raises(_rawffi.SegfaultException, a.__getitem__, 3) raises(_rawffi.SegfaultException, a.__setitem__, 3, 3) - - - Modified: pypy/dist/pypy/module/_rawffi/test/test_tracker.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/test/test_tracker.py (original) +++ pypy/dist/pypy/module/_rawffi/test/test_tracker.py Mon Feb 18 18:10:57 2008 @@ -24,6 +24,13 @@ s.free() assert _rawffi._num_of_allocated_objects() == 0 + def test_callback(self): + import _rawffi + c = _rawffi.CallbackPtr(lambda : 3, [], 'i') + assert _rawffi._num_of_allocated_objects() == 1 + c.free() + assert _rawffi._num_of_allocated_objects() == 0 + def teardown_class(cls): Tracker.DO_TRACING = False From arigo at codespeak.net Mon Feb 18 18:13:46 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 18 Feb 2008 18:13:46 +0100 (CET) Subject: [pypy-svn] r51599 - pypy/branch/unified-rtti Message-ID: <20080218171346.DA6C31684D8@codespeak.net> Author: arigo Date: Mon Feb 18 18:13:45 2008 New Revision: 51599 Added: pypy/branch/unified-rtti/ - copied from r51598, pypy/dist/ Log: A branch to try to unify the two "type" pointers that every RPython instance gets at the moment: the one from the GC and the vtable pointer. From arigo at codespeak.net Mon Feb 18 18:16:46 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 18 Feb 2008 18:16:46 +0100 (CET) Subject: [pypy-svn] r51600 - in pypy/branch/unified-rtti/pypy/rpython/lltypesystem: . test Message-ID: <20080218171646.3BEB01684D7@codespeak.net> Author: arigo Date: Mon Feb 18 18:16:45 2008 New Revision: 51600 Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_lltype.py Log: Tests and implementation for the new interface. (Nothing else fixed yet, of course) Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py Mon Feb 18 18:16:45 2008 @@ -147,9 +147,11 @@ def _inline_is_varsize(self, last): raise TypeError, "%r cannot be inlined in structure" % self - def _install_extras(self, adtmeths={}, hints={}): + def _install_extras(self, adtmeths={}, hints={}, runtime_type_info=None): self._adtmeths = frozendict(adtmeths) self._hints = frozendict(hints) + if runtime_type_info is not None: + _install_rtti(self, runtime_type_info) def __getattr__(self, name): adtmeth = self._adtmeths.get(name, NFOUND) @@ -207,7 +209,7 @@ if self._names: first = self._names[0] FIRSTTYPE = self._flds[first] - if (isinstance(FIRSTTYPE, (Struct, PyObjectType)) and + if (isinstance(FIRSTTYPE, FirstStructTypes) and self._gckind == FIRSTTYPE._gckind): return first, FIRSTTYPE return None, None @@ -281,34 +283,7 @@ n = 1 return _struct(self, n, initialization='example') -class RttiStruct(Struct): - _runtime_type_info = None - - def _attach_runtime_type_info_funcptr(self, funcptr, destrptr): - if self._runtime_type_info is None: - self._runtime_type_info = opaqueptr(RuntimeTypeInfo, name=self._name, about=self)._obj - if funcptr is not None: - T = typeOf(funcptr) - if (not isinstance(T, Ptr) or - not isinstance(T.TO, FuncType) or - len(T.TO.ARGS) != 1 or - T.TO.RESULT != Ptr(RuntimeTypeInfo) or - castable(T.TO.ARGS[0], Ptr(self)) < 0): - raise TypeError("expected a runtime type info function " - "implementation, got: %s" % funcptr) - self._runtime_type_info.query_funcptr = funcptr - if destrptr is not None : - T = typeOf(destrptr) - if (not isinstance(T, Ptr) or - not isinstance(T.TO, FuncType) or - len(T.TO.ARGS) != 1 or - T.TO.RESULT != Void or - castable(T.TO.ARGS[0], Ptr(self)) < 0): - raise TypeError("expected a destructor function " - "implementation, got: %s" % destrptr) - self._runtime_type_info.destructor_funcptr = destrptr - -class GcStruct(RttiStruct): +class GcStruct(Struct): _gckind = 'gc' STRUCT_BY_FLAVOR = {'raw': Struct, @@ -441,6 +416,19 @@ return [arg for arg in self.ARGS if arg is not Void] +class RuntimeTypeInfoType(ContainerType): + _gckind = 'raw' + + def _inline_is_varsize(self, last): + return False + + def _allocate(self, initialization, parent=None, parentindex=None): + return _rtti(initialization=initialization, + parent=parent, parentindex=parentindex) + +RuntimeTypeInfo = RuntimeTypeInfoType() + + class OpaqueType(ContainerType): _gckind = 'raw' @@ -466,8 +454,6 @@ def _allocate(self, initialization, parent=None, parentindex=None): return self._defl(parent=parent, parentindex=parentindex) -RuntimeTypeInfo = OpaqueType("RuntimeTypeInfo") - class GcOpaqueType(OpaqueType): _gckind = 'gc' @@ -491,6 +477,8 @@ PyObject = PyObjectType() +FirstStructTypes = (Struct, PyObjectType, RuntimeTypeInfoType) + class ForwardReference(ContainerType): _gckind = 'raw' def become(self, realcontainertype): @@ -768,8 +756,8 @@ % (CURTYPE, PTRTYPE)) if CURTYPE == PTRTYPE: return 0 - if (not isinstance(CURTYPE.TO, (Struct, PyObjectType)) or - not isinstance(PTRTYPE.TO, (Struct, PyObjectType))): + if (not isinstance(CURTYPE.TO, FirstStructTypes) or + not isinstance(PTRTYPE.TO, FirstStructTypes)): raise InvalidCast(CURTYPE, PTRTYPE) CURSTRUC = CURTYPE.TO PTRSTRUC = PTRTYPE.TO @@ -1653,6 +1641,24 @@ def __setattr__(self, attr, value): raise AttributeError("cannot change the attributes of %r" % (self,)) + +class _rtti(_parentable): + _kind = "rtti" + + def __init__(self, initialization=None, parent=None, parentindex=None): + _parentable.__init__(self, RuntimeTypeInfo) + if parent is not None: + self._setparentstructure(parent, parentindex) + + def _set_gctype(self, GCTYPE): + if hasattr(self, 'GCTYPE'): + raise TypeError("cannot use the same _rtti for several GC types") + if GCTYPE._gckind != 'gc': + raise TypeError("non-GC type %r cannot have an rtti" % (GCTYPE,)) + self._GCTYPE = GCTYPE + self.destructor_funcptr = nullptr(FuncType([Ptr(GCTYPE)], Void)) + + class _opaque(_parentable): def __init__(self, TYPE, parent=None, parentindex=None, **attrs): _parentable.__init__(self, TYPE) @@ -1726,6 +1732,8 @@ elif isinstance(T, OpaqueType): assert n is None o = _opaque(T, initialization=initialization) + elif T == RuntimeTypeInfo: + o = _rtti(initialization=initialization) else: raise TypeError, "malloc for Structs and Arrays only" if T._gckind != 'gc' and not immortal and flavor.startswith('gc'): @@ -1773,37 +1781,58 @@ assert oddint & 1, "only odd integers can be cast back to ptr" return _ptr(PTRTYPE, oddint, solid=True) -def attachRuntimeTypeInfo(GCSTRUCT, funcptr=None, destrptr=None): - if not isinstance(GCSTRUCT, RttiStruct): - raise TypeError, "expected a RttiStruct: %s" % GCSTRUCT - GCSTRUCT._attach_runtime_type_info_funcptr(funcptr, destrptr) - return _ptr(Ptr(RuntimeTypeInfo), GCSTRUCT._runtime_type_info) - -def getRuntimeTypeInfo(GCSTRUCT): - if not isinstance(GCSTRUCT, RttiStruct): - raise TypeError, "expected a RttiStruct: %s" % GCSTRUCT - if GCSTRUCT._runtime_type_info is None: - raise ValueError, ("no attached runtime type info for GcStruct %s" % - GCSTRUCT._name) - return _ptr(Ptr(RuntimeTypeInfo), GCSTRUCT._runtime_type_info) +def getRuntimeTypeInfo(TYPE, cache=None): + """Return the runtime_type_info attached to the GcStruct TYPE. + This is typically of type != Ptr(RuntimeTypeInfo) but castable + to Ptr(RuntimeTypeInfo). This raises TypeError if the TYPE has + no runtime_type_info, unless 'cache' is specified; in that case, + TYPE can be any GC type and a runtime_type_info is created for + it if it has none and stored in the cache to avoid mutating + the TYPE. + """ + if isinstance(TYPE, GcStruct) and hasattr(TYPE, '_rtti'): + return top_container(TYPE._rtti)._as_ptr() + if cache is None: + raise TypeError("%r has no runtime_type_info" % (TYPE,)) + try: + return cache[TYPE] + except KeyError: + rttiptr = malloc(RuntimeTypeInfo, immortal=True) + rttiptr._obj._set_gctype(TYPE) + cache[TYPE] = rttiptr + return rttiptr + +def _install_rtti(STRUCT, runtime_type_info): + if not isinstance(STRUCT, GcStruct): + raise TypeError("can only attach a runtime_type_info to a GcStruct") + if runtime_type_info is None: + runtime_type_info = malloc(RuntimeTypeInfo, immortal=True) + rttiptr = cast_pointer(Ptr(RuntimeTypeInfo), runtime_type_info) + # check that the attached info is compatible with the inlined parent + # structure type's attached info + name, SUBSTRUCT = STRUCT._first_struct() + if SUBSTRUCT is not None: + if not hasattr(SUBSTRUCT, '_rtti'): + raise TypeError("first inlined GcStruct has no runtime_type_info") + PARENT = typeOf(top_container(SUBSTRUCT._rtti)) + cast_pointer(Ptr(PARENT), runtime_type_info) # for checking + # ready + assert not hasattr(STRUCT, '_rtti') + STRUCT._rtti = rttiptr._obj + rttiptr._obj._set_gctype(STRUCT) def runtime_type_info(p): - T = typeOf(p) - if not isinstance(T, Ptr) or not isinstance(T.TO, RttiStruct): - raise TypeError, "runtime_type_info on non-RttiStruct pointer: %s" % p - struct = p._obj - top_parent = top_container(struct) - result = getRuntimeTypeInfo(top_parent._TYPE) - static_info = getRuntimeTypeInfo(T.TO) - query_funcptr = getattr(static_info._obj, 'query_funcptr', None) - if query_funcptr is not None: - T = typeOf(query_funcptr).TO.ARGS[0] - result2 = query_funcptr(cast_pointer(T, p)) - if result != result2: - raise RuntimeError, ("runtime type-info function for %s:\n" - " returned: %s,\n" - "should have been: %s" % (p, result2, result)) - return result + """This is a run-time operation that returns the exact type of 'p', + as a Ptr(RuntimeTypeInfo). + """ + # This might assume that 'p' points to a GcStruct with an explicitly + # specified runtime_type_info; it is unclear for now if all GCs will + # support this operation for other kinds of GC types. + GCTYPE = typeOf(p).TO + assert isinstance(GCTYPE, GcStruct), "XXX for now" + assert hasattr(GCTYPE, '_rtti'), "XXX for now" + EXACTTYPE = typeOf(top_container(p._obj)) + return EXACTTYPE._rtti._as_ptr() def isCompatibleType(TYPE1, TYPE2): return TYPE1._is_compatible(TYPE2) Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_lltype.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_lltype.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_lltype.py Mon Feb 18 18:16:45 2008 @@ -357,76 +357,55 @@ p = cast_pointer(Ptr(S), p1) assert p.s1.x == 5 -def test_getRuntimeTypeInfo(): - S = GcStruct('s', ('x', Signed)) - py.test.raises(ValueError, "getRuntimeTypeInfo(S)") - pinf0 = attachRuntimeTypeInfo(S) - assert pinf0._obj.about == S - pinf = getRuntimeTypeInfo(S) - assert pinf == pinf0 - pinf1 = getRuntimeTypeInfo(S) - assert pinf == pinf1 - Z = GcStruct('z', ('x', Unsigned)) - attachRuntimeTypeInfo(Z) - assert getRuntimeTypeInfo(Z) != pinf0 - Sbis = GcStruct('s', ('x', Signed)) - attachRuntimeTypeInfo(Sbis) - assert getRuntimeTypeInfo(Sbis) != pinf0 - assert Sbis != S # the attached runtime type info distinguishes them +def test_runtime_type_info(): + T1 = Struct('T1', ('base', RuntimeTypeInfo), ('u', Signed)) + t1 = malloc(T1, immortal=True) + t1.u = 21 + S1 = GcStruct('S1', ('x', Signed), runtime_type_info=t1) + + T2bogus = Struct('T2bogus', ('foobar', Signed)) + t2bogus = malloc(T2bogus, immortal=True) + py.test.raises(TypeError, "GcStruct('S2', runtime_type_info=t2bogus)") + + T2 = Struct('T2', ('base', T1), ('v', Signed)) + t2 = malloc(T2, immortal=True) + t2.base.u = 22 + t2.v = 33 + S2 = GcStruct('S2', ('parent', S1), ('y', Signed), runtime_type_info=t2) + + assert getRuntimeTypeInfo(S1) == t1 + assert getRuntimeTypeInfo(S2) == t2 + + x1 = malloc(S1) + x2 = malloc(S2) + p2 = x2.parent + assert runtime_type_info(x1) == t1.base + assert runtime_type_info(x2) == t2.base.base + assert runtime_type_info(p2) == t2.base.base + + S1bis = GcStruct('S1', ('x', Signed)) + assert S1bis != S1 # attached runtime type info distinguishes them + py.test.raises(TypeError, getRuntimeTypeInfo, S1bis) + + cache = {} + assert getRuntimeTypeInfo(S1, cache=cache) == t1 + t1bis = getRuntimeTypeInfo(S1bis, cache=cache) + assert t1bis != t1.base + assert getRuntimeTypeInfo(S1bis, cache=cache) == t1bis def test_getRuntimeTypeInfo_destrpointer(): - S = GcStruct('s', ('x', Signed)) + S = GcStruct('s', ('x', Signed), + runtime_type_info=malloc(RuntimeTypeInfo, immortal=True)) def f(s): s.x = 1 - def type_info_S(p): - return getRuntimeTypeInfo(S) - qp = functionptr(FuncType([Ptr(S)], Ptr(RuntimeTypeInfo)), - "type_info_S", - _callable=type_info_S) dp = functionptr(FuncType([Ptr(S)], Void), "destructor_funcptr", _callable=f) - pinf0 = attachRuntimeTypeInfo(S, qp, destrptr=dp) - assert pinf0._obj.about == S pinf = getRuntimeTypeInfo(S) - assert pinf == pinf0 - pinf1 = getRuntimeTypeInfo(S) - assert pinf == pinf1 + assert pinf._obj.destructor_funcptr == nullptr(FuncType([Ptr(S)], Void)) + pinf._obj.destructor_funcptr = dp assert pinf._obj.destructor_funcptr == dp - assert pinf._obj.query_funcptr == qp -def test_runtime_type_info(): - S = GcStruct('s', ('x', Signed)) - attachRuntimeTypeInfo(S) - s = malloc(S) - s.x = 0 - assert runtime_type_info(s) == getRuntimeTypeInfo(S) - S1 = GcStruct('s1', ('sub', S), ('x', Signed)) - attachRuntimeTypeInfo(S1) - s1 = malloc(S1) - s1.sub.x = 0 - s1.x = 0 - assert runtime_type_info(s1) == getRuntimeTypeInfo(S1) - assert runtime_type_info(s1.sub) == getRuntimeTypeInfo(S1) - assert runtime_type_info(cast_pointer(Ptr(S), s1)) == getRuntimeTypeInfo(S1) - def dynamic_type_info_S(p): - if p.x == 0: - return getRuntimeTypeInfo(S) - else: - return getRuntimeTypeInfo(S1) - fp = functionptr(FuncType([Ptr(S)], Ptr(RuntimeTypeInfo)), - "dynamic_type_info_S", - _callable=dynamic_type_info_S) - attachRuntimeTypeInfo(S, fp) - assert s.x == 0 - assert runtime_type_info(s) == getRuntimeTypeInfo(S) - s.x = 1 - py.test.raises(RuntimeError, "runtime_type_info(s)") - assert s1.sub.x == 0 - py.test.raises(RuntimeError, "runtime_type_info(s1.sub)") - s1.sub.x = 1 - assert runtime_type_info(s1.sub) == getRuntimeTypeInfo(S1) - def test_flavor_malloc(): def isweak(p, T): return p._weak and typeOf(p).TO == T From arigo at codespeak.net Mon Feb 18 20:34:39 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 18 Feb 2008 20:34:39 +0100 (CET) Subject: [pypy-svn] r51603 - in pypy/branch/unified-rtti/pypy: annotation rpython rpython/lltypesystem rpython/lltypesystem/test rpython/test Message-ID: <20080218193439.BE5EA1684E1@codespeak.net> Author: arigo Date: Mon Feb 18 20:34:37 2008 New Revision: 51603 Modified: pypy/branch/unified-rtti/pypy/annotation/builtin.py pypy/branch/unified-rtti/pypy/rpython/llinterp.py pypy/branch/unified-rtti/pypy/rpython/lltypesystem/exceptiondata.py pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lloperation.py pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py pypy/branch/unified-rtti/pypy/rpython/lltypesystem/opimpl.py pypy/branch/unified-rtti/pypy/rpython/lltypesystem/rclass.py pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_lltype.py pypy/branch/unified-rtti/pypy/rpython/rbuiltin.py pypy/branch/unified-rtti/pypy/rpython/rtyper.py pypy/branch/unified-rtti/pypy/rpython/test/test_rclass.py Log: In-progress: fixing lltypesystem.rclass. Modified: pypy/branch/unified-rtti/pypy/annotation/builtin.py ============================================================================== --- pypy/branch/unified-rtti/pypy/annotation/builtin.py (original) +++ pypy/branch/unified-rtti/pypy/annotation/builtin.py Mon Feb 18 20:34:37 2008 @@ -455,6 +455,12 @@ cast_p = lltype.cast_opaque_ptr(PtrT.const, s_p.ll_ptrtype._defl()) return SomePtr(ll_ptrtype=lltype.typeOf(cast_p)) +def runtime_type_info(s_p): + assert isinstance(s_p, SomePtr), ("runtime_type_info of non-pointer: %r" % + s_p) + res_p = lltype.runtime_type_info(s_p.ll_ptrtype._example()) + return SomePtr(ll_ptrtype=lltype.typeOf(res_p)) + def direct_fieldptr(s_p, s_fieldname): assert isinstance(s_p, SomePtr), "direct_* of non-pointer: %r" % s_p assert s_fieldname.is_constant() @@ -479,14 +485,6 @@ assert PtrT.is_constant() return SomePtr(ll_ptrtype=PtrT.const) -def getRuntimeTypeInfo(T): - assert T.is_constant() - return immutablevalue(lltype.getRuntimeTypeInfo(T.const)) - -def runtime_type_info(s_p): - assert isinstance(s_p, SomePtr), "runtime_type_info of non-pointer: %r" % s_p - return SomePtr(lltype.typeOf(lltype.runtime_type_info(s_p.ll_ptrtype._example()))) - def constPtr(T): assert T.is_constant() return immutablevalue(lltype.Ptr(T.const)) @@ -498,13 +496,12 @@ BUILTIN_ANALYZERS[lltype.nullptr] = nullptr BUILTIN_ANALYZERS[lltype.cast_pointer] = cast_pointer BUILTIN_ANALYZERS[lltype.cast_opaque_ptr] = cast_opaque_ptr +BUILTIN_ANALYZERS[lltype.runtime_type_info] = runtime_type_info BUILTIN_ANALYZERS[lltype.direct_fieldptr] = direct_fieldptr BUILTIN_ANALYZERS[lltype.direct_arrayitems] = direct_arrayitems BUILTIN_ANALYZERS[lltype.direct_ptradd] = direct_ptradd BUILTIN_ANALYZERS[lltype.cast_ptr_to_int] = cast_ptr_to_int BUILTIN_ANALYZERS[lltype.cast_int_to_ptr] = cast_int_to_ptr -BUILTIN_ANALYZERS[lltype.getRuntimeTypeInfo] = getRuntimeTypeInfo -BUILTIN_ANALYZERS[lltype.runtime_type_info] = runtime_type_info BUILTIN_ANALYZERS[lltype.Ptr] = constPtr # ootype Modified: pypy/branch/unified-rtti/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/llinterp.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/llinterp.py Mon Feb 18 20:34:37 2008 @@ -764,12 +764,6 @@ def op_gc_restore_exception(self, exc): raise NotImplementedError("gc_restore_exception") - def op_gc_call_rtti_destructor(self, rtti, addr): - if hasattr(rtti._obj, 'destructor_funcptr'): - d = rtti._obj.destructor_funcptr - obptr = addr.ref() - return self.op_direct_call(d, obptr) - def op_gc_deallocate(self, TYPE, addr): raise NotImplementedError("gc_deallocate") Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/exceptiondata.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/exceptiondata.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/exceptiondata.py Mon Feb 18 20:34:37 2008 @@ -53,11 +53,8 @@ #else: # assert cls.__module__ != 'exceptions', ( # "built-in exceptions should not grow attributes") - r_inst = rclass.getinstancerepr(rtyper, None) - r_inst.setup() default_excinst = malloc(self.lltype_of_exception_value.TO, immortal=True) - default_excinst.typeptr = r_inst.rclass.getvtable() # build the table in order base classes first, subclasses last sortedtable = [] Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lloperation.py Mon Feb 18 20:34:37 2008 @@ -390,7 +390,7 @@ 'gc_free': LLOp(), 'gc_fetch_exception': LLOp(), 'gc_restore_exception': LLOp(), - 'gc_call_rtti_destructor': LLOp(), + 'gc_runtime_type_info': LLOp(canfold=True), 'gc_deallocate': LLOp(), 'gc_push_alive_pyobj': LLOp(), 'gc_pop_alive_pyobj': LLOp(), Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py Mon Feb 18 20:34:37 2008 @@ -988,6 +988,8 @@ return adtmeth else: return getter(self) + if self._T == RuntimeTypeInfo: + return self._obj._getattr(field_name) raise AttributeError("%r instance has no field %r" % (self._T, field_name)) @@ -1003,6 +1005,9 @@ "expects %r\n" " got %r" % (self._T, field_name, T1, T2)) return + if self._T == RuntimeTypeInfo: + self._obj._setattr(field_name, val) + return raise AttributeError("%r instance has no field %r" % (self._T, field_name)) @@ -1644,6 +1649,7 @@ class _rtti(_parentable): _kind = "rtti" + destructor_funcptr = None def __init__(self, initialization=None, parent=None, parentindex=None): _parentable.__init__(self, RuntimeTypeInfo) @@ -1656,7 +1662,23 @@ if GCTYPE._gckind != 'gc': raise TypeError("non-GC type %r cannot have an rtti" % (GCTYPE,)) self._GCTYPE = GCTYPE - self.destructor_funcptr = nullptr(FuncType([Ptr(GCTYPE)], Void)) + + def _getattr(self, name): + if name == 'destructor_funcptr': + return self.destructor_funcptr + else: + raise AttributeError(name) + + def _setattr(self, name, value): + if name == 'destructor_funcptr': + T = typeOf(value).TO + if (not isinstance(T, FuncType) or len(T.ARGS) != 1 + or T.RESULT != Void or not isinstance(T.ARGS[0], Ptr) + or castable(T.ARGS[0], Ptr(self._GCTYPE)) < 0): + raise TypeError("bad type for destructor: %r" % (T,)) + self.destructor_funcptr = value + else: + raise AttributeError(name) class _opaque(_parentable): @@ -1782,16 +1804,15 @@ return _ptr(PTRTYPE, oddint, solid=True) def getRuntimeTypeInfo(TYPE, cache=None): - """Return the runtime_type_info attached to the GcStruct TYPE. - This is typically of type != Ptr(RuntimeTypeInfo) but castable - to Ptr(RuntimeTypeInfo). This raises TypeError if the TYPE has + """Return the runtime_type_info attached to the GcStruct TYPE, + as a Ptr(RuntimeTypeInfo). This raises TypeError if the TYPE has no runtime_type_info, unless 'cache' is specified; in that case, TYPE can be any GC type and a runtime_type_info is created for it if it has none and stored in the cache to avoid mutating the TYPE. """ if isinstance(TYPE, GcStruct) and hasattr(TYPE, '_rtti'): - return top_container(TYPE._rtti)._as_ptr() + return TYPE._rtti._as_ptr() if cache is None: raise TypeError("%r has no runtime_type_info" % (TYPE,)) try: @@ -1801,6 +1822,7 @@ rttiptr._obj._set_gctype(TYPE) cache[TYPE] = rttiptr return rttiptr +getRuntimeTypeInfo._annspecialcase_ = 'specialize:memo' def _install_rtti(STRUCT, runtime_type_info): if not isinstance(STRUCT, GcStruct): Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/opimpl.py Mon Feb 18 20:34:37 2008 @@ -313,6 +313,9 @@ ## assert type(x) is int ## return llmemory.cast_int_to_adr(x) +def op_gc_runtime_type_info(gcptr): + return lltype.runtime_type_info(gcptr) + def op_unichar_eq(x, y): assert isinstance(x, unicode) and len(x) == 1 Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/rclass.py Mon Feb 18 20:34:37 2008 @@ -12,8 +12,7 @@ from pypy.rpython.lltypesystem.lltype import \ Ptr, Struct, GcStruct, malloc, \ cast_pointer, cast_ptr_to_int, castable, nullptr, \ - RuntimeTypeInfo, getRuntimeTypeInfo, typeOf, \ - Array, Char, Void, attachRuntimeTypeInfo, \ + typeOf, Array, Char, Void, \ FuncType, Bool, Signed, functionptr, FuncType, PyObject from pypy.rpython.lltypesystem import lltype from pypy.rpython.robject import PyObjRepr, pyobj_repr @@ -27,8 +26,7 @@ # A root class "object" has: # # struct object_vtable { -# // struct object_vtable* parenttypeptr; not used any more -# RuntimeTypeInfo * rtti; +# RuntimeTypeInfo rtti; //GC-specific information # Signed subclassrange_min; //this is also the id of the class itself # Signed subclassrange_max; # array { char } * name; @@ -45,7 +43,7 @@ # The type of the instances is: # # struct object { // for the root class -# struct object_vtable* typeptr; +# some GC-specific fields including a ptr to the vtable # } # # struct X { @@ -55,22 +53,32 @@ # # there's also a nongcobject -OBJECT_VTABLE = lltype.ForwardReference() -CLASSTYPE = Ptr(OBJECT_VTABLE) -OBJECT = GcStruct('object', ('typeptr', CLASSTYPE), - hints = {'immutable': True, 'shouldntbenull': True}) +def ll_gettypeptr(p): + rtti = lltype.runtime_type_info(p) + return cast_pointer(CLASSTYPE, rtti) + +def ll_gettypeptr_nongc(p): + return p.typeptr + +OBJECT = lltype.GcForwardReference() OBJECTPTR = Ptr(OBJECT) -OBJECT_VTABLE.become(Struct('object_vtable', - #('parenttypeptr', CLASSTYPE), +OBJECT_VTABLE = Struct('object_vtable', + ('rtti', lltype.RuntimeTypeInfo), ('subclassrange_min', Signed), ('subclassrange_max', Signed), - ('rtti', Ptr(RuntimeTypeInfo)), ('name', Ptr(Array(Char))), ('instantiate', Ptr(FuncType([], OBJECTPTR))), - hints = {'immutable': True})) + hints = {'immutable': True}) +CLASSTYPE = Ptr(OBJECT_VTABLE) +OBJECT.become(GcStruct('object', + runtime_type_info = malloc(OBJECT_VTABLE, + immortal=True), + adtmeths = {'gettypeptr': ll_gettypeptr})) + # non-gc case -NONGCOBJECT = Struct('nongcobject', ('typeptr', CLASSTYPE)) -NONGCOBJECTPTR = Ptr(OBJECT) +NONGCOBJECT = Struct('nongcobject', ('typeptr', CLASSTYPE), + adtmeths = {'gettypeptr': ll_gettypeptr_nongc}) +NONGCOBJECTPTR = Ptr(NONGCOBJECT) OBJECT_BY_FLAVOR = {'gc': OBJECT, 'raw': NONGCOBJECT} @@ -80,12 +88,6 @@ 'stack': 'raw', } -def cast_vtable_to_typeptr(vtable): - while typeOf(vtable).TO != OBJECT_VTABLE: - vtable = vtable.super - return vtable - - class ClassRepr(AbstractClassRepr): def __init__(self, rtyper, classdef): AbstractClassRepr.__init__(self, rtyper, classdef) @@ -140,32 +142,15 @@ self.clsfields = clsfields self.pbcfields = pbcfields self.allmethods = allmethods - self.vtable = None - -# def convert_const(self, value): -# if not isinstance(value, (type, types.ClassType)): -# raise TyperError("not a class: %r" % (value,)) -# try: -# subclassdef = self.rtyper.annotator.getuserclasses()[value] -# except KeyError: -# raise TyperError("no classdef: %r" % (value,)) -# if self.classdef is not None: -# if self.classdef.commonbase(subclassdef) != self.classdef: -# raise TyperError("not a subclass of %r: %r" % ( -# self.classdef.cls, value)) -# # -# return getclassrepr(self.rtyper, subclassdef).getvtable() + self.vtable = malloc(self.vtable_type, immortal=True) + self.vtable_filled = False - def getvtable(self, cast_to_typeptr=True): + def getvtable(self): """Return a ptr to the vtable of this type.""" - if self.vtable is None: - self.vtable = malloc(self.vtable_type, immortal=True) + if not self.vtable_filled: self.setup_vtable(self.vtable, self) - # - vtable = self.vtable - if cast_to_typeptr: - vtable = cast_vtable_to_typeptr(vtable) - return vtable + self.vtable_filled = True + return cast_pointer(CLASSTYPE, self.vtable) getruntime = getvtable def setup_vtable(self, vtable, rsubcls): @@ -174,7 +159,6 @@ if self.classdef is None: # initialize the 'subclassrange_*' and 'name' fields if rsubcls.classdef is not None: - #vtable.parenttypeptr = rsubcls.rbase.getvtable() vtable.subclassrange_min = rsubcls.classdef.minid vtable.subclassrange_max = rsubcls.classdef.maxid else: #for the root class @@ -182,8 +166,6 @@ vtable.subclassrange_max = sys.maxint rinstance = getinstancerepr(self.rtyper, rsubcls.classdef) rinstance.setup() - if rinstance.gcflavor == 'gc': - vtable.rtti = getRuntimeTypeInfo(rinstance.object_type) if rsubcls.classdef is None: name = 'object' else: @@ -232,7 +214,9 @@ # to self's vtable type.""" def fromtypeptr(self, vcls, llops): - """Return the type pointer cast to self's vtable type.""" + """Return a RuntimeTypeInfo or Ptr(OBJECT_VTABLE) cast + to self's vtable type. + """ self.setup() castable(self.lowleveltype, vcls.concretetype) # sanity check return llops.genop('cast_pointer', [vcls], @@ -296,15 +280,15 @@ class InstanceRepr(AbstractInstanceRepr): def __init__(self, rtyper, classdef, gcflavor='gc'): AbstractInstanceRepr.__init__(self, rtyper, classdef) + self.gcflavor = gcflavor if classdef is None: - self.object_type = OBJECT_BY_FLAVOR[LLFLAVOR[gcflavor]] + self.object_type = self.get_base_object_type() else: ForwardRef = lltype.FORWARDREF_BY_FLAVOR[LLFLAVOR[gcflavor]] self.object_type = ForwardRef() self.prebuiltinstances = {} # { id(x): (x, _ptr) } self.lowleveltype = Ptr(self.object_type) - self.gcflavor = gcflavor def _setup_repr(self, llfields=None, hints=None, adtmeths=None): # NOTE: don't store mutable objects like the dicts below on 'self' @@ -315,7 +299,7 @@ fields = {} allinstancefields = {} if self.classdef is None: - fields['__class__'] = 'typeptr', get_type_repr(self.rtyper) + pass else: # instance attributes if llfields is None: @@ -347,18 +331,18 @@ if '_immutable_' in self.classdef.classdesc.classdict: hints = hints.copy() hints['immutable'] = True + kwds = {'hints': hints, + 'adtmeths': adtmeths} + if self.gcflavor == 'gc': + kwds['runtime_type_info'] = self.rclass.vtable object_type = MkStruct(self.classdef.name, ('super', self.rbase.object_type), - hints=hints, - adtmeths=adtmeths, - *llfields) + *llfields, **kwds) self.object_type.become(object_type) allinstancefields.update(self.rbase.allinstancefields) allinstancefields.update(fields) self.fields = fields self.allinstancefields = allinstancefields - if self.gcflavor == 'gc': - attachRuntimeTypeInfo(self.object_type) def _setup_repr_final(self): if self.gcflavor == 'gc': @@ -375,12 +359,14 @@ destrptr = functionptr(FUNCTYPE, graph.name, graph=graph, _callable=graph.func) - else: - destrptr = None - OBJECT = OBJECT_BY_FLAVOR[LLFLAVOR[self.gcflavor]] - self.rtyper.attachRuntimeTypeInfoFunc(self.object_type, - ll_runtime_type_info, - OBJECT, destrptr) + rtti = lltype.getRuntimeTypeInfo(self.object_type) + rtti = cast_pointer(Ptr(lltype.RuntimeTypeInfo), rtti) + rtti.destructor_funcptr = destrptr + + def get_base_object_type(self): + return OBJECT_BY_FLAVOR[LLFLAVOR[self.gcflavor]] + get_base_object_type._annspecialcase_ = 'specialize:memo' + def common_repr(self): # -> object or nongcobject reprs return getinstancerepr(self.rtyper, None, self.gcflavor) @@ -391,6 +377,7 @@ return cast_pointer(self.lowleveltype, result) def create_instance(self): + self.rclass.getvtable() # force it to be filled return malloc(self.object_type, flavor=self.gcflavor) def get_ll_hash_function(self): @@ -434,9 +421,23 @@ llattrvalue = r.convert_const(attrvalue) setattr(result, mangled_name, llattrvalue) else: - # OBJECT part - rclass = getclassrepr(self.rtyper, classdef) - result.typeptr = rclass.getvtable() + # OBJECT part: nothing to fill, unless we are a non-GC instance + if self.gcflavor != 'gc': + rclass = getclassrepr(self.rtyper, classdef) + result.typeptr = rclass.getvtable() + + def gettypeptr(self, vinst, llops): + if self.gcflavor == 'gc': + v_rtti = llops.genop('gc_runtime_type_info', [vinst], + resulttype = Ptr(lltype.RuntimeTypeInfo)) + return llops.genop('cast_pointer', [v_rtti], + resulttype = CLASSTYPE) + else: + v_base = llops.genop('cast_pointer', [vinst], + resulttype = NONGCOBJECTPTR) + c_typeptr = inputconst(Void, 'typeptr') + return llops.genop('getfield', [v_base, c_typeptr], + resulttype = CLASSTYPE) def getfieldrepr(self, attr): """Return the repr used for the given attribute.""" @@ -458,13 +459,16 @@ return llops.genop('getfield', [vinst, cname], resulttype=r) else: if self.classdef is None: - raise MissingRTypeAttribute(attr) + if attr == '__class__': + return self.gettypeptr(vinst, llops) + else: + raise MissingRTypeAttribute(attr) return self.rbase.getfield(vinst, attr, llops, force_cast=True, flags=flags) def setfield(self, vinst, attr, vvalue, llops, force_cast=False, flags={}): - """Write the given attribute (or __class__ for the type) of 'vinst'.""" + """Write the given attribute of 'vinst'.""" if attr in self.fields: mangled_name, r = self.fields[attr] cname = inputconst(Void, mangled_name) @@ -484,17 +488,14 @@ ctype = inputconst(Void, self.object_type) cflags = inputconst(Void, flags) vlist = [ctype, cflags] + self.rclass.getvtable() # force it to be filled vptr = llops.genop('malloc', vlist, resulttype = Ptr(self.object_type)) - ctypeptr = inputconst(CLASSTYPE, self.rclass.getvtable()) - self.setfield(vptr, '__class__', ctypeptr, llops) # initialize instance attributes from their defaults from the class if self.classdef is not None: flds = self.allinstancefields.keys() flds.sort() for fldname in flds: - if fldname == '__class__': - continue mangled_name, r = self.allinstancefields[fldname] if r.lowleveltype is Void: continue @@ -522,10 +523,13 @@ def rtype_getattr(self, hop): attr = hop.args_s[1].const vinst, vattr = hop.inputargs(self, Void) - if attr == '__class__' and hop.r_result.lowleveltype is Void: - # special case for when the result of '.__class__' is a constant - [desc] = hop.s_result.descriptions - return hop.inputconst(Void, desc.pyobj) + if attr == '__class__': + if hop.r_result.lowleveltype is Void: + # special case for when the result of '.__class__' is a const + [desc] = hop.s_result.descriptions + return hop.inputconst(Void, desc.pyobj) + else: + return self.getfield(vinst, '__class__', hop.llops) if attr in self.allinstancefields: return self.getfield(vinst, attr, hop.llops, flags=hop.args_s[0].flags) @@ -549,19 +553,21 @@ vinst, = hop.inputargs(self) return hop.genop('ptr_nonzero', [vinst], resulttype=Bool) - def ll_str(self, i): # doesn't work for non-gc classes! + def ll_str(self, i): from pypy.rpython.lltypesystem import rstr from pypy.rpython.lltypesystem.ll_str import ll_int2hex from pypy.rlib.rarithmetic import r_uint if not i: return rstr.null_str - instance = cast_pointer(OBJECTPTR, i) + BASEPTR = Ptr(self.get_base_object_type()) + instance = cast_pointer(BASEPTR, i) + vtable = instance.gettypeptr() uid = r_uint(cast_ptr_to_int(i)) - nameLen = len(instance.typeptr.name) + nameLen = len(vtable.name) nameString = rstr.mallocstr(nameLen-1) i = 0 while i < nameLen - 1: - nameString.chars[i] = instance.typeptr.name[i] + nameString.chars[i] = vtable.name[i] i += 1 res = rstr.instance_str_prefix res = rstr.ll_strconcat(res, nameString) @@ -667,7 +673,7 @@ # doesn't work for non-gc stuff! def ll_type(obj): - return cast_pointer(OBJECTPTR, obj).typeptr + return cast_pointer(OBJECTPTR, obj).gettypeptr() def ll_issubclass(subcls, cls): return cls.subclassrange_min <= subcls.subclassrange_min <= cls.subclassrange_max @@ -679,23 +685,20 @@ def ll_isinstance(obj, cls): # obj should be cast to OBJECT or NONGCOBJECT if not obj: return False - obj_cls = obj.typeptr + obj_cls = obj.gettypeptr() return ll_issubclass(obj_cls, cls) def ll_isinstance_const(obj, minid, maxid): if not obj: return False - return ll_issubclass_const(obj.typeptr, minid, maxid) + return ll_issubclass_const(obj.gettypeptr(), minid, maxid) def ll_isinstance_exact(obj, cls): if not obj: return False - obj_cls = obj.typeptr + obj_cls = obj.gettypeptr() return obj_cls == cls -def ll_runtime_type_info(obj): - return obj.typeptr.rtti - def ll_inst_hash(ins): if not ins: return 0 # for None @@ -710,10 +713,10 @@ def ll_inst_type(obj): if obj: - return obj.typeptr + return obj.gettypeptr() else: # type(None) -> NULL (for now) - return nullptr(typeOf(obj).TO.typeptr.TO) + return nullptr(CLASSTYPE.TO) def ll_both_none(ins1, ins2): return not ins1 and not ins2 Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_lltype.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_lltype.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_lltype.py Mon Feb 18 20:34:37 2008 @@ -373,8 +373,8 @@ t2.v = 33 S2 = GcStruct('S2', ('parent', S1), ('y', Signed), runtime_type_info=t2) - assert getRuntimeTypeInfo(S1) == t1 - assert getRuntimeTypeInfo(S2) == t2 + assert getRuntimeTypeInfo(S1) == t1.base + assert getRuntimeTypeInfo(S2) == t2.base.base x1 = malloc(S1) x2 = malloc(S2) @@ -388,7 +388,7 @@ py.test.raises(TypeError, getRuntimeTypeInfo, S1bis) cache = {} - assert getRuntimeTypeInfo(S1, cache=cache) == t1 + assert getRuntimeTypeInfo(S1, cache=cache) == t1.base t1bis = getRuntimeTypeInfo(S1bis, cache=cache) assert t1bis != t1.base assert getRuntimeTypeInfo(S1bis, cache=cache) == t1bis @@ -402,9 +402,16 @@ "destructor_funcptr", _callable=f) pinf = getRuntimeTypeInfo(S) - assert pinf._obj.destructor_funcptr == nullptr(FuncType([Ptr(S)], Void)) - pinf._obj.destructor_funcptr = dp - assert pinf._obj.destructor_funcptr == dp + assert pinf.destructor_funcptr is None + pinf.destructor_funcptr = dp + assert pinf.destructor_funcptr == dp + + S1 = GcStruct('s1', ('super', S), + runtime_type_info=malloc(RuntimeTypeInfo, immortal=True)) + pinf1 = getRuntimeTypeInfo(S1) + assert pinf1.destructor_funcptr is None + pinf1.destructor_funcptr = dp + assert pinf1.destructor_funcptr == dp def test_flavor_malloc(): def isweak(p, T): Modified: pypy/branch/unified-rtti/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/rbuiltin.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/rbuiltin.py Mon Feb 18 20:34:37 2008 @@ -476,7 +476,7 @@ def rtype_runtime_type_info(hop): assert isinstance(hop.args_r[0], rptr.PtrRepr) vlist = hop.inputargs(hop.args_r[0]) - return hop.genop('runtime_type_info', vlist, + return hop.genop('gc_runtime_type_info', vlist, resulttype = hop.r_result.lowleveltype) BUILTIN_TYPER[lltype.malloc] = rtype_malloc @@ -484,6 +484,7 @@ BUILTIN_TYPER[lltype.cast_primitive] = rtype_cast_primitive BUILTIN_TYPER[lltype.cast_pointer] = rtype_cast_pointer BUILTIN_TYPER[lltype.cast_opaque_ptr] = rtype_cast_opaque_ptr +BUILTIN_TYPER[lltype.runtime_type_info] = rtype_runtime_type_info BUILTIN_TYPER[lltype.direct_fieldptr] = rtype_direct_fieldptr BUILTIN_TYPER[lltype.direct_arrayitems] = rtype_direct_arrayitems BUILTIN_TYPER[lltype.direct_ptradd] = rtype_direct_ptradd @@ -491,9 +492,7 @@ BUILTIN_TYPER[lltype.cast_int_to_ptr] = rtype_cast_int_to_ptr BUILTIN_TYPER[lltype.typeOf] = rtype_const_result BUILTIN_TYPER[lltype.nullptr] = rtype_const_result -BUILTIN_TYPER[lltype.getRuntimeTypeInfo] = rtype_const_result BUILTIN_TYPER[lltype.Ptr] = rtype_const_result -BUILTIN_TYPER[lltype.runtime_type_info] = rtype_runtime_type_info BUILTIN_TYPER[rarithmetic.intmask] = rtype_intmask BUILTIN_TYPER[objectmodel.we_are_translated] = rtype_we_are_translated Modified: pypy/branch/unified-rtti/pypy/rpython/rtyper.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/rtyper.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/rtyper.py Mon Feb 18 20:34:37 2008 @@ -21,8 +21,8 @@ from pypy.rpython.lltypesystem.lltype import \ Signed, Unsigned, Float, Char, Bool, Void, \ LowLevelType, Ptr, ContainerType, \ - FuncType, functionptr, typeOf, RuntimeTypeInfo, \ - attachRuntimeTypeInfo, Primitive, Number + FuncType, functionptr, typeOf, \ + Primitive, Number from pypy.rpython.ootypesystem import ootype from pypy.translator.unsimplify import insert_empty_block from pypy.rpython.error import TyperError @@ -648,21 +648,6 @@ graph = self.annotate_helper(ll_function, argtypes) return self.getcallable(graph) - def attachRuntimeTypeInfoFunc(self, GCSTRUCT, func, ARG_GCSTRUCT=None, - destrptr=None): - self.call_all_setups() # compute ForwardReferences now - if ARG_GCSTRUCT is None: - ARG_GCSTRUCT = GCSTRUCT - args_s = [annmodel.SomePtr(Ptr(ARG_GCSTRUCT))] - graph = self.annotate_helper(func, args_s) - s = self.annotator.binding(graph.getreturnvar()) - if (not isinstance(s, annmodel.SomePtr) or - s.ll_ptrtype != Ptr(RuntimeTypeInfo)): - raise TyperError("runtime type info function %r returns %r, " - "excepted Ptr(RuntimeTypeInfo)" % (func, s)) - funcptr = self.getcallable(graph) - attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr) - # register operations from annotation model RPythonTyper._registeroperations(annmodel) Modified: pypy/branch/unified-rtti/pypy/rpython/test/test_rclass.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/test/test_rclass.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/test/test_rclass.py Mon Feb 18 20:34:37 2008 @@ -650,8 +650,7 @@ graph = graphof(t, f) TYPE = graph.startblock.operations[0].args[0].value RTTI = getRuntimeTypeInfo(TYPE) - queryptr = RTTI._obj.query_funcptr # should not raise - destrptr = RTTI._obj.destructor_funcptr + destrptr = RTTI.destructor_funcptr assert destrptr is not None def test_del_inheritance(self): @@ -681,19 +680,18 @@ t = TranslationContext() t.buildannotator().build_types(f, []) t.buildrtyper().specialize() - graph = graphof(t, f) - TYPEA = graph.startblock.operations[0].args[0].value + + from pypy.rpython.rclass import getinstancerepr + getcdef = t.annotator.bookkeeper.getuniqueclassdef + TYPEA = getinstancerepr(t.rtyper, getcdef(A)).lowleveltype.TO RTTIA = getRuntimeTypeInfo(TYPEA) - TYPEB = graph.startblock.operations[3].args[0].value + TYPEB = getinstancerepr(t.rtyper, getcdef(B)).lowleveltype.TO RTTIB = getRuntimeTypeInfo(TYPEB) - TYPEC = graph.startblock.operations[6].args[0].value + TYPEC = getinstancerepr(t.rtyper, getcdef(C)).lowleveltype.TO RTTIC = getRuntimeTypeInfo(TYPEC) - queryptra = RTTIA._obj.query_funcptr # should not raise - queryptrb = RTTIB._obj.query_funcptr # should not raise - queryptrc = RTTIC._obj.query_funcptr # should not raise - destrptra = RTTIA._obj.destructor_funcptr - destrptrb = RTTIB._obj.destructor_funcptr - destrptrc = RTTIC._obj.destructor_funcptr + destrptra = RTTIA.destructor_funcptr + destrptrb = RTTIB.destructor_funcptr + destrptrc = RTTIC.destructor_funcptr assert destrptra == destrptrc assert typeOf(destrptra).TO.ARGS[0] != typeOf(destrptrb).TO.ARGS[0] assert destrptra is not None From fijal at codespeak.net Mon Feb 18 21:40:27 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 21:40:27 +0100 (CET) Subject: [pypy-svn] r51604 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080218204027.1DAB41684DE@codespeak.net> Author: fijal Date: Mon Feb 18 21:40:24 2008 New Revision: 51604 Modified: pypy/dist/pypy/lib/app_test/ctypes/test_array.py Log: oops. Modified: pypy/dist/pypy/lib/app_test/ctypes/test_array.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_array.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_array.py Mon Feb 18 21:40:24 2008 @@ -113,12 +113,6 @@ class TestSophisticatedThings(BaseCTypesTestChecker): def test_array_of_structures(self): - import gc - gc.collect() - gc.collect() - gc.collect() - from _rawffi import _num_of_allocated_objects as _num - assert _num() == 3 class X(Structure): _fields_ = [('x', c_int), ('y', c_int)] From fijal at codespeak.net Mon Feb 18 21:40:43 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 21:40:43 +0100 (CET) Subject: [pypy-svn] r51605 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080218204043.7F98C1684DE@codespeak.net> Author: fijal Date: Mon Feb 18 21:40:42 2008 New Revision: 51605 Modified: pypy/dist/pypy/lib/app_test/ctypes/test_as_parameter.py Log: This passes. Modified: pypy/dist/pypy/lib/app_test/ctypes/test_as_parameter.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_as_parameter.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_as_parameter.py Mon Feb 18 21:40:42 2008 @@ -1,5 +1,6 @@ from ctypes import * import py +from support import BaseCTypesTestChecker def setup_module(mod): import conftest @@ -15,7 +16,7 @@ class POINT(Structure): _fields_ = [("x", c_int), ("y", c_int)] -class BasicTestWrap: +class BasicTestWrap(BaseCTypesTestChecker): def wrap(self, param): return param From fijal at codespeak.net Mon Feb 18 21:43:29 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 21:43:29 +0100 (CET) Subject: [pypy-svn] r51606 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080218204329.BC6B41684DB@codespeak.net> Author: fijal Date: Mon Feb 18 21:43:29 2008 New Revision: 51606 Modified: pypy/dist/pypy/lib/app_test/ctypes/support.py Log: This way we can test equality. Modified: pypy/dist/pypy/lib/app_test/ctypes/support.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/support.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/support.py Mon Feb 18 21:43:29 2008 @@ -7,7 +7,8 @@ pass else: import gc - gc.collect() + for _ in range(4): + gc.collect() cls.old_num = _rawffi._num_of_allocated_objects() def teardown_class(cls): @@ -21,4 +22,4 @@ for _ in range(4): gc.collect() # there is one reference coming from the byref() above - assert _rawffi._num_of_allocated_objects() <= cls.old_num + assert _rawffi._num_of_allocated_objects() == cls.old_num From fijal at codespeak.net Mon Feb 18 21:53:23 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 21:53:23 +0100 (CET) Subject: [pypy-svn] r51607 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080218205323.96B2F1684DE@codespeak.net> Author: fijal Date: Mon Feb 18 21:53:22 2008 New Revision: 51607 Modified: pypy/dist/pypy/lib/_ctypes/function.py Log: keepalive logic for functions. Modified: pypy/dist/pypy/lib/_ctypes/function.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/function.py (original) +++ pypy/dist/pypy/lib/_ctypes/function.py Mon Feb 18 21:53:22 2008 @@ -19,6 +19,7 @@ _restype_ = None _ffiletter = 'P' _ffishape = 'P' + _needs_free = False def _getargtypes(self): return self._argtypes_ @@ -49,6 +50,7 @@ argtypes = [arg._ffiletter for arg in self._argtypes_] restype = self._restype_._ffiletter self._ptr = _rawffi.CallbackPtr(argument, argtypes, restype) + self._needs_free = True self._buffer = self._ptr.byptr() elif isinstance(argument, tuple) and len(argument) == 2: import ctypes @@ -118,3 +120,11 @@ zip(argtypes, args)] except (UnicodeError, TypeError), e: raise ArgumentError(str(e)) + + def __del__(self): + if self._needs_free: + self._buffer.free() + self._buffer = None + self._ptr.free() + self._ptr = None + self._needs_free = False From fijal at codespeak.net Mon Feb 18 21:53:37 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 21:53:37 +0100 (CET) Subject: [pypy-svn] r51608 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080218205337.39A0E1684DE@codespeak.net> Author: fijal Date: Mon Feb 18 21:53:36 2008 New Revision: 51608 Modified: pypy/dist/pypy/lib/_ctypes/structure.py Log: Simplify a bit Modified: pypy/dist/pypy/lib/_ctypes/structure.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/structure.py (original) +++ pypy/dist/pypy/lib/_ctypes/structure.py Mon Feb 18 21:53:36 2008 @@ -178,8 +178,8 @@ except KeyError: return _CData.__getattribute__(self, name) offset = self.__class__._fieldtypes[name].offset - return fieldtype._CData_output(self._subarray(fieldtype, name), self, - offset) + suba = self._subarray(fieldtype, name) + return fieldtype._CData_output(suba, self, offset) def _get_buffer_for_param(self): return CArgObject(self._buffer.byptr()) From fijal at codespeak.net Mon Feb 18 21:53:49 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 21:53:49 +0100 (CET) Subject: [pypy-svn] r51609 - in pypy/dist/pypy/lib: _ctypes app_test/ctypes Message-ID: <20080218205349.544201684E2@codespeak.net> Author: fijal Date: Mon Feb 18 21:53:48 2008 New Revision: 51609 Modified: pypy/dist/pypy/lib/_ctypes/primitive.py pypy/dist/pypy/lib/app_test/ctypes/test_buffers.py pypy/dist/pypy/lib/app_test/ctypes/test_callbacks.py pypy/dist/pypy/lib/app_test/ctypes/test_cast.py Log: Keepalive logic for strings. Modified: pypy/dist/pypy/lib/_ctypes/primitive.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/primitive.py (original) +++ pypy/dist/pypy/lib/_ctypes/primitive.py Mon Feb 18 21:53:48 2008 @@ -2,7 +2,8 @@ SIMPLE_TYPE_CHARS = "cbBhHiIlLdfuzZqQPXOv" -from _ctypes.basics import _CData, _CDataMeta, cdata_from_address +from _ctypes.basics import _CData, _CDataMeta, cdata_from_address,\ + CArgObject from _ctypes.builtin import ConvMode class NULL(object): @@ -76,7 +77,7 @@ self._objects = value array = _rawffi.Array('c')(len(value)+1, value) value = array.buffer - # XXX free 'array' later + self._objects = {'0': CArgObject(array)} elif value is None: value = 0 self._buffer[0] = value @@ -99,7 +100,7 @@ self._objects = value array = _rawffi.Array('u')(len(value)+1, value) value = array.buffer - # XXX free 'array' later + self._objects = {'0': CArgObject(array)} elif value is None: value = 0 self._buffer[0] = value @@ -118,8 +119,7 @@ if isinstance(value, str): array = _rawffi.Array('c')(len(value)+1, value) value = array.buffer - self._objects = value - # XXX free 'array' later + self._objects = {'0': CArgObject(value)} elif value is None: value = 0 self._buffer[0] = value Modified: pypy/dist/pypy/lib/app_test/ctypes/test_buffers.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_buffers.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_buffers.py Mon Feb 18 21:53:48 2008 @@ -1,6 +1,7 @@ from ctypes import * +from support import BaseCTypesTestChecker -class TestStringBuffer: +class TestStringBuffer(BaseCTypesTestChecker): def test_buffer(self): b = create_string_buffer(32) Modified: pypy/dist/pypy/lib/app_test/ctypes/test_callbacks.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_callbacks.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_callbacks.py Mon Feb 18 21:53:48 2008 @@ -1,7 +1,8 @@ from ctypes import * import py +from support import BaseCTypesTestChecker -class TestCallbacks: +class TestCallbacks(BaseCTypesTestChecker): functype = CFUNCTYPE ## def tearDown(self): @@ -107,7 +108,7 @@ ################################################################ -class TestSampleCallbacks: +class TestSampleCallbacks(BaseCTypesTestChecker): def test_integrate(self): # Derived from some then non-working code, posted by David Foster Modified: pypy/dist/pypy/lib/app_test/ctypes/test_cast.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_cast.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_cast.py Mon Feb 18 21:53:48 2008 @@ -1,7 +1,8 @@ from ctypes import * import sys, py +from support import BaseCTypesTestChecker -class TestCast: +class TestCast(BaseCTypesTestChecker): def test_array2pointer(self): array = (c_int * 3)(42, 17, 2) @@ -30,9 +31,9 @@ assert [ptr[i] for i in range(3)] == [42, 17, 2] def test_p2a_objects(self): - py.test.skip("keepalive logic") + py.test.skip("We don't keep alive strings") array = (c_char_p * 5)() - assert array._objects == None + assert array._objects is None array[0] = "foo bar" assert array._objects == {'0': "foo bar"} @@ -60,6 +61,7 @@ def test_char_p(self): # This didn't work: bad argument to internal function s = c_char_p("hiho") + assert cast(cast(s, c_void_p), c_char_p).value == ( "hiho") From fijal at codespeak.net Mon Feb 18 21:56:48 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 21:56:48 +0100 (CET) Subject: [pypy-svn] r51610 - in pypy/dist/pypy/lib: _ctypes app_test/ctypes Message-ID: <20080218205648.ABB0A1684DC@codespeak.net> Author: fijal Date: Mon Feb 18 21:56:47 2008 New Revision: 51610 Modified: pypy/dist/pypy/lib/_ctypes/primitive.py pypy/dist/pypy/lib/app_test/ctypes/test_cfuncs.py pypy/dist/pypy/lib/app_test/ctypes/test_extra.py Log: a fix and two more tests pass Modified: pypy/dist/pypy/lib/_ctypes/primitive.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/primitive.py (original) +++ pypy/dist/pypy/lib/_ctypes/primitive.py Mon Feb 18 21:56:47 2008 @@ -119,7 +119,7 @@ if isinstance(value, str): array = _rawffi.Array('c')(len(value)+1, value) value = array.buffer - self._objects = {'0': CArgObject(value)} + self._objects = {'0': CArgObject(array)} elif value is None: value = 0 self._buffer[0] = value Modified: pypy/dist/pypy/lib/app_test/ctypes/test_cfuncs.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_cfuncs.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_cfuncs.py Mon Feb 18 21:56:47 2008 @@ -3,6 +3,7 @@ from ctypes import * import py +from support import BaseCTypesTestChecker def setup_module(mod): import conftest @@ -10,7 +11,7 @@ # this means you cannot run tests directly without invoking this mod.TestCFunctions._dll = CDLL(_ctypes_test) -class TestCFunctions: +class TestCFunctions(BaseCTypesTestChecker): def S(self): return c_longlong.in_dll(self._dll, "last_tf_arg_s").value Modified: pypy/dist/pypy/lib/app_test/ctypes/test_extra.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_extra.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_extra.py Mon Feb 18 21:56:47 2008 @@ -5,224 +5,226 @@ import py from ctypes import * +from support import BaseCTypesTestChecker -def test_primitive_pointer(): - x = c_int(5) - assert x.value == 5 - x.value = 6 - assert x.value == 6 - - p = pointer(x) # p ---> x = 6 - assert isinstance(p.contents, c_int) - p.contents.value += 1 - assert x.value == 7 # p ---> x = 7 - - y = c_int(12) - p.contents = y # p ---> y = 12 - p.contents.value += 2 # p ---> y = 14 - assert y.value == 14 - assert x.value == 7 - - pp = pointer(p) # pp ---> p ---> y = 14 - pp.contents.contents = x # pp ---> p ---> x = 7 - p.contents.value += 2 # pp ---> p ---> x = 9 - assert x.value == 9 - - assert isinstance(p[0], int) - p[0] += 1 # pp ---> p ---> x = 10 - assert x.value == 10 - z = c_int(86) - p[0] = z # pp ---> p ---> x = 86 (not z!) - assert x.value == 86 - z.value = 84 - assert x.value == 86 - - assert isinstance(pp[0], POINTER(c_int)) - assert pp[0].contents.value == x.value == 86 - pp[0].contents = z # pp ---> p ---> z = 84 - assert p.contents.value == z.value == 84 - -## *** the rest is commented out because it should work but occasionally -## *** trigger a ctypes bug (SourceForge bug #1467852). *** -## q = pointer(y) -## pp[0] = q # pp ---> p ---> y = 14 -## assert y.value == 14 # (^^^ not q! ) -## assert p.contents.value == 14 -## assert pp.contents.contents.value == 14 -## q.contents = x -## assert pp.contents.contents.value == 14 - - -def test_char_p(): - x = c_char_p("hello\x00world") - assert x.value == "hello" - x.value = "world" - assert x.value == "world" - - p = pointer(x) - assert p[0] == x.value == "world" - p[0] = "other" - assert x.value == p.contents.value == p[0] == "other" - - myarray = (c_char_p * 10)() - myarray[7] = "hello" - assert isinstance(myarray[7], str) - assert myarray[7] == "hello" - -def test_struct(): - class tagpoint(Structure): - _fields_ = [('x', c_int), - ('p', POINTER(c_short))] - - y = c_short(123) - z = c_short(-33) - s = tagpoint() - s.p.contents = z - assert s.p.contents.value == -33 - s.p = pointer(y) - assert s.p.contents.value == 123 - s.p.contents.value = 124 - assert y.value == 124 - - s = tagpoint(x=12) - assert s.x == 12 - s = tagpoint(17, p=pointer(z)) - assert s.x == 17 - assert s.p.contents.value == -33 - -def test_ptr_array(): - a = (POINTER(c_ushort) * 5)() - x = c_ushort(52) - y = c_ushort(1000) - - a[2] = pointer(x) - assert a[2].contents.value == 52 - a[2].contents.value += 1 - assert x.value == 53 - - a[3].contents = y - assert a[3].contents.value == 1000 - a[3].contents.value += 1 - assert y.value == 1001 - -def test_void_p(): - x = c_int(12) - p1 = cast(pointer(x), c_void_p) - p2 = cast(p1, POINTER(c_int)) - assert p2.contents.value == 12 - -def test_char_array(): - a = (c_char * 3)() - a[0] = 'x' - a[1] = 'y' - assert a.value == 'xy' - a[2] = 'z' - assert a.value == 'xyz' - - b = create_string_buffer(3) - assert type(b) is type(a) - assert len(b) == 3 - - b.value = "nxw" - assert b[0] == 'n' - assert b[1] == 'x' - assert b[2] == 'w' - - b.value = "?" - assert b[0] == '?' - assert b[1] == '\x00' - assert b[2] == 'w' - - class S(Structure): - _fields_ = [('p', POINTER(c_char))] - - s = S() - s.p = b - s.p.contents.value = '!' - assert b.value == '!' - - assert len(create_string_buffer(0)) == 0 - -def test_array(): - a = (c_int * 10)() - - class S(Structure): - _fields_ = [('p', POINTER(c_int))] - - s = S() - s.p = a - s.p.contents.value = 42 - assert a[0] == 42 - - a = (c_int * 5)(5, 6, 7) - assert list(a) == [5, 6, 7, 0, 0] - -def test_truth_value(): - p = POINTER(c_int)() - assert not p - p.contents = c_int(12) - assert p - # I can't figure out how to reset p to NULL... - - assert c_int(12) - assert not c_int(0) # a bit strange, if you ask me - assert c_int(-1) - assert not c_byte(0) - assert not c_char('\x00') # hum - assert not c_float(0.0) - assert not c_double(0.0) - assert not c_ulonglong(0) - assert c_ulonglong(2L**42) - - assert c_char_p("hello") - assert c_char_p("") - assert not c_char_p(None) - - assert not c_void_p() - -def test_sizeof(): - x = create_string_buffer(117) - assert sizeof(x) == 117 # assumes that chars are one byte each - x = (c_int * 42)() - assert sizeof(x) == 42 * sizeof(c_int) - -def test_convert_pointers(): - import conftest - _ctypes_test = str(conftest.sofile) - dll = CDLL(_ctypes_test) - func = dll._testfunc_p_p - func.restype = c_char_p - - # automatic conversions to c_char_p - func.argtypes = [c_char_p] - assert func("hello") == "hello" - assert func(c_char_p("hello")) == "hello" - assert func((c_char * 6)(*"hello")) == "hello" - assert func(create_string_buffer("hello")) == "hello" - - # automatic conversions to c_void_p - func.argtypes = [c_void_p] - assert func("hello") == "hello" - assert func(c_char_p("hello")) == "hello" - assert func((c_char * 6)(*"hello")) == "hello" - assert func((c_byte * 6)(104,101,108,108,111)) =="hello" - assert func(create_string_buffer("hello")) == "hello" - -def test_varsize_cast(): - import struct - N = struct.calcsize("l") - x = c_long() - p = cast(pointer(x), POINTER(c_ubyte*N)) - for i, c in enumerate(struct.pack("l", 12345678)): - p.contents[i] = ord(c) - assert x.value == 12345678 - -def test_cfunctype_inspection(): - T = CFUNCTYPE(c_int, c_ubyte) - # T.argtypes and T.restype don't work, must use a dummy instance - assert list(T().argtypes) == [c_ubyte] - assert T().restype == c_int - -def test_from_param(): - # other working cases of from_param - assert isinstance(c_void_p.from_param((c_int * 4)()), c_int*4) +class TestExtra(BaseCTypesTestChecker): + def test_primitive_pointer(self): + x = c_int(5) + assert x.value == 5 + x.value = 6 + assert x.value == 6 + + p = pointer(x) # p ---> x = 6 + assert isinstance(p.contents, c_int) + p.contents.value += 1 + assert x.value == 7 # p ---> x = 7 + + y = c_int(12) + p.contents = y # p ---> y = 12 + p.contents.value += 2 # p ---> y = 14 + assert y.value == 14 + assert x.value == 7 + + pp = pointer(p) # pp ---> p ---> y = 14 + pp.contents.contents = x # pp ---> p ---> x = 7 + p.contents.value += 2 # pp ---> p ---> x = 9 + assert x.value == 9 + + assert isinstance(p[0], int) + p[0] += 1 # pp ---> p ---> x = 10 + assert x.value == 10 + z = c_int(86) + p[0] = z # pp ---> p ---> x = 86 (not z!) + assert x.value == 86 + z.value = 84 + assert x.value == 86 + + assert isinstance(pp[0], POINTER(c_int)) + assert pp[0].contents.value == x.value == 86 + pp[0].contents = z # pp ---> p ---> z = 84 + assert p.contents.value == z.value == 84 + + ## *** the rest is commented out because it should work but occasionally + ## *** trigger a ctypes bug (SourceForge bug #1467852). *** + ## q = pointer(y) + ## pp[0] = q # pp ---> p ---> y = 14 + ## assert y.value == 14 # (^^^ not q! ) + ## assert p.contents.value == 14 + ## assert pp.contents.contents.value == 14 + ## q.contents = x + ## assert pp.contents.contents.value == 14 + + + def test_char_p(self): + x = c_char_p("hello\x00world") + assert x.value == "hello" + x.value = "world" + assert x.value == "world" + + p = pointer(x) + assert p[0] == x.value == "world" + p[0] = "other" + assert x.value == p.contents.value == p[0] == "other" + + myarray = (c_char_p * 10)() + myarray[7] = "hello" + assert isinstance(myarray[7], str) + assert myarray[7] == "hello" + + def test_struct(self): + class tagpoint(Structure): + _fields_ = [('x', c_int), + ('p', POINTER(c_short))] + + y = c_short(123) + z = c_short(-33) + s = tagpoint() + s.p.contents = z + assert s.p.contents.value == -33 + s.p = pointer(y) + assert s.p.contents.value == 123 + s.p.contents.value = 124 + assert y.value == 124 + + s = tagpoint(x=12) + assert s.x == 12 + s = tagpoint(17, p=pointer(z)) + assert s.x == 17 + assert s.p.contents.value == -33 + + def test_ptr_array(self): + a = (POINTER(c_ushort) * 5)() + x = c_ushort(52) + y = c_ushort(1000) + + a[2] = pointer(x) + assert a[2].contents.value == 52 + a[2].contents.value += 1 + assert x.value == 53 + + a[3].contents = y + assert a[3].contents.value == 1000 + a[3].contents.value += 1 + assert y.value == 1001 + + def test_void_p(self): + x = c_int(12) + p1 = cast(pointer(x), c_void_p) + p2 = cast(p1, POINTER(c_int)) + assert p2.contents.value == 12 + + def test_char_array(self): + a = (c_char * 3)() + a[0] = 'x' + a[1] = 'y' + assert a.value == 'xy' + a[2] = 'z' + assert a.value == 'xyz' + + b = create_string_buffer(3) + assert type(b) is type(a) + assert len(b) == 3 + + b.value = "nxw" + assert b[0] == 'n' + assert b[1] == 'x' + assert b[2] == 'w' + + b.value = "?" + assert b[0] == '?' + assert b[1] == '\x00' + assert b[2] == 'w' + + class S(Structure): + _fields_ = [('p', POINTER(c_char))] + + s = S() + s.p = b + s.p.contents.value = '!' + assert b.value == '!' + + assert len(create_string_buffer(0)) == 0 + + def test_array(self): + a = (c_int * 10)() + + class S(Structure): + _fields_ = [('p', POINTER(c_int))] + + s = S() + s.p = a + s.p.contents.value = 42 + assert a[0] == 42 + + a = (c_int * 5)(5, 6, 7) + assert list(a) == [5, 6, 7, 0, 0] + + def test_truth_value(self): + p = POINTER(c_int)() + assert not p + p.contents = c_int(12) + assert p + # I can't figure out how to reset p to NULL... + + assert c_int(12) + assert not c_int(0) # a bit strange, if you ask me + assert c_int(-1) + assert not c_byte(0) + assert not c_char('\x00') # hum + assert not c_float(0.0) + assert not c_double(0.0) + assert not c_ulonglong(0) + assert c_ulonglong(2L**42) + + assert c_char_p("hello") + assert c_char_p("") + assert not c_char_p(None) + + assert not c_void_p() + + def test_sizeof(self): + x = create_string_buffer(117) + assert sizeof(x) == 117 # assumes that chars are one byte each + x = (c_int * 42)() + assert sizeof(x) == 42 * sizeof(c_int) + + def test_convert_pointers(self): + import conftest + _ctypes_test = str(conftest.sofile) + dll = CDLL(_ctypes_test) + func = dll._testfunc_p_p + func.restype = c_char_p + + # automatic conversions to c_char_p + func.argtypes = [c_char_p] + assert func("hello") == "hello" + assert func(c_char_p("hello")) == "hello" + assert func((c_char * 6)(*"hello")) == "hello" + assert func(create_string_buffer("hello")) == "hello" + + # automatic conversions to c_void_p + func.argtypes = [c_void_p] + assert func("hello") == "hello" + assert func(c_char_p("hello")) == "hello" + assert func((c_char * 6)(*"hello")) == "hello" + assert func((c_byte * 6)(104,101,108,108,111)) =="hello" + assert func(create_string_buffer("hello")) == "hello" + + def test_varsize_cast(self): + import struct + N = struct.calcsize("l") + x = c_long() + p = cast(pointer(x), POINTER(c_ubyte*N)) + for i, c in enumerate(struct.pack("l", 12345678)): + p.contents[i] = ord(c) + assert x.value == 12345678 + + def test_cfunctype_inspection(self): + T = CFUNCTYPE(c_int, c_ubyte) + # T.argtypes and T.restype don't work, must use a dummy instance + assert list(T().argtypes) == [c_ubyte] + assert T().restype == c_int + + def test_from_param(self): + # other working cases of from_param + assert isinstance(c_void_p.from_param((c_int * 4)()), c_int*4) From fijal at codespeak.net Mon Feb 18 21:57:40 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 21:57:40 +0100 (CET) Subject: [pypy-svn] r51611 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080218205740.936531684DC@codespeak.net> Author: fijal Date: Mon Feb 18 21:57:39 2008 New Revision: 51611 Modified: pypy/dist/pypy/lib/app_test/ctypes/test_functions.py Log: This passes out of the box Modified: pypy/dist/pypy/lib/app_test/ctypes/test_functions.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_functions.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_functions.py Mon Feb 18 21:57:39 2008 @@ -29,7 +29,7 @@ class RECT(Structure): _fields_ = [("left", c_int), ("top", c_int), ("right", c_int), ("bottom", c_int)] -class TestFunctions:#(BaseCTypesTestChecker): +class TestFunctions(BaseCTypesTestChecker): def test_mro(self): # in Python 2.3, this raises TypeError: MRO conflict among bases classes, From fijal at codespeak.net Mon Feb 18 22:01:27 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 22:01:27 +0100 (CET) Subject: [pypy-svn] r51612 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080218210127.C4B2716840C@codespeak.net> Author: fijal Date: Mon Feb 18 22:01:24 2008 New Revision: 51612 Modified: pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py pypy/dist/pypy/lib/app_test/ctypes/test_memfunctions.py pypy/dist/pypy/lib/app_test/ctypes/test_pointers.py pypy/dist/pypy/lib/app_test/ctypes/test_sizes.py pypy/dist/pypy/lib/app_test/ctypes/test_structures.py pypy/dist/pypy/lib/app_test/ctypes/test_unicode.py Log: These tests are passing out of the box. This likely means that we don't have enough coverage Modified: pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py Mon Feb 18 22:01:24 2008 @@ -39,7 +39,7 @@ assert p._objects['1'].value == 3 def test_primitive(self): - assert c_char_p("abc")._objects == "abc" + assert c_char_p("abc")._objects['0']._buffer[0] == "a" assert c_int(3)._objects is None def test_pointer_to_pointer(self): Modified: pypy/dist/pypy/lib/app_test/ctypes/test_memfunctions.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_memfunctions.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_memfunctions.py Mon Feb 18 22:01:24 2008 @@ -2,8 +2,9 @@ import py import sys from ctypes import * +from support import BaseCTypesTestChecker -class TestMemFunctions: +class TestMemFunctions(BaseCTypesTestChecker): def test_memmove(self): # large buffers apparently increase the chance that the memory # is allocated in high address space. Modified: pypy/dist/pypy/lib/app_test/ctypes/test_pointers.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_pointers.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_pointers.py Mon Feb 18 22:01:24 2008 @@ -11,7 +11,7 @@ import conftest mod._ctypes_test = str(conftest.sofile) -class TestPointers: +class TestPointers(BaseCTypesTestChecker): def test_pointer_crash(self): class A(POINTER(c_ulong)): Modified: pypy/dist/pypy/lib/app_test/ctypes/test_sizes.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_sizes.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_sizes.py Mon Feb 18 22:01:24 2008 @@ -1,8 +1,9 @@ # Test specifically-sized containers. from ctypes import * +from support import BaseCTypesTestChecker -class TestSizes: +class TestSizes(BaseCTypesTestChecker): def test_8(self): assert 1 == sizeof(c_int8) assert 1 == sizeof(c_uint8) Modified: pypy/dist/pypy/lib/app_test/ctypes/test_structures.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_structures.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_structures.py Mon Feb 18 22:01:24 2008 @@ -349,7 +349,7 @@ assert "from_address" in dir(type(Structure)) assert "in_dll" in dir(type(Structure)) -class TestPointerMember:#(BaseCTypesTestChecker): +class TestPointerMember(BaseCTypesTestChecker): def test_1(self): # a Structure with a POINTER field Modified: pypy/dist/pypy/lib/app_test/ctypes/test_unicode.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_unicode.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_unicode.py Mon Feb 18 22:01:24 2008 @@ -1,6 +1,7 @@ # coding: latin-1 import ctypes import py +from support import BaseCTypesTestChecker try: ctypes.c_wchar @@ -14,7 +15,7 @@ mod.wcslen.argtypes = [ctypes.c_wchar_p] mod.func = dll._testfunc_p_p - class TestUnicode: + class TestUnicode(BaseCTypesTestChecker): def setup_method(self, method): self.prev_conv_mode = ctypes.set_conversion_mode("ascii", "strict") @@ -109,5 +110,3 @@ # is that correct? not sure. But with 'ignore', you get what you pay for.. assert buf[:] == "ab\0\0\0\0" -if __name__ == '__main__': - unittest.main() From fijal at codespeak.net Mon Feb 18 22:02:53 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 22:02:53 +0100 (CET) Subject: [pypy-svn] r51613 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080218210253.DE9DD16840C@codespeak.net> Author: fijal Date: Mon Feb 18 22:02:53 2008 New Revision: 51613 Modified: pypy/dist/pypy/lib/app_test/ctypes/test_array.py pypy/dist/pypy/lib/app_test/ctypes/test_array_in_pointer.py Log: Remove artifacts. Modified: pypy/dist/pypy/lib/app_test/ctypes/test_array.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_array.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_array.py Mon Feb 18 22:02:53 2008 @@ -2,7 +2,6 @@ import py from ctypes import * from support import BaseCTypesTestChecker -import _rawffi formats = "bBhHiIlLqQfd" Modified: pypy/dist/pypy/lib/app_test/ctypes/test_array_in_pointer.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_array_in_pointer.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_array_in_pointer.py Mon Feb 18 22:02:53 2008 @@ -26,7 +26,6 @@ class TestOne(BaseCTypesTestChecker): def test(self): - from _rawffi import _num_of_allocated_objects as _num # create an array of 4 values val_array = (Value * 4)() From fijal at codespeak.net Mon Feb 18 23:42:40 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 18 Feb 2008 23:42:40 +0100 (CET) Subject: [pypy-svn] r51614 - in pypy/dist/pypy/lib: _ctypes app_test/ctypes Message-ID: <20080218224240.DA9C316845E@codespeak.net> Author: fijal Date: Mon Feb 18 23:42:38 2008 New Revision: 51614 Modified: pypy/dist/pypy/lib/_ctypes/array.py pypy/dist/pypy/lib/_ctypes/structure.py pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py Log: array in structure. Test and a fix. Modified: pypy/dist/pypy/lib/_ctypes/array.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/array.py (original) +++ pypy/dist/pypy/lib/_ctypes/array.py Mon Feb 18 23:42:38 2008 @@ -2,7 +2,7 @@ import _rawffi from _ctypes.basics import _CData, cdata_from_address, _CDataMeta, sizeof,\ - keepalive_key, CArgObject + keepalive_key, store_reference, CArgObject def _create_unicode(buffer, maxlength): res = [] @@ -70,6 +70,14 @@ def _alignmentofinstances(self): return self._type_._alignmentofinstances() + def _CData_output(self, resarray, base=None, index=-1): + res = self.__new__(self) + ffiarray = self._ffiarray.fromaddress(resarray.buffer, self._length_) + res._buffer = ffiarray + res._base = base + res._index = index + return res.__ctypes_from_outparam__() + def array_get_slice_params(self, index): if index.step is not None: raise TypeError("3 arg slices not supported (for no reason)") @@ -128,7 +136,7 @@ return index = self._fix_index(index) if getattr(value, '_objects', None): - self._objects[keepalive_key(index)] = value._objects + store_reference(self, index, value._objects) arg = self._type_._CData_value(value) if not isinstance(self._type_._ffishape, tuple): self._buffer[index] = arg Modified: pypy/dist/pypy/lib/_ctypes/structure.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/structure.py (original) +++ pypy/dist/pypy/lib/_ctypes/structure.py Mon Feb 18 23:42:38 2008 @@ -60,7 +60,7 @@ pos = [0] * len(all_fields) fields = {} for i, (name, ctype) in enumerate(all_fields): - fields[name] = Field(name, pos[i], ctypes.sizeof(ctype), ctype) + fields[name] = Field(name, pos[i], ctypes.sizeof(ctype), ctype, i) if anon: resnames = [] for i, (name, value) in enumerate(all_fields): @@ -70,7 +70,8 @@ relpos = pos[i] + value._fieldtypes[subname].offset subvalue = value._fieldtypes[subname].ctype fields[subname] = Field(subname, relpos, - ctypes.sizeof(subvalue), subvalue) + ctypes.sizeof(subvalue), subvalue, + i) # XXX we never set rawfields here, let's wait for a test else: resnames.append(name) @@ -78,8 +79,8 @@ return names, rawfields, fields class Field(object): - def __init__(self, name, offset, size, ctype): - for k in ('name', 'offset', 'size', 'ctype'): + def __init__(self, name, offset, size, ctype, num): + for k in ('name', 'offset', 'size', 'ctype', 'num'): self.__dict__[k] = locals()[k] def __setattr__(self, name, value): @@ -177,7 +178,7 @@ fieldtype = self._fieldtypes[name].ctype except KeyError: return _CData.__getattribute__(self, name) - offset = self.__class__._fieldtypes[name].offset + offset = self.__class__._fieldtypes[name].num suba = self._subarray(fieldtype, name) return fieldtype._CData_output(suba, self, offset) Modified: pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_keepalive.py Mon Feb 18 23:42:38 2008 @@ -1,6 +1,7 @@ import py from ctypes import * +import sys class TestKeepalive: """ Tests whether various objects land in _objects @@ -39,6 +40,8 @@ assert p._objects['1'].value == 3 def test_primitive(self): + if not hasattr(sys, 'pypy_translation_info'): + py.test.skip("whitebox test") assert c_char_p("abc")._objects['0']._buffer[0] == "a" assert c_int(3)._objects is None @@ -74,3 +77,12 @@ assert a._objects['0:3']['1'] is s + def test_struct_with_inlined_array(self): + class S(Structure): + _fields_ = [('b', c_int), + ('a', POINTER(c_int) * 2)] + + s = S() + stuff = c_int(2) + s.a[1] = pointer(stuff) + assert s._objects == {'1:1': {'1': stuff}} From fijal at codespeak.net Tue Feb 19 00:19:33 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 19 Feb 2008 00:19:33 +0100 (CET) Subject: [pypy-svn] r51616 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080218231933.2ACAF1684C3@codespeak.net> Author: fijal Date: Tue Feb 19 00:19:26 2008 New Revision: 51616 Modified: pypy/dist/pypy/lib/app_test/ctypes/test_base.py Log: adapt the test to what ctypes really does. Modified: pypy/dist/pypy/lib/app_test/ctypes/test_base.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_base.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_base.py Tue Feb 19 00:19:26 2008 @@ -16,7 +16,7 @@ x = X() assert x.y._base is x - assert x.y._index == sizeof(POINTER(c_int)) + assert x.y._index == 1 def test_array(self): X = POINTER(c_int) * 24 From fijal at codespeak.net Tue Feb 19 00:32:23 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 19 Feb 2008 00:32:23 +0100 (CET) Subject: [pypy-svn] r51617 - in pypy/dist/pypy/lib: _ctypes app_test/ctypes Message-ID: <20080218233223.C77781684DB@codespeak.net> Author: fijal Date: Tue Feb 19 00:32:21 2008 New Revision: 51617 Modified: pypy/dist/pypy/lib/_ctypes/array.py pypy/dist/pypy/lib/_ctypes/basics.py pypy/dist/pypy/lib/_ctypes/structure.py pypy/dist/pypy/lib/app_test/ctypes/test_structures.py Log: * A bit of simplification * Forest of ifs what we support and what we don't in terms of args for arrays * Structure/Array inlining now shall work. Modified: pypy/dist/pypy/lib/_ctypes/array.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/array.py (original) +++ pypy/dist/pypy/lib/_ctypes/array.py Tue Feb 19 00:32:21 2008 @@ -3,6 +3,7 @@ from _ctypes.basics import _CData, cdata_from_address, _CDataMeta, sizeof,\ keepalive_key, store_reference, CArgObject +from _ctypes.builtin import _string_at_addr, _wstring_at_addr def _create_unicode(buffer, maxlength): res = [] @@ -71,6 +72,12 @@ return self._type_._alignmentofinstances() def _CData_output(self, resarray, base=None, index=-1): + # this seems to be a string if we're array of char, surprise! + from ctypes import c_char, c_wchar, c_char_p, c_wchar_p + if self._type_ is c_char: + return _rawffi.charp2string(resarray.buffer, self._length_) + if self._type_ is c_wchar: + xxx res = self.__new__(self) ffiarray = self._ffiarray.fromaddress(resarray.buffer, self._length_) res._buffer = ffiarray @@ -78,6 +85,22 @@ res._index = index return res.__ctypes_from_outparam__() + def _CData_value(self, value): + # array accepts very strange parameters as part of structure + # or function argument... + from ctypes import c_char, c_wchar + if issubclass(self._type_, (c_char, c_wchar)): + if isinstance(value, basestring): + if len(value) > self._length_: + raise ValueError("Invalid length") + return self(*value) + else: + if isinstance(value, tuple): + if len(value) > self._length_: + raise RuntimeError("Invalid length") + return self(*value) + return _CDataMeta._CData_value(self, value) + def array_get_slice_params(self, index): if index.step is not None: raise TypeError("3 arg slices not supported (for no reason)") @@ -158,6 +181,9 @@ def _get_buffer_for_param(self): return CArgObject(self._buffer.byptr()) + def _get_buffer_value(self): + return self._buffer.buffer + def __del__(self): if self._needs_free: self._buffer.free() Modified: pypy/dist/pypy/lib/_ctypes/basics.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/basics.py (original) +++ pypy/dist/pypy/lib/_ctypes/basics.py Tue Feb 19 00:32:21 2008 @@ -45,7 +45,7 @@ cobj = self.from_param(value) # we don't care here if this stuff will live afterwards, as we're # interested only in value anyway - return cobj._get_buffer_for_param()._buffer[0] + return cobj._get_buffer_value() def _CData_output(self, resarray, base=None, index=-1, needs_free=False): assert isinstance(resarray, _rawffi.ArrayInstance) @@ -99,6 +99,9 @@ def _get_buffer_for_param(self): return self + def _get_buffer_value(self): + return self._buffer[0] + def sizeof(tp): if not isinstance(tp, _CDataMeta): if isinstance(tp, _CData): Modified: pypy/dist/pypy/lib/_ctypes/structure.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/structure.py (original) +++ pypy/dist/pypy/lib/_ctypes/structure.py Tue Feb 19 00:32:21 2008 @@ -141,6 +141,11 @@ def _alignmentofinstances(self): return self._ffistruct.alignment + def _CData_value(self, value): + if isinstance(value, tuple): + return self(*value) + return _CDataMeta._CData_value(self, value) + def _CData_output(self, resarray, base=None, index=-1): res = self.__new__(self) ffistruct = self._ffistruct.fromaddress(resarray.buffer) @@ -169,7 +174,13 @@ if getattr(value, '_objects', None): key = keepalive_key(getattr(self.__class__, name).offset) store_reference(self, key, value._objects) - self._buffer.__setattr__(name, fieldtype._CData_value(value)) + arg = fieldtype._CData_value(value) + if isinstance(fieldtype._ffishape, tuple): + from ctypes import memmove + dest = self._buffer.fieldaddress(name) + memmove(dest, arg._buffer.buffer, fieldtype._ffishape[0]) + else: + self._buffer.__setattr__(name, arg) def __getattribute__(self, name): if name == '_fieldtypes': @@ -185,6 +196,9 @@ def _get_buffer_for_param(self): return CArgObject(self._buffer.byptr()) + def _get_buffer_value(self): + return self._buffer.buffer + def __del__(self): if self._needs_free: self._buffer.free() Modified: pypy/dist/pypy/lib/app_test/ctypes/test_structures.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_structures.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_structures.py Tue Feb 19 00:32:21 2008 @@ -161,7 +161,7 @@ # offset is always relative to the class... def test_packed(self): - py.test.skip("???") + py.test.skip("Custom alignment not supported") class X(Structure): _fields_ = [("a", c_byte), ("b", c_longlong)] @@ -203,7 +203,6 @@ raises(ValueError, type(Structure), "X", (Structure,), d) def test_initializers(self): - py.test.skip("Structures with inlined arrays") class Person(Structure): _fields_ = [("name", c_char*6), ("age", c_int)] @@ -235,7 +234,6 @@ raises(TypeError, setattr, POINT, "_fields_", [("x", 1), ("y", 2)]) def test_intarray_fields(self): - py.test.skip("Structures with inlined arrays") class SomeInts(Structure): _fields_ = [("a", c_int * 4)] @@ -247,7 +245,6 @@ raises(RuntimeError, SomeInts, (1, 2, 3, 4, 5)) def test_nested_initializers(self): - py.test.skip("???") # test initializing nested structures class Phone(Structure): _fields_ = [("areacode", c_char*6), @@ -266,7 +263,7 @@ assert p.age == 5 def test_structures_with_wchar(self): - py.test.skip("Inlined array") + py.test.skip("need unicode support on _rawffi level") try: c_wchar except NameError: @@ -287,7 +284,7 @@ raises(ValueError, PersonW, u"1234567890123") def test_init_errors(self): - py.test.skip("???") + py.test.skip("Very fragile errors") class Phone(Structure): _fields_ = [("areacode", c_char*6), ("number", c_char*12)] @@ -334,7 +331,7 @@ ## (AttributeError, "class must define a '_fields_' attribute")) def test_abstract_class(self): - py.test.skip("???") + py.test.skip("What is _abstract_?") class X(Structure): _abstract_ = "something" # try 'X()' @@ -405,7 +402,7 @@ def test_vice_versa(self): - py.test.skip("???") + py.test.skip("I don't understand this test at all") class First(Structure): pass class Second(Structure): From fijal at codespeak.net Tue Feb 19 00:33:56 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 19 Feb 2008 00:33:56 +0100 (CET) Subject: [pypy-svn] r51618 - pypy/dist/pypy/doc/discussion Message-ID: <20080218233356.B32BD1684DB@codespeak.net> Author: fijal Date: Tue Feb 19 00:33:55 2008 New Revision: 51618 Modified: pypy/dist/pypy/doc/discussion/ctypes_todo.txt Log: status update Modified: pypy/dist/pypy/doc/discussion/ctypes_todo.txt ============================================================================== --- pypy/dist/pypy/doc/discussion/ctypes_todo.txt (original) +++ pypy/dist/pypy/doc/discussion/ctypes_todo.txt Tue Feb 19 00:33:55 2008 @@ -16,12 +16,9 @@ - raw structure/arrays as function arguments/result types are not supported (even on _rawffi level) - - not all combinations of arrays/structures inlined in other - arrays/structures work. - - bitfields are not implemented -* keepalive is not there (everything is kept alive forever basically) + - byteorder is not implemented * as all stuff is applevel, we cannot have it really fast right now. From arigo at codespeak.net Tue Feb 19 10:11:24 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 19 Feb 2008 10:11:24 +0100 (CET) Subject: [pypy-svn] r51621 - in pypy/dist/pypy: rpython/memory rpython/tool rpython/tool/test translator/c/src translator/c/test translator/llvm Message-ID: <20080219091124.7D19916843C@codespeak.net> Author: arigo Date: Tue Feb 19 10:11:22 2008 New Revision: 51621 Modified: pypy/dist/pypy/rpython/memory/lltypelayout.py pypy/dist/pypy/rpython/tool/rffi_platform.py pypy/dist/pypy/rpython/tool/test/test_rffi_platform.py pypy/dist/pypy/translator/c/src/mem.h pypy/dist/pypy/translator/c/test/test_newgc.py pypy/dist/pypy/translator/llvm/database.py Log: Figured out that 8-bytes alignment is not really necessary; gcc uses 4-bytes alignment even for 'double'. This checkin adds logic for detecting the alignment. Modified: pypy/dist/pypy/rpython/memory/lltypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/lltypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/lltypelayout.py Tue Feb 19 10:11:22 2008 @@ -2,7 +2,7 @@ import struct -memory_alignment = 4 +memory_alignment = struct.calcsize("P") primitive_to_fmt = {lltype.Signed: "l", lltype.Unsigned: "L", Modified: pypy/dist/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/dist/pypy/rpython/tool/rffi_platform.py Tue Feb 19 10:11:22 2008 @@ -63,6 +63,24 @@ setattr(CConfig, k, v) return configure(CConfig)['SIZE'] +def memory_alignment(): + """Return the alignment (in bytes) of memory allocations. + This is enough to make sure a structure with pointers and 'double' + fields is properly aligned.""" + global _memory_alignment + if _memory_alignment is None: + S = getstruct('struct memory_alignment_test', """ + struct memory_alignment_test { + double d; + void* p; + }; + """, []) + result = S._hints['align'] + assert result & (result-1) == 0, "not a power of two??" + _memory_alignment = result + return _memory_alignment +_memory_alignment = None + # ____________________________________________________________ # # General interface Modified: pypy/dist/pypy/rpython/tool/test/test_rffi_platform.py ============================================================================== --- pypy/dist/pypy/rpython/tool/test/test_rffi_platform.py (original) +++ pypy/dist/pypy/rpython/tool/test/test_rffi_platform.py Tue Feb 19 10:11:22 2008 @@ -203,3 +203,8 @@ def test_sizeof(): assert rffi_platform.sizeof("char", ExternalCompilationInfo()) == 1 + +def test_memory_alignment(): + a = rffi_platform.memory_alignment() + print a + assert a % struct.calcsize("P") == 0 Modified: pypy/dist/pypy/translator/c/src/mem.h ============================================================================== --- pypy/dist/pypy/translator/c/src/mem.h (original) +++ pypy/dist/pypy/translator/c/src/mem.h Tue Feb 19 10:11:22 2008 @@ -3,8 +3,17 @@ /*** C header subsection: operations on LowLevelTypes ***/ /* alignment for arena-based garbage collectors: the following line - enforces an alignment of sizeof(double). */ -#define MEMORY_ALIGNMENT sizeof(double) + enforces an alignment that should be enough for any structure + containing pointers and 'double' fields. */ +struct rpy_memory_alignment_test1 { + double d; + void* p; +}; +struct rpy_memory_alignment_test2 { + char c; + struct rpy_memory_alignment_test1 s; +}; +#define MEMORY_ALIGNMENT offsetof(struct rpy_memory_alignment_test2, s) #define ROUND_UP_FOR_ALLOCATION(x) \ (((x) + (MEMORY_ALIGNMENT-1)) & ~(MEMORY_ALIGNMENT-1)) Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Tue Feb 19 10:11:22 2008 @@ -784,9 +784,9 @@ assert res == 42 def test_object_alignment(self): - # all objects returned by the GC should be aligned on a 8-bytes - # boundary, or whatever sizeof(double) is on this platform + # all objects returned by the GC should be properly aligned. from pypy.rpython.lltypesystem import rffi + from pypy.rpython.tool import rffi_platform mylist = ['a', 'bc', '84139871', 'ajkdh', '876'] def f(): result = 0 @@ -800,10 +800,7 @@ fn = self.getcompiled(f) res = fn() - import struct - expected_alignment = struct.calcsize("d") - assert (expected_alignment & (expected_alignment-1)) == 0, ( - "sizeof(double) not a power of two") + expected_alignment = rffi_platform.memory_alignment() assert (res & (expected_alignment-1)) == 0 def test_void_list(self): Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Tue Feb 19 10:11:22 2008 @@ -576,10 +576,13 @@ def repr_offset(self, value): if isinstance(value, llarena.RoundedUpForAllocation): # XXX not supported when used in a CompositeOffset + from pypy.rpython.tool import rffi_platform + align = rffi_platform.memory_alignment() r_basesize = self.repr_offset(value.basesize) # Note that the following expression is known to crash 'llc'; # you may need to upgrade llvm. - return "and(i32 add(i32 %s, i32 7), i32 -8)" % r_basesize + return "and(i32 add(i32 %s, i32 %d), i32 %d)" % ( + r_basesize, align-1, ~(align-1)) from_, indices, to = self.get_offset(value, []) From arigo at codespeak.net Tue Feb 19 12:09:40 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 19 Feb 2008 12:09:40 +0100 (CET) Subject: [pypy-svn] r51622 - pypy/branch/no-forw-ptr Message-ID: <20080219110940.50BCD16843C@codespeak.net> Author: arigo Date: Tue Feb 19 12:09:38 2008 New Revision: 51622 Added: pypy/branch/no-forw-ptr/ - copied from r51621, pypy/dist/ Log: A branch in which we try to remove the 'forw' field in the header of all objects in the SemiSpaceGC. From arigo at codespeak.net Tue Feb 19 12:30:15 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 19 Feb 2008 12:30:15 +0100 (CET) Subject: [pypy-svn] r51623 - in pypy/branch/no-forw-ptr/pypy/rpython: lltypesystem lltypesystem/test memory Message-ID: <20080219113015.0AD981684D1@codespeak.net> Author: arigo Date: Tue Feb 19 12:30:14 2008 New Revision: 51623 Modified: pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/llarena.py pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/llmemory.py pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/lltype.py pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/test/test_llarena.py pypy/branch/no-forw-ptr/pypy/rpython/memory/gcheader.py Log: This seems to be the amount of hacking needed to allow an arena to replace an object with another, and still have the old address work in order to go to the new object's header. This works only to some extend -- many operations on the old address still raise RuntimeError as before, in the hope that it will not allow nasty bugs to go unnoticed. Modified: pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/llarena.py Tue Feb 19 12:30:14 2008 @@ -1,4 +1,4 @@ -import array +import array, weakref from pypy.rpython.lltypesystem import lltype, llmemory # An "arena" is a large area of memory which can hold a number of @@ -15,6 +15,7 @@ class Arena(object): object_arena_location = {} # {container: (arena, offset)} + old_object_arena_location = weakref.WeakKeyDictionary() def __init__(self, nbytes, zero): self.nbytes = nbytes @@ -29,7 +30,7 @@ if size is None: stop = self.nbytes else: - stop = start + size + stop = start + llmemory.raw_malloc_usage(size) assert 0 <= start <= stop <= self.nbytes for offset, ptr in self.objectptrs.items(): size = self.objectsizes[offset] @@ -79,9 +80,7 @@ addr2 = size._raw_malloc([], zero=zero) pattern = 'X' + 'x'*(bytes-1) self.usagemap[offset:offset+bytes] = array.array('c', pattern) - self.objectptrs[offset] = addr2.ptr - self.objectsizes[offset] = bytes - Arena.object_arena_location[addr2.ptr._obj] = self, offset + self.setobject(addr2, offset, bytes) # common case: 'size' starts with a GCHeaderOffset. In this case # we can also remember that the real object starts after the header. while isinstance(size, RoundedUpForAllocation): @@ -91,12 +90,17 @@ objaddr = addr2 + size.offsets[0] hdrbytes = llmemory.raw_malloc_usage(size.offsets[0]) objoffset = offset + hdrbytes - assert objoffset not in self.objectptrs - self.objectptrs[objoffset] = objaddr.ptr - self.objectsizes[objoffset] = bytes - hdrbytes - Arena.object_arena_location[objaddr.ptr._obj] = self, objoffset + self.setobject(objaddr, objoffset, bytes - hdrbytes) return addr2 + def setobject(self, objaddr, offset, bytes): + assert offset not in self.objectptrs + self.objectptrs[offset] = objaddr.ptr + self.objectsizes[offset] = bytes + container = objaddr.ptr._obj + Arena.object_arena_location[container] = self, offset + Arena.old_object_arena_location[container] = self, offset + class fakearenaaddress(llmemory.fakeaddress): def __init__(self, arena, offset): @@ -205,6 +209,30 @@ return self.arena._getid() + self.offset +def _getfakearenaaddress(addr): + """Logic to handle test_replace_object_with_stub().""" + if isinstance(addr, fakearenaaddress): + return addr + else: + assert isinstance(addr, llmemory.fakeaddress) + assert addr, "NULL address" + # it must be possible to use the address of an already-freed + # arena object + obj = addr.ptr._getobj(check=False) + return _oldobj_to_address(obj) + +def _oldobj_to_address(obj): + obj = obj._normalizedcontainer(check=False) + try: + arena, offset = Arena.old_object_arena_location[obj] + except KeyError: + if obj._was_freed(): + msg = "taking address of %r, but it was freed" + else: + msg = "taking address of %r, but it is not in an arena" + raise RuntimeError(msg % (obj,)) + return arena.getaddr(offset) + class RoundedUpForAllocation(llmemory.AddressOffset): """A size that is rounded up in order to preserve alignment of objects following it. For arenas containing heterogenous objects. @@ -247,7 +275,7 @@ """Free all objects in the arena, which can then be reused. The arena is filled with zeroes if 'zero' is True. This can also be used on a subrange of the arena.""" - assert isinstance(arena_addr, fakearenaaddress) + arena_addr = _getfakearenaaddress(arena_addr) arena_addr.arena.reset(zero, arena_addr.offset, size) def arena_reserve(addr, size, check_alignment=True): @@ -256,7 +284,7 @@ overlap. The size must be symbolic; in non-translated version this is used to know what type of lltype object to allocate.""" from pypy.rpython.memory.lltypelayout import memory_alignment - assert isinstance(addr, fakearenaaddress) + addr = _getfakearenaaddress(addr) if check_alignment and (addr.offset & (memory_alignment-1)) != 0: raise ArenaError("object at offset %d would not be correctly aligned" % (addr.offset,)) Modified: pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/llmemory.py Tue Feb 19 12:30:14 2008 @@ -339,7 +339,9 @@ # NOTE: the 'ptr' in the addresses must be normalized. # Use cast_ptr_to_adr() instead of directly fakeaddress() if unsure. def __init__(self, ptr): - self.ptr = ptr or None # null ptr => None + if ptr is not None and ptr._obj0 is None: + ptr = None # null ptr => None + self.ptr = ptr def __repr__(self): if self.ptr is None: @@ -374,8 +376,8 @@ def __eq__(self, other): if isinstance(other, fakeaddress): - obj1 = self.ptr - obj2 = other.ptr + obj1 = self._fixup().ptr + obj2 = other._fixup().ptr if obj1 is not None: obj1 = obj1._obj if obj2 is not None: obj2 = obj2._obj return obj1 == obj2 @@ -412,8 +414,9 @@ return self.ptr def _cast_to_ptr(self, EXPECTED_TYPE): - if self: - return cast_any_ptr(EXPECTED_TYPE, self.ptr) + addr = self._fixup() + if addr: + return cast_any_ptr(EXPECTED_TYPE, addr.ptr) else: return lltype.nullptr(EXPECTED_TYPE.TO) @@ -423,6 +426,14 @@ else: return 0 + def _fixup(self): + if self.ptr is not None and self.ptr._was_freed(): + # hack to support llarena.test_replace_object_with_stub() + from pypy.rpython.lltypesystem import llarena + return llarena._getfakearenaaddress(self) + else: + return self + # ____________________________________________________________ class NullAddressError(Exception): Modified: pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/lltype.py Tue Feb 19 12:30:14 2008 @@ -968,7 +968,7 @@ self._set_solid(solid) self._set_obj0(obj0) - def _getobj(self): + def _getobj(self, check=True): obj = self._obj0 if obj is not None: if self._weak: @@ -977,12 +977,17 @@ raise RuntimeError("accessing already garbage collected %r" % (self._T,)) if isinstance(obj, _container): - obj._check() + if check: + obj._check() elif isinstance(obj, str) and obj.startswith("delayed!"): raise DelayedPointer return obj _obj = property(_getobj) + def _was_freed(self): + return (self._obj0 is not None and + self._getobj(check=False)._was_freed()) + def __getattr__(self, field_name): # ! can only return basic or ptr ! if isinstance(self._T, Struct): if field_name in self._T._flds: @@ -1178,6 +1183,10 @@ from pypy.rpython.lltypesystem import llmemory if isinstance(self._T, FuncType): return llmemory.fakeaddress(self) + elif self._was_freed(): + # hack to support llarena.test_replace_object_with_stub() + from pypy.rpython.lltypesystem import llarena + return llarena._oldobj_to_address(self._getobj(check=False)) elif isinstance(self._obj, _subarray): return llmemory.fakeaddress(self) ## # return an address built as an offset in the whole array @@ -1192,8 +1201,8 @@ def _as_ptr(self): return self - def _as_obj(self): - return self._obj + def _as_obj(self, check=True): + return self._getobj(check=check) def _expose(self, offset, val): """XXX A nice docstring here""" @@ -1258,13 +1267,13 @@ class _container(object): __slots__ = () - def _parentstructure(self): + def _parentstructure(self, check=True): return None def _check(self): pass def _as_ptr(self): return _ptr(Ptr(self._TYPE), self, True) - def _as_obj(self): + def _as_obj(self, check=True): return self def _normalizedcontainer(self): return self @@ -1295,7 +1304,16 @@ self._storage = None def _was_freed(self): - return self._storage is None + if self._storage is None: + return True + if self._wrparent is None: + return False + parent = self._wrparent() + if parent is None: + raise RuntimeError("accessing sub%s %r,\n" + "but already garbage collected parent %r" + % (self._kind, self, self._parent_type)) + return parent._was_freed() def _setparentstructure(self, parent, parentindex): self._wrparent = weakref.ref(parent) @@ -1307,14 +1325,15 @@ # keep strong reference to parent, we share the same allocation self._keepparent = parent - def _parentstructure(self): + def _parentstructure(self, check=True): if self._wrparent is not None: parent = self._wrparent() if parent is None: raise RuntimeError("accessing sub%s %r,\n" "but already garbage collected parent %r" % (self._kind, self, self._parent_type)) - parent._check() + if check: + parent._check() return parent return None @@ -1323,14 +1342,15 @@ raise RuntimeError("accessing freed %r" % self._TYPE) self._parentstructure() - def _normalizedcontainer(self): + def _normalizedcontainer(self, check=True): # if we are the first inlined substructure of a structure, # return the whole (larger) structure instead container = self while True: - parent, index = parentlink(container) + parent = container._parentstructure(check=check) if parent is None: break + index = container._parent_index T = typeOf(parent) if (not isinstance(T, Struct) or T._first_struct()[0] != index or isinstance(T, FixedSizeArray)): @@ -1525,7 +1545,7 @@ def __repr__(self): return '<_subarray at %r in %r>' % (self._parent_index, - self._parentstructure()) + self._parentstructure(check=False)) def getlength(self): assert isinstance(self._TYPE, FixedSizeArray) Modified: pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/test/test_llarena.py Tue Feb 19 12:30:14 2008 @@ -165,6 +165,8 @@ plist.append(llmemory.cast_adr_to_ptr(b, SPTR)) # clear blist[1] and blist[2] but not blist[0] nor blist[3] arena_reset(blist[1], llmemory.raw_malloc_usage(precomputed_size)*2, False) + py.test.raises(RuntimeError, "plist[1].x") # marked as freed + py.test.raises(RuntimeError, "plist[2].x") # marked as freed # re-reserve object at index 1 and 2 blist[1] = reserve(1) blist[2] = reserve(2) @@ -173,6 +175,10 @@ assert plist[3].x == 103 py.test.raises(RuntimeError, "plist[1].x") # marked as freed py.test.raises(RuntimeError, "plist[2].x") # marked as freed + # but we can still cast the old ptrs to addresses, which compare equal + # to the new ones we gotq + assert llmemory.cast_ptr_to_adr(plist[1]) == blist[1] + assert llmemory.cast_ptr_to_adr(plist[2]) == blist[2] # check via addresses assert (blist[0] + llmemory.offsetof(SX, 'x')).signed[0] == 100 assert (blist[3] + llmemory.offsetof(SX, 'x')).signed[0] == 103 @@ -204,6 +210,45 @@ assert llmemory.cast_adr_to_int(a) == llmemory.cast_adr_to_int(a1) assert llmemory.cast_adr_to_int(a+1) == llmemory.cast_adr_to_int(a1) + 1 +def test_replace_object_with_stub(): + from pypy.rpython.memory.gcheader import GCHeaderBuilder + HDR = lltype.Struct('HDR', ('x', lltype.Signed)) + S = lltype.GcStruct('S', ('y', lltype.Signed), ('z', lltype.Signed)) + STUB = lltype.GcStruct('STUB', ('t', lltype.Char)) + gcheaderbuilder = GCHeaderBuilder(HDR) + size_gc_header = gcheaderbuilder.size_gc_header + + a = arena_malloc(50, True) + hdraddr = a + 12 + arena_reserve(hdraddr, size_gc_header + llmemory.sizeof(S)) + hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR)) + hdr.x = 42 + obj = llmemory.cast_adr_to_ptr(hdraddr + size_gc_header, lltype.Ptr(S)) + obj.y = -5 + obj.z = -6 + + hdraddr = llmemory.cast_ptr_to_adr(obj) - size_gc_header + arena_reset(hdraddr, size_gc_header + llmemory.sizeof(S), False) + arena_reserve(hdraddr, size_gc_header + llmemory.sizeof(STUB)) + + # check that it possible to reach the newly reserved HDR+STUB + # via the header of the old 'obj' pointer, both via the existing + # 'hdraddr': + hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR)) + hdr.x = 46 + stub = llmemory.cast_adr_to_ptr(hdraddr + size_gc_header, lltype.Ptr(STUB)) + stub.t = '!' + + # and via a (now-invalid) pointer to the old 'obj': (this is needed + # because during a garbage collection there are still pointers to + # the old 'obj' around to be fixed) + hdraddr = llmemory.cast_ptr_to_adr(obj) - size_gc_header + hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR)) + assert hdr.x == 46 + stub = llmemory.cast_adr_to_ptr(hdraddr + size_gc_header, + lltype.Ptr(STUB)) + assert stub.t == '!' + def test_llinterpreted(): from pypy.rpython.test.test_llinterp import interpret Modified: pypy/branch/no-forw-ptr/pypy/rpython/memory/gcheader.py ============================================================================== --- pypy/branch/no-forw-ptr/pypy/rpython/memory/gcheader.py (original) +++ pypy/branch/no-forw-ptr/pypy/rpython/memory/gcheader.py Tue Feb 19 12:30:14 2008 @@ -16,17 +16,17 @@ def header_of_object(self, gcptr): # XXX hackhackhack - gcptr = gcptr._as_obj() + gcptr = gcptr._as_obj(check=False) if isinstance(gcptr, llmemory._gctransformed_wref): - return self.obj2header[gcptr._ptr._as_obj()] + return self.obj2header[gcptr._ptr._as_obj(check=False)] return self.obj2header[gcptr] def object_from_header(headerptr): - return header2obj[headerptr._as_obj()] + return header2obj[headerptr._as_obj(check=False)] object_from_header = staticmethod(object_from_header) def get_header(self, gcptr): - return self.obj2header.get(gcptr._as_obj(), None) + return self.obj2header.get(gcptr._as_obj(check=False), None) def attach_header(self, gcptr, headerptr): gcobj = gcptr._as_obj() From arigo at codespeak.net Tue Feb 19 12:32:14 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 19 Feb 2008 12:32:14 +0100 (CET) Subject: [pypy-svn] r51624 - pypy/branch/no-forw-ptr/pypy/rpython/memory/gc Message-ID: <20080219113214.5B18C1684D6@codespeak.net> Author: arigo Date: Tue Feb 19 12:32:13 2008 New Revision: 51624 Modified: pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/semispace.py Log: Remove the 'forw' pointer from the header. test_gc.TestSemiSpaceGC passes :-) Modified: pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/semispace.py Tue Feb 19 12:32:13 2008 @@ -15,8 +15,9 @@ TYPEID_MASK = 0xffff first_gcflag = 1 << 16 -GCFLAG_IMMORTAL = first_gcflag -GCFLAG_FINALIZATION_ORDERING = first_gcflag << 1 +GCFLAG_FORWARDED = first_gcflag +GCFLAG_IMMORTAL = first_gcflag << 1 +GCFLAG_FINALIZATION_ORDERING = first_gcflag << 2 DEBUG_PRINT = False memoryError = MemoryError() @@ -26,12 +27,14 @@ inline_simple_malloc = True inline_simple_malloc_varsize = True needs_zero_gc_pointers = False - first_unused_gcflag = first_gcflag << 2 + first_unused_gcflag = first_gcflag << 3 total_collection_time = 0.0 total_collection_count = 0 - HDR = lltype.Struct('header', ('forw', llmemory.Address), - ('tid', lltype.Signed)) + HDR = lltype.Struct('header', ('tid', lltype.Signed)) + FORWARDSTUB = lltype.GcStruct('forwarding_stub', + ('forw', llmemory.Address)) + FORWARDSTUBPTR = lltype.Ptr(FORWARDSTUB) def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096, max_space_size=sys.maxint//2+1): @@ -289,15 +292,13 @@ root.address[0] = self.copy(root.address[0]) def copy(self, obj): - # Objects not living the GC heap have all been initialized by - # setting their 'forw' address so that it points to themselves. - # The logic below will thus simply return 'obj' if 'obj' is prebuilt. if self.is_forwarded(obj): #llop.debug_print(lltype.Void, obj, "already copied to", self.get_forwarding_address(obj)) return self.get_forwarding_address(obj) else: newaddr = self.free - totalsize = self.size_gc_header() + self.get_size(obj) + objsize = self.get_size(obj) + totalsize = self.size_gc_header() + objsize llarena.arena_reserve(newaddr, totalsize) raw_memcopy(obj - self.size_gc_header(), newaddr, totalsize) self.free += totalsize @@ -305,7 +306,7 @@ #llop.debug_print(lltype.Void, obj, "copied to", newobj, # "tid", self.header(obj).tid, # "size", totalsize) - self.set_forwarding_address(obj, newobj) + self.set_forwarding_address(obj, newobj, objsize) return newobj def trace_and_copy(self, obj): @@ -316,14 +317,35 @@ pointer.address[0] = self.copy(pointer.address[0]) def is_forwarded(self, obj): - return self.header(obj).forw != NULL + return self.header(obj).tid & GCFLAG_FORWARDED != 0 + # note: all prebuilt objects also have this flag set def get_forwarding_address(self, obj): - return self.header(obj).forw + tid = self.header(obj).tid + if tid & GCFLAG_IMMORTAL: + return obj # prebuilt objects are "forwarded" to themselves + else: + stub = llmemory.cast_adr_to_ptr(obj, self.FORWARDSTUBPTR) + return stub.forw - def set_forwarding_address(self, obj, newobj): - gc_info = self.header(obj) - gc_info.forw = newobj + def set_forwarding_address(self, obj, newobj, objsize): + # To mark an object as forwarded, we set the GCFLAG_FORWARDED and + # overwrite the object with a FORWARDSTUB. Doing so is a bit + # long-winded on llarena, but it all melts down to two memory + # writes after translation to C. + size_gc_header = self.size_gc_header() + stubsize = llmemory.sizeof(self.FORWARDSTUB) + tid = self.header(obj).tid + ll_assert(tid & GCFLAG_IMMORTAL == 0, "unexpected GCFLAG_IMMORTAL") + ll_assert(tid & GCFLAG_FORWARDED == 0, "unexpected GCFLAG_FORWARDED") + # replace the object at 'obj' with a FORWARDSTUB. + hdraddr = obj - size_gc_header + llarena.arena_reset(hdraddr, size_gc_header + objsize, False) + llarena.arena_reserve(hdraddr, size_gc_header + stubsize) + hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(self.HDR)) + hdr.tid = tid | GCFLAG_FORWARDED + stub = llmemory.cast_adr_to_ptr(obj, self.FORWARDSTUBPTR) + stub.forw = newobj def get_size(self, obj): typeid = self.get_type_id(obj) @@ -340,26 +362,23 @@ return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) def get_type_id(self, addr): - return self.header(addr).tid & TYPEID_MASK + tid = self.header(addr).tid + ll_assert(tid & GCFLAG_FORWARDED == 0, "get_type_id on forwarded obj") + # Forwarded objects are overwritten with a FORWARDSTUB. Although + # calling get_type_id() on a forwarded object works by itself, + # we catch it as an error because it's likely that what is then + # done with the typeid is bogus. + return tid & TYPEID_MASK def init_gc_object(self, addr, typeid, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - #hdr.forw = NULL -- unneeded, the space is initially filled with zero hdr.tid = typeid | flags def init_gc_object_immortal(self, addr, typeid, flags=0): - # immortal objects always have forward to themselves hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.tid = typeid | flags | GCFLAG_IMMORTAL - self.init_forwarding(addr + self.gcheaderbuilder.size_gc_header) - - def init_forwarding(self, obj): - hdr = self.header(obj) - if hdr.tid & GCFLAG_IMMORTAL: - hdr.forw = obj # prebuilt objects point to themselves, - # so that a collection does not move them - else: - hdr.forw = NULL + hdr.tid = typeid | flags | GCFLAG_IMMORTAL | GCFLAG_FORWARDED + # immortal objects always have GCFLAG_FORWARDED set; + # see get_forwarding_address(). def deal_with_objects_with_finalizers(self, scan): # walk over list of objects with finalizers @@ -371,6 +390,7 @@ new_with_finalizer = self.AddressDeque() marked = self.AddressDeque() pending = self.AddressStack() + self.tmpstack = self.AddressStack() while self.objects_with_finalizers.non_empty(): x = self.objects_with_finalizers.popleft() ll_assert(self._finalization_state(x) != 1, @@ -385,11 +405,9 @@ state = self._finalization_state(y) if state == 0: self._bump_finalization_state_from_0_to_1(y) + self.trace(y, self._append_if_nonnull, pending) elif state == 2: - self._bump_finalization_state_from_2_to_3(y) - else: - continue # don't need to recurse inside y - self.trace(y, self._append_if_nonnull, pending) + self._recursively_bump_finalization_state_from_2_to_3(y) scan = self._recursively_bump_finalization_state_from_1_to_2( x, scan) @@ -403,16 +421,11 @@ # we must also fix the state from 2 to 3 here, otherwise # we leave the GCFLAG_FINALIZATION_ORDERING bit behind # which will confuse the next collection - pending.append(x) - while pending.non_empty(): - y = pending.pop() - state = self._finalization_state(y) - if state == 2: - self._bump_finalization_state_from_2_to_3(y) - self.trace(y, self._append_if_nonnull, pending) + self._recursively_bump_finalization_state_from_2_to_3(x) else: new_with_finalizer.append(newx) + self.tmpstack.delete() pending.delete() marked.delete() self.objects_with_finalizers.delete() @@ -445,12 +458,19 @@ hdr = self.header(obj) hdr.tid |= GCFLAG_FINALIZATION_ORDERING - def _bump_finalization_state_from_2_to_3(self, obj): + def _recursively_bump_finalization_state_from_2_to_3(self, obj): ll_assert(self._finalization_state(obj) == 2, "unexpected finalization state != 2") newobj = self.get_forwarding_address(obj) - hdr = self.header(newobj) - hdr.tid &= ~GCFLAG_FINALIZATION_ORDERING + pending = self.tmpstack + ll_assert(not pending.non_empty(), "tmpstack not empty") + pending.append(newobj) + while pending.non_empty(): + y = pending.pop() + hdr = self.header(y) + if hdr.tid & GCFLAG_FINALIZATION_ORDERING: # state 2 ? + hdr.tid &= ~GCFLAG_FINALIZATION_ORDERING # change to state 3 + self.trace(y, self._append_if_nonnull, pending) def _recursively_bump_finalization_state_from_1_to_2(self, obj, scan): # recursively convert objects from state 1 to state 2. From arigo at codespeak.net Tue Feb 19 12:59:27 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 19 Feb 2008 12:59:27 +0100 (CET) Subject: [pypy-svn] r51625 - pypy/branch/no-forw-ptr/pypy/rpython/memory/gc Message-ID: <20080219115927.997011684DB@codespeak.net> Author: arigo Date: Tue Feb 19 12:59:26 2008 New Revision: 51625 Modified: pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/semispace.py Log: Move this initialization code to translation-time instead of run-time. Modified: pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/semispace.py Tue Feb 19 12:59:26 2008 @@ -44,6 +44,11 @@ self.gcheaderbuilder = GCHeaderBuilder(self.HDR) self.AddressStack = get_address_stack(chunk_size) self.AddressDeque = get_address_deque(chunk_size) + self.objects_with_finalizers = self.AddressDeque() + self.run_finalizers = self.AddressDeque() + self.objects_with_weakrefs = self.AddressStack() + self.finalizer_lock_count = 0 + self.red_zone = 0 def setup(self): if DEBUG_PRINT: @@ -55,11 +60,6 @@ self.fromspace = llarena.arena_malloc(self.space_size, True) ll_assert(bool(self.fromspace), "couldn't allocate fromspace") self.free = self.tospace - self.objects_with_finalizers = self.AddressDeque() - self.run_finalizers = self.AddressDeque() - self.objects_with_weakrefs = self.AddressStack() - self.finalizer_lock_count = 0 - self.red_zone = 0 def disable_finalizers(self): self.finalizer_lock_count += 1 From arigo at codespeak.net Tue Feb 19 13:03:50 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 19 Feb 2008 13:03:50 +0100 (CET) Subject: [pypy-svn] r51626 - pypy/branch/no-forw-ptr/pypy/rpython/memory Message-ID: <20080219120350.5A0281684DB@codespeak.net> Author: arigo Date: Tue Feb 19 13:03:49 2008 New Revision: 51626 Modified: pypy/branch/no-forw-ptr/pypy/rpython/memory/support.py Log: Translation fix for prebuilt AddressStack and AddressDeque. Modified: pypy/branch/no-forw-ptr/pypy/rpython/memory/support.py ============================================================================== --- pypy/branch/no-forw-ptr/pypy/rpython/memory/support.py (original) +++ pypy/branch/no-forw-ptr/pypy/rpython/memory/support.py Tue Feb 19 13:03:49 2008 @@ -26,7 +26,10 @@ def get(self): if not self.free_list: - return lltype.malloc(CHUNK, flavor="raw") + # we zero-initialize the chunks to make the translation + # backends happy, but we don't need to do it at run-time. + zero = not we_are_translated() + return lltype.malloc(CHUNK, flavor="raw", zero=zero) result = self.free_list self.free_list = result.next From arigo at codespeak.net Tue Feb 19 13:19:30 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 19 Feb 2008 13:19:30 +0100 (CET) Subject: [pypy-svn] r51627 - pypy/branch/no-forw-ptr/pypy/rpython/memory/gc Message-ID: <20080219121930.63C5D1684E4@codespeak.net> Author: arigo Date: Tue Feb 19 13:19:29 2008 New Revision: 51627 Modified: pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/semispace.py Log: Fix the assert. Apparently only GenerationGC tests trigger this bug. Modified: pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/semispace.py Tue Feb 19 13:19:29 2008 @@ -363,9 +363,10 @@ def get_type_id(self, addr): tid = self.header(addr).tid - ll_assert(tid & GCFLAG_FORWARDED == 0, "get_type_id on forwarded obj") - # Forwarded objects are overwritten with a FORWARDSTUB. Although - # calling get_type_id() on a forwarded object works by itself, + ll_assert(tid & (GCFLAG_FORWARDED|GCFLAG_IMMORTAL) != GCFLAG_FORWARDED, + "get_type_id on forwarded obj") + # Non-prebuilt forwarded objects are overwritten with a FORWARDSTUB. + # Although calling get_type_id() on a forwarded object works by itself, # we catch it as an error because it's likely that what is then # done with the typeid is bogus. return tid & TYPEID_MASK From arigo at codespeak.net Tue Feb 19 13:20:06 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 19 Feb 2008 13:20:06 +0100 (CET) Subject: [pypy-svn] r51628 - in pypy/branch/no-forw-ptr/pypy/rpython: lltypesystem memory/gc Message-ID: <20080219122006.7AEC71684E4@codespeak.net> Author: arigo Date: Tue Feb 19 13:20:06 2008 New Revision: 51628 Modified: pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/llarena.py pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/generation.py Log: Fix the GenerationGC for the removal of the .forw pointers. Modified: pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/llarena.py Tue Feb 19 13:20:06 2008 @@ -152,6 +152,7 @@ return True def compare_with_fakeaddr(self, other): + other = other._fixup() if not other: return None, None obj = other.ptr._obj Modified: pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/generation.py Tue Feb 19 13:20:06 2008 @@ -22,8 +22,8 @@ class GenerationGC(SemiSpaceGC): """A basic generational GC: it's a SemiSpaceGC with an additional nursery for young objects. A write barrier is used to ensure that - old objects that contain pointers to young objects are in a linked - list, chained to each other via their 'forw' header field. + old objects that contain pointers to young objects are recorded in + a list. """ inline_simple_malloc = True inline_simple_malloc_varsize = True @@ -44,19 +44,16 @@ self.initial_nursery_size = nursery_size self.auto_nursery_size = auto_nursery_size self.min_nursery_size = min_nursery_size - - def setup(self): - SemiSpaceGC.setup(self) - self.reset_nursery() - self.old_objects_pointing_to_young = NULL - # ^^^ the head of a linked list inside the old objects space; it + self.old_objects_pointing_to_young = self.AddressStack() + # ^^^ a list of addresses inside the old objects space; it # may contain static prebuilt objects as well. More precisely, # it lists exactly the old and static objects whose - # GCFLAG_NO_YOUNG_PTRS bit is not set. The 'forw' header field - # of such objects is abused for this linked list; it needs to be - # reset to its correct value when GCFLAG_NO_YOUNG_PTRS is set - # again at the start of a collection. + # GCFLAG_NO_YOUNG_PTRS bit is not set. self.young_objects_with_weakrefs = self.AddressStack() + self.reset_nursery() + + def setup(self): + SemiSpaceGC.setup(self) self.set_nursery_size(self.initial_nursery_size) # the GC is fully setup now. The rest can make use of it. if self.auto_nursery_size: @@ -226,14 +223,11 @@ # the next usage. def reset_young_gcflags(self): - obj = self.old_objects_pointing_to_young - while obj: + oldlist = self.old_objects_pointing_to_young + while oldlist.non_empty(): + obj = oldlist.pop() hdr = self.header(obj) hdr.tid |= GCFLAG_NO_YOUNG_PTRS - nextobj = hdr.forw - self.init_forwarding(obj) - obj = nextobj - self.old_objects_pointing_to_young = NULL def weakrefs_grow_older(self): while self.young_objects_with_weakrefs.non_empty(): @@ -278,19 +272,15 @@ def collect_oldrefs_to_nursery(self): # Follow the old_objects_pointing_to_young list and move the - # young objects they point to out of the nursery. The 'forw' - # fields are reset to their correct value along the way. + # young objects they point to out of the nursery. count = 0 - obj = self.old_objects_pointing_to_young - while obj: + oldlist = self.old_objects_pointing_to_young + while oldlist.non_empty(): count += 1 - nextobj = self.header(obj).forw - self.init_forwarding(obj) + obj = oldlist.pop() self.trace_and_drag_out_of_nursery(obj) - obj = nextobj if DEBUG_PRINT: llop.debug_print(lltype.Void, "collect_oldrefs_to_nursery", count) - self.old_objects_pointing_to_young = NULL def collect_roots_in_nursery(self): # we don't need to trace prebuilt GcStructs during a minor collect: @@ -362,8 +352,7 @@ "nursery object with GCFLAG_NO_YOUNG_PTRS") oldhdr = self.header(addr_struct) if self.is_in_nursery(addr): - oldhdr.forw = self.old_objects_pointing_to_young - self.old_objects_pointing_to_young = addr_struct + self.old_objects_pointing_to_young.append(addr_struct) oldhdr.tid &= ~GCFLAG_NO_YOUNG_PTRS if oldhdr.tid & GCFLAG_NO_HEAP_PTRS: self.move_to_static_roots(addr_struct) From arigo at codespeak.net Tue Feb 19 13:42:25 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 19 Feb 2008 13:42:25 +0100 (CET) Subject: [pypy-svn] r51629 - pypy/dist/pypy/translator/llvm/test Message-ID: <20080219124225.8318B1684E6@codespeak.net> Author: arigo Date: Tue Feb 19 13:42:25 2008 New Revision: 51629 Modified: pypy/dist/pypy/translator/llvm/test/test_newgc.py Log: Fix test to account for the reduced header and alignment. Modified: pypy/dist/pypy/translator/llvm/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_newgc.py Tue Feb 19 13:42:25 2008 @@ -5,7 +5,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena def test_gc_offsets(): - STRUCT = lltype.GcStruct('S1', ('x', lltype.Signed)) + STRUCT = lltype.GcStruct('S1', ('x', lltype.Signed), ('y', lltype.Char)) ARRAY = lltype.GcArray(lltype.Signed) s1 = llarena.round_up_for_allocation(llmemory.sizeof(STRUCT)) s2 = llmemory.offsetof(STRUCT, 'x') @@ -25,10 +25,10 @@ i3 = (res // 10000) % 100 i4 = (res // 100) % 100 i5 = (res // 1) % 100 - assert i1 % 8 == 0 + assert i1 % 4 == 0 assert 12 <= i1 <= 24 assert 8 <= i2 <= i1 - 4 - assert 8 <= i3 <= 16 + assert 4 <= i3 <= 12 assert i4 == i5 assert i3 + 4 <= i5 From arigo at codespeak.net Tue Feb 19 13:53:40 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 19 Feb 2008 13:53:40 +0100 (CET) Subject: [pypy-svn] r51630 - pypy/branch/no-forw-ptr/pypy/rpython/memory/gc Message-ID: <20080219125340.EE5BE1684DF@codespeak.net> Author: arigo Date: Tue Feb 19 13:53:39 2008 New Revision: 51630 Modified: pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/semispace.py Log: Ouch! Can't prebuild these address lists, because we call delete() on them later on :-) Modified: pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/semispace.py Tue Feb 19 13:53:39 2008 @@ -44,9 +44,6 @@ self.gcheaderbuilder = GCHeaderBuilder(self.HDR) self.AddressStack = get_address_stack(chunk_size) self.AddressDeque = get_address_deque(chunk_size) - self.objects_with_finalizers = self.AddressDeque() - self.run_finalizers = self.AddressDeque() - self.objects_with_weakrefs = self.AddressStack() self.finalizer_lock_count = 0 self.red_zone = 0 @@ -60,6 +57,9 @@ self.fromspace = llarena.arena_malloc(self.space_size, True) ll_assert(bool(self.fromspace), "couldn't allocate fromspace") self.free = self.tospace + self.objects_with_finalizers = self.AddressDeque() + self.run_finalizers = self.AddressDeque() + self.objects_with_weakrefs = self.AddressStack() def disable_finalizers(self): self.finalizer_lock_count += 1 From fijal at codespeak.net Tue Feb 19 13:56:59 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 19 Feb 2008 13:56:59 +0100 (CET) Subject: [pypy-svn] r51631 - pypy/extradoc/talk/pycon2008 Message-ID: <20080219125659.606E31684E6@codespeak.net> Author: fijal Date: Tue Feb 19 13:56:58 2008 New Revision: 51631 Added: pypy/extradoc/talk/pycon2008/sprint.txt (contents, props changed) Log: Some sprint info Added: pypy/extradoc/talk/pycon2008/sprint.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/pycon2008/sprint.txt Tue Feb 19 13:56:58 2008 @@ -0,0 +1,41 @@ +A bit late sprint announcement, but I've been waiting till I get +the US visa to make sure that the sprint happens. + +Project: PyPy (http://codespeak.net/pypy) + +Project description: PyPy is a python compiler written in Python. It's also +a very flexible compiler toolchain which can compile so called RPython +(restricted Python, a subset of Python without dynamic features) into +a variety of platforms including C/POSIX, JVM and CLI. PyPy also features +set of experimental features, like different gcs or different threading +models, which makes it a good platform to experiment with python language. + +Sprint leader: Maciej Fijalkowski (fijall at gmail.com) + XXX holger? + +Attendees: they'll be listed on our page when they submit their info + XXX link + +Instructions: Suggestions for contributors would be to come to #pypy +at irc.freenode.net in order to create a codespeak account. In any case, +repository and all documentation are available on the website. + +Some instructions: This sprint is newcomer-friendly sprint. We'll present +PyPy tutorial and code-walk on sunday afternoon. List of task is as usual +very long and depending on attendees. It's suggested that people would +come to an IRC and chat a bit to have a clue what is feasible on a sprint +and what is not. For example, we can work on: + +* JVM backend bindings for Java libraries +* Extending pypy necessary for software X to run +* Adding more modules to pypy +* Sketching ctypes-based software, killing dependency on C modules + written using C/CPython API. +* Optimizing certain parts of pypy for certain micro-benchmarks. +* XXX whatever... + +Software requirements: That depends on a thing that people would like to +work on, but in general pygame, libgc and libffi are very very usefull (although +not needed). PyPy checkout, gcc and such things are a must. For people +wanting to work on particular backends, aproprietly Java, Mono (or .NET), +spidermonkey are needed. From arigo at codespeak.net Tue Feb 19 14:16:38 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 19 Feb 2008 14:16:38 +0100 (CET) Subject: [pypy-svn] r51632 - in pypy/dist/pypy/rpython: lltypesystem memory/gc memory/gctransform memory/test Message-ID: <20080219131638.667C91684E6@codespeak.net> Author: arigo Date: Tue Feb 19 14:16:37 2008 New Revision: 51632 Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/gc/marksweep.py pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Log: Kill the GC operation 'x_become', only implemented in the mark-n-sweep GC and not completely working. Modified: pypy/dist/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lloperation.py Tue Feb 19 14:16:37 2008 @@ -403,10 +403,6 @@ 'gc_x_clone': LLOp(canraise=(MemoryError, RuntimeError), canunwindgc=True), 'gc_x_size_header': LLOp(), - # this one is even more experimental; only implemented with the - # Mark&Sweep GC, and likely only useful when combined with - # stackless: - 'gc_x_become': LLOp(canraise=(RuntimeError,), canunwindgc=True), # for llvm.gcroot() support. can change at any time 'llvm_frameaddress': LLOp(sideeffects=False), Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Tue Feb 19 14:16:37 2008 @@ -111,9 +111,6 @@ def x_clone(self, clonedata): raise RuntimeError("no support for x_clone in the GC") - def x_become(self, target_addr, source_addr): - raise RuntimeError("no support for x_become in the GC") - def trace(self, obj, callback, arg): """Enumerate the locations inside the given obj that can contain GC pointers. For each such location, callback(pointer, arg) is Modified: pypy/dist/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/dist/pypy/rpython/memory/gc/marksweep.py Tue Feb 19 14:16:37 2008 @@ -673,176 +673,6 @@ # reinstall the pool that was current at the beginning of x_clone() clonedata.pool = self.x_swap_pool(curpool) - def add_reachable_to_stack2(self, obj, objects): - size_gc_header = self.gcheaderbuilder.size_gc_header - gc_info = obj - size_gc_header - hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - if hdr.typeid & 1: - return - self.trace(obj, self._add_reachable_and_rename, objects) - - def _add_reachable_and_rename(self, pointer, objects): - obj = pointer.address[0] - if obj: - if obj == self.x_become_target_addr: - obj = pointer.address[0] = self.x_become_source_addr - objects.append(obj) - - def x_become(self, target_addr, source_addr): - # 1. mark from the roots, and also the objects that objects-with-del - # point to (using the list of malloced_objects_with_finalizer) - # 2. walk the list of objects-without-del and free the ones not marked - # 3. walk the list of objects-with-del and for the ones not marked: - # call __del__, move the object to the list of object-without-del - import time - from pypy.rpython.lltypesystem.lloperation import llop - if DEBUG_PRINT: - llop.debug_print(lltype.Void, 'collecting...') - start_time = time.time() - size_gc_header = self.gcheaderbuilder.size_gc_header -## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, -## size_gc_header) - - # push the roots on the mark stack - objects = self.AddressStack() # mark stack - self._mark_stack = objects - # the last sweep did not clear the mark bit of static roots, - # since they are not in the malloced_objects list - self.root_walker.walk_roots( - MarkSweepGC._mark_root_and_clear_bit, # stack roots - MarkSweepGC._mark_root_and_clear_bit, # static in prebuilt non-gc - MarkSweepGC._mark_root_and_clear_bit) # static in prebuilt gc - - # from this point onwards, no more mallocs should be possible - old_malloced = self.bytes_malloced - self.bytes_malloced = 0 - curr_heap_size = 0 - freed_size = 0 - - self.x_become_target_addr = target_addr - self.x_become_source_addr = source_addr - - # mark objects reachable by objects with a finalizer, but not those - # themselves. add their size to curr_heap_size, since they always - # survive the collection - hdr = self.malloced_objects_with_finalizer - while hdr: - next = hdr.next - typeid = hdr.typeid >> 1 - gc_info = llmemory.cast_ptr_to_adr(hdr) - obj = gc_info + size_gc_header - self.add_reachable_to_stack2(obj, objects) - addr = llmemory.cast_ptr_to_adr(hdr) - size = self.fixed_size(typeid) - if self.is_varsize(typeid): - length = (obj + self.varsize_offset_to_length(typeid)).signed[0] - size += self.varsize_item_sizes(typeid) * length - estimate = raw_malloc_usage(size_gc_header + size) - curr_heap_size += estimate - hdr = next - - # mark thinks on the mark stack and put their descendants onto the - # stack until the stack is empty - while objects.non_empty(): #mark - curr = objects.pop() - self.add_reachable_to_stack2(curr, objects) - gc_info = curr - size_gc_header - hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - if hdr.typeid & 1: - continue - hdr.typeid = hdr.typeid | 1 - objects.delete() - # also mark self.curpool - if self.curpool: - gc_info = llmemory.cast_ptr_to_adr(self.curpool) - size_gc_header - hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - hdr.typeid = hdr.typeid | 1 - - # sweep: delete objects without del if they are not marked - # unmark objects without del that are marked - firstpoolnode = lltype.malloc(self.POOLNODE, flavor='raw') - firstpoolnode.linkedlist = self.malloced_objects - firstpoolnode.nextnode = self.poolnodes - prevpoolnode = lltype.nullptr(self.POOLNODE) - poolnode = firstpoolnode - while poolnode: #sweep - ppnext = llmemory.cast_ptr_to_adr(poolnode) - ppnext += llmemory.offsetof(self.POOLNODE, 'linkedlist') - hdr = poolnode.linkedlist - while hdr: #sweep - typeid = hdr.typeid >> 1 - next = hdr.next - addr = llmemory.cast_ptr_to_adr(hdr) - size = self.fixed_size(typeid) - if self.is_varsize(typeid): - length = (addr + size_gc_header + self.varsize_offset_to_length(typeid)).signed[0] - size += self.varsize_item_sizes(typeid) * length - estimate = raw_malloc_usage(size_gc_header + size) - if hdr.typeid & 1: - hdr.typeid = hdr.typeid & (~1) - ppnext.address[0] = addr - ppnext = llmemory.cast_ptr_to_adr(hdr) - ppnext += llmemory.offsetof(self.HDR, 'next') - curr_heap_size += estimate - else: - freed_size += estimate - raw_free(addr) - hdr = next - ppnext.address[0] = llmemory.NULL - next = poolnode.nextnode - if not poolnode.linkedlist and prevpoolnode: - # completely empty node - prevpoolnode.nextnode = next - lltype.free(poolnode, flavor='raw') - else: - prevpoolnode = poolnode - poolnode = next - self.malloced_objects = firstpoolnode.linkedlist - self.poolnodes = firstpoolnode.nextnode - lltype.free(firstpoolnode, flavor='raw') - - if curr_heap_size > self.bytes_malloced_threshold: - self.bytes_malloced_threshold = curr_heap_size - end_time = time.time() - self.total_collection_time += end_time - start_time - if DEBUG_PRINT: - llop.debug_print(lltype.Void, - " malloced since previous collection:", - old_malloced, "bytes") - llop.debug_print(lltype.Void, - " heap usage at start of collection: ", - self.heap_usage + old_malloced, "bytes") - llop.debug_print(lltype.Void, - " freed: ", - freed_size, "bytes") - llop.debug_print(lltype.Void, - " new heap usage: ", - curr_heap_size, "bytes") - llop.debug_print(lltype.Void, - " total time spent collecting: ", - self.total_collection_time, "seconds") -## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, -## size_gc_header) - assert self.heap_usage + old_malloced == curr_heap_size + freed_size - - # call finalizers if needed - self.heap_usage = curr_heap_size - hdr = self.malloced_objects_with_finalizer - self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR) - while hdr: - next = hdr.next - if hdr.typeid & 1: - hdr.next = self.malloced_objects_with_finalizer - self.malloced_objects_with_finalizer = hdr - hdr.typeid = hdr.typeid & (~1) - else: - obj = llmemory.cast_ptr_to_adr(hdr) + size_gc_header - finalizer = self.getfinalizer(hdr.typeid >> 1) - finalizer(obj) - hdr.next = self.malloced_objects - self.malloced_objects = hdr - hdr = next - class PrintingMarkSweepGC(MarkSweepGC): _alloc_flavor_ = "raw" Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Tue Feb 19 14:16:37 2008 @@ -23,7 +23,7 @@ class CollectAnalyzer(graphanalyze.GraphAnalyzer): def operation_is_true(self, op): - if op.opname in ('gc__collect', 'gc_x_become'): + if op.opname == 'gc__collect': return True if op.opname in ('malloc', 'malloc_varsize'): flags = op.args[1].value @@ -316,11 +316,6 @@ annmodel.s_None, minimal_transform = False) - self.x_become_ptr = getfn( - GCClass.x_become.im_func, - [s_gc, annmodel.SomeAddress(), annmodel.SomeAddress()], - annmodel.s_None) - annhelper.finish() # at this point, annotate all mix-level helpers annhelper.backend_optimize() @@ -549,15 +544,6 @@ [c_result], resultvar=op.result) - def gct_gc_x_become(self, hop): - op = hop.spaceop - [v_target, v_source] = op.args - livevars = self.push_roots(hop) - hop.genop("direct_call", - [self.x_become_ptr, self.c_const_gc, v_target, v_source], - resultvar=op.result) - self.pop_roots(hop, livevars) - def gct_zero_gc_pointers_inside(self, hop): if self.needs_zero_gc_pointers: v_ob = hop.spaceop.args[0] Modified: pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/dist/pypy/rpython/memory/test/test_transformed_gc.py Tue Feb 19 14:16:37 2008 @@ -729,26 +729,6 @@ def test_instances(self): py.test.skip("fails for a stupid reasons") - def test_x_become(self): - from pypy.rlib import objectmodel - S = lltype.GcStruct("S", ('x', lltype.Signed)) - def f(): - x = lltype.malloc(S) - x.x = 10 - y = lltype.malloc(S) - y.x = 20 - z = x - llop.gc_x_become(lltype.Void, - llmemory.cast_ptr_to_adr(x), - llmemory.cast_ptr_to_adr(y)) - # keep 'y' alive until the x_become() is finished, because in - # theory it could go away as soon as only its address is present - objectmodel.keepalive_until_here(y) - return z.x - run = self.runner(f) - res = run([]) - assert res == 20 - class TestPrintingGC(GenericGCTests): gcname = "statistics" From arigo at codespeak.net Tue Feb 19 15:02:52 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 19 Feb 2008 15:02:52 +0100 (CET) Subject: [pypy-svn] r51633 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20080219140252.537161684EC@codespeak.net> Author: arigo Date: Tue Feb 19 15:02:50 2008 New Revision: 51633 Modified: pypy/dist/pypy/rpython/memory/gc/generation.py Log: Recover a GCFLAG bit that was "lost" by mistake. (Not really important.) Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Tue Feb 19 15:02:50 2008 @@ -11,11 +11,11 @@ # in the nursery. It is initially set on all prebuilt and old objects, # and gets cleared by the write_barrier() when we write in them a # pointer to a young object. -GCFLAG_NO_YOUNG_PTRS = SemiSpaceGC.first_unused_gcflag << 1 +GCFLAG_NO_YOUNG_PTRS = SemiSpaceGC.first_unused_gcflag << 0 # The following flag is set for static roots which are not on the list # of static roots yet, but will appear with write barrier -GCFLAG_NO_HEAP_PTRS = SemiSpaceGC.first_unused_gcflag << 2 +GCFLAG_NO_HEAP_PTRS = SemiSpaceGC.first_unused_gcflag << 1 DEBUG_PRINT = False @@ -29,7 +29,7 @@ inline_simple_malloc_varsize = True needs_write_barrier = True prebuilt_gc_objects_are_static_roots = False - first_unused_gcflag = SemiSpaceGC.first_unused_gcflag << 3 + first_unused_gcflag = SemiSpaceGC.first_unused_gcflag << 2 def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, nursery_size=128, From arigo at codespeak.net Tue Feb 19 16:31:38 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 19 Feb 2008 16:31:38 +0100 (CET) Subject: [pypy-svn] r51634 - pypy/extradoc/talk/pycon2008 Message-ID: <20080219153138.D41A11684EC@codespeak.net> Author: arigo Date: Tue Feb 19 16:31:37 2008 New Revision: 51634 Modified: pypy/extradoc/talk/pycon2008/sprint.txt Log: Only typos Modified: pypy/extradoc/talk/pycon2008/sprint.txt ============================================================================== --- pypy/extradoc/talk/pycon2008/sprint.txt (original) +++ pypy/extradoc/talk/pycon2008/sprint.txt Tue Feb 19 16:31:37 2008 @@ -7,8 +7,8 @@ a very flexible compiler toolchain which can compile so called RPython (restricted Python, a subset of Python without dynamic features) into a variety of platforms including C/POSIX, JVM and CLI. PyPy also features -set of experimental features, like different gcs or different threading -models, which makes it a good platform to experiment with python language. +a set of experimental features, like different GCs or different threading +models, which makes it a good platform to experiment with the python language. Sprint leader: Maciej Fijalkowski (fijall at gmail.com) XXX holger? @@ -21,21 +21,21 @@ repository and all documentation are available on the website. Some instructions: This sprint is newcomer-friendly sprint. We'll present -PyPy tutorial and code-walk on sunday afternoon. List of task is as usual +a PyPy tutorial and code-walk on Sunday afternoon. The list of task is as usual very long and depending on attendees. It's suggested that people would come to an IRC and chat a bit to have a clue what is feasible on a sprint and what is not. For example, we can work on: * JVM backend bindings for Java libraries -* Extending pypy necessary for software X to run +* Extending pypy as necessary for software X to run * Adding more modules to pypy * Sketching ctypes-based software, killing dependency on C modules written using C/CPython API. * Optimizing certain parts of pypy for certain micro-benchmarks. * XXX whatever... -Software requirements: That depends on a thing that people would like to -work on, but in general pygame, libgc and libffi are very very usefull (although -not needed). PyPy checkout, gcc and such things are a must. For people -wanting to work on particular backends, aproprietly Java, Mono (or .NET), +Software requirements: the details depends on what people would like to +work on, but in general pygame, libgc and libffi are very very useful (although +not needed). A PyPy checkout, gcc and such things are a must. For people +wanting to work on particular backends, appropriately Java, Mono (or .NET), spidermonkey are needed. From fijal at codespeak.net Tue Feb 19 16:34:30 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 19 Feb 2008 16:34:30 +0100 (CET) Subject: [pypy-svn] r51635 - in pypy/extradoc: sprintinfo/pycon2008 talk/pycon2008 Message-ID: <20080219153430.DD86A1684EE@codespeak.net> Author: fijal Date: Tue Feb 19 16:34:30 2008 New Revision: 51635 Added: pypy/extradoc/sprintinfo/pycon2008/ pypy/extradoc/sprintinfo/pycon2008/sprint.txt - copied unchanged from r51634, pypy/extradoc/talk/pycon2008/sprint.txt Removed: pypy/extradoc/talk/pycon2008/sprint.txt Log: Move sprintinfo to sprintinfo From fijal at codespeak.net Tue Feb 19 16:35:57 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 19 Feb 2008 16:35:57 +0100 (CET) Subject: [pypy-svn] r51636 - pypy/extradoc/sprintinfo/pycon2008 Message-ID: <20080219153557.CC64216841A@codespeak.net> Author: fijal Date: Tue Feb 19 16:35:57 2008 New Revision: 51636 Modified: pypy/extradoc/sprintinfo/pycon2008/sprint.txt Log: Few clarifications Modified: pypy/extradoc/sprintinfo/pycon2008/sprint.txt ============================================================================== --- pypy/extradoc/sprintinfo/pycon2008/sprint.txt (original) +++ pypy/extradoc/sprintinfo/pycon2008/sprint.txt Tue Feb 19 16:35:57 2008 @@ -1,5 +1,6 @@ -A bit late sprint announcement, but I've been waiting till I get -the US visa to make sure that the sprint happens. +Place: Pycon 2008 + +Dates: 17th March 2008 - 20th March 2008 Project: PyPy (http://codespeak.net/pypy) @@ -11,10 +12,9 @@ models, which makes it a good platform to experiment with the python language. Sprint leader: Maciej Fijalkowski (fijall at gmail.com) - XXX holger? -Attendees: they'll be listed on our page when they submit their info - XXX link +Attendees: list will be available at +http://codespeak.net/pypy/extradoc/sprintinfo/pycon2008/people.txt Instructions: Suggestions for contributors would be to come to #pypy at irc.freenode.net in order to create a codespeak account. In any case, @@ -32,7 +32,7 @@ * Sketching ctypes-based software, killing dependency on C modules written using C/CPython API. * Optimizing certain parts of pypy for certain micro-benchmarks. -* XXX whatever... +* Whatever else.... Software requirements: the details depends on what people would like to work on, but in general pygame, libgc and libffi are very very useful (although From fijal at codespeak.net Tue Feb 19 16:36:43 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 19 Feb 2008 16:36:43 +0100 (CET) Subject: [pypy-svn] r51637 - pypy/extradoc/sprintinfo/pycon2008 Message-ID: <20080219153643.E1A0C1684DD@codespeak.net> Author: fijal Date: Tue Feb 19 16:36:43 2008 New Revision: 51637 Added: pypy/extradoc/sprintinfo/pycon2008/people.txt - copied unchanged from r51634, pypy/extradoc/sprintinfo/leysin-winter-2008/people.txt Log: add people.txt From fijal at codespeak.net Tue Feb 19 16:37:50 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 19 Feb 2008 16:37:50 +0100 (CET) Subject: [pypy-svn] r51638 - pypy/extradoc/sprintinfo/pycon2008 Message-ID: <20080219153750.52E4B1684DB@codespeak.net> Author: fijal Date: Tue Feb 19 16:37:49 2008 New Revision: 51638 Modified: pypy/extradoc/sprintinfo/pycon2008/people.txt Log: update Modified: pypy/extradoc/sprintinfo/pycon2008/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/pycon2008/people.txt (original) +++ pypy/extradoc/sprintinfo/pycon2008/people.txt Tue Feb 19 16:37:49 2008 @@ -1,28 +1,27 @@ -People coming to the Leysin sprint Winter 2008 +People coming to the Pycon 2008 sprint ================================================== People who have a ``?`` in their arrive/depart or accomodation column are known to be coming but there are no details available yet from them. +People on the following list were present at previous sprints: ==================== ============== ======================= Name Arrive/Depart Accomodation ==================== ============== ======================= -Armin Rigo -- private -Carl Friedrich Bolz 12th-19th? Ermina -Antonio Cuni 14th-19th? Ermina -Maciej Fijalkowski 11th-20th Ermina (arrive at 10pm) -Toby Watson 12th-19th Ermina -Paul deGrandis 12th-19th Ermina ==================== ============== ======================= -People on the following list were present at previous sprints: - ==================== ============== ===================== Name Arrive/Depart Accomodation ==================== ============== ===================== +Armin Rigo ? ? +Carl Friedrich Bolz ? ? +Antonio Cuni ? ? +Maciej Fijalkowski ? ? +Toby Watson ? ? +Paul deGrandis ? ? Christian Tismer ? ? Michael Hudson ? ? Anders Lehmann ? ? @@ -30,7 +29,7 @@ Lene Wagner ? ? Amaury Forgeot d'Arc ? ? Valentino Volonghi ? ? -Boris Feigin ? ? +Boris Feigin ? ? Andrew Thompson ? ? Bert Freudenberg ? ? Beatrice Duering ? ? From fijal at codespeak.net Tue Feb 19 16:40:46 2008 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 19 Feb 2008 16:40:46 +0100 (CET) Subject: [pypy-svn] r51639 - pypy/extradoc/sprintinfo/pycon2008 Message-ID: <20080219154046.29A721684DB@codespeak.net> Author: fijal Date: Tue Feb 19 16:40:45 2008 New Revision: 51639 Modified: pypy/extradoc/sprintinfo/pycon2008/sprint.txt Log: update Modified: pypy/extradoc/sprintinfo/pycon2008/sprint.txt ============================================================================== --- pypy/extradoc/sprintinfo/pycon2008/sprint.txt (original) +++ pypy/extradoc/sprintinfo/pycon2008/sprint.txt Tue Feb 19 16:40:45 2008 @@ -11,7 +11,7 @@ a set of experimental features, like different GCs or different threading models, which makes it a good platform to experiment with the python language. -Sprint leader: Maciej Fijalkowski (fijall at gmail.com) +Sprint leader: Maciej Fijalkowski (fijal at merlinux.de) Attendees: list will be available at http://codespeak.net/pypy/extradoc/sprintinfo/pycon2008/people.txt From cfbolz at codespeak.net Tue Feb 19 16:48:36 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 19 Feb 2008 16:48:36 +0100 (CET) Subject: [pypy-svn] r51640 - pypy/dist/pypy/rlib/parsing Message-ID: <20080219154836.03D191684F1@codespeak.net> Author: cfbolz Date: Tue Feb 19 16:48:35 2008 New Revision: 51640 Modified: pypy/dist/pypy/rlib/parsing/deterministic.py pypy/dist/pypy/rlib/parsing/ebnfparse.py Log: emit comments that the code is auto-generated Modified: pypy/dist/pypy/rlib/parsing/deterministic.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/deterministic.py (original) +++ pypy/dist/pypy/rlib/parsing/deterministic.py Tue Feb 19 16:48:35 2008 @@ -266,6 +266,7 @@ from pypy.rlib.parsing.codebuilder import Codebuilder result = Codebuilder() result.start_block("def recognize(runner, i):") + result.emit("#auto-generated code, don't edit") result.emit("assert i >= 0") result.emit("input = runner.text") result.emit("state = 0") Modified: pypy/dist/pypy/rlib/parsing/ebnfparse.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/ebnfparse.py (original) +++ pypy/dist/pypy/rlib/parsing/ebnfparse.py Tue Feb 19 16:48:35 2008 @@ -298,6 +298,7 @@ for i in range(len(self.rules)): self.create_visit_method(i) self.start_block("def transform(self, tree):") + self.emit("#auto-generated code, don't edit") self.emit("assert isinstance(tree, Nonterminal)") startsymbol = self.rules[0].nonterminal self.emit("assert tree.symbol == %r" % (startsymbol, )) @@ -339,6 +340,7 @@ rule = self.rules[index] change = self.changes[index] self.start_block("def visit_%s(self, node):" % (rule.nonterminal, )) + self.emit("#auto-generated code, don't edit") if len(change) == 0: self.emit("return [node]") self.end_block(rule.nonterminal) From arigo at codespeak.net Tue Feb 19 17:23:50 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 19 Feb 2008 17:23:50 +0100 (CET) Subject: [pypy-svn] r51641 - in pypy/branch/no-forw-ptr/pypy: rpython/lltypesystem rpython/memory/gc rpython/memory/gctransform rpython/memory/test translator/llvm/test Message-ID: <20080219162350.3FEEB1684EA@codespeak.net> Author: arigo Date: Tue Feb 19 17:23:49 2008 New Revision: 51641 Modified: pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/lloperation.py pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/base.py pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/generation.py pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/marksweep.py pypy/branch/no-forw-ptr/pypy/rpython/memory/gctransform/framework.py pypy/branch/no-forw-ptr/pypy/rpython/memory/test/test_transformed_gc.py pypy/branch/no-forw-ptr/pypy/translator/llvm/test/test_newgc.py Log: Merge trunk into branch, r51622:51634. Modified: pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/no-forw-ptr/pypy/rpython/lltypesystem/lloperation.py Tue Feb 19 17:23:49 2008 @@ -403,10 +403,6 @@ 'gc_x_clone': LLOp(canraise=(MemoryError, RuntimeError), canunwindgc=True), 'gc_x_size_header': LLOp(), - # this one is even more experimental; only implemented with the - # Mark&Sweep GC, and likely only useful when combined with - # stackless: - 'gc_x_become': LLOp(canraise=(RuntimeError,), canunwindgc=True), # for llvm.gcroot() support. can change at any time 'llvm_frameaddress': LLOp(sideeffects=False), Modified: pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/base.py Tue Feb 19 17:23:49 2008 @@ -111,9 +111,6 @@ def x_clone(self, clonedata): raise RuntimeError("no support for x_clone in the GC") - def x_become(self, target_addr, source_addr): - raise RuntimeError("no support for x_become in the GC") - def trace(self, obj, callback, arg): """Enumerate the locations inside the given obj that can contain GC pointers. For each such location, callback(pointer, arg) is Modified: pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/generation.py Tue Feb 19 17:23:49 2008 @@ -11,11 +11,11 @@ # in the nursery. It is initially set on all prebuilt and old objects, # and gets cleared by the write_barrier() when we write in them a # pointer to a young object. -GCFLAG_NO_YOUNG_PTRS = SemiSpaceGC.first_unused_gcflag << 1 +GCFLAG_NO_YOUNG_PTRS = SemiSpaceGC.first_unused_gcflag << 0 # The following flag is set for static roots which are not on the list # of static roots yet, but will appear with write barrier -GCFLAG_NO_HEAP_PTRS = SemiSpaceGC.first_unused_gcflag << 2 +GCFLAG_NO_HEAP_PTRS = SemiSpaceGC.first_unused_gcflag << 1 DEBUG_PRINT = False @@ -29,7 +29,7 @@ inline_simple_malloc_varsize = True needs_write_barrier = True prebuilt_gc_objects_are_static_roots = False - first_unused_gcflag = SemiSpaceGC.first_unused_gcflag << 3 + first_unused_gcflag = SemiSpaceGC.first_unused_gcflag << 2 def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, nursery_size=128, Modified: pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/branch/no-forw-ptr/pypy/rpython/memory/gc/marksweep.py Tue Feb 19 17:23:49 2008 @@ -673,176 +673,6 @@ # reinstall the pool that was current at the beginning of x_clone() clonedata.pool = self.x_swap_pool(curpool) - def add_reachable_to_stack2(self, obj, objects): - size_gc_header = self.gcheaderbuilder.size_gc_header - gc_info = obj - size_gc_header - hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - if hdr.typeid & 1: - return - self.trace(obj, self._add_reachable_and_rename, objects) - - def _add_reachable_and_rename(self, pointer, objects): - obj = pointer.address[0] - if obj: - if obj == self.x_become_target_addr: - obj = pointer.address[0] = self.x_become_source_addr - objects.append(obj) - - def x_become(self, target_addr, source_addr): - # 1. mark from the roots, and also the objects that objects-with-del - # point to (using the list of malloced_objects_with_finalizer) - # 2. walk the list of objects-without-del and free the ones not marked - # 3. walk the list of objects-with-del and for the ones not marked: - # call __del__, move the object to the list of object-without-del - import time - from pypy.rpython.lltypesystem.lloperation import llop - if DEBUG_PRINT: - llop.debug_print(lltype.Void, 'collecting...') - start_time = time.time() - size_gc_header = self.gcheaderbuilder.size_gc_header -## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, -## size_gc_header) - - # push the roots on the mark stack - objects = self.AddressStack() # mark stack - self._mark_stack = objects - # the last sweep did not clear the mark bit of static roots, - # since they are not in the malloced_objects list - self.root_walker.walk_roots( - MarkSweepGC._mark_root_and_clear_bit, # stack roots - MarkSweepGC._mark_root_and_clear_bit, # static in prebuilt non-gc - MarkSweepGC._mark_root_and_clear_bit) # static in prebuilt gc - - # from this point onwards, no more mallocs should be possible - old_malloced = self.bytes_malloced - self.bytes_malloced = 0 - curr_heap_size = 0 - freed_size = 0 - - self.x_become_target_addr = target_addr - self.x_become_source_addr = source_addr - - # mark objects reachable by objects with a finalizer, but not those - # themselves. add their size to curr_heap_size, since they always - # survive the collection - hdr = self.malloced_objects_with_finalizer - while hdr: - next = hdr.next - typeid = hdr.typeid >> 1 - gc_info = llmemory.cast_ptr_to_adr(hdr) - obj = gc_info + size_gc_header - self.add_reachable_to_stack2(obj, objects) - addr = llmemory.cast_ptr_to_adr(hdr) - size = self.fixed_size(typeid) - if self.is_varsize(typeid): - length = (obj + self.varsize_offset_to_length(typeid)).signed[0] - size += self.varsize_item_sizes(typeid) * length - estimate = raw_malloc_usage(size_gc_header + size) - curr_heap_size += estimate - hdr = next - - # mark thinks on the mark stack and put their descendants onto the - # stack until the stack is empty - while objects.non_empty(): #mark - curr = objects.pop() - self.add_reachable_to_stack2(curr, objects) - gc_info = curr - size_gc_header - hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - if hdr.typeid & 1: - continue - hdr.typeid = hdr.typeid | 1 - objects.delete() - # also mark self.curpool - if self.curpool: - gc_info = llmemory.cast_ptr_to_adr(self.curpool) - size_gc_header - hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - hdr.typeid = hdr.typeid | 1 - - # sweep: delete objects without del if they are not marked - # unmark objects without del that are marked - firstpoolnode = lltype.malloc(self.POOLNODE, flavor='raw') - firstpoolnode.linkedlist = self.malloced_objects - firstpoolnode.nextnode = self.poolnodes - prevpoolnode = lltype.nullptr(self.POOLNODE) - poolnode = firstpoolnode - while poolnode: #sweep - ppnext = llmemory.cast_ptr_to_adr(poolnode) - ppnext += llmemory.offsetof(self.POOLNODE, 'linkedlist') - hdr = poolnode.linkedlist - while hdr: #sweep - typeid = hdr.typeid >> 1 - next = hdr.next - addr = llmemory.cast_ptr_to_adr(hdr) - size = self.fixed_size(typeid) - if self.is_varsize(typeid): - length = (addr + size_gc_header + self.varsize_offset_to_length(typeid)).signed[0] - size += self.varsize_item_sizes(typeid) * length - estimate = raw_malloc_usage(size_gc_header + size) - if hdr.typeid & 1: - hdr.typeid = hdr.typeid & (~1) - ppnext.address[0] = addr - ppnext = llmemory.cast_ptr_to_adr(hdr) - ppnext += llmemory.offsetof(self.HDR, 'next') - curr_heap_size += estimate - else: - freed_size += estimate - raw_free(addr) - hdr = next - ppnext.address[0] = llmemory.NULL - next = poolnode.nextnode - if not poolnode.linkedlist and prevpoolnode: - # completely empty node - prevpoolnode.nextnode = next - lltype.free(poolnode, flavor='raw') - else: - prevpoolnode = poolnode - poolnode = next - self.malloced_objects = firstpoolnode.linkedlist - self.poolnodes = firstpoolnode.nextnode - lltype.free(firstpoolnode, flavor='raw') - - if curr_heap_size > self.bytes_malloced_threshold: - self.bytes_malloced_threshold = curr_heap_size - end_time = time.time() - self.total_collection_time += end_time - start_time - if DEBUG_PRINT: - llop.debug_print(lltype.Void, - " malloced since previous collection:", - old_malloced, "bytes") - llop.debug_print(lltype.Void, - " heap usage at start of collection: ", - self.heap_usage + old_malloced, "bytes") - llop.debug_print(lltype.Void, - " freed: ", - freed_size, "bytes") - llop.debug_print(lltype.Void, - " new heap usage: ", - curr_heap_size, "bytes") - llop.debug_print(lltype.Void, - " total time spent collecting: ", - self.total_collection_time, "seconds") -## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, -## size_gc_header) - assert self.heap_usage + old_malloced == curr_heap_size + freed_size - - # call finalizers if needed - self.heap_usage = curr_heap_size - hdr = self.malloced_objects_with_finalizer - self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR) - while hdr: - next = hdr.next - if hdr.typeid & 1: - hdr.next = self.malloced_objects_with_finalizer - self.malloced_objects_with_finalizer = hdr - hdr.typeid = hdr.typeid & (~1) - else: - obj = llmemory.cast_ptr_to_adr(hdr) + size_gc_header - finalizer = self.getfinalizer(hdr.typeid >> 1) - finalizer(obj) - hdr.next = self.malloced_objects - self.malloced_objects = hdr - hdr = next - class PrintingMarkSweepGC(MarkSweepGC): _alloc_flavor_ = "raw" Modified: pypy/branch/no-forw-ptr/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/no-forw-ptr/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/no-forw-ptr/pypy/rpython/memory/gctransform/framework.py Tue Feb 19 17:23:49 2008 @@ -23,7 +23,7 @@ class CollectAnalyzer(graphanalyze.GraphAnalyzer): def operation_is_true(self, op): - if op.opname in ('gc__collect', 'gc_x_become'): + if op.opname == 'gc__collect': return True if op.opname in ('malloc', 'malloc_varsize'): flags = op.args[1].value @@ -316,11 +316,6 @@ annmodel.s_None, minimal_transform = False) - self.x_become_ptr = getfn( - GCClass.x_become.im_func, - [s_gc, annmodel.SomeAddress(), annmodel.SomeAddress()], - annmodel.s_None) - annhelper.finish() # at this point, annotate all mix-level helpers annhelper.backend_optimize() @@ -549,15 +544,6 @@ [c_result], resultvar=op.result) - def gct_gc_x_become(self, hop): - op = hop.spaceop - [v_target, v_source] = op.args - livevars = self.push_roots(hop) - hop.genop("direct_call", - [self.x_become_ptr, self.c_const_gc, v_target, v_source], - resultvar=op.result) - self.pop_roots(hop, livevars) - def gct_zero_gc_pointers_inside(self, hop): if self.needs_zero_gc_pointers: v_ob = hop.spaceop.args[0] Modified: pypy/branch/no-forw-ptr/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/no-forw-ptr/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/no-forw-ptr/pypy/rpython/memory/test/test_transformed_gc.py Tue Feb 19 17:23:49 2008 @@ -729,26 +729,6 @@ def test_instances(self): py.test.skip("fails for a stupid reasons") - def test_x_become(self): - from pypy.rlib import objectmodel - S = lltype.GcStruct("S", ('x', lltype.Signed)) - def f(): - x = lltype.malloc(S) - x.x = 10 - y = lltype.malloc(S) - y.x = 20 - z = x - llop.gc_x_become(lltype.Void, - llmemory.cast_ptr_to_adr(x), - llmemory.cast_ptr_to_adr(y)) - # keep 'y' alive until the x_become() is finished, because in - # theory it could go away as soon as only its address is present - objectmodel.keepalive_until_here(y) - return z.x - run = self.runner(f) - res = run([]) - assert res == 20 - class TestPrintingGC(GenericGCTests): gcname = "statistics" Modified: pypy/branch/no-forw-ptr/pypy/translator/llvm/test/test_newgc.py ============================================================================== --- pypy/branch/no-forw-ptr/pypy/translator/llvm/test/test_newgc.py (original) +++ pypy/branch/no-forw-ptr/pypy/translator/llvm/test/test_newgc.py Tue Feb 19 17:23:49 2008 @@ -5,7 +5,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena def test_gc_offsets(): - STRUCT = lltype.GcStruct('S1', ('x', lltype.Signed)) + STRUCT = lltype.GcStruct('S1', ('x', lltype.Signed), ('y', lltype.Char)) ARRAY = lltype.GcArray(lltype.Signed) s1 = llarena.round_up_for_allocation(llmemory.sizeof(STRUCT)) s2 = llmemory.offsetof(STRUCT, 'x') @@ -25,10 +25,10 @@ i3 = (res // 10000) % 100 i4 = (res // 100) % 100 i5 = (res // 1) % 100 - assert i1 % 8 == 0 + assert i1 % 4 == 0 assert 12 <= i1 <= 24 assert 8 <= i2 <= i1 - 4 - assert 8 <= i3 <= 16 + assert 4 <= i3 <= 12 assert i4 == i5 assert i3 + 4 <= i5 From jared.grubb at codespeak.net Tue Feb 19 17:25:39 2008 From: jared.grubb at codespeak.net (jared.grubb at codespeak.net) Date: Tue, 19 Feb 2008 17:25:39 +0100 (CET) Subject: [pypy-svn] r51642 - in pypy/dist/pypy/rlib/parsing: . test Message-ID: <20080219162539.19F321683BC@codespeak.net> Author: jared.grubb Date: Tue Feb 19 17:25:38 2008 New Revision: 51642 Modified: pypy/dist/pypy/rlib/parsing/lexer.py pypy/dist/pypy/rlib/parsing/test/test_lexer.py Log: rlib.parser.lexer: add copy() methods, add a few comments, minor code changes; test_lexer: add tests for copy() methods Modified: pypy/dist/pypy/rlib/parsing/lexer.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/lexer.py (original) +++ pypy/dist/pypy/rlib/parsing/lexer.py Tue Feb 19 17:25:38 2008 @@ -8,6 +8,9 @@ self.source = source self.source_pos = source_pos + def copy(self): + return Token(self.name, self.source, self.source_pos) + def __eq__(self, other): # for testing only return self.__dict__ == other.__dict__ @@ -20,10 +23,14 @@ return "Token(%r, %r, %r)" % (self.name, self.source, self.source_pos) class SourcePos(object): + """An object to record position in source code.""" def __init__(self, i, lineno, columnno): - self.i = i - self.lineno = lineno - self.columnno = columnno + self.i = i # index in source string + self.lineno = lineno # line number in source + self.columnno = columnno # column in line + + def copy(self): + return SourcePos(self.i, self.lineno, self.columnno) def __eq__(self, other): # for testing only @@ -46,7 +53,6 @@ self.automaton.optimize() # XXX not sure whether this is a good idea if ignore is None: ignore = [] - self.ignore = [] for ign in ignore: assert ign in names self.ignore = dict.fromkeys(ignore) @@ -57,8 +63,8 @@ self.ignore, eof) def tokenize(self, text, eof=False): - r = LexingDFARunner(self.matcher, self.automaton, text, - self.ignore, eof) + """Return a list of Token's from text.""" + r = self.get_runner(text, eof) result = [] while 1: try: @@ -105,27 +111,26 @@ def find_next_token(self): while 1: self.state = 0 - i = self.last_matched_index + 1 - start = i + start = self.last_matched_index + 1 assert start >= 0 - if i == len(self.text): - if self.eof: - self.last_matched_index += 1 - return self.make_token(i, -1, "", eof=True) - else: - raise StopIteration - if i >= len(self.text) + 1: + + # Handle end of file situation + if start == len(self.text) and self.eof: + self.last_matched_index += 1 + return self.make_token(start, -1, "", eof=True) + elif start >= len(self.text): raise StopIteration - i = self.inner_loop(i) + + i = self.inner_loop(start) if i < 0: i = ~i - if start == self.last_matched_index + 1: + stop = self.last_matched_index + 1 + assert stop >= 0 + if start == stop: source_pos = SourcePos(i - 1, self.lineno, self.columnno) raise deterministic.LexerError(self.text, self.state, source_pos) - stop = self.last_matched_index + 1 - assert stop >= 0 - source = self.text[start: stop] + source = self.text[start:stop] result = self.make_token(start, self.last_matched_index, source) self.adjust_position(source) if self.ignore_token(self.last_matched_state): @@ -146,10 +151,10 @@ raise deterministic.LexerError(self.text, self.state, source_pos) def adjust_position(self, token): - lineno = self.lineno - columnno = self.columnno - self.lineno += token.count("\n") - if lineno == self.lineno: + """Update the line# and col# as a result of this token.""" + newlines = token.count("\n") + self.lineno += newlines + if newlines==0: self.columnno += len(token) else: self.columnno = token.rfind("\n") Modified: pypy/dist/pypy/rlib/parsing/test/test_lexer.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/test/test_lexer.py (original) +++ pypy/dist/pypy/rlib/parsing/test/test_lexer.py Tue Feb 19 17:25:38 2008 @@ -3,7 +3,6 @@ from pypy.rlib.parsing.regex import * from pypy.rlib.parsing import deterministic - class TestDirectLexer(object): def get_lexer(self, rexs, names, ignore=None): return Lexer(rexs, names, ignore) @@ -133,3 +132,27 @@ tok = runner.find_next_token() assert tok.name == "WHITE" py.test.raises(deterministic.LexerError, runner.find_next_token) + +class TestSourcePos(object): + def test_copy(self): + base = SourcePos(1, 2, 3) + attributes = {'i':4, 'lineno': 5, 'columnno': 6} + for attr, new_val in attributes.iteritems(): + copy = base.copy() + assert base==copy + setattr(copy, attr, new_val) # change one attribute + assert base!=copy + +class TestToken(object): + def test_copy(self): + base = Token('test', 'spource', SourcePos(1,2,3)) + attributes = {'name': 'xxx', 'source': 'yyy', 'source_pos': SourcePos(4,5,6)} + for attr, new_val in attributes.iteritems(): + copy = base.copy() + assert base==copy + setattr(copy, attr, new_val) # change one attribute + assert base!=copy + # copy() is not deep... verify this. + copy = base.copy() + copy.source_pos.i = 0 # changes base too + assert base==copy From arigo at codespeak.net Tue Feb 19 18:14:03 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 19 Feb 2008 18:14:03 +0100 (CET) Subject: [pypy-svn] r51643 - pypy/branch/gc_hash Message-ID: <20080219171403.CF4A21683F1@codespeak.net> Author: arigo Date: Tue Feb 19 18:14:02 2008 New Revision: 51643 Added: pypy/branch/gc_hash/ - copied from r51642, pypy/branch/no-forw-ptr/ Log: The no-forw-ptr branch seems to work; benchmarking is inconsistent so I'll merge it a bit later when I've checked some more things. In the meantime here is a branch of the branch in which to experiment with delegating the hash_cache support to the GC. From arigo at codespeak.net Tue Feb 19 18:14:55 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 19 Feb 2008 18:14:55 +0100 (CET) Subject: [pypy-svn] r51644 - in pypy/branch/gc_hash/pypy/rpython/lltypesystem: . test Message-ID: <20080219171455.048031683FE@codespeak.net> Author: arigo Date: Tue Feb 19 18:14:55 2008 New Revision: 51644 Modified: pypy/branch/gc_hash/pypy/rpython/lltypesystem/lltype.py pypy/branch/gc_hash/pypy/rpython/lltypesystem/test/test_lltype.py Log: hash_cache support in lltype for GcStructs, including the possibility to ask for prebuilt gcstructs to have a specific hash value. Modified: pypy/branch/gc_hash/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/gc_hash/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/gc_hash/pypy/rpython/lltypesystem/lltype.py Tue Feb 19 18:14:55 2008 @@ -310,6 +310,24 @@ class GcStruct(RttiStruct): _gckind = 'gc' + _hash_cache = False + + def _install_extras(self, hash_cache=False, **extras): + RttiStruct._install_extras(self, **extras) + if hash_cache: + if self._hash_cache_level() is not None: + raise TypeError("in %r, parent structure %r has already got " + "a hash_cache" % (self, + self._hash_cache_level())) + self._hash_cache = True + + def _hash_cache_level(self): + TYPE = self + while not TYPE._hash_cache: + _, TYPE = TYPE._first_struct() + if TYPE is None: + return None + return TYPE STRUCT_BY_FLAVOR = {'raw': Struct, 'gc': GcStruct} @@ -1358,8 +1376,10 @@ container = parent return container -def _struct_variety(flds, cache={}): - flds = list(flds) +def _struct_variety(TYPE, cache={}): + flds = list(TYPE._names) + if TYPE._hash_cache: + flds.append('_hash_cache_') flds.sort() tag = tuple(flds) try: @@ -1369,11 +1389,6 @@ __slots__ = flds cache[tag] = _struct1 return _struct1 - -#for pickling support: -def _get_empty_instance_of_struct_variety(flds): - cls = _struct_variety(flds) - return object.__new__(cls) class _struct(_parentable): _kind = "structure" @@ -1381,7 +1396,7 @@ __slots__ = () def __new__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): - my_variety = _struct_variety(TYPE._names) + my_variety = _struct_variety(TYPE) return object.__new__(my_variety) def __init__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): @@ -1825,6 +1840,37 @@ "should have been: %s" % (p, result2, result)) return result +def _ptr2hashcontainer(p): + T = typeOf(p) + if not isinstance(T, Ptr) or not isinstance(T.TO, GcStruct): + raise TypeError, "Ptr(GcStruct) expected, got %r" % (T,) + TYPE = T.TO._hash_cache_level() + if TYPE is None: + raise TypeError, "hash_gc_object: %r has no hash_cache" % (T.TO,) + if not p: + return None + container = cast_pointer(Ptr(TYPE), p)._obj + return container + +def hash_gc_object(p): + container = _ptr2hashcontainer(p) + if container is None: + return 0 # hash(NULL) + try: + return container._hash_cache_ + except AttributeError: + result = container._hash_cache_ = intmask(id(container)) + return result + +def init_hash_gc_object(p, value): + """For a prebuilt object p, initialize its hash value to 'value'.""" + container = _ptr2hashcontainer(p) + if container is None: + raise ValueError("cannot change hash(NULL)!") + if hasattr(container, '_hash_cache_'): + raise ValueError("the hash of %r was already computed" % (p,)) + container._hash_cache_ = intmask(value) + def isCompatibleType(TYPE1, TYPE2): return TYPE1._is_compatible(TYPE2) Modified: pypy/branch/gc_hash/pypy/rpython/lltypesystem/test/test_lltype.py ============================================================================== --- pypy/branch/gc_hash/pypy/rpython/lltypesystem/test/test_lltype.py (original) +++ pypy/branch/gc_hash/pypy/rpython/lltypesystem/test/test_lltype.py Tue Feb 19 18:14:55 2008 @@ -727,3 +727,22 @@ setattr(s, word, i) for i, word in enumerate(words): assert getattr(s, word) == i + +def test_hash_gc_object(): + S = GcStruct('S', ('x', Signed), hash_cache=False) + S2 = GcStruct('S2', ('super', S), hash_cache=True) + S3 = GcStruct('S3', ('super', S2)) + py.test.raises(TypeError, "GcStruct('S4', ('super', S3), hash_cache=True)") + + s3 = malloc(S3) + hash3 = hash_gc_object(s3) + assert hash3 == hash_gc_object(s3) + assert hash3 == hash_gc_object(s3.super) + py.test.raises(TypeError, hash_gc_object, s3.super.super) + py.test.raises(ValueError, init_hash_gc_object, s3, -123) + + s3 = malloc(S3) + init_hash_gc_object(s3, -123) + assert -123 == hash_gc_object(s3) + assert -123 == hash_gc_object(s3.super) + py.test.raises(TypeError, hash_gc_object, s3.super.super) From xoraxax at codespeak.net Tue Feb 19 18:17:59 2008 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Tue, 19 Feb 2008 18:17:59 +0100 (CET) Subject: [pypy-svn] r51645 - pypy/extradoc/sprintinfo/pycon2008 Message-ID: <20080219171759.8530D1683F4@codespeak.net> Author: xoraxax Date: Tue Feb 19 18:17:59 2008 New Revision: 51645 Modified: pypy/extradoc/sprintinfo/pycon2008/people.txt Log: I think its confusing without this line Modified: pypy/extradoc/sprintinfo/pycon2008/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/pycon2008/people.txt (original) +++ pypy/extradoc/sprintinfo/pycon2008/people.txt Tue Feb 19 18:17:59 2008 @@ -13,6 +13,9 @@ ==================== ============== ======================= ==================== ============== ======================= + +People on the following list were present at previous sprints: + ==================== ============== ===================== Name Arrive/Depart Accomodation ==================== ============== ===================== From arigo at codespeak.net Tue Feb 19 18:20:51 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 19 Feb 2008 18:20:51 +0100 (CET) Subject: [pypy-svn] r51646 - in pypy/branch/gc_hash/pypy: annotation rpython rpython/lltypesystem rpython/test Message-ID: <20080219172051.D21A916841A@codespeak.net> Author: arigo Date: Tue Feb 19 18:20:51 2008 New Revision: 51646 Modified: pypy/branch/gc_hash/pypy/annotation/builtin.py pypy/branch/gc_hash/pypy/rpython/llinterp.py pypy/branch/gc_hash/pypy/rpython/lltypesystem/llheap.py pypy/branch/gc_hash/pypy/rpython/lltypesystem/lloperation.py pypy/branch/gc_hash/pypy/rpython/lltypesystem/lltype.py pypy/branch/gc_hash/pypy/rpython/lltypesystem/rclass.py pypy/branch/gc_hash/pypy/rpython/rbuiltin.py pypy/branch/gc_hash/pypy/rpython/test/test_rclass.py pypy/branch/gc_hash/pypy/rpython/test/test_rptr.py Log: Add the gc_hash operation and use hash_cache in rclass.py. Modified: pypy/branch/gc_hash/pypy/annotation/builtin.py ============================================================================== --- pypy/branch/gc_hash/pypy/annotation/builtin.py (original) +++ pypy/branch/gc_hash/pypy/annotation/builtin.py Tue Feb 19 18:20:51 2008 @@ -487,6 +487,11 @@ assert isinstance(s_p, SomePtr), "runtime_type_info of non-pointer: %r" % s_p return SomePtr(lltype.typeOf(lltype.runtime_type_info(s_p.ll_ptrtype._example()))) +def hash_gc_object(s_p): + assert isinstance(s_p, SomePtr), "hash_gc_object of non-pointer: %r" % s_p + lltype.hash_gc_object(s_p.ll_ptrtype._defl()) # for the checks + return SomeInteger() + def constPtr(T): assert T.is_constant() return immutablevalue(lltype.Ptr(T.const)) @@ -505,6 +510,7 @@ BUILTIN_ANALYZERS[lltype.cast_int_to_ptr] = cast_int_to_ptr BUILTIN_ANALYZERS[lltype.getRuntimeTypeInfo] = getRuntimeTypeInfo BUILTIN_ANALYZERS[lltype.runtime_type_info] = runtime_type_info +BUILTIN_ANALYZERS[lltype.hash_gc_object] = hash_gc_object BUILTIN_ANALYZERS[lltype.Ptr] = constPtr # ootype Modified: pypy/branch/gc_hash/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/gc_hash/pypy/rpython/llinterp.py (original) +++ pypy/branch/gc_hash/pypy/rpython/llinterp.py Tue Feb 19 18:20:51 2008 @@ -791,6 +791,9 @@ def op_gc_id(self, v_ptr): return self.heap.gc_id(v_ptr) + def op_gc_hash(self, v_ptr): + return self.heap.gc_hash(v_ptr) + def op_gc_set_max_heap_size(self, maxsize): raise NotImplementedError("gc_set_max_heap_size") Modified: pypy/branch/gc_hash/pypy/rpython/lltypesystem/llheap.py ============================================================================== --- pypy/branch/gc_hash/pypy/rpython/lltypesystem/llheap.py (original) +++ pypy/branch/gc_hash/pypy/rpython/lltypesystem/llheap.py Tue Feb 19 18:20:51 2008 @@ -16,6 +16,7 @@ inneraddr.ref()[0] = newvalue from pypy.rpython.lltypesystem.lltype import cast_ptr_to_int as gc_id +from pypy.rpython.lltypesystem.lltype import hash_gc_object as gc_hash def weakref_create_getlazy(objgetter): return weakref_create(objgetter()) Modified: pypy/branch/gc_hash/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/gc_hash/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/gc_hash/pypy/rpython/lltypesystem/lloperation.py Tue Feb 19 18:20:51 2008 @@ -396,6 +396,7 @@ 'gc_pop_alive_pyobj': LLOp(), 'gc_reload_possibly_moved': LLOp(), 'gc_id': LLOp(canraise=(MemoryError,), sideeffects=False), + 'gc_hash': LLOp(sideeffects=False), 'gc_set_max_heap_size': LLOp(), # experimental operations in support of thread cloning, only # implemented by the Mark&Sweep GC Modified: pypy/branch/gc_hash/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/gc_hash/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/gc_hash/pypy/rpython/lltypesystem/lltype.py Tue Feb 19 18:20:51 2008 @@ -143,6 +143,7 @@ class ContainerType(LowLevelType): _adtmeths = {} + _hash_cache = False def _inline_is_varsize(self, last): raise TypeError, "%r cannot be inlined in structure" % self @@ -310,7 +311,6 @@ class GcStruct(RttiStruct): _gckind = 'gc' - _hash_cache = False def _install_extras(self, hash_cache=False, **extras): RttiStruct._install_extras(self, **extras) @@ -1859,7 +1859,7 @@ try: return container._hash_cache_ except AttributeError: - result = container._hash_cache_ = intmask(id(container)) + result = container._hash_cache_ = cast_ptr_to_int(p) return result def init_hash_gc_object(p, value): Modified: pypy/branch/gc_hash/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/gc_hash/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/gc_hash/pypy/rpython/lltypesystem/rclass.py Tue Feb 19 18:20:51 2008 @@ -330,10 +330,7 @@ llfields.append((mangled_name, r.lowleveltype)) # # hash() support - if self.rtyper.needs_hash_support(self.classdef): - from pypy.rpython import rint - fields['_hash_cache_'] = 'hash_cache', rint.signed_repr - llfields.append(('hash_cache', Signed)) + hash_cache = self.rtyper.needs_hash_support(self.classdef) self.rbase = getinstancerepr(self.rtyper, self.classdef.basedef, self.gcflavor) @@ -351,6 +348,7 @@ ('super', self.rbase.object_type), hints=hints, adtmeths=adtmeths, + hash_cache=hash_cache, *llfields) self.object_type.become(object_type) allinstancefields.update(self.rbase.allinstancefields) @@ -402,7 +400,7 @@ except AttributeError: INSPTR = self.lowleveltype def _ll_hash_function(ins): - return ll_inst_hash(cast_pointer(INSPTR, ins)) + return lltype.hash_gc_object(cast_pointer(INSPTR, ins)) self._ll_hash_function = _ll_hash_function return _ll_hash_function else: @@ -417,8 +415,6 @@ for name, (mangled_name, r) in self.fields.items(): if r.lowleveltype is Void: llattrvalue = None - elif name == '_hash_cache_': # hash() support - llattrvalue = hash(value) else: try: attrvalue = getattr(value, name) @@ -433,6 +429,10 @@ else: llattrvalue = r.convert_const(attrvalue) setattr(result, mangled_name, llattrvalue) + # hash support + if self.rtyper.needs_hash_support(self.classdef): + llhashvalue = hash(value) + lltype.init_hash_gc_object(result, llhashvalue) else: # OBJECT part rclass = getclassrepr(self.rtyper, classdef) @@ -498,10 +498,7 @@ mangled_name, r = self.allinstancefields[fldname] if r.lowleveltype is Void: continue - if fldname == '_hash_cache_': - value = Constant(0, Signed) - else: - value = self.classdef.classdesc.read_attribute(fldname, None) + value = self.classdef.classdesc.read_attribute(fldname, None) if value is not None: cvalue = inputconst(r.lowleveltype, r.convert_desc_or_const(value)) @@ -696,18 +693,6 @@ def ll_runtime_type_info(obj): return obj.typeptr.rtti -def ll_inst_hash(ins): - if not ins: - return 0 # for None - cached = ins.hash_cache - if cached == 0: - # XXX this should ideally be done in a GC-dependent way: we only - # need a hash_cache for moving GCs, and we only need the '~' to - # avoid Boehm keeping the object alive if the value is passed - # around - cached = ins.hash_cache = ~cast_ptr_to_int(ins) - return cached - def ll_inst_type(obj): if obj: return obj.typeptr Modified: pypy/branch/gc_hash/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/branch/gc_hash/pypy/rpython/rbuiltin.py (original) +++ pypy/branch/gc_hash/pypy/rpython/rbuiltin.py Tue Feb 19 18:20:51 2008 @@ -479,6 +479,12 @@ return hop.genop('runtime_type_info', vlist, resulttype = hop.r_result.lowleveltype) +def rtype_hash_gc_object(hop): + assert isinstance(hop.args_r[0], rptr.PtrRepr) + vlist = hop.inputargs(hop.args_r[0]) + return hop.genop('gc_hash', vlist, + resulttype = lltype.Signed) + BUILTIN_TYPER[lltype.malloc] = rtype_malloc BUILTIN_TYPER[lltype.free] = rtype_free BUILTIN_TYPER[lltype.cast_primitive] = rtype_cast_primitive @@ -494,6 +500,7 @@ BUILTIN_TYPER[lltype.getRuntimeTypeInfo] = rtype_const_result BUILTIN_TYPER[lltype.Ptr] = rtype_const_result BUILTIN_TYPER[lltype.runtime_type_info] = rtype_runtime_type_info +BUILTIN_TYPER[lltype.hash_gc_object] = rtype_hash_gc_object BUILTIN_TYPER[rarithmetic.intmask] = rtype_intmask BUILTIN_TYPER[objectmodel.we_are_translated] = rtype_we_are_translated Modified: pypy/branch/gc_hash/pypy/rpython/test/test_rclass.py ============================================================================== --- pypy/branch/gc_hash/pypy/rpython/test/test_rclass.py (original) +++ pypy/branch/gc_hash/pypy/rpython/test/test_rclass.py Tue Feb 19 18:20:51 2008 @@ -403,10 +403,8 @@ res = self.interpret(f, []) # xxx this is too precise, checking the exact implementation - if isinstance(self, OORtypeMixin): - assert res.item0 == res.item1 - else: - assert res.item0 == ~res.item1 + # but it works on top of lltype.py and ootype.py + assert res.item0 == res.item1 # the following property is essential on top of the lltypesystem # otherwise prebuilt dictionaries are broken. It's not that # relevant on top of the ootypesystem though. Modified: pypy/branch/gc_hash/pypy/rpython/test/test_rptr.py ============================================================================== --- pypy/branch/gc_hash/pypy/rpython/test/test_rptr.py (original) +++ pypy/branch/gc_hash/pypy/rpython/test/test_rptr.py Tue Feb 19 18:20:51 2008 @@ -313,3 +313,25 @@ res = interpret(f, []) assert res == 1 +def test_hash_gc_object(): + T = GcStruct('T', ('x', Signed)) + t1 = malloc(T) + t2 = malloc(T) + hash1 = hash_gc_object(t1) + def gethash(p): # indirection to prevent constant-folding by the flow space + return hash_gc_object(p) + def f(): + t3 = malloc(T) + hash1 = gethash(t1) + hash2 = gethash(t2) + hash3 = gethash(t3) + assert hash1 == gethash(t1) + assert hash2 == gethash(t2) + assert hash3 == gethash(t3) + assert hash1 == hash_gc_object(t1) + assert hash2 == hash_gc_object(t2) + assert hash3 == hash_gc_object(t3) + return hash1 + + res = interpret(f, []) + assert res == hash1 From arigo at codespeak.net Tue Feb 19 18:21:52 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 19 Feb 2008 18:21:52 +0100 (CET) Subject: [pypy-svn] r51647 - pypy/branch/gc_hash/pypy/rpython/test Message-ID: <20080219172152.E57E716841A@codespeak.net> Author: arigo Date: Tue Feb 19 18:21:52 2008 New Revision: 51647 Modified: pypy/branch/gc_hash/pypy/rpython/test/test_rptr.py Log: Fix test. Modified: pypy/branch/gc_hash/pypy/rpython/test/test_rptr.py ============================================================================== --- pypy/branch/gc_hash/pypy/rpython/test/test_rptr.py (original) +++ pypy/branch/gc_hash/pypy/rpython/test/test_rptr.py Tue Feb 19 18:21:52 2008 @@ -314,7 +314,7 @@ assert res == 1 def test_hash_gc_object(): - T = GcStruct('T', ('x', Signed)) + T = GcStruct('T', ('x', Signed), hash_cache=True) t1 = malloc(T) t2 = malloc(T) hash1 = hash_gc_object(t1) From pypy-svn at codespeak.net Tue Feb 19 19:44:54 2008 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Tue, 19 Feb 2008 19:44:54 +0100 (CET) Subject: [pypy-svn] February 70% OFF Message-ID: <20080219124425.8930.qmail@ppp85-141-242-71.pppoe.mtu-net.ru> An HTML attachment was scrubbed... URL: From cfbolz at codespeak.net Tue Feb 19 21:24:45 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 19 Feb 2008 21:24:45 +0100 (CET) Subject: [pypy-svn] r51653 - pypy/dist/pypy/rlib/parsing Message-ID: <20080219202445.54D24168401@codespeak.net> Author: cfbolz Date: Tue Feb 19 21:24:42 2008 New Revision: 51653 Modified: pypy/dist/pypy/rlib/parsing/ebnfparse.py pypy/dist/pypy/rlib/parsing/pypackrat.py pypy/dist/pypy/rlib/parsing/regexparse.py Log: regenerate all those files, to get the comments about auto-generated code Modified: pypy/dist/pypy/rlib/parsing/ebnfparse.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/ebnfparse.py (original) +++ pypy/dist/pypy/rlib/parsing/ebnfparse.py Tue Feb 19 21:24:42 2008 @@ -448,11 +448,13 @@ # generated code between this line and its other occurence class EBNFToAST(object): def visit_file(self, node): + #auto-generated code, don't edit children = [] children.extend(self.visit_list(node.children[0])) children.extend([node.children[1]]) return [Nonterminal(node.symbol, children)] def visit__plus_symbol0(self, node): + #auto-generated code, don't edit length = len(node.children) if length == 1: children = [] @@ -465,17 +467,20 @@ children.extend(expr[0].children) return [Nonterminal(node.symbol, children)] def visit_list(self, node): + #auto-generated code, don't edit children = [] expr = self.visit__plus_symbol0(node.children[0]) assert len(expr) == 1 children.extend(expr[0].children) return [Nonterminal(node.symbol, children)] def visit_element(self, node): + #auto-generated code, don't edit length = len(node.children) if node.children[0].symbol == 'production': return self.visit_production(node.children[0]) return self.visit_regex(node.children[0]) def visit_regex(self, node): + #auto-generated code, don't edit children = [] children.extend([node.children[0]]) children.extend([node.children[1]]) @@ -483,10 +488,12 @@ children.extend([node.children[3]]) return [Nonterminal(node.symbol, children)] def visit__maybe_symbol0(self, node): + #auto-generated code, don't edit children = [] children.extend(self.visit_body(node.children[0])) return [Nonterminal(node.symbol, children)] def visit_production(self, node): + #auto-generated code, don't edit length = len(node.children) if length == 3: children = [] @@ -503,6 +510,7 @@ children.extend([node.children[3]]) return [Nonterminal(node.symbol, children)] def visit__star_symbol1(self, node): + #auto-generated code, don't edit length = len(node.children) if length == 2: children = [] @@ -515,6 +523,7 @@ children.extend(expr[0].children) return [Nonterminal(node.symbol, children)] def visit_body(self, node): + #auto-generated code, don't edit length = len(node.children) if length == 1: children = [] @@ -527,6 +536,7 @@ children.extend(self.visit_expansion(node.children[1])) return [Nonterminal(node.symbol, children)] def visit__plus_symbol1(self, node): + #auto-generated code, don't edit length = len(node.children) if length == 1: children = [] @@ -539,12 +549,14 @@ children.extend(expr[0].children) return [Nonterminal(node.symbol, children)] def visit_expansion(self, node): + #auto-generated code, don't edit children = [] expr = self.visit__plus_symbol1(node.children[0]) assert len(expr) == 1 children.extend(expr[0].children) return [Nonterminal(node.symbol, children)] def visit_decorated(self, node): + #auto-generated code, don't edit length = len(node.children) if length == 1: return self.visit_enclosed(node.children[0]) @@ -563,6 +575,7 @@ children.extend([node.children[1]]) return [Nonterminal(node.symbol, children)] def visit_enclosed(self, node): + #auto-generated code, don't edit length = len(node.children) if length == 1: return self.visit_primary(node.children[0]) @@ -586,6 +599,7 @@ children.extend([node.children[2]]) return [Nonterminal(node.symbol, children)] def visit_primary(self, node): + #auto-generated code, don't edit length = len(node.children) if node.children[0].symbol == 'NONTERMINALNAME': children = [] @@ -599,6 +613,7 @@ children.extend([node.children[0]]) return [Nonterminal(node.symbol, children)] def transform(self, tree): + #auto-generated code, don't edit assert isinstance(tree, Nonterminal) assert tree.symbol == 'file' r = self.visit_file(tree) Modified: pypy/dist/pypy/rlib/parsing/pypackrat.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/pypackrat.py (original) +++ pypy/dist/pypy/rlib/parsing/pypackrat.py Tue Feb 19 21:24:42 2008 @@ -2640,7 +2640,10 @@ self._pos = _choice13 raise BacktrackException _upto = _runner.last_matched_index + 1 - _result = self._inputstream[self._pos: _upto] + _pos = self._pos + assert _pos >= 0 + assert _upto >= 0 + _result = self._inputstream[_pos: _upto] self._pos = _upto return _result def _regex1006631623(self): @@ -2651,7 +2654,10 @@ self._pos = _choice14 raise BacktrackException _upto = _runner.last_matched_index + 1 - _result = self._inputstream[self._pos: _upto] + _pos = self._pos + assert _pos >= 0 + assert _upto >= 0 + _result = self._inputstream[_pos: _upto] self._pos = _upto return _result def _regex528667127(self): @@ -2662,7 +2668,10 @@ self._pos = _choice15 raise BacktrackException _upto = _runner.last_matched_index + 1 - _result = self._inputstream[self._pos: _upto] + _pos = self._pos + assert _pos >= 0 + assert _upto >= 0 + _result = self._inputstream[_pos: _upto] self._pos = _upto return _result def _regex291086639(self): @@ -2673,7 +2682,10 @@ self._pos = _choice16 raise BacktrackException _upto = _runner.last_matched_index + 1 - _result = self._inputstream[self._pos: _upto] + _pos = self._pos + assert _pos >= 0 + assert _upto >= 0 + _result = self._inputstream[_pos: _upto] self._pos = _upto return _result def _regex1074651696(self): @@ -2684,7 +2696,10 @@ self._pos = _choice17 raise BacktrackException _upto = _runner.last_matched_index + 1 - _result = self._inputstream[self._pos: _upto] + _pos = self._pos + assert _pos >= 0 + assert _upto >= 0 + _result = self._inputstream[_pos: _upto] self._pos = _upto return _result def _regex1124192327(self): @@ -2695,7 +2710,10 @@ self._pos = _choice18 raise BacktrackException _upto = _runner.last_matched_index + 1 - _result = self._inputstream[self._pos: _upto] + _pos = self._pos + assert _pos >= 0 + assert _upto >= 0 + _result = self._inputstream[_pos: _upto] self._pos = _upto return _result def _regex1979538501(self): @@ -2706,7 +2724,10 @@ self._pos = _choice19 raise BacktrackException _upto = _runner.last_matched_index + 1 - _result = self._inputstream[self._pos: _upto] + _pos = self._pos + assert _pos >= 0 + assert _upto >= 0 + _result = self._inputstream[_pos: _upto] self._pos = _upto return _result class _Runner(object): @@ -2717,6 +2738,7 @@ self.last_matched_index = -1 self.state = -1 def recognize_299149370(runner, i): + #auto-generated code, don't edit assert i >= 0 input = runner.text state = 0 @@ -2724,43 +2746,44 @@ if state == 0: runner.last_matched_index = i - 1 runner.last_matched_state = state - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 0 return i - if char == ' ': + if char == '\n': state = 1 - elif char == '\n': + elif char == ' ': state = 2 else: break if state == 1: - if i < len(input): + runner.last_matched_index = i - 1 + runner.last_matched_state = state + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 1 - return ~i - if char == ' ': + return i + if char == '\n': + state = 1 + continue + elif char == ' ': state = 1 continue - elif char == '\n': - state = 2 else: break if state == 2: - runner.last_matched_index = i - 1 - runner.last_matched_state = state - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 2 - return i + return ~i if char == '\n': - state = 2 + state = 1 continue elif char == ' ': state = 2 @@ -2778,52 +2801,53 @@ runner.state = state return ~i def recognize_1006631623(runner, i): + #auto-generated code, don't edit assert i >= 0 input = runner.text state = 0 while 1: if state == 0: - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 0 return ~i if char == '`': - state = 1 + state = 3 else: break - if state == 1: - if i < len(input): + if state == 2: + try: char = input[i] i += 1 - else: - runner.state = 1 + except IndexError: + runner.state = 2 return ~i - if '\x00' <= char <= '[': - state = 1 - continue - elif ']' <= char <= '_': - state = 1 - continue - elif 'a' <= char <= '\xff': - state = 1 - continue - elif char == '\\': - state = 2 - elif char == '`': + if '\x00' <= char <= '\xff': state = 3 else: break - if state == 2: - if i < len(input): + if state == 3: + try: char = input[i] i += 1 - else: - runner.state = 2 + except IndexError: + runner.state = 3 return ~i - if '\x00' <= char <= '\xff': + if char == '`': state = 1 + elif char == '\\': + state = 2 + continue + elif ']' <= char <= '_': + state = 3 + continue + elif '\x00' <= char <= '[': + state = 3 + continue + elif 'a' <= char <= '\xff': + state = 3 continue else: break @@ -2838,56 +2862,57 @@ runner.state = state return ~i def recognize_528667127(runner, i): + #auto-generated code, don't edit assert i >= 0 input = runner.text state = 0 while 1: if state == 0: - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 0 return ~i if char == ' ': state = 0 continue elif char == '#': - state = 1 + state = 2 else: break if state == 1: - if i < len(input): + runner.last_matched_index = i - 1 + runner.last_matched_state = state + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 1 - return ~i - if '\x00' <= char <= '\t': - state = 1 - continue - elif '\x0b' <= char <= '\xff': - state = 1 + return i + if char == ' ': + state = 0 continue - elif char == '\n': + elif char == '#': state = 2 else: break if state == 2: - runner.last_matched_index = i - 1 - runner.last_matched_state = state - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 2 - return i - if char == ' ': - state = 0 - continue - elif char == '#': + return ~i + if char == '\n': state = 1 continue + elif '\x00' <= char <= '\t': + state = 2 + continue + elif '\x0b' <= char <= '\xff': + state = 2 + continue else: break runner.last_matched_state = state @@ -2901,39 +2926,40 @@ runner.state = state return ~i def recognize_291086639(runner, i): + #auto-generated code, don't edit assert i >= 0 input = runner.text state = 0 while 1: if state == 0: - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 0 return ~i if char == '{': - state = 1 + state = 2 else: break - if state == 1: - if i < len(input): + if state == 2: + try: char = input[i] i += 1 - else: - runner.state = 1 + except IndexError: + runner.state = 2 return ~i - if '\x00' <= char <= '\t': + if char == '}': state = 1 + elif '\x00' <= char <= '\t': + state = 2 continue elif '\x0b' <= char <= '|': - state = 1 + state = 2 continue elif '~' <= char <= '\xff': - state = 1 - continue - elif char == '}': state = 2 + continue else: break runner.last_matched_state = state @@ -2947,20 +2973,21 @@ runner.state = state return ~i def recognize_1074651696(runner, i): + #auto-generated code, don't edit assert i >= 0 input = runner.text state = 0 while 1: if state == 0: - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 0 return ~i - if 'A' <= char <= 'Z': + if char == '_': state = 1 - elif char == '_': + elif 'A' <= char <= 'Z': state = 1 elif 'a' <= char <= 'z': state = 1 @@ -2969,19 +2996,19 @@ if state == 1: runner.last_matched_index = i - 1 runner.last_matched_state = state - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 1 return i - if '0' <= char <= '9': + if char == '_': state = 1 continue - elif 'A' <= char <= 'Z': + elif '0' <= char <= '9': state = 1 continue - elif char == '_': + elif 'A' <= char <= 'Z': state = 1 continue elif 'a' <= char <= 'z': @@ -3000,15 +3027,16 @@ runner.state = state return ~i def recognize_1124192327(runner, i): + #auto-generated code, don't edit assert i >= 0 input = runner.text state = 0 while 1: if state == 0: - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 0 return ~i if char == "'": @@ -3016,10 +3044,10 @@ else: break if state == 1: - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 1 return ~i if '\x00' <= char <= '&': @@ -3043,32 +3071,33 @@ runner.state = state return ~i def recognize_1979538501(runner, i): + #auto-generated code, don't edit assert i >= 0 input = runner.text state = 0 while 1: if state == 0: - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 0 return ~i if char == '#': state = 1 + elif char == ' ': + state = 2 elif char == '\t': state = 2 elif char == '\n': state = 2 - elif char == ' ': - state = 2 else: break if state == 1: - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 1 return ~i if '\x00' <= char <= '\t': Modified: pypy/dist/pypy/rlib/parsing/regexparse.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/regexparse.py (original) +++ pypy/dist/pypy/rlib/parsing/regexparse.py Tue Feb 19 21:24:42 2008 @@ -1277,7 +1277,10 @@ self._pos = _choice0 raise BacktrackException _upto = _runner.last_matched_index + 1 - _result = self._inputstream[self._pos: _upto] + _pos = self._pos + assert _pos >= 0 + assert _upto >= 0 + _result = self._inputstream[_pos: _upto] self._pos = _upto return _result def _regex1323868075(self): @@ -1288,7 +1291,10 @@ self._pos = _choice1 raise BacktrackException _upto = _runner.last_matched_index + 1 - _result = self._inputstream[self._pos: _upto] + _pos = self._pos + assert _pos >= 0 + assert _upto >= 0 + _result = self._inputstream[_pos: _upto] self._pos = _upto return _result def _regex1380912319(self): @@ -1299,7 +1305,10 @@ self._pos = _choice2 raise BacktrackException _upto = _runner.last_matched_index + 1 - _result = self._inputstream[self._pos: _upto] + _pos = self._pos + assert _pos >= 0 + assert _upto >= 0 + _result = self._inputstream[_pos: _upto] self._pos = _upto return _result class _Runner(object): @@ -1310,34 +1319,35 @@ self.last_matched_index = -1 self.state = -1 def recognize_1166214427(runner, i): + #auto-generated code, don't edit assert i >= 0 input = runner.text state = 0 while 1: if state == 0: - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 0 return ~i - if '1' <= char <= '9': + if char == '0': state = 1 - elif char == '0': + elif '1' <= char <= '9': state = 2 else: break - if state == 1: + if state == 2: runner.last_matched_index = i - 1 runner.last_matched_state = state - if i < len(input): + try: char = input[i] i += 1 - else: - runner.state = 1 + except IndexError: + runner.state = 2 return i if '0' <= char <= '9': - state = 1 + state = 2 continue else: break @@ -1352,27 +1362,28 @@ runner.state = state return ~i def recognize_1323868075(runner, i): + #auto-generated code, don't edit assert i >= 0 input = runner.text state = 0 while 1: if state == 0: - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 0 return ~i - if '\x00' <= char <= "'": + if char == '\\': state = 1 elif '/' <= char <= '>': state = 1 elif '@' <= char <= 'Z': state = 1 - elif char == '\\': - state = 1 elif '_' <= char <= 'z': state = 1 + elif '\x00' <= char <= "'": + state = 1 elif '~' <= char <= '\xff': state = 1 else: @@ -1388,65 +1399,70 @@ runner.state = state return ~i def recognize_1380912319(runner, i): + #auto-generated code, don't edit assert i >= 0 input = runner.text state = 0 while 1: if state == 0: - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 0 return ~i if char == '\\': - state = 1 + state = 4 else: break if state == 1: - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 1 return ~i - if char == 'x': - state = 2 - elif '\x00' <= char <= 'w': + if 'A' <= char <= 'F': state = 3 - elif 'y' <= char <= '\xff': + elif 'a' <= char <= 'f': + state = 3 + elif '0' <= char <= '9': state = 3 else: break if state == 2: runner.last_matched_index = i - 1 runner.last_matched_state = state - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 2 return i - if '0' <= char <= '9': - state = 4 - elif 'A' <= char <= 'F': - state = 4 + if 'A' <= char <= 'F': + state = 1 + continue elif 'a' <= char <= 'f': - state = 4 + state = 1 + continue + elif '0' <= char <= '9': + state = 1 + continue else: break if state == 4: - if i < len(input): + try: char = input[i] i += 1 - else: + except IndexError: runner.state = 4 return ~i - if '0' <= char <= '9': - state = 3 - elif 'A' <= char <= 'F': + if char == 'x': + state = 2 + continue + elif '\x00' <= char <= 'w': state = 3 - elif 'a' <= char <= 'f': + elif 'y' <= char <= '\xff': state = 3 else: break @@ -1490,6 +1506,7 @@ + def test_generate(): f = py.magic.autopath() oldcontent = f.read() From cfbolz at codespeak.net Tue Feb 19 23:09:17 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 19 Feb 2008 23:09:17 +0100 (CET) Subject: [pypy-svn] r51654 - pypy/dist/pypy/rlib/parsing Message-ID: <20080219220917.43ED81683FB@codespeak.net> Author: cfbolz Date: Tue Feb 19 23:09:15 2008 New Revision: 51654 Modified: pypy/dist/pypy/rlib/parsing/lexer.py Log: fix two typos and add an assert that would have caught the typos Modified: pypy/dist/pypy/rlib/parsing/lexer.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/lexer.py (original) +++ pypy/dist/pypy/rlib/parsing/lexer.py Tue Feb 19 23:09:15 2008 @@ -131,14 +131,14 @@ raise deterministic.LexerError(self.text, self.state, source_pos) source = self.text[start:stop] - result = self.make_token(start, self.last_matched_index, source) + result = self.make_token(start, self.last_matched_state, source) self.adjust_position(source) if self.ignore_token(self.last_matched_state): continue return result if self.last_matched_index == i - 1: source = self.text[start: ] - result = self.make_token(start, self.last_matched_index, source) + result = self.make_token(start, self.last_matched_state, source) self.adjust_position(source) if self.ignore_token(self.last_matched_state): if self.eof: @@ -192,6 +192,7 @@ return self.automaton.names[state] in self.ignore def make_token(self, index, state, text, eof=False): + assert (eof and state == -1) or 0 <= state < len(self.automaton.names) source_pos = SourcePos(index, self.lineno, self.columnno) if eof: return Token("EOF", "EOF", source_pos) From arigo at codespeak.net Wed Feb 20 10:21:18 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 20 Feb 2008 10:21:18 +0100 (CET) Subject: [pypy-svn] r51673 - in pypy/branch/gc_hash/pypy: rpython/memory/test translator/c/test Message-ID: <20080220092118.CE869168440@codespeak.net> Author: arigo Date: Wed Feb 20 10:21:18 2008 New Revision: 51673 Modified: pypy/branch/gc_hash/pypy/rpython/memory/test/snippet.py pypy/branch/gc_hash/pypy/rpython/memory/test/test_gc.py pypy/branch/gc_hash/pypy/translator/c/test/test_boehm.py pypy/branch/gc_hash/pypy/translator/c/test/test_newgc.py pypy/branch/gc_hash/pypy/translator/c/test/test_typed.py Log: Intermediate check-in: add hash test. Modified: pypy/branch/gc_hash/pypy/rpython/memory/test/snippet.py ============================================================================== --- pypy/branch/gc_hash/pypy/rpython/memory/test/snippet.py (original) +++ pypy/branch/gc_hash/pypy/rpython/memory/test/snippet.py Wed Feb 20 10:21:18 2008 @@ -1,7 +1,44 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem.lloperation import llop -class SemiSpaceGCTests: +class AnyGCTests(object): + + def test_hash(self): + class A(object): + pass + class B(A): + pass + class C(B): + pass + c1 = C() + hashc1 = hash(c1) + def gethash(x): # indirection to prevent constant-folding + return hash(x) + def f(): + error = 0 + b1 = B() + c2 = C() + hashb1 = hash(b1) + hashc2 = hash(c2) + if hashb1 != hash(b1): error += 1 + if hashc1 != hash(c1): error += 2 + if hashc2 != hash(c2): error += 4 + if hashb1 != gethash(b1): error += 10 + if hashc1 != gethash(c1): error += 20 + if hashc2 != gethash(c2): error += 40 + llop.gc__collect(lltype.Void) + if hashb1 != hash(b1): error += 100 + if hashc1 != hash(c1): error += 200 + if hashc2 != hash(c2): error += 400 + if hashb1 != gethash(b1): error += 1000 + if hashc1 != gethash(c1): error += 2000 + if hashc2 != gethash(c2): error += 4000 + return error + res = self.run(f) + assert res == 0 + + +class SemiSpaceGCTests(AnyGCTests): large_tests_ok = False def test_finalizer_order(self): Modified: pypy/branch/gc_hash/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/gc_hash/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/gc_hash/pypy/rpython/memory/test/test_gc.py Wed Feb 20 10:21:18 2008 @@ -19,7 +19,7 @@ print >>sys.stdout, strmsg -class GCTest(object): +class GCTest(snippet.AnyGCTests): GC_PARAMS = {} def setup_class(cls): Modified: pypy/branch/gc_hash/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/branch/gc_hash/pypy/translator/c/test/test_boehm.py (original) +++ pypy/branch/gc_hash/pypy/translator/c/test/test_boehm.py Wed Feb 20 10:21:18 2008 @@ -1,6 +1,7 @@ import py from pypy.translator.translator import TranslationContext from pypy.rpython.lltypesystem import lltype +from pypy.rpython.memory.test import snippet from pypy.translator.tool.cbuild import check_boehm_presence from pypy.translator.c.genc import CExtModuleBuilder from pypy import conftest @@ -9,7 +10,7 @@ if not check_boehm_presence(): py.test.skip("Boehm GC not present") -class AbstractGCTestClass(object): +class AbstractGCTestClass(snippet.AnyGCTests): gcpolicy = "boehm" stacklessgc = False @@ -46,6 +47,12 @@ return cbuilder.get_entry_point(isolated=True) return compile() + # interface for snippet.py + large_tests_ok = True + def run(self, func): + fn = self.getcompiled(func) + return fn() + class TestUsingBoehm(AbstractGCTestClass): gcpolicy = "boehm" Modified: pypy/branch/gc_hash/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/gc_hash/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/gc_hash/pypy/translator/c/test/test_newgc.py Wed Feb 20 10:21:18 2008 @@ -282,12 +282,6 @@ gcpolicy = "marksweep" should_be_moving = False - # interface for snippet.py - large_tests_ok = True - def run(self, func): - fn = self.getcompiled(func) - return fn() - def test_empty_collect(self): def f(): llop.gc__collect(lltype.Void) Modified: pypy/branch/gc_hash/pypy/translator/c/test/test_typed.py ============================================================================== --- pypy/branch/gc_hash/pypy/translator/c/test/test_typed.py (original) +++ pypy/branch/gc_hash/pypy/translator/c/test/test_typed.py Wed Feb 20 10:21:18 2008 @@ -589,7 +589,7 @@ res = f() # xxx this is too precise, checking the exact implementation - assert res[0] == ~res[1] + assert res[0] == res[1] assert res[2] == hash(c) assert res[3] == hash(d) From arigo at codespeak.net Wed Feb 20 10:42:05 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 20 Feb 2008 10:42:05 +0100 (CET) Subject: [pypy-svn] r51675 - in pypy/dist/pypy/rpython: lltypesystem lltypesystem/test memory memory/gc Message-ID: <20080220094205.43C3B168447@codespeak.net> Author: arigo Date: Wed Feb 20 10:42:04 2008 New Revision: 51675 Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py pypy/dist/pypy/rpython/lltypesystem/llmemory.py pypy/dist/pypy/rpython/lltypesystem/lltype.py pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py pypy/dist/pypy/rpython/memory/gc/generation.py pypy/dist/pypy/rpython/memory/gc/semispace.py pypy/dist/pypy/rpython/memory/gcheader.py pypy/dist/pypy/rpython/memory/support.py Log: Merge the n-forw-ptr branch, which removes the 'forw' field from the HDR of all the objects allocated by the SemiSpaceGC. in llarena, llmemory, lltype: This seems to be the amount of hacking needed to allow an arena to replace an object with another, and still have the old address work in order to go to the new object's header. This works only to some extend -- many operations on the old address still raise RuntimeError as before, in the hope that it will not allow nasty bugs to go unnoticed. in semispace, generation: Remove 'forw', replace it with a flag. Move some initialization code to translation-time instead of run-time. Can't prebuild the address lists in generation.py, because we call delete() on them later on :-) in support: Translation fix for prebuilt AddressStack and AddressDeque. Modified: pypy/dist/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llarena.py Wed Feb 20 10:42:04 2008 @@ -1,4 +1,4 @@ -import array +import array, weakref from pypy.rpython.lltypesystem import lltype, llmemory # An "arena" is a large area of memory which can hold a number of @@ -15,6 +15,7 @@ class Arena(object): object_arena_location = {} # {container: (arena, offset)} + old_object_arena_location = weakref.WeakKeyDictionary() def __init__(self, nbytes, zero): self.nbytes = nbytes @@ -29,7 +30,7 @@ if size is None: stop = self.nbytes else: - stop = start + size + stop = start + llmemory.raw_malloc_usage(size) assert 0 <= start <= stop <= self.nbytes for offset, ptr in self.objectptrs.items(): size = self.objectsizes[offset] @@ -79,9 +80,7 @@ addr2 = size._raw_malloc([], zero=zero) pattern = 'X' + 'x'*(bytes-1) self.usagemap[offset:offset+bytes] = array.array('c', pattern) - self.objectptrs[offset] = addr2.ptr - self.objectsizes[offset] = bytes - Arena.object_arena_location[addr2.ptr._obj] = self, offset + self.setobject(addr2, offset, bytes) # common case: 'size' starts with a GCHeaderOffset. In this case # we can also remember that the real object starts after the header. while isinstance(size, RoundedUpForAllocation): @@ -91,12 +90,17 @@ objaddr = addr2 + size.offsets[0] hdrbytes = llmemory.raw_malloc_usage(size.offsets[0]) objoffset = offset + hdrbytes - assert objoffset not in self.objectptrs - self.objectptrs[objoffset] = objaddr.ptr - self.objectsizes[objoffset] = bytes - hdrbytes - Arena.object_arena_location[objaddr.ptr._obj] = self, objoffset + self.setobject(objaddr, objoffset, bytes - hdrbytes) return addr2 + def setobject(self, objaddr, offset, bytes): + assert offset not in self.objectptrs + self.objectptrs[offset] = objaddr.ptr + self.objectsizes[offset] = bytes + container = objaddr.ptr._obj + Arena.object_arena_location[container] = self, offset + Arena.old_object_arena_location[container] = self, offset + class fakearenaaddress(llmemory.fakeaddress): def __init__(self, arena, offset): @@ -148,6 +152,7 @@ return True def compare_with_fakeaddr(self, other): + other = other._fixup() if not other: return None, None obj = other.ptr._obj @@ -205,6 +210,30 @@ return self.arena._getid() + self.offset +def _getfakearenaaddress(addr): + """Logic to handle test_replace_object_with_stub().""" + if isinstance(addr, fakearenaaddress): + return addr + else: + assert isinstance(addr, llmemory.fakeaddress) + assert addr, "NULL address" + # it must be possible to use the address of an already-freed + # arena object + obj = addr.ptr._getobj(check=False) + return _oldobj_to_address(obj) + +def _oldobj_to_address(obj): + obj = obj._normalizedcontainer(check=False) + try: + arena, offset = Arena.old_object_arena_location[obj] + except KeyError: + if obj._was_freed(): + msg = "taking address of %r, but it was freed" + else: + msg = "taking address of %r, but it is not in an arena" + raise RuntimeError(msg % (obj,)) + return arena.getaddr(offset) + class RoundedUpForAllocation(llmemory.AddressOffset): """A size that is rounded up in order to preserve alignment of objects following it. For arenas containing heterogenous objects. @@ -247,7 +276,7 @@ """Free all objects in the arena, which can then be reused. The arena is filled with zeroes if 'zero' is True. This can also be used on a subrange of the arena.""" - assert isinstance(arena_addr, fakearenaaddress) + arena_addr = _getfakearenaaddress(arena_addr) arena_addr.arena.reset(zero, arena_addr.offset, size) def arena_reserve(addr, size, check_alignment=True): @@ -256,7 +285,7 @@ overlap. The size must be symbolic; in non-translated version this is used to know what type of lltype object to allocate.""" from pypy.rpython.memory.lltypelayout import memory_alignment - assert isinstance(addr, fakearenaaddress) + addr = _getfakearenaaddress(addr) if check_alignment and (addr.offset & (memory_alignment-1)) != 0: raise ArenaError("object at offset %d would not be correctly aligned" % (addr.offset,)) Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py Wed Feb 20 10:42:04 2008 @@ -339,7 +339,9 @@ # NOTE: the 'ptr' in the addresses must be normalized. # Use cast_ptr_to_adr() instead of directly fakeaddress() if unsure. def __init__(self, ptr): - self.ptr = ptr or None # null ptr => None + if ptr is not None and ptr._obj0 is None: + ptr = None # null ptr => None + self.ptr = ptr def __repr__(self): if self.ptr is None: @@ -374,8 +376,8 @@ def __eq__(self, other): if isinstance(other, fakeaddress): - obj1 = self.ptr - obj2 = other.ptr + obj1 = self._fixup().ptr + obj2 = other._fixup().ptr if obj1 is not None: obj1 = obj1._obj if obj2 is not None: obj2 = obj2._obj return obj1 == obj2 @@ -412,8 +414,9 @@ return self.ptr def _cast_to_ptr(self, EXPECTED_TYPE): - if self: - return cast_any_ptr(EXPECTED_TYPE, self.ptr) + addr = self._fixup() + if addr: + return cast_any_ptr(EXPECTED_TYPE, addr.ptr) else: return lltype.nullptr(EXPECTED_TYPE.TO) @@ -423,6 +426,14 @@ else: return 0 + def _fixup(self): + if self.ptr is not None and self.ptr._was_freed(): + # hack to support llarena.test_replace_object_with_stub() + from pypy.rpython.lltypesystem import llarena + return llarena._getfakearenaaddress(self) + else: + return self + # ____________________________________________________________ class NullAddressError(Exception): Modified: pypy/dist/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lltype.py Wed Feb 20 10:42:04 2008 @@ -968,7 +968,7 @@ self._set_solid(solid) self._set_obj0(obj0) - def _getobj(self): + def _getobj(self, check=True): obj = self._obj0 if obj is not None: if self._weak: @@ -977,12 +977,17 @@ raise RuntimeError("accessing already garbage collected %r" % (self._T,)) if isinstance(obj, _container): - obj._check() + if check: + obj._check() elif isinstance(obj, str) and obj.startswith("delayed!"): raise DelayedPointer return obj _obj = property(_getobj) + def _was_freed(self): + return (self._obj0 is not None and + self._getobj(check=False)._was_freed()) + def __getattr__(self, field_name): # ! can only return basic or ptr ! if isinstance(self._T, Struct): if field_name in self._T._flds: @@ -1178,6 +1183,10 @@ from pypy.rpython.lltypesystem import llmemory if isinstance(self._T, FuncType): return llmemory.fakeaddress(self) + elif self._was_freed(): + # hack to support llarena.test_replace_object_with_stub() + from pypy.rpython.lltypesystem import llarena + return llarena._oldobj_to_address(self._getobj(check=False)) elif isinstance(self._obj, _subarray): return llmemory.fakeaddress(self) ## # return an address built as an offset in the whole array @@ -1192,8 +1201,8 @@ def _as_ptr(self): return self - def _as_obj(self): - return self._obj + def _as_obj(self, check=True): + return self._getobj(check=check) def _expose(self, offset, val): """XXX A nice docstring here""" @@ -1258,13 +1267,13 @@ class _container(object): __slots__ = () - def _parentstructure(self): + def _parentstructure(self, check=True): return None def _check(self): pass def _as_ptr(self): return _ptr(Ptr(self._TYPE), self, True) - def _as_obj(self): + def _as_obj(self, check=True): return self def _normalizedcontainer(self): return self @@ -1295,7 +1304,16 @@ self._storage = None def _was_freed(self): - return self._storage is None + if self._storage is None: + return True + if self._wrparent is None: + return False + parent = self._wrparent() + if parent is None: + raise RuntimeError("accessing sub%s %r,\n" + "but already garbage collected parent %r" + % (self._kind, self, self._parent_type)) + return parent._was_freed() def _setparentstructure(self, parent, parentindex): self._wrparent = weakref.ref(parent) @@ -1307,14 +1325,15 @@ # keep strong reference to parent, we share the same allocation self._keepparent = parent - def _parentstructure(self): + def _parentstructure(self, check=True): if self._wrparent is not None: parent = self._wrparent() if parent is None: raise RuntimeError("accessing sub%s %r,\n" "but already garbage collected parent %r" % (self._kind, self, self._parent_type)) - parent._check() + if check: + parent._check() return parent return None @@ -1323,14 +1342,15 @@ raise RuntimeError("accessing freed %r" % self._TYPE) self._parentstructure() - def _normalizedcontainer(self): + def _normalizedcontainer(self, check=True): # if we are the first inlined substructure of a structure, # return the whole (larger) structure instead container = self while True: - parent, index = parentlink(container) + parent = container._parentstructure(check=check) if parent is None: break + index = container._parent_index T = typeOf(parent) if (not isinstance(T, Struct) or T._first_struct()[0] != index or isinstance(T, FixedSizeArray)): @@ -1525,7 +1545,7 @@ def __repr__(self): return '<_subarray at %r in %r>' % (self._parent_index, - self._parentstructure()) + self._parentstructure(check=False)) def getlength(self): assert isinstance(self._TYPE, FixedSizeArray) Modified: pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/test/test_llarena.py Wed Feb 20 10:42:04 2008 @@ -165,6 +165,8 @@ plist.append(llmemory.cast_adr_to_ptr(b, SPTR)) # clear blist[1] and blist[2] but not blist[0] nor blist[3] arena_reset(blist[1], llmemory.raw_malloc_usage(precomputed_size)*2, False) + py.test.raises(RuntimeError, "plist[1].x") # marked as freed + py.test.raises(RuntimeError, "plist[2].x") # marked as freed # re-reserve object at index 1 and 2 blist[1] = reserve(1) blist[2] = reserve(2) @@ -173,6 +175,10 @@ assert plist[3].x == 103 py.test.raises(RuntimeError, "plist[1].x") # marked as freed py.test.raises(RuntimeError, "plist[2].x") # marked as freed + # but we can still cast the old ptrs to addresses, which compare equal + # to the new ones we gotq + assert llmemory.cast_ptr_to_adr(plist[1]) == blist[1] + assert llmemory.cast_ptr_to_adr(plist[2]) == blist[2] # check via addresses assert (blist[0] + llmemory.offsetof(SX, 'x')).signed[0] == 100 assert (blist[3] + llmemory.offsetof(SX, 'x')).signed[0] == 103 @@ -204,6 +210,45 @@ assert llmemory.cast_adr_to_int(a) == llmemory.cast_adr_to_int(a1) assert llmemory.cast_adr_to_int(a+1) == llmemory.cast_adr_to_int(a1) + 1 +def test_replace_object_with_stub(): + from pypy.rpython.memory.gcheader import GCHeaderBuilder + HDR = lltype.Struct('HDR', ('x', lltype.Signed)) + S = lltype.GcStruct('S', ('y', lltype.Signed), ('z', lltype.Signed)) + STUB = lltype.GcStruct('STUB', ('t', lltype.Char)) + gcheaderbuilder = GCHeaderBuilder(HDR) + size_gc_header = gcheaderbuilder.size_gc_header + + a = arena_malloc(50, True) + hdraddr = a + 12 + arena_reserve(hdraddr, size_gc_header + llmemory.sizeof(S)) + hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR)) + hdr.x = 42 + obj = llmemory.cast_adr_to_ptr(hdraddr + size_gc_header, lltype.Ptr(S)) + obj.y = -5 + obj.z = -6 + + hdraddr = llmemory.cast_ptr_to_adr(obj) - size_gc_header + arena_reset(hdraddr, size_gc_header + llmemory.sizeof(S), False) + arena_reserve(hdraddr, size_gc_header + llmemory.sizeof(STUB)) + + # check that it possible to reach the newly reserved HDR+STUB + # via the header of the old 'obj' pointer, both via the existing + # 'hdraddr': + hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR)) + hdr.x = 46 + stub = llmemory.cast_adr_to_ptr(hdraddr + size_gc_header, lltype.Ptr(STUB)) + stub.t = '!' + + # and via a (now-invalid) pointer to the old 'obj': (this is needed + # because during a garbage collection there are still pointers to + # the old 'obj' around to be fixed) + hdraddr = llmemory.cast_ptr_to_adr(obj) - size_gc_header + hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR)) + assert hdr.x == 46 + stub = llmemory.cast_adr_to_ptr(hdraddr + size_gc_header, + lltype.Ptr(STUB)) + assert stub.t == '!' + def test_llinterpreted(): from pypy.rpython.test.test_llinterp import interpret Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Wed Feb 20 10:42:04 2008 @@ -22,8 +22,8 @@ class GenerationGC(SemiSpaceGC): """A basic generational GC: it's a SemiSpaceGC with an additional nursery for young objects. A write barrier is used to ensure that - old objects that contain pointers to young objects are in a linked - list, chained to each other via their 'forw' header field. + old objects that contain pointers to young objects are recorded in + a list. """ inline_simple_malloc = True inline_simple_malloc_varsize = True @@ -44,19 +44,16 @@ self.initial_nursery_size = nursery_size self.auto_nursery_size = auto_nursery_size self.min_nursery_size = min_nursery_size - - def setup(self): - SemiSpaceGC.setup(self) - self.reset_nursery() - self.old_objects_pointing_to_young = NULL - # ^^^ the head of a linked list inside the old objects space; it + self.old_objects_pointing_to_young = self.AddressStack() + # ^^^ a list of addresses inside the old objects space; it # may contain static prebuilt objects as well. More precisely, # it lists exactly the old and static objects whose - # GCFLAG_NO_YOUNG_PTRS bit is not set. The 'forw' header field - # of such objects is abused for this linked list; it needs to be - # reset to its correct value when GCFLAG_NO_YOUNG_PTRS is set - # again at the start of a collection. + # GCFLAG_NO_YOUNG_PTRS bit is not set. self.young_objects_with_weakrefs = self.AddressStack() + self.reset_nursery() + + def setup(self): + SemiSpaceGC.setup(self) self.set_nursery_size(self.initial_nursery_size) # the GC is fully setup now. The rest can make use of it. if self.auto_nursery_size: @@ -226,14 +223,11 @@ # the next usage. def reset_young_gcflags(self): - obj = self.old_objects_pointing_to_young - while obj: + oldlist = self.old_objects_pointing_to_young + while oldlist.non_empty(): + obj = oldlist.pop() hdr = self.header(obj) hdr.tid |= GCFLAG_NO_YOUNG_PTRS - nextobj = hdr.forw - self.init_forwarding(obj) - obj = nextobj - self.old_objects_pointing_to_young = NULL def weakrefs_grow_older(self): while self.young_objects_with_weakrefs.non_empty(): @@ -278,19 +272,15 @@ def collect_oldrefs_to_nursery(self): # Follow the old_objects_pointing_to_young list and move the - # young objects they point to out of the nursery. The 'forw' - # fields are reset to their correct value along the way. + # young objects they point to out of the nursery. count = 0 - obj = self.old_objects_pointing_to_young - while obj: + oldlist = self.old_objects_pointing_to_young + while oldlist.non_empty(): count += 1 - nextobj = self.header(obj).forw - self.init_forwarding(obj) + obj = oldlist.pop() self.trace_and_drag_out_of_nursery(obj) - obj = nextobj if DEBUG_PRINT: llop.debug_print(lltype.Void, "collect_oldrefs_to_nursery", count) - self.old_objects_pointing_to_young = NULL def collect_roots_in_nursery(self): # we don't need to trace prebuilt GcStructs during a minor collect: @@ -362,8 +352,7 @@ "nursery object with GCFLAG_NO_YOUNG_PTRS") oldhdr = self.header(addr_struct) if self.is_in_nursery(addr): - oldhdr.forw = self.old_objects_pointing_to_young - self.old_objects_pointing_to_young = addr_struct + self.old_objects_pointing_to_young.append(addr_struct) oldhdr.tid &= ~GCFLAG_NO_YOUNG_PTRS if oldhdr.tid & GCFLAG_NO_HEAP_PTRS: self.move_to_static_roots(addr_struct) Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Wed Feb 20 10:42:04 2008 @@ -15,8 +15,9 @@ TYPEID_MASK = 0xffff first_gcflag = 1 << 16 -GCFLAG_IMMORTAL = first_gcflag -GCFLAG_FINALIZATION_ORDERING = first_gcflag << 1 +GCFLAG_FORWARDED = first_gcflag +GCFLAG_IMMORTAL = first_gcflag << 1 +GCFLAG_FINALIZATION_ORDERING = first_gcflag << 2 DEBUG_PRINT = False memoryError = MemoryError() @@ -26,12 +27,14 @@ inline_simple_malloc = True inline_simple_malloc_varsize = True needs_zero_gc_pointers = False - first_unused_gcflag = first_gcflag << 2 + first_unused_gcflag = first_gcflag << 3 total_collection_time = 0.0 total_collection_count = 0 - HDR = lltype.Struct('header', ('forw', llmemory.Address), - ('tid', lltype.Signed)) + HDR = lltype.Struct('header', ('tid', lltype.Signed)) + FORWARDSTUB = lltype.GcStruct('forwarding_stub', + ('forw', llmemory.Address)) + FORWARDSTUBPTR = lltype.Ptr(FORWARDSTUB) def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096, max_space_size=sys.maxint//2+1): @@ -41,6 +44,8 @@ self.gcheaderbuilder = GCHeaderBuilder(self.HDR) self.AddressStack = get_address_stack(chunk_size) self.AddressDeque = get_address_deque(chunk_size) + self.finalizer_lock_count = 0 + self.red_zone = 0 def setup(self): if DEBUG_PRINT: @@ -55,8 +60,6 @@ self.objects_with_finalizers = self.AddressDeque() self.run_finalizers = self.AddressDeque() self.objects_with_weakrefs = self.AddressStack() - self.finalizer_lock_count = 0 - self.red_zone = 0 def disable_finalizers(self): self.finalizer_lock_count += 1 @@ -289,15 +292,13 @@ root.address[0] = self.copy(root.address[0]) def copy(self, obj): - # Objects not living the GC heap have all been initialized by - # setting their 'forw' address so that it points to themselves. - # The logic below will thus simply return 'obj' if 'obj' is prebuilt. if self.is_forwarded(obj): #llop.debug_print(lltype.Void, obj, "already copied to", self.get_forwarding_address(obj)) return self.get_forwarding_address(obj) else: newaddr = self.free - totalsize = self.size_gc_header() + self.get_size(obj) + objsize = self.get_size(obj) + totalsize = self.size_gc_header() + objsize llarena.arena_reserve(newaddr, totalsize) raw_memcopy(obj - self.size_gc_header(), newaddr, totalsize) self.free += totalsize @@ -305,7 +306,7 @@ #llop.debug_print(lltype.Void, obj, "copied to", newobj, # "tid", self.header(obj).tid, # "size", totalsize) - self.set_forwarding_address(obj, newobj) + self.set_forwarding_address(obj, newobj, objsize) return newobj def trace_and_copy(self, obj): @@ -316,14 +317,35 @@ pointer.address[0] = self.copy(pointer.address[0]) def is_forwarded(self, obj): - return self.header(obj).forw != NULL + return self.header(obj).tid & GCFLAG_FORWARDED != 0 + # note: all prebuilt objects also have this flag set def get_forwarding_address(self, obj): - return self.header(obj).forw + tid = self.header(obj).tid + if tid & GCFLAG_IMMORTAL: + return obj # prebuilt objects are "forwarded" to themselves + else: + stub = llmemory.cast_adr_to_ptr(obj, self.FORWARDSTUBPTR) + return stub.forw - def set_forwarding_address(self, obj, newobj): - gc_info = self.header(obj) - gc_info.forw = newobj + def set_forwarding_address(self, obj, newobj, objsize): + # To mark an object as forwarded, we set the GCFLAG_FORWARDED and + # overwrite the object with a FORWARDSTUB. Doing so is a bit + # long-winded on llarena, but it all melts down to two memory + # writes after translation to C. + size_gc_header = self.size_gc_header() + stubsize = llmemory.sizeof(self.FORWARDSTUB) + tid = self.header(obj).tid + ll_assert(tid & GCFLAG_IMMORTAL == 0, "unexpected GCFLAG_IMMORTAL") + ll_assert(tid & GCFLAG_FORWARDED == 0, "unexpected GCFLAG_FORWARDED") + # replace the object at 'obj' with a FORWARDSTUB. + hdraddr = obj - size_gc_header + llarena.arena_reset(hdraddr, size_gc_header + objsize, False) + llarena.arena_reserve(hdraddr, size_gc_header + stubsize) + hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(self.HDR)) + hdr.tid = tid | GCFLAG_FORWARDED + stub = llmemory.cast_adr_to_ptr(obj, self.FORWARDSTUBPTR) + stub.forw = newobj def get_size(self, obj): typeid = self.get_type_id(obj) @@ -340,26 +362,24 @@ return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) def get_type_id(self, addr): - return self.header(addr).tid & TYPEID_MASK + tid = self.header(addr).tid + ll_assert(tid & (GCFLAG_FORWARDED|GCFLAG_IMMORTAL) != GCFLAG_FORWARDED, + "get_type_id on forwarded obj") + # Non-prebuilt forwarded objects are overwritten with a FORWARDSTUB. + # Although calling get_type_id() on a forwarded object works by itself, + # we catch it as an error because it's likely that what is then + # done with the typeid is bogus. + return tid & TYPEID_MASK def init_gc_object(self, addr, typeid, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - #hdr.forw = NULL -- unneeded, the space is initially filled with zero hdr.tid = typeid | flags def init_gc_object_immortal(self, addr, typeid, flags=0): - # immortal objects always have forward to themselves hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.tid = typeid | flags | GCFLAG_IMMORTAL - self.init_forwarding(addr + self.gcheaderbuilder.size_gc_header) - - def init_forwarding(self, obj): - hdr = self.header(obj) - if hdr.tid & GCFLAG_IMMORTAL: - hdr.forw = obj # prebuilt objects point to themselves, - # so that a collection does not move them - else: - hdr.forw = NULL + hdr.tid = typeid | flags | GCFLAG_IMMORTAL | GCFLAG_FORWARDED + # immortal objects always have GCFLAG_FORWARDED set; + # see get_forwarding_address(). def deal_with_objects_with_finalizers(self, scan): # walk over list of objects with finalizers @@ -371,6 +391,7 @@ new_with_finalizer = self.AddressDeque() marked = self.AddressDeque() pending = self.AddressStack() + self.tmpstack = self.AddressStack() while self.objects_with_finalizers.non_empty(): x = self.objects_with_finalizers.popleft() ll_assert(self._finalization_state(x) != 1, @@ -385,11 +406,9 @@ state = self._finalization_state(y) if state == 0: self._bump_finalization_state_from_0_to_1(y) + self.trace(y, self._append_if_nonnull, pending) elif state == 2: - self._bump_finalization_state_from_2_to_3(y) - else: - continue # don't need to recurse inside y - self.trace(y, self._append_if_nonnull, pending) + self._recursively_bump_finalization_state_from_2_to_3(y) scan = self._recursively_bump_finalization_state_from_1_to_2( x, scan) @@ -403,16 +422,11 @@ # we must also fix the state from 2 to 3 here, otherwise # we leave the GCFLAG_FINALIZATION_ORDERING bit behind # which will confuse the next collection - pending.append(x) - while pending.non_empty(): - y = pending.pop() - state = self._finalization_state(y) - if state == 2: - self._bump_finalization_state_from_2_to_3(y) - self.trace(y, self._append_if_nonnull, pending) + self._recursively_bump_finalization_state_from_2_to_3(x) else: new_with_finalizer.append(newx) + self.tmpstack.delete() pending.delete() marked.delete() self.objects_with_finalizers.delete() @@ -445,12 +459,19 @@ hdr = self.header(obj) hdr.tid |= GCFLAG_FINALIZATION_ORDERING - def _bump_finalization_state_from_2_to_3(self, obj): + def _recursively_bump_finalization_state_from_2_to_3(self, obj): ll_assert(self._finalization_state(obj) == 2, "unexpected finalization state != 2") newobj = self.get_forwarding_address(obj) - hdr = self.header(newobj) - hdr.tid &= ~GCFLAG_FINALIZATION_ORDERING + pending = self.tmpstack + ll_assert(not pending.non_empty(), "tmpstack not empty") + pending.append(newobj) + while pending.non_empty(): + y = pending.pop() + hdr = self.header(y) + if hdr.tid & GCFLAG_FINALIZATION_ORDERING: # state 2 ? + hdr.tid &= ~GCFLAG_FINALIZATION_ORDERING # change to state 3 + self.trace(y, self._append_if_nonnull, pending) def _recursively_bump_finalization_state_from_1_to_2(self, obj, scan): # recursively convert objects from state 1 to state 2. Modified: pypy/dist/pypy/rpython/memory/gcheader.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcheader.py (original) +++ pypy/dist/pypy/rpython/memory/gcheader.py Wed Feb 20 10:42:04 2008 @@ -16,17 +16,17 @@ def header_of_object(self, gcptr): # XXX hackhackhack - gcptr = gcptr._as_obj() + gcptr = gcptr._as_obj(check=False) if isinstance(gcptr, llmemory._gctransformed_wref): - return self.obj2header[gcptr._ptr._as_obj()] + return self.obj2header[gcptr._ptr._as_obj(check=False)] return self.obj2header[gcptr] def object_from_header(headerptr): - return header2obj[headerptr._as_obj()] + return header2obj[headerptr._as_obj(check=False)] object_from_header = staticmethod(object_from_header) def get_header(self, gcptr): - return self.obj2header.get(gcptr._as_obj(), None) + return self.obj2header.get(gcptr._as_obj(check=False), None) def attach_header(self, gcptr, headerptr): gcobj = gcptr._as_obj() Modified: pypy/dist/pypy/rpython/memory/support.py ============================================================================== --- pypy/dist/pypy/rpython/memory/support.py (original) +++ pypy/dist/pypy/rpython/memory/support.py Wed Feb 20 10:42:04 2008 @@ -26,7 +26,10 @@ def get(self): if not self.free_list: - return lltype.malloc(CHUNK, flavor="raw") + # we zero-initialize the chunks to make the translation + # backends happy, but we don't need to do it at run-time. + zero = not we_are_translated() + return lltype.malloc(CHUNK, flavor="raw", zero=zero) result = self.free_list self.free_list = result.next From arigo at codespeak.net Wed Feb 20 10:42:24 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 20 Feb 2008 10:42:24 +0100 (CET) Subject: [pypy-svn] r51676 - pypy/branch/no-forw-ptr Message-ID: <20080220094224.1064F168447@codespeak.net> Author: arigo Date: Wed Feb 20 10:42:23 2008 New Revision: 51676 Removed: pypy/branch/no-forw-ptr/ Log: Remove merged branch. From arigo at codespeak.net Wed Feb 20 11:42:18 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 20 Feb 2008 11:42:18 +0100 (CET) Subject: [pypy-svn] r51677 - in pypy/dist/pypy: rlib rpython/memory/gc rpython/memory/gctransform translator/c/test translator/goal translator/sandbox Message-ID: <20080220104218.3E526168454@codespeak.net> Author: arigo Date: Wed Feb 20 11:42:17 2008 New Revision: 51677 Modified: pypy/dist/pypy/rlib/rgc.py pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/gc/semispace.py pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/translator/c/test/test_newgc.py pypy/dist/pypy/translator/goal/targetpypystandalone.py pypy/dist/pypy/translator/sandbox/pypy_interact.py Log: A trivial but working implementation of rgc.set_max_heap_size() for SemiSpaceGC. Modified: pypy/dist/pypy/rlib/rgc.py ============================================================================== --- pypy/dist/pypy/rlib/rgc.py (original) +++ pypy/dist/pypy/rlib/rgc.py Wed Feb 20 11:42:17 2008 @@ -7,7 +7,8 @@ def set_max_heap_size(nbytes): """Limit the heap size to n bytes. - So far only implemented by the Boehm GC.""" + So far only implemented by the Boehm GC and the semispace/generation GCs. + """ pass # ____________________________________________________________ Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Wed Feb 20 11:42:17 2008 @@ -105,6 +105,9 @@ def id(self, ptr): return lltype.cast_ptr_to_int(ptr) + def set_max_heap_size(self, size): + pass + def x_swap_pool(self, newpool): return newpool Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Wed Feb 20 11:42:17 2008 @@ -187,6 +187,14 @@ self.space_size = newsize return True # success + def set_max_heap_size(self, size): + # Set the maximum semispace size. Note that the logic above will + # round this number *up* to the next power of two. Also, this is + # the size of one semispace only, so actual usage can be the double + # during a collection. Finally, note that this will never shrink + # an already-allocated heap. + self.max_space_size = size + def collect(self): self.semispace_collect() # the indirection is required by the fact that collect() is referred Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Wed Feb 20 11:42:17 2008 @@ -275,6 +275,11 @@ else: self.id_ptr = None + self.set_max_heap_size_ptr = getfn(GCClass.set_max_heap_size.im_func, + [s_gc, + annmodel.SomeInteger(nonneg=True)], + annmodel.s_None) + if GCClass.needs_write_barrier: self.write_barrier_ptr = getfn(GCClass.write_barrier.im_func, [s_gc, annmodel.SomeAddress(), @@ -603,6 +608,12 @@ else: hop.rename('cast_ptr_to_int') # works nicely for non-moving GCs + def gct_gc_set_max_heap_size(self, hop): + [v_size] = hop.spaceop.args + hop.genop("direct_call", [self.set_max_heap_size_ptr, + self.c_const_gc, + v_size]) + def transform_generic_set(self, hop): from pypy.objspace.flow.model import Constant opname = hop.spaceop.opname Modified: pypy/dist/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/c/test/test_newgc.py Wed Feb 20 11:42:17 2008 @@ -875,6 +875,26 @@ res = fn() assert res == -2 + def test_gc_set_max_heap_size(self): + def g(n): + return 'x' * n + def fn(): + # the semispace size starts at 8MB for now, so setting a + # smaller limit has no effect + from pypy.rlib import rgc + rgc.set_max_heap_size(20000000) # almost 20 MB + s1 = s2 = s3 = None + try: + s1 = g(400000) # ~ 400 KB + s2 = g(4000000) # ~ 4 MB + s3 = g(40000000) # ~ 40 MB + except MemoryError: + pass + return (s1 is not None) + (s2 is not None) + (s3 is not None) + c_fn = self.getcompiled(fn) + res = c_fn() + assert res == 2 + class TestGenerationalGC(TestSemiSpaceGC): gcpolicy = "generation" Modified: pypy/dist/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/dist/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/dist/pypy/translator/goal/targetpypystandalone.py Wed Feb 20 11:42:17 2008 @@ -35,7 +35,9 @@ # debug(" argv -> " + arg) if len(argv) > 2 and argv[1] == '--heapsize': # Undocumented option, handled at interp-level. - # It has silently no effect in non-Boehm translations. + # It has silently no effect with some GCs. + # It works in Boehm and in the semispace or generational GCs + # (but see comments in semispace.py:set_max_heap_size()). # At the moment this option exists mainly to support sandboxing. from pypy.rlib import rgc rgc.set_max_heap_size(int(argv[2])) Modified: pypy/dist/pypy/translator/sandbox/pypy_interact.py ============================================================================== --- pypy/dist/pypy/translator/sandbox/pypy_interact.py (original) +++ pypy/dist/pypy/translator/sandbox/pypy_interact.py Wed Feb 20 11:42:17 2008 @@ -10,7 +10,8 @@ which is the virtual current dir (always read-only for now) --heapsize=N limit memory usage to N bytes, or kilo- mega- giga-bytes with the 'k', 'm' or 'g' suffix respectively. - ATM this only works with PyPy translated with Boehm. + ATM this only works with PyPy translated with Boehm or + the semispace or generation GCs. --timeout=N limit execution time to N (real-time) seconds. --log=FILE log all user input into the FILE From mwh at codespeak.net Wed Feb 20 11:51:03 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 20 Feb 2008 11:51:03 +0100 (CET) Subject: [pypy-svn] r51678 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20080220105103.C19E0168455@codespeak.net> Author: mwh Date: Wed Feb 20 11:51:01 2008 New Revision: 51678 Modified: pypy/dist/pypy/rpython/memory/gctransform/boehm.py pypy/dist/pypy/rpython/memory/gctransform/refcounting.py Log: remove some unused imports (playing with a fun pyflakes/flymake hack) Modified: pypy/dist/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/boehm.py Wed Feb 20 11:51:01 2008 @@ -2,8 +2,6 @@ from pypy.rpython.memory.gctransform.support import type_contains_pyobjs, \ get_rtti, _static_deallocator_body_for_type, LLTransformerOp, ll_call_destructor from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rpython import rmodel -from pypy.rlib.rarithmetic import ovfcheck from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem.lloperation import llop Modified: pypy/dist/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/refcounting.py Wed Feb 20 11:51:01 2008 @@ -1,4 +1,3 @@ -import py from pypy.rpython.memory.gctransform.transform import GCTransformer, mallocHelpers from pypy.rpython.memory.gctransform.support import find_gc_ptrs_in_type, \ get_rtti, _static_deallocator_body_for_type, LLTransformerOp, ll_call_destructor From mwh at codespeak.net Wed Feb 20 12:19:07 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 20 Feb 2008 12:19:07 +0100 (CET) Subject: [pypy-svn] r51681 - pypy/dist/pypy/annotation Message-ID: <20080220111907.16E73168418@codespeak.net> Author: mwh Date: Wed Feb 20 12:19:06 2008 New Revision: 51681 Modified: pypy/dist/pypy/annotation/annrpython.py pypy/dist/pypy/annotation/binaryop.py pypy/dist/pypy/annotation/bookkeeper.py pypy/dist/pypy/annotation/builtin.py pypy/dist/pypy/annotation/model.py pypy/dist/pypy/annotation/policy.py pypy/dist/pypy/annotation/signature.py pypy/dist/pypy/annotation/specialize.py pypy/dist/pypy/annotation/unaryop.py Log: clean up some unused imports Modified: pypy/dist/pypy/annotation/annrpython.py ============================================================================== --- pypy/dist/pypy/annotation/annrpython.py (original) +++ pypy/dist/pypy/annotation/annrpython.py Wed Feb 20 12:19:06 2008 @@ -1,10 +1,10 @@ from __future__ import generators -from types import ClassType, FunctionType +from types import FunctionType from pypy.tool.ansi_print import ansi_log, raise_nicer_exception from pypy.annotation import model as annmodel from pypy.tool.pairtype import pair -from pypy.annotation.bookkeeper import Bookkeeper, getbookkeeper +from pypy.annotation.bookkeeper import Bookkeeper from pypy.annotation import signature from pypy.objspace.flow.model import Variable, Constant from pypy.objspace.flow.model import FunctionGraph @@ -26,7 +26,7 @@ import pypy.rpython.ootypesystem.bltregistry # has side effects import pypy.rpython.extfuncregistry # has side effects import pypy.rlib.nonconst # has side effects - + if translator is None: # interface for tests from pypy.translator.translator import TranslationContext Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Wed Feb 20 12:19:06 2008 @@ -18,14 +18,11 @@ from pypy.annotation.model import isdegenerated, TLS from pypy.annotation.model import read_can_only_throw from pypy.annotation.model import add_knowntypedata, merge_knowntypedata -from pypy.annotation.model import lltype_to_annotation from pypy.annotation.model import SomeGenericCallable from pypy.annotation.model import SomeExternalInstance, SomeUnicodeString from pypy.annotation.bookkeeper import getbookkeeper from pypy.objspace.flow.model import Variable, Constant -from pypy.annotation.listdef import ListDef from pypy.rlib import rarithmetic -from pypy.rpython import extregistry # convenience only! def immutablevalue(x): Modified: pypy/dist/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/dist/pypy/annotation/bookkeeper.py (original) +++ pypy/dist/pypy/annotation/bookkeeper.py Wed Feb 20 12:19:06 2008 @@ -8,17 +8,17 @@ from pypy.objspace.flow.model import Constant from pypy.annotation.model import SomeString, SomeChar, SomeFloat, \ SomePtr, unionof, SomeInstance, SomeDict, SomeBuiltin, SomePBC, \ - SomeInteger, SomeExternalObject, SomeOOInstance, TLS, SomeAddress, \ + SomeInteger, SomeOOInstance, TLS, SomeAddress, \ SomeUnicodeCodePoint, SomeOOStaticMeth, s_None, s_ImpossibleValue, \ SomeLLADTMeth, SomeBool, SomeTuple, SomeOOClass, SomeImpossibleValue, \ SomeUnicodeString, SomeList, SomeObject, HarmlesslyBlocked, \ SomeWeakRef, lltype_to_annotation -from pypy.annotation.classdef import ClassDef, InstanceSource -from pypy.annotation.listdef import ListDef, MOST_GENERAL_LISTDEF -from pypy.annotation.dictdef import DictDef, MOST_GENERAL_DICTDEF +from pypy.annotation.classdef import InstanceSource +from pypy.annotation.listdef import ListDef +from pypy.annotation.dictdef import DictDef from pypy.annotation import description from pypy.annotation.signature import annotationoftype -from pypy.interpreter.argument import Arguments, ArgErr +from pypy.interpreter.argument import Arguments from pypy.rlib.objectmodel import r_dict, Symbolic from pypy.tool.algo.unionfind import UnionFind from pypy.rpython.lltypesystem import lltype, llmemory Modified: pypy/dist/pypy/annotation/builtin.py ============================================================================== --- pypy/dist/pypy/annotation/builtin.py (original) +++ pypy/dist/pypy/annotation/builtin.py Wed Feb 20 12:19:06 2008 @@ -8,7 +8,6 @@ from pypy.annotation.model import SomeUnicodeCodePoint, SomeAddress from pypy.annotation.model import SomeFloat, unionof, SomeUnicodeString from pypy.annotation.model import SomePBC, SomeInstance, SomeDict -from pypy.annotation.model import SomeExternalObject from pypy.annotation.model import SomeWeakRef from pypy.annotation.model import annotation_to_lltype, lltype_to_annotation, ll_to_annotation from pypy.annotation.model import add_knowntypedata @@ -360,7 +359,7 @@ return SomeObject() # collect all functions -import __builtin__, exceptions +import __builtin__ BUILTIN_ANALYZERS = {} for name, value in globals().items(): if name.startswith('builtin_'): Modified: pypy/dist/pypy/annotation/model.py ============================================================================== --- pypy/dist/pypy/annotation/model.py (original) +++ pypy/dist/pypy/annotation/model.py Wed Feb 20 12:19:06 2008 @@ -33,11 +33,9 @@ from pypy.tool import descriptor from pypy.tool.pairtype import pair, extendabletype from pypy.tool.tls import tlsobject -from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong, base_int +from pypy.rlib.rarithmetic import r_uint, r_ulonglong, base_int from pypy.rlib.rarithmetic import r_singlefloat import inspect, weakref -from sys import maxint -from pypy.annotation.description import FunctionDesc DEBUG = True # set to False to disable recording of debugging information TLS = tlsobject() Modified: pypy/dist/pypy/annotation/policy.py ============================================================================== --- pypy/dist/pypy/annotation/policy.py (original) +++ pypy/dist/pypy/annotation/policy.py Wed Feb 20 12:19:06 2008 @@ -4,10 +4,8 @@ from pypy.annotation.specialize import memo # for some reason, model must be imported first, # or we create a cycle. -from pypy.annotation import model as annmodel from pypy.annotation.bookkeeper import getbookkeeper from pypy.annotation.signature import Sig -import types class BasicAnnotatorPolicy(object): Modified: pypy/dist/pypy/annotation/signature.py ============================================================================== --- pypy/dist/pypy/annotation/signature.py (original) +++ pypy/dist/pypy/annotation/signature.py Wed Feb 20 12:19:06 2008 @@ -1,10 +1,9 @@ import types from pypy.annotation.model import SomeBool, SomeInteger, SomeString,\ - SomeFloat, SomeList, SomeDict, s_None, SomeExternalObject,\ + SomeFloat, SomeList, SomeDict, s_None, \ SomeObject, SomeInstance, SomeTuple, lltype_to_annotation,\ unionof, SomeUnicodeString -from pypy.annotation.classdef import ClassDef, InstanceSource from pypy.annotation.listdef import ListDef, MOST_GENERAL_LISTDEF from pypy.annotation.dictdef import DictDef, MOST_GENERAL_DICTDEF @@ -38,7 +37,6 @@ def _compute_annotation(t, bookkeeper=None): from pypy.rpython.lltypesystem import lltype - from pypy.annotation.bookkeeper import getbookkeeper from pypy.rpython import extregistry if isinstance(t, SomeObject): return t @@ -65,7 +63,6 @@ return annotationoftype(t, bookkeeper) def annotationoftype(t, bookkeeper=False): - from pypy.annotation.builtin import BUILTIN_ANALYZERS from pypy.rpython import extregistry """The most precise SomeValue instance that contains all Modified: pypy/dist/pypy/annotation/specialize.py ============================================================================== --- pypy/dist/pypy/annotation/specialize.py (original) +++ pypy/dist/pypy/annotation/specialize.py Wed Feb 20 12:19:06 2008 @@ -1,5 +1,4 @@ # specialization support -import types import py from pypy.tool.uid import uid from pypy.tool.sourcetools import func_with_new_name Modified: pypy/dist/pypy/annotation/unaryop.py ============================================================================== --- pypy/dist/pypy/annotation/unaryop.py (original) +++ pypy/dist/pypy/annotation/unaryop.py Wed Feb 20 12:19:06 2008 @@ -4,7 +4,7 @@ from pypy.annotation.model import \ SomeObject, SomeInteger, SomeBool, SomeString, SomeChar, SomeList, \ - SomeDict, SomeUnicodeCodePoint, SomeTuple, SomeImpossibleValue, \ + SomeDict, SomeTuple, SomeImpossibleValue, \ SomeInstance, SomeBuiltin, SomeFloat, SomeIterator, SomePBC, \ SomeExternalObject, SomeTypedAddressAccess, SomeAddress, \ s_ImpossibleValue, s_Bool, s_None, \ @@ -14,7 +14,6 @@ from pypy.annotation import builtin from pypy.annotation.binaryop import _clone ## XXX where to put this? from pypy.rpython import extregistry -from pypy.annotation.signature import annotation # convenience only! def immutablevalue(x): From mwh at codespeak.net Wed Feb 20 12:38:30 2008 From: mwh at codespeak.net (mwh at codespeak.net) Date: Wed, 20 Feb 2008 12:38:30 +0100 (CET) Subject: [pypy-svn] r51682 - pypy/dist/pypy/annotation Message-ID: <20080220113830.3D925168457@codespeak.net> Author: mwh Date: Wed Feb 20 12:38:29 2008 New Revision: 51682 Modified: pypy/dist/pypy/annotation/policy.py Log: now i read the comment, maybe this one should go back... Modified: pypy/dist/pypy/annotation/policy.py ============================================================================== --- pypy/dist/pypy/annotation/policy.py (original) +++ pypy/dist/pypy/annotation/policy.py Wed Feb 20 12:38:29 2008 @@ -4,6 +4,7 @@ from pypy.annotation.specialize import memo # for some reason, model must be imported first, # or we create a cycle. +from pypy.annotation import model as annmodel from pypy.annotation.bookkeeper import getbookkeeper from pypy.annotation.signature import Sig From arigo at codespeak.net Wed Feb 20 12:39:20 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 20 Feb 2008 12:39:20 +0100 (CET) Subject: [pypy-svn] r51683 - in pypy/dist/pypy/translator/c/gcc: . test Message-ID: <20080220113920.712CD168457@codespeak.net> Author: arigo Date: Wed Feb 20 12:39:18 2008 New Revision: 51683 Modified: pypy/dist/pypy/translator/c/gcc/test/test_asmgcroot.py pypy/dist/pypy/translator/c/gcc/trackgcroot.py Log: Improve the asmgcc tests by checking that the sorting done in asmgcroot.py works correctly. So far, at least on Linux, the addresses were automatically sorted in these tests (but not e.g. in faassen builds of PyPy, due to gcc profiling). Modified: pypy/dist/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/dist/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/dist/pypy/translator/c/gcc/test/test_asmgcroot.py Wed Feb 20 12:39:18 2008 @@ -38,6 +38,7 @@ cbuilder = CStandaloneBuilder(t, main, config=config) c_source_filename = cbuilder.generate_source( defines = cbuilder.DEBUG_DEFINES) + self.patch_makefile(cbuilder.targetdir) if conftest.option.view: t.view() exe_name = cbuilder.compile() @@ -63,6 +64,24 @@ return int(result) return run + def patch_makefile(self, targetdir): + # for testing, patch the Makefile to add the -r option to + # trackgcroot.py. + makefile = targetdir.join('Makefile') + f = makefile.open() + lines = f.readlines() + f.close() + found = False + for i in range(len(lines)): + if 'trackgcroot.py' in lines[i]: + lines[i] = lines[i].replace('trackgcroot.py', + 'trackgcroot.py -r') + found = True + assert found + f = makefile.open('w') + f.writelines(lines) + f.close() + class TestAsmGCRootWithSemiSpaceGC(AbstractTestAsmGCRoot, test_newgc.TestSemiSpaceGC): Modified: pypy/dist/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/dist/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/dist/pypy/translator/c/gcc/trackgcroot.py Wed Feb 20 12:39:18 2008 @@ -1,6 +1,6 @@ #! /usr/bin/env python -import re, sys, os +import re, sys, os, random r_functionstart = re.compile(r"\t.type\s+(\w+),\s*[@]function\s*$") r_functionend = re.compile(r"\t.size\s+(\w+),\s*[.]-(\w+)\s*$") @@ -26,11 +26,12 @@ class GcRootTracker(object): - def __init__(self, verbose=0): + def __init__(self, verbose=0, shuffle=False): self.gcmaptable = [] self.verbose = verbose self.seen_main = False self.files_seen = 0 + self.shuffle = shuffle # to debug the sorting logic in asmgcroot.py def dump(self, output): assert self.seen_main @@ -129,7 +130,11 @@ if self.verbose > 1: for label, state in table: print >> sys.stderr, label, '\t', format_callshape(state) - self.gcmaptable.extend(compress_gcmaptable(table)) + table = compress_gcmaptable(table) + if self.shuffle and random.random() < 0.5: + self.gcmaptable[:0] = table + else: + self.gcmaptable.extend(table) self.seen_main |= tracker.is_main return tracker.lines @@ -973,12 +978,18 @@ if __name__ == '__main__': - if sys.argv and sys.argv[1] == '-v': - del sys.argv[1] - verbose = sys.maxint - else: - verbose = 1 - tracker = GcRootTracker(verbose=verbose) + verbose = 1 + shuffle = False + while len(sys.argv) > 1: + if sys.argv[1] == '-v': + del sys.argv[1] + verbose = sys.maxint + elif sys.argv[1] == '-r': + del sys.argv[1] + shuffle = True + else: + break + tracker = GcRootTracker(verbose=verbose, shuffle=shuffle) for fn in sys.argv[1:]: tmpfn = fn + '.TMP' f = open(fn, 'r') From arigo at codespeak.net Wed Feb 20 12:42:14 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 20 Feb 2008 12:42:14 +0100 (CET) Subject: [pypy-svn] r51684 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20080220114214.7B791168457@codespeak.net> Author: arigo Date: Wed Feb 20 12:42:14 2008 New Revision: 51684 Modified: pypy/dist/pypy/rpython/memory/gctransform/asmgcroot.py Log: Replace the naive sorting algorithm with a call to qsort() from the libc, but only called lazily if the binary search fails. Best-of-both-worlds result: in non-gcc-profiled builds, the table is already sorted, so binary search will always succeed; in gcc-profiled builds, sorting is needed anyway, and qsort() should be much better than the previous algorithm. Modified: pypy/dist/pypy/rpython/memory/gctransform/asmgcroot.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/asmgcroot.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/asmgcroot.py Wed Feb 20 12:42:14 2008 @@ -52,17 +52,6 @@ self._asm_callback = _asm_callback self._shape_decompressor = ShapeDecompressor() - def setup_root_walker(self): - # The gcmap table is a list of entries, two machine words each: - # void *SafePointAddress; - # int Shape; - # Here, i.e. when the program starts, we sort it - # in-place on the SafePointAddress to allow for more - # efficient searches. - gcmapstart = llop.llvm_gcmapstart(llmemory.Address) - gcmapend = llop.llvm_gcmapend(llmemory.Address) - insertion_sort(gcmapstart, gcmapend) - def walk_stack_roots(self, collect_stack_root): gcdata = self.gcdata gcdata._gc_collect_stack_root = collect_stack_root @@ -147,13 +136,13 @@ # gcmapstart = llop.llvm_gcmapstart(llmemory.Address) gcmapend = llop.llvm_gcmapend(llmemory.Address) - item = binary_search(gcmapstart, gcmapend, retaddr) - if item.address[0] != retaddr: - # 'retaddr' not exactly found. Check that 'item' the start of a - # compressed range containing 'retaddr'. - if retaddr > item.address[0] and item.signed[1] < 0: - pass # ok - else: + item = search_in_gcmap(gcmapstart, gcmapend, retaddr) + if not item: + # the item may have been not found because the array was + # not sorted. Sort it and try again. + sort_gcmap(gcmapstart, gcmapend) + item = search_in_gcmap(gcmapstart, gcmapend, retaddr) + if not item: llop.debug_fatalerror(lltype.Void, "cannot find gc roots!") return False # @@ -243,30 +232,33 @@ count -= middleindex return start -def insertion_sort(start, end): - """Sort an array of pairs of addresses. - - This is an insertion sort, so it's slowish unless the array is mostly - sorted already (which is what I expect, but XXX check this). - """ - # XXX this should check that it's not changing the relative order - # of entry and the following entry in case it's a compressed "range" - # entry, i.e. "entry.signed[1] < 0". - next = start - while next < end: - # assuming the interval from start (included) to next (excluded) - # to be already sorted, move the next element back into the array - # until it reaches its proper place. - addr1 = next.address[0] - addr2 = next.address[1] - scan = next - while scan > start and addr1 < scan.address[-2]: - scan.address[0] = scan.address[-2] - scan.address[1] = scan.address[-1] - scan -= arrayitemsize - scan.address[0] = addr1 - scan.address[1] = addr2 - next += arrayitemsize +def search_in_gcmap(gcmapstart, gcmapend, retaddr): + item = binary_search(gcmapstart, gcmapend, retaddr) + if item.address[0] == retaddr: + return item # found + # 'retaddr' not exactly found. Check that 'item' is the start of a + # compressed range that includes 'retaddr'. + if retaddr > item.address[0] and item.signed[1] < 0: + return item # ok + else: + return llmemory.NULL # failed + +def sort_gcmap(gcmapstart, gcmapend): + count = (gcmapend - gcmapstart) // arrayitemsize + qsort(gcmapstart, + rffi.cast(rffi.SIZE_T, count), + rffi.cast(rffi.SIZE_T, arrayitemsize), + llhelper(QSORT_CALLBACK_PTR, _compare_gcmap_entries)) + +def _compare_gcmap_entries(addr1, addr2): + key1 = addr1.address[0] + key2 = addr2.address[0] + if key1 < key2: + return -1 + elif key1 == key2: + return 0 + else: + return 1 # ____________________________________________________________ @@ -336,3 +328,14 @@ sandboxsafe=True, _nowrapper=True) c_asm_gcroot = Constant(pypy_asm_gcroot, lltype.typeOf(pypy_asm_gcroot)) + +QSORT_CALLBACK_PTR = lltype.Ptr(lltype.FuncType([llmemory.Address, + llmemory.Address], rffi.INT)) +qsort = rffi.llexternal('qsort', + [llmemory.Address, + rffi.SIZE_T, + rffi.SIZE_T, + QSORT_CALLBACK_PTR], + lltype.Void, + sandboxsafe=True, + _nowrapper=True) From arigo at codespeak.net Wed Feb 20 14:23:51 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 20 Feb 2008 14:23:51 +0100 (CET) Subject: [pypy-svn] r51685 - pypy/branch/gc-in-genc Message-ID: <20080220132351.2B191168459@codespeak.net> Author: arigo Date: Wed Feb 20 14:23:50 2008 New Revision: 51685 Added: pypy/branch/gc-in-genc/ - copied from r51684, pypy/dist/ Log: Yet another branch, in which to start an attempt to try to cleanup the interactions between genc and the 'gc'-kind types and objects. From cfbolz at codespeak.net Wed Feb 20 14:38:28 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 20 Feb 2008 14:38:28 +0100 (CET) Subject: [pypy-svn] r51686 - in pypy/dist/pypy/module/_rawffi: . test Message-ID: <20080220133828.0DF70168469@codespeak.net> Author: cfbolz Date: Wed Feb 20 14:38:27 2008 New Revision: 51686 Modified: pypy/dist/pypy/module/_rawffi/structure.py pypy/dist/pypy/module/_rawffi/test/test__rawffi.py Log: make the __call__ of Structure not take any arguments, since it is not needed Modified: pypy/dist/pypy/module/_rawffi/structure.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/structure.py (original) +++ pypy/dist/pypy/module/_rawffi/structure.py Wed Feb 20 14:38:27 2008 @@ -69,14 +69,9 @@ raise OperationError(space.w_AttributeError, space.wrap( "C Structure has no attribute %s" % attr)) - def descr_call(self, space, __args__): - args_w, kwargs_w = __args__.unpack() - if args_w: - raise OperationError( - space.w_TypeError, - space.wrap("Structure accepts only keyword arguments as field initializers")) - return space.wrap(W_StructureInstance(space, self, 0, kwargs_w)) - descr_call.unwrap_spec = ['self', ObjSpace, Arguments] + def descr_call(self, space): + return space.wrap(W_StructureInstance(space, self, 0)) + descr_call.unwrap_spec = ['self', ObjSpace] def descr_repr(self, space): fieldnames = ' '.join(["'%s'" % name for name, _ in self.fields]) @@ -86,7 +81,7 @@ descr_repr.unwrap_spec = ['self', ObjSpace] def fromaddress(self, space, address): - return space.wrap(W_StructureInstance(space, self, address, None)) + return space.wrap(W_StructureInstance(space, self, address)) fromaddress.unwrap_spec = ['self', ObjSpace, r_uint] def descr_fieldoffset(self, space, attr): @@ -129,12 +124,9 @@ cast_pos._annspecialcase_ = 'specialize:arg(2)' class W_StructureInstance(W_DataInstance): - def __init__(self, space, shape, address, fieldinits_w): + def __init__(self, space, shape, address): W_DataInstance.__init__(self, space, shape.size, address) self.shape = shape - if fieldinits_w: - for field, w_value in fieldinits_w.iteritems(): - self.setattr(space, field, w_value) def descr_repr(self, space): addr = rffi.cast(lltype.Unsigned, self.ll_buffer) Modified: pypy/dist/pypy/module/_rawffi/test/test__rawffi.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/test/test__rawffi.py (original) +++ pypy/dist/pypy/module/_rawffi/test/test__rawffi.py Wed Feb 20 14:38:27 2008 @@ -280,8 +280,14 @@ lib = _rawffi.CDLL(self.lib_name) inner = lib.ptr("inner_struct_elem", ['P'], 'c') X = _rawffi.Structure([('x1', 'i'), ('x2', 'h'), ('x3', 'c'), ('next', 'P')]) - next = X(next=0, x3='x') - x = X(next=next, x1=1, x2=2, x3='x') + next = X() + next.next = 0 + next.x3 = 'x' + x = X() + x.next = next + x.x1 = 1 + x.x2 = 2 + x.x3 = 'x' assert X.fromaddress(x.next).x3 == 'x' x.free() next.free() @@ -319,7 +325,8 @@ lib = _rawffi.CDLL(self.lib_name) A = _rawffi.Array('P') X = _rawffi.Structure([('x1', 'i'), ('x2', 'h'), ('x3', 'c'), ('next', 'P')]) - x = X(x2=3) + x = X() + x.x2 = 3 a = A(3) a[1] = x get_array_elem_s = lib.ptr('get_array_elem_s', ['P', 'i'], 'P') @@ -421,7 +428,9 @@ def test_setattr_struct(self): import _rawffi X = _rawffi.Structure([('value1', 'i'), ('value2', 'i')]) - x = X(value1=1, value2=2) + x = X() + x.value1 = 1 + x.value2 = 2 assert x.value1 == 1 assert x.value2 == 2 x.value1 = 3 @@ -454,7 +463,8 @@ assert a.shape is A a.free() S = _rawffi.Structure([('v1', 'i')]) - s = S(v1=3) + s = S() + s.v1 = 3 assert s.shape is S s.free() From cfbolz at codespeak.net Wed Feb 20 14:54:58 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 20 Feb 2008 14:54:58 +0100 (CET) Subject: [pypy-svn] r51687 - pypy/dist/pypy/module/_rawffi/test Message-ID: <20080220135458.C42A1168466@codespeak.net> Author: cfbolz Date: Wed Feb 20 14:54:57 2008 New Revision: 51687 Modified: pypy/dist/pypy/module/_rawffi/test/test_tracker.py Log: make this test robust against other tests leaking Modified: pypy/dist/pypy/module/_rawffi/test/test_tracker.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/test/test_tracker.py (original) +++ pypy/dist/pypy/module/_rawffi/test/test_tracker.py Wed Feb 20 14:54:57 2008 @@ -10,26 +10,27 @@ def test_array(self): import _rawffi - assert _rawffi._num_of_allocated_objects() == 0 + oldnum = _rawffi._num_of_allocated_objects() a = _rawffi.Array('c')(3) - assert _rawffi._num_of_allocated_objects() == 1 + assert _rawffi._num_of_allocated_objects() - oldnum == 1 a.free() - assert _rawffi._num_of_allocated_objects() == 0 + assert _rawffi._num_of_allocated_objects() - oldnum == 0 def test_structure(self): import _rawffi - assert _rawffi._num_of_allocated_objects() == 0 + oldnum = _rawffi._num_of_allocated_objects() s = _rawffi.Structure([('a', 'i'), ('b', 'i')])() - assert _rawffi._num_of_allocated_objects() == 1 + assert _rawffi._num_of_allocated_objects() - oldnum == 1 s.free() - assert _rawffi._num_of_allocated_objects() == 0 + assert _rawffi._num_of_allocated_objects() - oldnum == 0 def test_callback(self): import _rawffi + oldnum = _rawffi._num_of_allocated_objects() c = _rawffi.CallbackPtr(lambda : 3, [], 'i') - assert _rawffi._num_of_allocated_objects() == 1 + assert _rawffi._num_of_allocated_objects() - oldnum== 1 c.free() - assert _rawffi._num_of_allocated_objects() == 0 + assert _rawffi._num_of_allocated_objects() - oldnum== 0 def teardown_class(cls): Tracker.DO_TRACING = False From cfbolz at codespeak.net Wed Feb 20 15:09:29 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 20 Feb 2008 15:09:29 +0100 (CET) Subject: [pypy-svn] r51688 - in pypy/dist/pypy/module/_rawffi: . test Message-ID: <20080220140929.C9E851684D1@codespeak.net> Author: cfbolz Date: Wed Feb 20 15:09:29 2008 New Revision: 51688 Modified: pypy/dist/pypy/module/_rawffi/interp_rawffi.py pypy/dist/pypy/module/_rawffi/structure.py pypy/dist/pypy/module/_rawffi/test/test__rawffi.py pypy/dist/pypy/module/_rawffi/tracker.py Log: add a possibility to have structures automatically free their buffers Modified: pypy/dist/pypy/module/_rawffi/interp_rawffi.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/interp_rawffi.py (original) +++ pypy/dist/pypy/module/_rawffi/interp_rawffi.py Wed Feb 20 15:09:29 2008 @@ -214,13 +214,15 @@ def free(self, space): if not self.ll_buffer: raise segfault_exception(space, "freeing NULL pointer") + self._free() + free.unwrap_spec = ['self', ObjSpace] + + def _free(self): if tracker.DO_TRACING: ll_buf = rffi.cast(rffi.INT, self.ll_buffer) tracker.trace_free(ll_buf) lltype.free(self.ll_buffer, flavor='raw') self.ll_buffer = lltype.nullptr(rffi.VOIDP.TO) - free.unwrap_spec = ['self', ObjSpace] - def unwrap_truncate_int(TP, space, w_arg): if space.is_true(space.isinstance(w_arg, space.w_int)): Modified: pypy/dist/pypy/module/_rawffi/structure.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/structure.py (original) +++ pypy/dist/pypy/module/_rawffi/structure.py Wed Feb 20 15:09:29 2008 @@ -69,9 +69,21 @@ raise OperationError(space.w_AttributeError, space.wrap( "C Structure has no attribute %s" % attr)) - def descr_call(self, space): + def descr_call(self, space, __args__): + args_w, kwargs_w = __args__.unpack() + if args_w: + raise OperationError(space.w_TypeError, + space.wrap("too many arguments")) + autofree = False + if 'autofree' in kwargs_w: + autofree = space.is_true(kwargs_w.pop('autofree')) + if len(kwargs_w): + raise OperationError(space.w_TypeError, + space.wrap("unknown keyword argument")) + if autofree: + return space.wrap(W_StructureInstanceAutoFree(space, self)) return space.wrap(W_StructureInstance(space, self, 0)) - descr_call.unwrap_spec = ['self', ObjSpace] + descr_call.unwrap_spec = ['self', ObjSpace, Arguments] def descr_repr(self, space): fieldnames = ' '.join(["'%s'" % name for name, _ in self.fields]) @@ -169,3 +181,11 @@ ) W_StructureInstance.typedef.acceptable_as_base_class = False +class W_StructureInstanceAutoFree(W_StructureInstance): + def __init__(self, space, shape): + W_StructureInstance.__init__(self, space, shape, 0) + + def __del__(self): + if self.ll_buffer: + self._free() + Modified: pypy/dist/pypy/module/_rawffi/test/test__rawffi.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/test/test__rawffi.py (original) +++ pypy/dist/pypy/module/_rawffi/test/test__rawffi.py Wed Feb 20 15:09:29 2008 @@ -4,6 +4,7 @@ from pypy.translator.tool.cbuild import compile_c_module, \ ExternalCompilationInfo from pypy.module._rawffi.interp_rawffi import TYPEMAP +from pypy.module._rawffi.tracker import Tracker import os, sys, py @@ -610,3 +611,28 @@ a.free() raises(_rawffi.SegfaultException, a.__getitem__, 3) raises(_rawffi.SegfaultException, a.__setitem__, 3, 3) + + +class AppTestAutoFree: + def setup_class(cls): + space = gettestobjspace(usemodules=('_rawffi', 'struct')) + cls.space = space + cls.w_sizes_and_alignments = space.wrap(dict( + [(k, (v.c_size, v.c_alignment)) for k,v in TYPEMAP.iteritems()])) + Tracker.DO_TRACING = True + + def test_structure_autofree(self): + import gc, _rawffi + S = _rawffi.Structure([('x', 'i')]) + oldnum = _rawffi._num_of_allocated_objects() + s = S(autofree=True) + s.x = 3 + s = None + gc.collect() + gc.collect() + gc.collect() + assert oldnum == _rawffi._num_of_allocated_objects() + + def teardown_class(cls): + Tracker.DO_TRACING = False + Modified: pypy/dist/pypy/module/_rawffi/tracker.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/tracker.py (original) +++ pypy/dist/pypy/module/_rawffi/tracker.py Wed Feb 20 15:09:29 2008 @@ -12,7 +12,7 @@ self.alloced = {} def trace_allocation(self, address, obj): - self.alloced[address] = obj + self.alloced[address] = True def trace_free(self, address): del self.alloced[address] From cfbolz at codespeak.net Wed Feb 20 15:24:35 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 20 Feb 2008 15:24:35 +0100 (CET) Subject: [pypy-svn] r51689 - in pypy/dist/pypy/module/_rawffi: . test Message-ID: <20080220142435.33479168445@codespeak.net> Author: cfbolz Date: Wed Feb 20 15:24:34 2008 New Revision: 51689 Modified: pypy/dist/pypy/module/_rawffi/array.py pypy/dist/pypy/module/_rawffi/test/test__rawffi.py Log: support autofrees for arrays too Modified: pypy/dist/pypy/module/_rawffi/array.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/array.py (original) +++ pypy/dist/pypy/module/_rawffi/array.py Wed Feb 20 15:24:34 2008 @@ -32,13 +32,25 @@ self.space = space self.itemtp = itemtp - def allocate(self, space, length): + def allocate(self, space, length, autofree=False): + if autofree: + return W_ArrayInstanceAutoFree(space, self, length) return W_ArrayInstance(space, self, length) - def descr_call(self, space, length, w_iterable=None): - result = self.allocate(space, length) - if not space.is_w(w_iterable, space.w_None): - items_w = space.unpackiterable(w_iterable) + def descr_call(self, space, length, __args__): + args_w, kwargs_w = __args__.unpack() + if len(args_w) > 1: + raise OperationError(space.w_TypeError, + space.wrap("too many arguments")) + autofree = False + if 'autofree' in kwargs_w: + autofree = space.is_true(kwargs_w.pop('autofree')) + if len(kwargs_w): + raise OperationError(space.w_TypeError, + space.wrap("unknown keyword argument")) + result = self.allocate(space, length, autofree) + if len(args_w) == 1: + items_w = space.unpackiterable(args_w[0]) iterlength = len(items_w) if iterlength > length: raise OperationError(space.w_ValueError, @@ -91,7 +103,7 @@ __new__ = interp2app(descr_new_array, unwrap_spec=[ObjSpace, W_Root, W_Root]), __call__ = interp2app(W_Array.descr_call, - unwrap_spec=['self', ObjSpace, int, W_Root]), + unwrap_spec=['self', ObjSpace, int, Arguments]), __repr__ = interp2app(W_Array.descr_repr), fromaddress = interp2app(W_Array.fromaddress), gettypecode = interp2app(W_Array.descr_gettypecode), @@ -155,3 +167,13 @@ itemaddress = interp2app(W_ArrayInstance.descr_itemaddress), ) W_ArrayInstance.typedef.acceptable_as_base_class = False + + +class W_ArrayInstanceAutoFree(W_ArrayInstance): + def __init__(self, space, shape, length): + W_ArrayInstance.__init__(self, space, shape, length, 0) + + def __del__(self): + if self.ll_buffer: + self._free() + Modified: pypy/dist/pypy/module/_rawffi/test/test__rawffi.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/test/test__rawffi.py (original) +++ pypy/dist/pypy/module/_rawffi/test/test__rawffi.py Wed Feb 20 15:24:34 2008 @@ -629,10 +629,19 @@ s.x = 3 s = None gc.collect() - gc.collect() + assert oldnum == _rawffi._num_of_allocated_objects() + + def test_array_autofree(self): + import gc, _rawffi + oldnum = _rawffi._num_of_allocated_objects() + + A = _rawffi.Array('c') + a = A(6, 'xxyxx\x00', autofree=True) + assert _rawffi.charp2string(a.buffer) == 'xxyxx' + a = None gc.collect() assert oldnum == _rawffi._num_of_allocated_objects() + def teardown_class(cls): Tracker.DO_TRACING = False - From arigo at codespeak.net Wed Feb 20 15:50:51 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 20 Feb 2008 15:50:51 +0100 (CET) Subject: [pypy-svn] r51690 - in pypy/branch/gc-in-genc/pypy: rpython/lltypesystem rpython/memory/gctransform translator/c Message-ID: <20080220145051.06DC0168457@codespeak.net> Author: arigo Date: Wed Feb 20 15:50:49 2008 New Revision: 51690 Modified: pypy/branch/gc-in-genc/pypy/rpython/lltypesystem/llmemory.py pypy/branch/gc-in-genc/pypy/rpython/memory/gctransform/framework.py pypy/branch/gc-in-genc/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/gc-in-genc/pypy/rpython/memory/gctransform/transform.py pypy/branch/gc-in-genc/pypy/translator/c/database.py pypy/branch/gc-in-genc/pypy/translator/c/funcgen.py pypy/branch/gc-in-genc/pypy/translator/c/gc.py pypy/branch/gc-in-genc/pypy/translator/c/node.py Log: In-progress. The plan is to ask the GC transformer to turn GC types into raw types with extra fields for the headers, and GC objects into raw objects. I hope this is not going to make the "translate.py" memory requirements go through the roof again. Modified: pypy/branch/gc-in-genc/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/gc-in-genc/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/gc-in-genc/pypy/rpython/lltypesystem/llmemory.py Wed Feb 20 15:50:49 2008 @@ -103,7 +103,7 @@ while True: src = cast_adr_to_ptr(srcadr, PTR) dst = cast_adr_to_ptr(dstadr, PTR) - _reccopy(src, dst) + reccopy(src, dst) repeat -= 1 if repeat <= 0: break @@ -162,7 +162,7 @@ PTR = lltype.Ptr(self.TYPE) src = cast_adr_to_ptr(srcadr, PTR) dst = cast_adr_to_ptr(dstadr, PTR) - _reccopy(src, dst) + reccopy(src, dst) class CompositeOffset(AddressOffset): @@ -285,7 +285,7 @@ return cast_ptr_to_adr(headerptr) def raw_memcopy(self, srcadr, dstadr): - _reccopy(srcadr.ptr, dstadr.ptr) + reccopy(srcadr.ptr, dstadr.ptr) class GCHeaderAntiOffset(AddressOffset): def __init__(self, gcheaderbuilder): @@ -698,7 +698,7 @@ return lltype.cast_pointer(EXPECTED_TYPE, ptr) -def _reccopy(source, dest): +def reccopy(source, dest): # copy recursively a structure or array onto another. T = lltype.typeOf(source).TO assert T == lltype.typeOf(dest).TO @@ -709,7 +709,7 @@ if isinstance(ITEMTYPE, lltype.ContainerType): subsrc = source._obj.getitem(i)._as_ptr() subdst = dest._obj.getitem(i)._as_ptr() - _reccopy(subsrc, subdst) + reccopy(subsrc, subdst) else: # this is a hack XXX de-hack this llvalue = source._obj.getitem(i, uninitialized_ok=True) @@ -720,7 +720,7 @@ if isinstance(FIELDTYPE, lltype.ContainerType): subsrc = source._obj._getattr(name)._as_ptr() subdst = dest._obj._getattr(name)._as_ptr() - _reccopy(subsrc, subdst) + reccopy(subsrc, subdst) else: # this is a hack XXX de-hack this llvalue = source._obj._getattr(name, uninitialized_ok=True) Modified: pypy/branch/gc-in-genc/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/gc-in-genc/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/gc-in-genc/pypy/rpython/memory/gctransform/framework.py Wed Feb 20 15:50:49 2008 @@ -332,11 +332,7 @@ self.c_const_gc = rmodel.inputconst(r_gc, self.gcdata.gc) self.needs_zero_gc_pointers = GCClass.needs_zero_gc_pointers - HDR = self._gc_HDR = self.gcdata.gc.gcheaderbuilder.HDR - self._gc_fields = fields = [] - for fldname in HDR._names: - FLDTYPE = getattr(HDR, fldname) - fields.append(('_' + fldname, FLDTYPE)) + self.HDR = self.gcdata.gc.gcheaderbuilder.HDR def build_root_walker(self): return ShadowStackRootWalker(self) @@ -351,12 +347,9 @@ def finalizer_funcptr_for_type(self, TYPE): return self.layoutbuilder.finalizer_funcptr_for_type(TYPE) - def gc_fields(self): - return self._gc_fields - def gc_field_values_for(self, obj): hdr = self.gcdata.gc.gcheaderbuilder.header_of_object(obj) - HDR = self._gc_HDR + HDR = self.HDR return [getattr(hdr, fldname) for fldname in HDR._names] def finish_tables(self): Modified: pypy/branch/gc-in-genc/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/gc-in-genc/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/gc-in-genc/pypy/rpython/memory/gctransform/refcounting.py Wed Feb 20 15:50:49 2008 @@ -174,14 +174,12 @@ lltype.typeOf(dealloc_fptr), dealloc_fptr) hop.genop("direct_call", [cdealloc_fptr, v_addr]) - def consider_constant(self, TYPE, value): - if value is not lltype.top_container(value): - return - if isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)): - p = value._as_ptr() - if not self.gcheaderbuilder.get_header(p): - hdr = self.gcheaderbuilder.new_header(p) - hdr.refcount = sys.maxint // 2 + def build_gc_header(self, p): + hdr = self.gcheaderbuilder.get_header(p) + if not hdr: + hdr = self.gcheaderbuilder.new_header(p) + hdr.refcount = sys.maxint // 2 + return hdr def static_deallocation_funcptr_for_type(self, TYPE): if TYPE in self.static_deallocator_funcptrs: Modified: pypy/branch/gc-in-genc/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/gc-in-genc/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/gc-in-genc/pypy/rpython/memory/gctransform/transform.py Wed Feb 20 15:50:49 2008 @@ -486,6 +486,9 @@ self.stack_malloc_fixedsize_ptr = self.inittime_helper( ll_stack_malloc_fixedsize, [lltype.Signed], llmemory.Address) + self.raw_type_cache = {} + self.raw_obj_cache = {} + def gct_malloc(self, hop): TYPE = hop.spaceop.result.concretetype.TO assert not TYPE._is_varsize() @@ -611,3 +614,119 @@ hop.genop('raw_free', [v]) else: assert False, "%s has no support for free with flavor %r" % (self, flavor) + + # __________ data types and prebuilt objects __________ + # + # The logic below converts GC types to non-GC types, and prebuilt GC + # objects to corresponding prebuilt non-GC objects. It is not recursive + # in the sense that if a GcStruct contains a GC pointer, its convertion + # as a raw Struct still contains a GC pointer. + + def needs_gcheader(self, T): + assert isinstance(T, lltype.ContainerType) and T._gckind == 'gc' + if isinstance(T, lltype.GcStruct): + if T._first_struct() != (None, None): + return False # gcheader already in the first field + return True + + def get_raw_type_for_gc_type(self, T): + """For a 'gc'-kind type T, returns a corresponding 'raw'-kind type + which should be used by the backends. + """ + try: + return self.raw_type_cache[T] + except KeyError: + if isinstance(T, (lltype.GcStruct, lltype.GcArray)): + # default logic: insert a header of type self.HDR + rawfields = [] + if self.needs_gcheader(T): + rawfields.append(('gc__hdr', self.HDR)) # xxx name clash? + if isinstance(T, lltype.GcStruct): + for name in T._names: + FIELD = T._flds[name] + if (isinstance(FIELD, lltype.ContainerType) + and FIELD._gckind == 'gc'): + FIELD = self.get_raw_type_for_gc_type(FIELD) + rawfields.append((name, FIELD)) + else: + rawfields.append(('array', lltype.Array(T.OF))) + kwds = {'hints': {'raw_for_gc': True}} + RAWT = lltype.Struct(T._name, *rawfields, **kwds) + else: + raise TypeError("not supported by %s: %r" % ( + self.__class__.__name__, T)) + self.raw_type_cache[T] = RAWT + return RAWT + + def transform_prebuilt_gc_object(self, container): + try: + return self.raw_obj_cache[container] + except KeyError: + pass + # if 'container' is inlined in a larger object, convert that + # and get a reference to the corresponding part in the convertion + parent, parentindex = lltype.parentlink(container) + if parent is not None: + rawparent = self.transform_prebuilt_gc_object(parent) + if isinstance(parentindex, str): + result = rawparent._getattr(parentindex) + else: + result = rawparent.getitem(parentindex) + else: + T = lltype.typeOf(container) + if isinstance(T, (lltype.GcStruct, lltype.GcArray)): + # allocate the equivalent raw structure + RAWT = self.get_raw_type_for_gc_type(T) + if T._is_varsize(): + length = container.getlength() + else: + length = None + result = lltype.malloc(RAWT, length, immortal=True)._as_obj() + # copy the header and structure fields or array items in place + self._copy_fields_from_gc_to_raw(container, result) + else: + raise TypeError("not supported by %s: %r" % ( + self.__class__.__name__, T)) + self.raw_obj_cache[container] = result + return result + + def _copy_fields_from_gc_to_raw(self, container, raw): + T = lltype.typeOf(container) + if self.needs_gcheader(T): + hdr = self.build_gc_header(lltype.top_container(container)) + llmemory.reccopy(hdr, raw.gc__hdr._as_ptr()) + if isinstance(T, lltype.GcStruct): + for name in T._names: + FIELD = T._flds[name] + src = container._getattr(name) + if isinstance(FIELD, lltype.ContainerType): + dst = raw._getattr(name) + if FIELD._gckind == 'gc': + # an inlined GC substructure: recursively convert it + self._copy_fields_from_gc_to_raw(src, dst) + else: + # an inlined raw substructure: copy it over + llmemory.reccopy(src._as_ptr(), dst._as_ptr()) + else: + # a non-container value + setattr(raw, name, src) + else: + assert isinstance(T, lltype.GcArray) + dst = raw.array + assert dst.getlength() == container.getlength() + if isinstance(T.OF, lltype.ContainerType): + # array of containers - reccopy each item + for i in range(container.getlength()): + llmemory.reccopy(container.getitem(i)._as_ptr(), + dst.getitem(i)._as_ptr()) + else: + # array of non-containers - copy each item + for i in range(container.getlength()): + dst.setitem(i, container.getitem(i)) + + def consider_constant(self, T, container): + if not T._hints.get('raw_for_gc', False): + self.consider_constant_nongc(T, container) + + def consider_constant_nongc(self, T, container): + "To be overridden." Modified: pypy/branch/gc-in-genc/pypy/translator/c/database.py ============================================================================== --- pypy/branch/gc-in-genc/pypy/translator/c/database.py (original) +++ pypy/branch/gc-in-genc/pypy/translator/c/database.py Wed Feb 20 15:50:49 2008 @@ -22,9 +22,8 @@ # ____________________________________________________________ class LowLevelDatabase(object): - gctransformer = None - def __init__(self, translator=None, standalone=False, + def __init__(self, translator, standalone=False, gcpolicyclass=None, stacklesstransformer=None, thread_enabled=False, @@ -54,12 +53,8 @@ self.late_initializations = [] self.namespace = CNameManager() - if translator is None or translator.rtyper is None: - self.exctransformer = None - else: - self.exctransformer = translator.getexceptiontransformer() - if translator is not None: - self.gctransformer = self.gcpolicy.transformerclass(translator) + self.exctransformer = translator.getexceptiontransformer() + self.gctransformer = self.gcpolicy.transformerclass(translator) self.completed = False self.instrument_ncounter = 0 @@ -71,8 +66,16 @@ else: key = T, varlength try: - node = self.structdefnodes[key] + return self.structdefnodes[key] except KeyError: + pass + if not isinstance(T, ContainerType): + raise TypeError("ContainerType expected, got %r" % (T,)) + if T._gckind == 'gc': + RAWT = self.gctransformer.get_raw_type_for_gc_type(T) + assert RAWT._gckind == 'raw' + node = self.gettypedefnode(RAWT, varlength) + else: if isinstance(T, Struct): if isinstance(T, FixedSizeArray): node = FixedSizeArrayDefNode(self, T) @@ -83,15 +86,13 @@ node = BareBoneArrayDefNode(self, T, varlength) else: node = ArrayDefNode(self, T, varlength) - elif isinstance(T, OpaqueType) and T.hints.get("render_structure", False): + elif (isinstance(T, OpaqueType) and + T.hints.get("render_structure", False)): node = ExtTypeOpaqueDefNode(self, T) - elif T == WeakRef: - REALT = self.gcpolicy.get_real_weakref_type() - node = self.gettypedefnode(REALT) else: raise NoCorrespondingNode("don't know about %r" % (T,)) - self.structdefnodes[key] = node self.pendingsetupnodes.append(node) + self.structdefnodes[key] = node return node def gettype(self, T, varlength=1, who_asks=None, argnames=[]): @@ -144,21 +145,24 @@ else: raise Exception("don't know about type %r" % (T,)) - def getcontainernode(self, container, _dont_write_c_code=True, **buildkwds): + def getcontainernode(self, container): #, **buildkwds): try: node = self.containernodes[container] except KeyError: T = typeOf(container) + if T._gckind == 'gc': + gct = self.gctransformer + rawcontainer = gct.transform_prebuilt_gc_object(container) + node = self.getcontainernode(rawcontainer) + self.containernodes[container] = node + return node + if isinstance(T, (lltype.Array, lltype.Struct)): - if hasattr(self.gctransformer, 'consider_constant'): - self.gctransformer.consider_constant(T, container) + self.gctransformer.consider_constant(T, container) + nodefactory = ContainerNodeFactory[T.__class__] - node = nodefactory(self, T, container, **buildkwds) + node = nodefactory(self, T, container) #, **buildkwds) self.containernodes[container] = node - # _dont_write_c_code should only be False for a hack in - # weakrefnode_factory() - if not _dont_write_c_code: - return node kind = getattr(node, 'nodekind', '?') self.containerstats[kind] = self.containerstats.get(kind, 0) + 1 self.containerlist.append(node) @@ -278,15 +282,13 @@ # steps with calls to the next 'finish' function from the following # list: finish_callbacks = [] - if self.gctransformer: - finish_callbacks.append(('GC transformer: finished helpers', - self.gctransformer.finish_helpers)) + finish_callbacks.append(('GC transformer: finished helpers', + self.gctransformer.finish_helpers)) if self.stacklesstransformer: finish_callbacks.append(('Stackless transformer: finished', self.stacklesstransformer.finish)) - if self.gctransformer: - finish_callbacks.append(('GC transformer: finished tables', - self.gctransformer.finish_tables)) + finish_callbacks.append(('GC transformer: finished tables', + self.gctransformer.finish_tables)) def add_dependencies(newdependencies): for value in newdependencies: Modified: pypy/branch/gc-in-genc/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/gc-in-genc/pypy/translator/c/funcgen.py (original) +++ pypy/branch/gc-in-genc/pypy/translator/c/funcgen.py Wed Feb 20 15:50:49 2008 @@ -47,11 +47,9 @@ if db.stacklesstransformer: db.stacklesstransformer.transform_graph(graph) # apply the exception transformation - if self.db.exctransformer: - self.db.exctransformer.create_exception_handling(self.graph) + self.db.exctransformer.create_exception_handling(self.graph) # apply the gc transformation - if self.db.gctransformer: - self.db.gctransformer.transform_graph(self.graph) + self.db.gctransformer.transform_graph(self.graph) #self.graph.show() self.collect_var_and_types() Modified: pypy/branch/gc-in-genc/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/gc-in-genc/pypy/translator/c/gc.py (original) +++ pypy/branch/gc-in-genc/pypy/translator/c/gc.py Wed Feb 20 15:50:49 2008 @@ -33,9 +33,6 @@ def array_gcheader_initdata(self, defnode): return self.common_gcheader_initdata(defnode) - def struct_after_definition(self, defnode): - return [] - def gc_libraries(self): return [] Modified: pypy/branch/gc-in-genc/pypy/translator/c/node.py ============================================================================== --- pypy/branch/gc-in-genc/pypy/translator/c/node.py (original) +++ pypy/branch/gc-in-genc/pypy/translator/c/node.py Wed Feb 20 15:50:49 2008 @@ -1,7 +1,7 @@ from __future__ import generators from pypy.rpython.lltypesystem.lltype import \ Struct, Array, FixedSizeArray, FuncType, PyObjectType, typeOf, \ - GcStruct, GcArray, RttiStruct, ContainerType, \ + ContainerType, \ parentlink, Ptr, PyObject, Void, OpaqueType, Float, \ RuntimeTypeInfo, getRuntimeTypeInfo, Char, _subarray from pypy.rpython.lltypesystem import llmemory @@ -15,29 +15,11 @@ from pypy.translator.c import extfunc from pypy.translator.tool.cbuild import ExternalCompilationInfo -def needs_gcheader(T): - if not isinstance(T, ContainerType): - return False - if T._gckind != 'gc': - return False - if isinstance(T, GcStruct): - if T._first_struct() != (None, None): - return False # gcheader already in the first field - return True - -class defaultproperty(object): - def __init__(self, fget): - self.fget = fget - def __get__(self, obj, cls=None): - if obj is None: - return self - else: - return self.fget(obj) - class StructDefNode: typetag = 'struct' def __init__(self, db, STRUCT, varlength=1): + assert STRUCT._gckind == 'raw' self.db = db self.STRUCT = STRUCT self.LLTYPE = STRUCT @@ -51,7 +33,6 @@ with_number = False if STRUCT._hints.get('union'): self.typetag = 'union' - assert STRUCT._gckind == 'raw' # not supported: "GcUnion" if STRUCT._hints.get('typedef'): self.typetag = '' assert STRUCT._hints.get('external') @@ -77,9 +58,6 @@ db = self.db STRUCT = self.STRUCT varlength = self.varlength - if needs_gcheader(self.STRUCT): - for fname, T in db.gcpolicy.struct_gcheader_definition(self): - self.fields.append((fname, db.gettype(T, who_asks=self))) for name in STRUCT._names: T = self.c_struct_field_type(name) if name == STRUCT._arrayfld: @@ -88,22 +66,6 @@ else: typename = db.gettype(T, who_asks=self) self.fields.append((self.c_struct_field_name(name), typename)) - self.gcinfo # force it to be computed - - def computegcinfo(self): - # let the gcpolicy do its own setup - self.gcinfo = None # unless overwritten below - rtti = None - STRUCT = self.STRUCT - if isinstance(STRUCT, RttiStruct): - try: - rtti = getRuntimeTypeInfo(STRUCT) - except ValueError: - pass - if self.varlength == 1: - self.db.gcpolicy.struct_setup(self, rtti) - return self.gcinfo - gcinfo = defaultproperty(computegcinfo) def gettype(self): return '%s %s @' % (self.typetag, self.name) @@ -162,8 +124,6 @@ if is_empty: yield '\t' + 'char _dummy; /* this struct is empty */' yield '};' - for line in self.db.gcpolicy.struct_after_definition(self): - yield line def visitor_lines(self, prefix, on_field): STRUCT = self.STRUCT @@ -195,12 +155,12 @@ typetag = 'struct' def __init__(self, db, ARRAY, varlength=1): + assert ARRAY._gckind == 'raw' self.db = db self.ARRAY = ARRAY self.LLTYPE = ARRAY original_varlength = varlength - self.gcfields = [] - + if ARRAY._hints.get('isrpystring'): varlength += 1 # for the NUL char terminator at the end of the string self.varlength = varlength @@ -221,20 +181,8 @@ return # setup() was already called, likely by __init__ db = self.db ARRAY = self.ARRAY - self.gcinfo # force it to be computed - if needs_gcheader(ARRAY): - for fname, T in db.gcpolicy.array_gcheader_definition(self): - self.gcfields.append((fname, db.gettype(T, who_asks=self))) self.itemtypename = db.gettype(ARRAY.OF, who_asks=self) - def computegcinfo(self): - # let the gcpolicy do its own setup - self.gcinfo = None # unless overwritten below - if self.varlength == 1: - self.db.gcpolicy.array_setup(self) - return self.gcinfo - gcinfo = defaultproperty(computegcinfo) - def gettype(self): return '%s %s @' % (self.typetag, self.name) @@ -256,10 +204,7 @@ return 'RPyItem(%s, %s)' % (baseexpr, indexexpr) def definition(self): - gcpolicy = self.db.gcpolicy yield 'struct %s {' % self.name - for fname, typename in self.gcfields: - yield '\t' + cdecl(typename, fname) + ';' if not self.ARRAY._hints.get('nolength', False): yield '\tlong length;' line = '%s;' % cdecl(self.itemtypename, 'items[%d]'% self.varlength) @@ -312,7 +257,6 @@ Implemented directly as a C array instead of a struct with an items field. rffi kind of expects such arrays to be 'bare' C arrays. """ - gcinfo = None name = None forward_decl = None @@ -359,7 +303,6 @@ class FixedSizeArrayDefNode: - gcinfo = None name = None typetag = 'struct' @@ -530,10 +473,6 @@ data = [] - if needs_gcheader(self.T): - for i, thing in enumerate(self.db.gcpolicy.struct_gcheader_initdata(self)): - data.append(('gcheader%d'%i, thing)) - for name in self.T._names: data.append((name, getattr(self.obj, name))) @@ -581,13 +520,6 @@ def initializationexpr(self, decoration=''): defnode = self.db.gettypedefnode(self.T) yield '{' - if needs_gcheader(self.T): - for i, thing in enumerate(self.db.gcpolicy.array_gcheader_initdata(self)): - lines = generic_initializationexpr(self.db, thing, - 'gcheader%d'%i, - '%sgcheader%d' % (decoration, i)) - for line in lines: - yield line if self.T._hints.get('nolength', False): length = '' else: @@ -903,23 +835,12 @@ def implementation(self): return [] -def weakrefnode_factory(db, T, obj): - assert isinstance(obj, llmemory._wref) - ptarget = obj._dereference() - wrapper = db.gcpolicy.convert_weakref_to(ptarget) - container = wrapper._obj - obj._converted_weakref = container # hack for genllvm :-/ - return db.getcontainernode(container, _dont_write_c_code=False) - ContainerNodeFactory = { Struct: StructNode, - GcStruct: StructNode, Array: ArrayNode, - GcArray: ArrayNode, FixedSizeArray: FixedSizeArrayNode, FuncType: FuncNode, OpaqueType: opaquenode_factory, PyObjectType: PyObjectNode, - llmemory._WeakRefType: weakrefnode_factory, } From cfbolz at codespeak.net Wed Feb 20 16:02:09 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 20 Feb 2008 16:02:09 +0100 (CET) Subject: [pypy-svn] r51691 - pypy/dist/pypy/module/_rawffi Message-ID: <20080220150209.BB4CE168460@codespeak.net> Author: cfbolz Date: Wed Feb 20 16:02:09 2008 New Revision: 51691 Modified: pypy/dist/pypy/module/_rawffi/array.py pypy/dist/pypy/module/_rawffi/structure.py Log: be less clever and hopefully fix translation Modified: pypy/dist/pypy/module/_rawffi/array.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/array.py (original) +++ pypy/dist/pypy/module/_rawffi/array.py Wed Feb 20 16:02:09 2008 @@ -37,20 +37,10 @@ return W_ArrayInstanceAutoFree(space, self, length) return W_ArrayInstance(space, self, length) - def descr_call(self, space, length, __args__): - args_w, kwargs_w = __args__.unpack() - if len(args_w) > 1: - raise OperationError(space.w_TypeError, - space.wrap("too many arguments")) - autofree = False - if 'autofree' in kwargs_w: - autofree = space.is_true(kwargs_w.pop('autofree')) - if len(kwargs_w): - raise OperationError(space.w_TypeError, - space.wrap("unknown keyword argument")) + def descr_call(self, space, length, w_items=None, autofree=False): result = self.allocate(space, length, autofree) - if len(args_w) == 1: - items_w = space.unpackiterable(args_w[0]) + if not space.is_w(w_items, space.w_None): + items_w = space.unpackiterable(w_items) iterlength = len(items_w) if iterlength > length: raise OperationError(space.w_ValueError, @@ -103,7 +93,7 @@ __new__ = interp2app(descr_new_array, unwrap_spec=[ObjSpace, W_Root, W_Root]), __call__ = interp2app(W_Array.descr_call, - unwrap_spec=['self', ObjSpace, int, Arguments]), + unwrap_spec=['self', ObjSpace, int, W_Root, int]), __repr__ = interp2app(W_Array.descr_repr), fromaddress = interp2app(W_Array.fromaddress), gettypecode = interp2app(W_Array.descr_gettypecode), Modified: pypy/dist/pypy/module/_rawffi/structure.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/structure.py (original) +++ pypy/dist/pypy/module/_rawffi/structure.py Wed Feb 20 16:02:09 2008 @@ -69,21 +69,11 @@ raise OperationError(space.w_AttributeError, space.wrap( "C Structure has no attribute %s" % attr)) - def descr_call(self, space, __args__): - args_w, kwargs_w = __args__.unpack() - if args_w: - raise OperationError(space.w_TypeError, - space.wrap("too many arguments")) - autofree = False - if 'autofree' in kwargs_w: - autofree = space.is_true(kwargs_w.pop('autofree')) - if len(kwargs_w): - raise OperationError(space.w_TypeError, - space.wrap("unknown keyword argument")) + def descr_call(self, space, autofree=False): if autofree: return space.wrap(W_StructureInstanceAutoFree(space, self)) return space.wrap(W_StructureInstance(space, self, 0)) - descr_call.unwrap_spec = ['self', ObjSpace, Arguments] + descr_call.unwrap_spec = ['self', ObjSpace, int] def descr_repr(self, space): fieldnames = ' '.join(["'%s'" % name for name, _ in self.fields]) From arigo at codespeak.net Wed Feb 20 16:12:34 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 20 Feb 2008 16:12:34 +0100 (CET) Subject: [pypy-svn] r51692 - pypy/branch/gc-in-genc/pypy/translator/c Message-ID: <20080220151234.10F851683DA@codespeak.net> Author: arigo Date: Wed Feb 20 16:12:33 2008 New Revision: 51692 Modified: pypy/branch/gc-in-genc/pypy/translator/c/database.py pypy/branch/gc-in-genc/pypy/translator/c/gc.py Log: Some genc tests start to pass. I didn't think they would any time soon... Modified: pypy/branch/gc-in-genc/pypy/translator/c/database.py ============================================================================== --- pypy/branch/gc-in-genc/pypy/translator/c/database.py (original) +++ pypy/branch/gc-in-genc/pypy/translator/c/database.py Wed Feb 20 16:12:33 2008 @@ -75,6 +75,9 @@ RAWT = self.gctransformer.get_raw_type_for_gc_type(T) assert RAWT._gckind == 'raw' node = self.gettypedefnode(RAWT, varlength) + # the next line is hopefully temporary, pending the + # unified-rtti branch + self.gcpolicy.fixup_rawtype_rtti(T, node) else: if isinstance(T, Struct): if isinstance(T, FixedSizeArray): @@ -145,7 +148,7 @@ else: raise Exception("don't know about type %r" % (T,)) - def getcontainernode(self, container): #, **buildkwds): + def getcontainernode(self, container, **buildkwds): try: node = self.containernodes[container] except KeyError: @@ -161,7 +164,7 @@ self.gctransformer.consider_constant(T, container) nodefactory = ContainerNodeFactory[T.__class__] - node = nodefactory(self, T, container) #, **buildkwds) + node = nodefactory(self, T, container, **buildkwds) self.containernodes[container] = node kind = getattr(node, 'nodekind', '?') self.containerstats[kind] = self.containerstats.get(kind, 0) + 1 Modified: pypy/branch/gc-in-genc/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/gc-in-genc/pypy/translator/c/gc.py (original) +++ pypy/branch/gc-in-genc/pypy/translator/c/gc.py Wed Feb 20 16:12:33 2008 @@ -54,6 +54,16 @@ def rtti_type(self): return '' + def fixup_rawtype_rtti(self, GCTYPE, rawdefnode): + # hopefully temporary, pending the unified-rtti branch + rawdefnode.gcinfo = None + if isinstance(GCTYPE, lltype.RttiStruct): + try: + rtti = lltype.getRuntimeTypeInfo(GCTYPE) + except ValueError: + rtti = None + self.struct_setup(rawdefnode, rtti) + def OP_GC_PUSH_ALIVE_PYOBJ(self, funcgen, op): expr = funcgen.expr(op.args[0]) if expr == 'NULL': From arigo at codespeak.net Wed Feb 20 16:14:29 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 20 Feb 2008 16:14:29 +0100 (CET) Subject: [pypy-svn] r51693 - pypy/branch/gc-in-genc/pypy/translator/c Message-ID: <20080220151429.71223168472@codespeak.net> Author: arigo Date: Wed Feb 20 16:14:28 2008 New Revision: 51693 Modified: pypy/branch/gc-in-genc/pypy/translator/c/gc.py Log: Kill a bit more code. Modified: pypy/branch/gc-in-genc/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/gc-in-genc/pypy/translator/c/gc.py (original) +++ pypy/branch/gc-in-genc/pypy/translator/c/gc.py Wed Feb 20 16:14:28 2008 @@ -15,24 +15,6 @@ self.db = db self.thread_enabled = thread_enabled - def common_gcheader_definition(self, defnode): - return [] - - def common_gcheader_initdata(self, defnode): - return [] - - def struct_gcheader_definition(self, defnode): - return self.common_gcheader_definition(defnode) - - def struct_gcheader_initdata(self, defnode): - return self.common_gcheader_initdata(defnode) - - def array_gcheader_definition(self, defnode): - return self.common_gcheader_definition(defnode) - - def array_gcheader_initdata(self, defnode): - return self.common_gcheader_initdata(defnode) - def gc_libraries(self): return [] @@ -48,9 +30,6 @@ def struct_setup(self, structdefnode, rtti): return None - def array_setup(self, arraydefnode): - return None - def rtti_type(self): return '' @@ -86,22 +65,6 @@ class RefcountingGcPolicy(BasicGcPolicy): transformerclass = refcounting.RefcountingGCTransformer - def common_gcheader_definition(self, defnode): - if defnode.db.gctransformer is not None: - HDR = defnode.db.gctransformer.HDR - return [(name, HDR._flds[name]) for name in HDR._names] - else: - return [] - - def common_gcheader_initdata(self, defnode): - if defnode.db.gctransformer is not None: - gct = defnode.db.gctransformer - hdr = gct.gcheaderbuilder.header_of_object(top_container(defnode.obj)) - HDR = gct.HDR - return [getattr(hdr, fldname) for fldname in HDR._names] - else: - return [] - # for structs def struct_setup(self, structdefnode, rtti): @@ -112,11 +75,6 @@ structdefnode.gcinfo = RefcountingInfo() structdefnode.gcinfo.static_deallocator = structdefnode.db.get(fptr) - # for arrays - - def array_setup(self, arraydefnode): - pass - # for rtti node def rtti_type(self): @@ -171,9 +129,6 @@ class BoehmGcPolicy(BasicGcPolicy): transformerclass = boehm.BoehmGCTransformer - def array_setup(self, arraydefnode): - pass - def struct_setup(self, structdefnode, rtti): pass @@ -275,9 +230,6 @@ fptr = gctransf.finalizer_funcptr_for_type(structdefnode.STRUCT) self.db.get(fptr) - def array_setup(self, arraydefnode): - pass - def rtti_type(self): return FrameworkGcRuntimeTypeInfo_OpaqueNode.typename From cfbolz at codespeak.net Wed Feb 20 16:26:43 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 20 Feb 2008 16:26:43 +0100 (CET) Subject: [pypy-svn] r51694 - pypy/dist/pypy/module/_rawffi Message-ID: <20080220152643.CDD751684DB@codespeak.net> Author: cfbolz Date: Wed Feb 20 16:26:43 2008 New Revision: 51694 Modified: pypy/dist/pypy/module/_rawffi/tracker.py Log: use None instead of True Modified: pypy/dist/pypy/module/_rawffi/tracker.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/tracker.py (original) +++ pypy/dist/pypy/module/_rawffi/tracker.py Wed Feb 20 16:26:43 2008 @@ -12,7 +12,7 @@ self.alloced = {} def trace_allocation(self, address, obj): - self.alloced[address] = True + self.alloced[address] = None def trace_free(self, address): del self.alloced[address] From arigo at codespeak.net Wed Feb 20 16:36:32 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 20 Feb 2008 16:36:32 +0100 (CET) Subject: [pypy-svn] r51695 - in pypy/dist/pypy/rpython: . test Message-ID: <20080220153632.22F3A1684D6@codespeak.net> Author: arigo Date: Wed Feb 20 16:36:31 2008 New Revision: 51695 Modified: pypy/dist/pypy/rpython/rint.py pypy/dist/pypy/rpython/test/test_rdict.py Log: Test and fix. Modified: pypy/dist/pypy/rpython/rint.py ============================================================================== --- pypy/dist/pypy/rpython/rint.py (original) +++ pypy/dist/pypy/rpython/rint.py Wed Feb 20 16:36:31 2008 @@ -297,7 +297,7 @@ def get_ll_dummyval_obj(self, rtyper, s_value): # if >= 0, then all negative values are special - if s_value.nonneg and not s_value.unsigned: + if s_value.nonneg and self.lowleveltype is Signed: return signed_repr # whose ll_dummy_value is -1 else: return None Modified: pypy/dist/pypy/rpython/test/test_rdict.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rdict.py (original) +++ pypy/dist/pypy/rpython/test/test_rdict.py Wed Feb 20 16:36:31 2008 @@ -670,6 +670,19 @@ assert hasattr(DICT.entries.TO.OF, 'f_everused') # all ints can be zero assert hasattr(DICT.entries.TO.OF, 'f_valid') # no dummy available + def test_opt_boolean_has_no_dummy(self): + def f(n): + d = {n: True} + d[-87] = True + del d[n] + return len(d.copy()), d[-87], d + res = self.interpret(f, [5]) + assert res.item0 == 1 + assert res.item1 is True + DICT = lltype.typeOf(res.item2).TO + assert hasattr(DICT.entries.TO.OF, 'f_everused') # all ints can be zero + assert hasattr(DICT.entries.TO.OF, 'f_valid') # no dummy available + def test_opt_multiple_identical_dicts(self): def f(n): s = "x" * n From arigo at codespeak.net Wed Feb 20 16:47:56 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 20 Feb 2008 16:47:56 +0100 (CET) Subject: [pypy-svn] r51696 - pypy/branch/gc-in-genc/pypy/rpython/memory/gctransform Message-ID: <20080220154756.E63B816846C@codespeak.net> Author: arigo Date: Wed Feb 20 16:47:53 2008 New Revision: 51696 Modified: pypy/branch/gc-in-genc/pypy/rpython/memory/gctransform/transform.py Log: One more test passes. Modified: pypy/branch/gc-in-genc/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/gc-in-genc/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/gc-in-genc/pypy/rpython/memory/gctransform/transform.py Wed Feb 20 16:47:53 2008 @@ -648,10 +648,12 @@ and FIELD._gckind == 'gc'): FIELD = self.get_raw_type_for_gc_type(FIELD) rawfields.append((name, FIELD)) + rawname = T._name else: rawfields.append(('array', lltype.Array(T.OF))) + rawname = 'gcarray' kwds = {'hints': {'raw_for_gc': True}} - RAWT = lltype.Struct(T._name, *rawfields, **kwds) + RAWT = lltype.Struct(rawname, *rawfields, **kwds) else: raise TypeError("not supported by %s: %r" % ( self.__class__.__name__, T)) @@ -678,7 +680,11 @@ # allocate the equivalent raw structure RAWT = self.get_raw_type_for_gc_type(T) if T._is_varsize(): - length = container.getlength() + if isinstance(T, lltype.Struct): + array = container._getattr(T._arrayfld) + else: + array = container + length = array.getlength() else: length = None result = lltype.malloc(RAWT, length, immortal=True)._as_obj() From cfbolz at codespeak.net Wed Feb 20 21:03:14 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 20 Feb 2008 21:03:14 +0100 (CET) Subject: [pypy-svn] r51701 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080220200314.ED9BA1684F8@codespeak.net> Author: cfbolz Date: Wed Feb 20 21:03:13 2008 New Revision: 51701 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: factor out hannotate function to use from upcoming portal tests Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Wed Feb 20 21:03:13 2008 @@ -34,6 +34,37 @@ novirtualcontainer=True, entrypoint_returns_red=False) + +def hannotate(func, values, policy=None, inline=None, backendoptimize=False, + portal=None, type_system="lltype"): + # build the normal ll graphs for ll_function + t = TranslationContext() + a = t.buildannotator() + argtypes = getargtypes(a, values) + a.build_types(func, argtypes) + rtyper = t.buildrtyper(type_system = type_system) + rtyper.specialize() + if inline: + auto_inlining(t, threshold=inline) + if backendoptimize: + from pypy.translator.backendopt.all import backend_optimizations + backend_optimizations(t, inline_threshold=inline or 0) + if portal is None: + portal = func + + if hasattr(policy, "seetranslator"): + policy.seetranslator(t) + graph1 = graphof(t, portal) + # build hint annotator types + hannotator = HintAnnotator(base_translator=t, policy=policy) + hs = hannotator.build_types(graph1, [SomeLLAbstractConstant(v.concretetype, + {OriginFlags(): True}) + for v in graph1.getargs()]) + hannotator.simplify() + if conftest.option.view: + hannotator.translator.view() + return hs, hannotator, rtyper + class AbstractInterpretationTest(object): RGenOp = LLRGenOp @@ -48,6 +79,7 @@ del cls._cache del cls._cache_order + def serialize(self, func, values, policy=None, inline=None, backendoptimize=False): key = func, backendoptimize @@ -63,33 +95,14 @@ if len(self._cache_order) >= 3: del self._cache[self._cache_order.pop(0)] # build the normal ll graphs for ll_function - t = TranslationContext() - a = t.buildannotator() - argtypes = getargtypes(a, values) - a.build_types(func, argtypes) - rtyper = t.buildrtyper(type_system = self.type_system) - rtyper.specialize() - self.rtyper = rtyper - if inline: - from pypy.translator.backendopt.inline import auto_inlining - auto_inlining(t, threshold=inline) - if backendoptimize: - from pypy.translator.backendopt.all import backend_optimizations - backend_optimizations(t) - graph1 = graphof(t, func) - - # build hint annotator types if policy is None: policy = P_OOPSPEC_NOVIRTUAL - hannotator = HintAnnotator(base_translator=t, policy=policy) - hs = hannotator.build_types(graph1, [SomeLLAbstractConstant(v.concretetype, - {OriginFlags(): True}) - for v in graph1.getargs()]) - hannotator.simplify() - t = hannotator.translator + hs, hannotator, rtyper = hannotate(func, values, policy, inline, + backendoptimize, + type_system=self.type_system) + self.rtyper = rtyper self.hannotator = hannotator - if conftest.option.view: - t.view() + t = hannotator.translator graph2 = graphof(t, func) self.graph = graph2 writer = BytecodeWriter(t, hannotator, self.RGenOp) From cfbolz at codespeak.net Wed Feb 20 22:17:04 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 20 Feb 2008 22:17:04 +0100 (CET) Subject: [pypy-svn] r51707 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080220211704.2A64216850E@codespeak.net> Author: cfbolz Date: Wed Feb 20 22:17:03 2008 New Revision: 51707 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vdict.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vlist.py Log: cleaning up test infrastructure a bit Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Wed Feb 20 22:17:03 2008 @@ -18,6 +18,10 @@ from pypy import conftest P_NOVIRTUAL = HintAnnotatorPolicy(novirtualcontainer=True) +P_OOPSPEC = HintAnnotatorPolicy(novirtualcontainer=True, oopspec=True) +P_OOPSPEC_NOVIRTUAL = HintAnnotatorPolicy(oopspec=True, + novirtualcontainer=True, + entrypoint_returns_red=False) def getargtypes(annotator, values): return [annotation(annotator, x) for x in values] @@ -30,9 +34,6 @@ t = annmodel.lltype_to_annotation(T) return a.typeannotation(t) -P_OOPSPEC_NOVIRTUAL = HintAnnotatorPolicy(oopspec=True, - novirtualcontainer=True, - entrypoint_returns_red=False) def hannotate(func, values, policy=None, inline=None, backendoptimize=False, @@ -65,12 +66,11 @@ hannotator.translator.view() return hs, hannotator, rtyper -class AbstractInterpretationTest(object): +class InterpretationTest(object): RGenOp = LLRGenOp def setup_class(cls): - from pypy.jit.timeshifter.test.conftest import option cls.on_llgraph = cls.RGenOp is LLRGenOp cls._cache = {} cls._cache_order = [] @@ -80,38 +80,31 @@ del cls._cache_order - def serialize(self, func, values, policy=None, - inline=None, backendoptimize=False): - key = func, backendoptimize - try: - cache, argtypes = self._cache[key] - except KeyError: - pass - else: - self.__dict__.update(cache) - assert argtypes == getargtypes(self.rtyper.annotator, values) - return self.writer, self.jitcode, self.argcolors - + def _serialize(self, func, values, policy=None, + inline=None, backendoptimize=False, + portal=None): if len(self._cache_order) >= 3: del self._cache[self._cache_order.pop(0)] # build the normal ll graphs for ll_function if policy is None: policy = P_OOPSPEC_NOVIRTUAL + if portal is None: + portal = func hs, hannotator, rtyper = hannotate(func, values, policy, inline, - backendoptimize, + backendoptimize, portal, type_system=self.type_system) self.rtyper = rtyper - self.hannotator = hannotator + self.hintannotator = hannotator t = hannotator.translator - graph2 = graphof(t, func) + graph2 = graphof(t, portal) self.graph = graph2 writer = BytecodeWriter(t, hannotator, self.RGenOp) jitcode = writer.make_bytecode(graph2) - rtyper.specialize_more_blocks() + # the bytecode writer can ask for llhelpers about lists and dicts + rtyper.specialize_more_blocks() argcolors = [] # make residual functype - ha = self.hannotator RESTYPE = originalconcretetype(hannotator.binding(graph2.getreturnvar())) ARGS = [] for var in graph2.getargs(): @@ -128,10 +121,26 @@ self.jitcode = jitcode self.argcolors = argcolors + + def serialize(self, func, values, policy=None, + inline=None, backendoptimize=False, + portal=None): + key = func, backendoptimize + try: + cache, argtypes = self._cache[key] + except KeyError: + pass + else: + self.__dict__.update(cache) + assert argtypes == getargtypes(self.rtyper.annotator, values) + return self.writer, self.jitcode, self.argcolors + if len(self._cache_order) >= 3: + del self._cache[self._cache_order.pop(0)] + self._serialize(func, values, policy, inline, backendoptimize, portal) cache = self.__dict__.copy() - self._cache[key] = cache, getargtypes(rtyper.annotator, values) + self._cache[key] = cache, getargtypes(self.rtyper.annotator, values) self._cache_order.append(key) - return writer, jitcode, argcolors + return self.writer, self.jitcode, self.argcolors def interpret(self, ll_function, values, opt_consts=[], *args, **kwds): if hasattr(ll_function, 'convert_arguments'): @@ -191,14 +200,18 @@ res = llinterp.eval_graph(graph, residualargs) return res + def get_residual_graph(self): + return self.residual_graph + def check_insns(self, expected=None, **counts): - self.insns = summary(self.residual_graph) + self.insns = summary(self.get_residual_graph()) if expected is not None: assert self.insns == expected for opname, count in counts.items(): assert self.insns.get(opname, 0) == count -class SimpleTests(AbstractInterpretationTest): + +class SimpleTests(InterpretationTest): def test_simple_fixed(self): py.test.skip("green return") def ll_function(x, y): Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vdict.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vdict.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vdict.py Wed Feb 20 22:17:03 2008 @@ -1,13 +1,10 @@ import py from pypy.jit.hintannotator.policy import HintAnnotatorPolicy -from pypy.jit.rainbow.test.test_interpreter import AbstractInterpretationTest +from pypy.jit.rainbow.test.test_interpreter import InterpretationTest, P_OOPSPEC from pypy.rlib.jit import hint -P_OOPSPEC = HintAnnotatorPolicy(novirtualcontainer = True, - oopspec = True) - -class TestVDict(AbstractInterpretationTest): +class TestVDict(InterpretationTest): type_system = "lltype" def test_vdict(self): Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vlist.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vlist.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_vlist.py Wed Feb 20 22:17:03 2008 @@ -1,12 +1,11 @@ import py from pypy.jit.hintannotator.policy import HintAnnotatorPolicy -from pypy.jit.rainbow.test.test_interpreter import AbstractInterpretationTest +from pypy.jit.rainbow.test.test_interpreter import InterpretationTest, P_OOPSPEC from pypy.rlib.jit import hint -P_OOPSPEC = HintAnnotatorPolicy(novirtualcontainer=True, oopspec=True) -class TestVList(AbstractInterpretationTest): +class TestVList(InterpretationTest): type_system = "lltype" def test_vlist(self): From cfbolz at codespeak.net Wed Feb 20 22:56:57 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 20 Feb 2008 22:56:57 +0100 (CET) Subject: [pypy-svn] r51709 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080220215657.3A0FC16850E@codespeak.net> Author: cfbolz Date: Wed Feb 20 22:56:56 2008 New Revision: 51709 Added: pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py (contents, props changed) pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py (contents, props changed) Log: start with portal tests. Then do the simplest thing that makes the tests pass, which is doing nothing, in this case :-) Added: pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py ============================================================================== --- (empty file) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py Wed Feb 20 22:56:56 2008 @@ -0,0 +1,12 @@ +# graph transformations for transforming the portal graph(s) + +class PortalRewriter(object): + def __init__(self, hintannotator, rtyper, RGenOp): + self.hintannotator = hintannotator + self.rtyper = rtyper + self.RGenOp = RGenOp + + def rewrite(self, origportalgraph, view=False): + self.origportalgraph = origportalgraph + self.view = view + self.readportalgraph = None Added: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py ============================================================================== --- (empty file) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Wed Feb 20 22:56:56 2008 @@ -0,0 +1,149 @@ +import py + +from pypy import conftest +from pypy.translator.translator import graphof +from pypy.jit.timeshifter.test.test_timeshift import TestLLType as TSTestLLType, getargtypes +from pypy.jit.rainbow.portal import PortalRewriter +from pypy.jit.rainbow.test.test_interpreter import P_NOVIRTUAL, StopAtXPolicy +from pypy.jit.rainbow.test.test_interpreter import hannotate, InterpretationTest +from pypy.jit.rainbow.test.test_vlist import P_OOPSPEC +from pypy.rpython.llinterp import LLInterpreter +from pypy.rpython.lltypesystem import lltype +from pypy.objspace.flow.model import summary +from pypy.rlib.jit import hint +from pypy.jit.codegen.llgraph.rgenop import RGenOp as LLRGenOp + +class PortalTest(InterpretationTest): + RGenOp = LLRGenOp + small = True + + def _timeshift_from_portal(self, main, portal, main_args, + inline=None, policy=None, + backendoptimize=False): + # decode the 'values' if they are specified as strings + if hasattr(main, 'convert_arguments'): + assert len(main.convert_arguments) == len(main_args) + main_args = [decoder(value) for decoder, value in zip( + main.convert_arguments, + main_args)] + key = main, portal, inline, policy, backendoptimize + try: + cache, argtypes = self._cache[key] + except KeyError: + pass + else: + self.__dict__.update(cache) + assert argtypes == getargtypes(self.rtyper.annotator, main_args) + return main_args + + self._serialize(main, main_args, portal=portal, + policy=policy, inline=inline, + backendoptimize=backendoptimize) + + t = self.rtyper.annotator.translator + self.maingraph = graphof(t, main) + + # rewire the original portal + + rewriter = PortalRewriter(self.hintannotator, self.rtyper, self.RGenOp) + origportalgraph = graphof(t, portal) + rewriter.rewrite(origportalgraph=origportalgraph, + view = conftest.option.view and self.small) + + if conftest.option.view and self.small: + t.view() + self.readportalgraph = rewriter.readportalgraph + + # Populate the cache + if len(self._cache_order) >= 3: + del self._cache[self._cache_order.pop(0)] + cache = self.__dict__.copy() + self._cache[key] = cache, getargtypes(self.rtyper.annotator, main_args) + self._cache_order.append(key) + return main_args + + + def timeshift_from_portal(self, main, portal, main_args, + inline=None, policy=None, + backendoptimize=False): + main_args = self._timeshift_from_portal(main, portal, main_args, + inline=inline, policy=policy, + backendoptimize=backendoptimize) + self.main_args = main_args + self.main_is_portal = main is portal + llinterp = LLInterpreter(self.rtyper) + res = llinterp.eval_graph(self.maingraph, main_args) + return res + + def get_residual_graph(self): + llinterp = LLInterpreter(self.rtyper) + if self.main_is_portal: + residual_graph = llinterp.eval_graph(self.readportalgraph, + self.main_args)._obj.graph + else: + residual_graphs = llinterp.eval_graph(self.readallportalsgraph, []) + assert residual_graphs.ll_length() == 1 + residual_graph = residual_graphs.ll_getitem_fast(0)._obj.graph + return residual_graph + + def count_direct_calls(self): + residual_graph = self.get_residual_graph() + calls = {} + for block in residual_graph.iterblocks(): + for op in block.operations: + if op.opname == 'direct_call': + graph = getattr(op.args[0].value._obj, 'graph', None) + calls[graph] = calls.get(graph, 0) + 1 + return calls + + +class TestPortal(PortalTest): + type_system = "lltype" + + def test_simple(self): + + def main(code, x): + return evaluate(code, x) + + def evaluate(y, x): + hint(y, concrete=True) + z = y+x + return z + + res = self.timeshift_from_portal(main, evaluate, [3, 2]) + assert res == 5 + + res = self.timeshift_from_portal(main, evaluate, [3, 5]) + assert res == 8 + + res = self.timeshift_from_portal(main, evaluate, [4, 7]) + assert res == 11 + + def test_main_as_portal(self): + def main(x): + return x + + res = self.timeshift_from_portal(main, main, [42]) + assert res == 42 + + def test_multiple_portal_calls(self): + py.test.skip("promote not implemented") + def ll_function(n): + hint(None, global_merge_point=True) + k = n + if k > 5: + k //= 2 + k = hint(k, promote=True) + k *= 17 + return hint(k, variable=True) + + res = self.timeshift_from_portal(ll_function, ll_function, [4], + policy=P_NOVIRTUAL) + assert res == 68 + self.check_insns(int_floordiv=1, int_mul=0) + + res = self.timeshift_from_portal(ll_function, ll_function, [4], + policy=P_NOVIRTUAL) + assert res == 68 + self.check_insns(int_floordiv=1, int_mul=0) + From cfbolz at codespeak.net Wed Feb 20 22:59:16 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 20 Feb 2008 22:59:16 +0100 (CET) Subject: [pypy-svn] r51710 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080220215916.9851916850E@codespeak.net> Author: cfbolz Date: Wed Feb 20 22:59:16 2008 New Revision: 51710 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/ (props changed) pypy/branch/jit-refactoring/pypy/jit/rainbow/test/ (props changed) pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Log: fixeol Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Wed Feb 20 22:59:16 2008 @@ -112,12 +112,15 @@ res = self.timeshift_from_portal(main, evaluate, [3, 2]) assert res == 5 + self.check_insns({"int_add": 1}) res = self.timeshift_from_portal(main, evaluate, [3, 5]) assert res == 8 + self.check_insns({"int_add": 1}) res = self.timeshift_from_portal(main, evaluate, [4, 7]) assert res == 11 + self.check_insns({"int_add": 1}) def test_main_as_portal(self): def main(x): @@ -125,6 +128,7 @@ res = self.timeshift_from_portal(main, main, [42]) assert res == 42 + self.check_insns({}) def test_multiple_portal_calls(self): py.test.skip("promote not implemented") From cfbolz at codespeak.net Thu Feb 21 00:39:05 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 21 Feb 2008 00:39:05 +0100 (CET) Subject: [pypy-svn] r51713 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080220233905.282AE16850B@codespeak.net> Author: cfbolz Date: Thu Feb 21 00:39:04 2008 New Revision: 51713 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: move finish_jitstate to a more commonly accessible place than a test Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Thu Feb 21 00:39:04 2008 @@ -71,6 +71,20 @@ self.bytecode_loop() return self.jitstate + def finish_jitstate(self, graphsigtoken): + jitstate = self.jitstate + exceptiondesc = self.exceptiondesc + returnbox = rtimeshift.getreturnbox(jitstate) + gv_ret = returnbox.getgenvar(jitstate) + builder = jitstate.curbuilder + for virtualizable_box in jitstate.virtualizables: + assert isinstance(virtualizable_box, rvalue.PtrRedBox) + content = virtualizable_box.content + assert isinstance(content, rcontainer.VirtualizableStruct) + content.store_back(jitstate) + exceptiondesc.store_global_excdata(jitstate) + jitstate.curbuilder.finish_and_return(graphsigtoken, gv_ret) + def bytecode_loop(self): while 1: bytecode = self.load_2byte() Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Thu Feb 21 00:39:04 2008 @@ -156,17 +156,6 @@ jitstate = rtimeshift.JITState(builder, None, writer.exceptiondesc.null_exc_type_box, writer.exceptiondesc.null_exc_value_box) - def ll_finish_jitstate(jitstate, exceptiondesc, graphsigtoken): - returnbox = rtimeshift.getreturnbox(jitstate) - gv_ret = returnbox.getgenvar(jitstate) - builder = jitstate.curbuilder - for virtualizable_box in jitstate.virtualizables: - assert isinstance(virtualizable_box, rvalue.PtrRedBox) - content = virtualizable_box.content - assert isinstance(content, rcontainer.VirtualizableStruct) - content.store_back(jitstate) - exceptiondesc.store_global_excdata(jitstate) - jitstate.curbuilder.finish_and_return(graphsigtoken, gv_ret) # build arguments greenargs = [] redargs = [] @@ -188,8 +177,7 @@ red_i += 1 jitstate = writer.interpreter.run(jitstate, jitcode, greenargs, redargs) if jitstate is not None: - ll_finish_jitstate(jitstate, writer.interpreter.exceptiondesc, - sigtoken) + writer.interpreter.finish_jitstate(sigtoken) builder.end() generated = gv_generated.revealconst(lltype.Ptr(self.RESIDUAL_FUNCTYPE)) graph = generated._obj.graph From cfbolz at codespeak.net Thu Feb 21 11:20:54 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 21 Feb 2008 11:20:54 +0100 (CET) Subject: [pypy-svn] r51715 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080221102054.193041684FA@codespeak.net> Author: cfbolz Date: Thu Feb 21 11:20:53 2008 New Revision: 51715 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: attach the RGenOp instance to the interpreter Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Thu Feb 21 11:20:53 2008 @@ -48,13 +48,14 @@ STOP = STOP() class JitInterpreter(object): - def __init__(self, exceptiondesc): + def __init__(self, exceptiondesc, RGenOp): self.exceptiondesc = exceptiondesc self.opcode_implementations = [] self.opcode_descs = [] self.opname_to_index = {} self.jitstate = None self.queue = None + self.rgenop = RGenOp() self._add_implemented_opcodes() def run(self, jitstate, bytecode, greenargs, redargs, @@ -516,8 +517,7 @@ genconst = self.get_greenarg() arg = genconst.revealconst(opdesc.ARGS[i]) args += (arg, ) - rgenop = self.jitstate.curbuilder.rgenop - result = rgenop.genconst(opdesc.llop(*args)) + result = self.rgenop.genconst(opdesc.llop(*args)) self.green_result(result) elif color == "red": if opdesc.nb_args == 1: Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Thu Feb 21 11:20:53 2008 @@ -149,7 +149,7 @@ ll_function.convert_arguments, values)] writer, jitcode, argcolors = self.serialize(ll_function, values, **kwds) - rgenop = writer.RGenOp() + rgenop = writer.interpreter.rgenop sigtoken = rgenop.sigToken(self.RESIDUAL_FUNCTYPE) builder, gv_generated, inputargs_gv = rgenop.newgraph(sigtoken, "generated") builder.start_writing() From arigo at codespeak.net Thu Feb 21 11:35:15 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 11:35:15 +0100 (CET) Subject: [pypy-svn] r51716 - in pypy/branch/unified-rtti/pypy: rpython rpython/lltypesystem rpython/lltypesystem/test rpython/memory/gctransform rpython/test translator/c Message-ID: <20080221103515.6B56E16850A@codespeak.net> Author: arigo Date: Thu Feb 21 11:35:14 2008 New Revision: 51716 Modified: pypy/branch/unified-rtti/pypy/rpython/llinterp.py pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py pypy/branch/unified-rtti/pypy/rpython/lltypesystem/rclass.py pypy/branch/unified-rtti/pypy/rpython/lltypesystem/rtagged.py pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_lltype.py pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_rtagged.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/support.py pypy/branch/unified-rtti/pypy/rpython/test/test_exception.py pypy/branch/unified-rtti/pypy/rpython/test/test_llann.py pypy/branch/unified-rtti/pypy/rpython/test/test_nongc.py pypy/branch/unified-rtti/pypy/rpython/test/test_rlist.py pypy/branch/unified-rtti/pypy/rpython/test/tool.py pypy/branch/unified-rtti/pypy/translator/c/gc.py pypy/branch/unified-rtti/pypy/translator/c/node.py Log: Some general progress. Modified: pypy/branch/unified-rtti/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/llinterp.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/llinterp.py Thu Feb 21 11:35:14 2008 @@ -507,7 +507,7 @@ if ll_exc is None: raise LLFatalError(msg) else: - ll_exc_type = lltype.cast_pointer(rclass.OBJECTPTR, ll_exc).typeptr + ll_exc_type = rclass.ll_type(ll_exc) raise LLFatalError(msg, LLException(ll_exc_type, ll_exc)) def op_debug_llinterpcall(self, pythonfunction, *args_ll): Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py Thu Feb 21 11:35:14 2008 @@ -1805,7 +1805,7 @@ def getRuntimeTypeInfo(TYPE, cache=None): """Return the runtime_type_info attached to the GcStruct TYPE, - as a Ptr(RuntimeTypeInfo). This raises TypeError if the TYPE has + as a Ptr(RuntimeTypeInfo). This returns None if the TYPE has no runtime_type_info, unless 'cache' is specified; in that case, TYPE can be any GC type and a runtime_type_info is created for it if it has none and stored in the cache to avoid mutating @@ -1814,7 +1814,7 @@ if isinstance(TYPE, GcStruct) and hasattr(TYPE, '_rtti'): return TYPE._rtti._as_ptr() if cache is None: - raise TypeError("%r has no runtime_type_info" % (TYPE,)) + return None try: return cache[TYPE] except KeyError: Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/rclass.py Thu Feb 21 11:35:14 2008 @@ -14,7 +14,7 @@ cast_pointer, cast_ptr_to_int, castable, nullptr, \ typeOf, Array, Char, Void, \ FuncType, Bool, Signed, functionptr, FuncType, PyObject -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, rffi from pypy.rpython.robject import PyObjRepr, pyobj_repr from pypy.rpython.extregistry import ExtRegistryEntry from pypy.annotation import model as annmodel @@ -29,7 +29,7 @@ # RuntimeTypeInfo rtti; //GC-specific information # Signed subclassrange_min; //this is also the id of the class itself # Signed subclassrange_max; -# array { char } * name; +# char * name; # struct object * instantiate(); # } # @@ -66,13 +66,17 @@ ('rtti', lltype.RuntimeTypeInfo), ('subclassrange_min', Signed), ('subclassrange_max', Signed), - ('name', Ptr(Array(Char))), + ('name', rffi.CCHARP), ('instantiate', Ptr(FuncType([], OBJECTPTR))), hints = {'immutable': True}) CLASSTYPE = Ptr(OBJECT_VTABLE) +root_object_vtable = malloc(OBJECT_VTABLE, immortal=True, zero=True) +root_object_vtable.subclassrange_min = 0 +root_object_vtable.subclassrange_max = sys.maxint +root_object_vtable.name = rffi.str2charp("object") + OBJECT.become(GcStruct('object', - runtime_type_info = malloc(OBJECT_VTABLE, - immortal=True), + runtime_type_info = root_object_vtable, adtmeths = {'gettypeptr': ll_gettypeptr})) # non-gc case @@ -94,6 +98,8 @@ if classdef is None: # 'object' root type self.vtable_type = OBJECT_VTABLE + self.vtable = root_object_vtable + self.vtable_filled = True else: self.vtable_type = lltype.ForwardReference() self.lowleveltype = Ptr(self.vtable_type) @@ -139,11 +145,11 @@ *llfields, **kwds) self.vtable_type.become(vtable_type) allmethods.update(self.rbase.allmethods) + self.vtable = malloc(self.vtable_type, immortal=True) + self.vtable_filled = False self.clsfields = clsfields self.pbcfields = pbcfields self.allmethods = allmethods - self.vtable = malloc(self.vtable_type, immortal=True) - self.vtable_filled = False def getvtable(self): """Return a ptr to the vtable of this type.""" @@ -158,22 +164,12 @@ given subclass.""" if self.classdef is None: # initialize the 'subclassrange_*' and 'name' fields - if rsubcls.classdef is not None: - vtable.subclassrange_min = rsubcls.classdef.minid - vtable.subclassrange_max = rsubcls.classdef.maxid - else: #for the root class - vtable.subclassrange_min = 0 - vtable.subclassrange_max = sys.maxint + assert rsubcls.classdef is not None + vtable.subclassrange_min = rsubcls.classdef.minid + vtable.subclassrange_max = rsubcls.classdef.maxid rinstance = getinstancerepr(self.rtyper, rsubcls.classdef) rinstance.setup() - if rsubcls.classdef is None: - name = 'object' - else: - name = rsubcls.classdef.shortname - vtable.name = malloc(Array(Char), len(name)+1, immortal=True) - for i in range(len(name)): - vtable.name[i] = name[i] - vtable.name[len(name)] = '\x00' + vtable.name = rffi.str2charp(rsubcls.classdef.shortname) if hasattr(rsubcls.classdef, 'my_instantiate_graph'): graph = rsubcls.classdef.my_instantiate_graph vtable.instantiate = self.rtyper.getcallable(graph) @@ -488,9 +484,17 @@ ctype = inputconst(Void, self.object_type) cflags = inputconst(Void, flags) vlist = [ctype, cflags] - self.rclass.getvtable() # force it to be filled + vtable = self.rclass.getvtable() # don't remove this line, + # it forces the vtable to be filled vptr = llops.genop('malloc', vlist, resulttype = Ptr(self.object_type)) + if self.gcflavor != 'gc': + # fill in 'typeptr' + vbase = llops.genop('cast_pointer', [vptr], + resulttype=NONGCOBJECTPTR) + ctypeptr = inputconst(Void, 'typeptr') + cvalue = inputconst(CLASSTYPE, vtable) + llops.genop('setfield', [vbase, ctypeptr, cvalue]) # initialize instance attributes from their defaults from the class if self.classdef is not None: flds = self.allinstancefields.keys() @@ -562,13 +566,15 @@ BASEPTR = Ptr(self.get_base_object_type()) instance = cast_pointer(BASEPTR, i) vtable = instance.gettypeptr() + nameLen = 0 + while vtable.name[nameLen] != '\x00': + nameLen += 1 + nameString = rstr.mallocstr(nameLen) + j = 0 + while j < nameLen: + nameString.chars[j] = vtable.name[j] + j += 1 uid = r_uint(cast_ptr_to_int(i)) - nameLen = len(vtable.name) - nameString = rstr.mallocstr(nameLen-1) - i = 0 - while i < nameLen - 1: - nameString.chars[i] = vtable.name[i] - i += 1 res = rstr.instance_str_prefix res = rstr.ll_strconcat(res, nameString) res = rstr.ll_strconcat(res, rstr.instance_str_infix) Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/rtagged.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/rtagged.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/rtagged.py Thu Feb 21 11:35:14 2008 @@ -17,7 +17,6 @@ def _setup_repr(self): InstanceRepr._setup_repr(self) flds = self.allinstancefields.keys() - flds.remove('__class__') if self.is_parent: if flds: raise TyperError("%r is a base class of an UnboxedValue," @@ -67,7 +66,7 @@ cunboxedcls = inputconst(CLASSTYPE, unboxedclass_repr.getvtable()) if self.is_parent: # If the lltype of vinst shows that it cannot be a tagged value, - # we can directly read the typeptr. Otherwise, call a helper that + # we can fall back to gettypeptr(). Otherwise, call a helper that # checks if the tag bit is set in the pointer. unboxedinstance_repr = getinstancerepr(self.rtyper, self.unboxedclassdef) @@ -75,18 +74,14 @@ lltype.castable(unboxedinstance_repr.lowleveltype, vinst.concretetype) except lltype.InvalidCast: - can_be_tagged = False + # cannot be tagged + return self.gettypeptr(vinst, llops) else: - can_be_tagged = True - vinst = llops.genop('cast_pointer', [vinst], - resulttype=self.common_repr()) - if can_be_tagged: + # normal case: must check + vinst = llops.genop('cast_pointer', [vinst], + resulttype=self.common_repr()) return llops.gendirectcall(ll_unboxed_getclass, vinst, cunboxedcls) - else: - ctypeptr = inputconst(lltype.Void, 'typeptr') - return llops.genop('getfield', [vinst, ctypeptr], - resulttype = CLASSTYPE) else: return cunboxedcls @@ -145,7 +140,7 @@ if lltype.cast_ptr_to_int(instance) & 1: return class_if_unboxed else: - return instance.typeptr + return instance.gettypeptr() def ll_unboxed_isinstance_const(obj, minid, maxid, answer_if_unboxed): if not obj: @@ -153,4 +148,4 @@ if lltype.cast_ptr_to_int(obj) & 1: return answer_if_unboxed else: - return ll_issubclass_const(obj.typeptr, minid, maxid) + return ll_issubclass_const(obj.gettypeptr(), minid, maxid) Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_lltype.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_lltype.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_lltype.py Thu Feb 21 11:35:14 2008 @@ -385,7 +385,7 @@ S1bis = GcStruct('S1', ('x', Signed)) assert S1bis != S1 # attached runtime type info distinguishes them - py.test.raises(TypeError, getRuntimeTypeInfo, S1bis) + assert getRuntimeTypeInfo(S1bis) is None cache = {} assert getRuntimeTypeInfo(S1, cache=cache) == t1.base Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_rtagged.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_rtagged.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_rtagged.py Thu Feb 21 11:35:14 2008 @@ -210,7 +210,9 @@ t = interp.typer.annotator.translator ggraph = graphof(t, g) - assert summary(ggraph) == {'cast_pointer': 2, 'getfield': 2} + s = summary(ggraph) + assert s == {'gc_runtime_type_info': 1, 'getfield': 1, + 'cast_pointer': s['cast_pointer']} # any number of them is ok res = interp.eval_graph(graph, [-1000]) assert res == 68 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py Thu Feb 21 11:35:14 2008 @@ -1,6 +1,6 @@ from pypy.rpython.memory.gctransform.transform import GCTransformer, mallocHelpers from pypy.rpython.memory.gctransform.support import type_contains_pyobjs, \ - get_rtti, _static_deallocator_body_for_type, LLTransformerOp, ll_call_destructor + _static_deallocator_body_for_type, LLTransformerOp, ll_call_destructor from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython import rmodel from pypy.rlib.rarithmetic import ovfcheck Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py Thu Feb 21 11:35:14 2008 @@ -1,6 +1,6 @@ from pypy.rpython.memory.gctransform.transform import GCTransformer from pypy.rpython.memory.gctransform.support import find_gc_ptrs_in_type, \ - get_rtti, ll_call_destructor, type_contains_pyobjs, var_ispyobj + ll_call_destructor, type_contains_pyobjs, var_ispyobj from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython import rmodel from pypy.rpython.memory import gctypelayout Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py Thu Feb 21 11:35:14 2008 @@ -1,7 +1,7 @@ import py from pypy.rpython.memory.gctransform.transform import GCTransformer, mallocHelpers from pypy.rpython.memory.gctransform.support import find_gc_ptrs_in_type, \ - get_rtti, _static_deallocator_body_for_type, LLTransformerOp, ll_call_destructor + _static_deallocator_body_for_type, LLTransformerOp, ll_call_destructor from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem.lloperation import llop from pypy.translator.backendopt.support import var_needsgc @@ -31,10 +31,14 @@ ## print ' '*i, a, repr(b)[:100-i-len(a)], id(b) ADDRESS_VOID_FUNC = lltype.FuncType([llmemory.Address], lltype.Void) +RTTIPTR = lltype.Ptr(lltype.RuntimeTypeInfo) class RefcountingGCTransformer(GCTransformer): - HDR = lltype.Struct("header", ("refcount", lltype.Signed)) + # xxx not all objects need a typeptr, but refcounting is a bit + # deprecated now, being not efficient at all + HDR = lltype.Struct("header", ("refcount", lltype.Signed), + ("typeptr", RTTIPTR)) def __init__(self, translator): super(RefcountingGCTransformer, self).__init__(translator, inline=True) @@ -51,21 +55,13 @@ if adr: gcheader = llmemory.cast_adr_to_ptr(adr - gc_header_offset, HDRPTR) gcheader.refcount = gcheader.refcount + 1 - def ll_decref(adr, dealloc): + def ll_decref(adr): if adr: gcheader = llmemory.cast_adr_to_ptr(adr - gc_header_offset, HDRPTR) refcount = gcheader.refcount - 1 gcheader.refcount = refcount if refcount == 0: - dealloc(adr) - def ll_decref_simple(adr): - if adr: - gcheader = llmemory.cast_adr_to_ptr(adr - gc_header_offset, HDRPTR) - refcount = gcheader.refcount - 1 - if refcount == 0: - llop.gc_free(lltype.Void, adr) - else: - gcheader.refcount = refcount + gcheader.typeptr.xxx(adr) def ll_no_pointer_dealloc(adr): llop.gc_free(lltype.Void, adr) @@ -259,7 +255,7 @@ return self.dynamic_deallocator_funcptrs[TYPE] #print_call_chain(self) - rtti = get_rtti(TYPE) + rtti = lltype.getRuntimeTypeInfo(TYPE) if rtti is None: p = self.static_deallocation_funcptr_for_type(TYPE) self.dynamic_deallocator_funcptrs[TYPE] = p Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/support.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/support.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/support.py Thu Feb 21 11:35:14 2008 @@ -47,14 +47,6 @@ else: return False -def get_rtti(TYPE): - if isinstance(TYPE, lltype.RttiStruct): - try: - return lltype.getRuntimeTypeInfo(TYPE) - except ValueError: - pass - return None - def _static_deallocator_body_for_type(v, TYPE, depth=1): if isinstance(TYPE, lltype.Array): inner = list(_static_deallocator_body_for_type('v_%i'%depth, TYPE.OF, depth+1)) Modified: pypy/branch/unified-rtti/pypy/rpython/test/test_exception.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/test/test_exception.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/test/test_exception.py Thu Feb 21 11:35:14 2008 @@ -40,21 +40,22 @@ excdata = t.rtyper.getexceptiondata() getcdef = t.annotator.bookkeeper.getuniqueclassdef + class_reprs = t.rtyper.class_reprs #t.view() ovferr_inst = excdata.fn_pyexcclass2exc(pyobjectptr(OverflowError)) classdef = getcdef(OverflowError) - assert ovferr_inst.typeptr == t.rtyper.class_reprs[classdef].getvtable() + assert ovferr_inst.gettypeptr() == class_reprs[classdef].getvtable() taberr_inst = excdata.fn_pyexcclass2exc(pyobjectptr(TabError)) classdef = getcdef(StandardError) # most precise class seen - assert taberr_inst.typeptr == t.rtyper.class_reprs[classdef].getvtable() + assert taberr_inst.gettypeptr() == class_reprs[classdef].getvtable() myerr_inst = excdata.fn_pyexcclass2exc(pyobjectptr(MyException)) - assert myerr_inst.typeptr == t.rtyper.class_reprs[None].getvtable() + assert myerr_inst.gettypeptr() == class_reprs[None].getvtable() strgerr_inst = excdata.fn_pyexcclass2exc(pyobjectptr(MyStrangeException)) - assert strgerr_inst.typeptr == t.rtyper.class_reprs[None].getvtable() + assert strgerr_inst.gettypeptr() == class_reprs[None].getvtable() class BaseTestException(BaseRtypingTest): def test_exception_with_arg(self): Modified: pypy/branch/unified-rtti/pypy/rpython/test/test_llann.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/test/test_llann.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/test/test_llann.py Thu Feb 21 11:35:14 2008 @@ -341,18 +341,18 @@ assert s.items[1].const == 3 def test_getRuntimeTypeInfo(self): - S = GcStruct('s', ('x', Signed)) - attachRuntimeTypeInfo(S) + rtti = malloc(RuntimeTypeInfo, immortal=True) + S = GcStruct('s', ('x', Signed), runtime_type_info = rtti) def llf(): return getRuntimeTypeInfo(S) s = self.annotate(llf, []) assert isinstance(s, annmodel.SomePtr) assert s.ll_ptrtype == Ptr(RuntimeTypeInfo) - assert s.const == getRuntimeTypeInfo(S) + assert s.const == getRuntimeTypeInfo(S) == rtti def test_runtime_type_info(self): - S = GcStruct('s', ('x', Signed)) - attachRuntimeTypeInfo(S) + rtti = malloc(RuntimeTypeInfo, immortal=True) + S = GcStruct('s', ('x', Signed), runtime_type_info = rtti) def llf(p): return runtime_type_info(p) s = self.annotate(llf, [annmodel.SomePtr(Ptr(S))]) Modified: pypy/branch/unified-rtti/pypy/rpython/test/test_nongc.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/test/test_nongc.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/test/test_nongc.py Thu Feb 21 11:35:14 2008 @@ -66,17 +66,15 @@ assert (Bdef, 'raw') in rtyper.instance_reprs assert (Bdef, 'gc') not in rtyper.instance_reprs -def test_unsupported(): - class A: +def test_nongc_str(): + class AFooBar: _alloc_flavor_ = "raw" def f(): - return str(A()) - a = RPythonAnnotator() - #does not raise: - s = a.build_types(f, []) - assert s.knowntype == str - rtyper = RPythonTyper(a) - py.test.raises(TypeError,rtyper.specialize) # results in an invalid cast + return str(AFooBar()) + res = interpret(f, []) + res = ''.join(res.chars) + print res + assert 'AFooBar' in res def test_isinstance(): class A: Modified: pypy/branch/unified-rtti/pypy/rpython/test/test_rlist.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/test/test_rlist.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/test/test_rlist.py Thu Feb 21 11:35:14 2008 @@ -496,8 +496,6 @@ return None res = self.interpret(f, [1]) assert self.class_name(res) == 'A' - #''.join(res.super.typeptr.name) == 'A\00' - def test_reverse(self): def dummyfn(): Modified: pypy/branch/unified-rtti/pypy/rpython/test/tool.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/test/tool.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/test/tool.py Thu Feb 21 11:35:14 2008 @@ -73,7 +73,7 @@ return fnptr._obj._callable def class_name(self, value): - return "".join(value.super.typeptr.name)[:-1] + return "".join(value.super.gettypeptr().name)[:-1] def read_attr(self, value, attr_name): value = value._obj Modified: pypy/branch/unified-rtti/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/gc.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/gc.py Thu Feb 21 11:35:14 2008 @@ -2,7 +2,7 @@ from pypy.translator.c.support import cdecl from pypy.translator.c.node import ContainerNode from pypy.rpython.lltypesystem.lltype import \ - typeOf, Ptr, ContainerType, RttiStruct, \ + typeOf, Ptr, ContainerType, \ RuntimeTypeInfo, getRuntimeTypeInfo, top_container from pypy.rpython.memory.gctransform import \ refcounting, boehm, framework, stacklessframework, llvmgcroot, asmgcroot @@ -140,7 +140,7 @@ def __init__(self, db, T, obj): assert T == RuntimeTypeInfo - assert isinstance(obj.about, RttiStruct) + assert isinstance(obj.about, GcStruct) self.db = db self.T = T self.obj = obj @@ -221,7 +221,7 @@ def __init__(self, db, T, obj): assert T == RuntimeTypeInfo - assert isinstance(obj.about, RttiStruct) + assert isinstance(obj.about, GcStruct) self.db = db self.T = T self.obj = obj Modified: pypy/branch/unified-rtti/pypy/translator/c/node.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/node.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/node.py Thu Feb 21 11:35:14 2008 @@ -1,7 +1,7 @@ from __future__ import generators from pypy.rpython.lltypesystem.lltype import \ Struct, Array, FixedSizeArray, FuncType, PyObjectType, typeOf, \ - GcStruct, GcArray, RttiStruct, ContainerType, \ + GcStruct, GcArray, ContainerType, \ parentlink, Ptr, PyObject, Void, OpaqueType, Float, \ RuntimeTypeInfo, getRuntimeTypeInfo, Char, _subarray from pypy.rpython.lltypesystem import llmemory @@ -95,7 +95,8 @@ self.gcinfo = None # unless overwritten below rtti = None STRUCT = self.STRUCT - if isinstance(STRUCT, RttiStruct): + if isinstance(STRUCT, GcStruct): + XXX try: rtti = getRuntimeTypeInfo(STRUCT) except ValueError: From cfbolz at codespeak.net Thu Feb 21 11:42:17 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 21 Feb 2008 11:42:17 +0100 (CET) Subject: [pypy-svn] r51717 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080221104217.B34E4168510@codespeak.net> Author: cfbolz Date: Thu Feb 21 11:42:17 2008 New Revision: 51717 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: use genconst instead of building global constants Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Thu Feb 21 11:42:17 2008 @@ -20,7 +20,7 @@ type_system = self.rtyper.type_system.name self.exceptiondesc = exception.ExceptionDesc( RGenOp, etrafo, type_system, False) - self.interpreter = JitInterpreter(self.exceptiondesc) + self.interpreter = JitInterpreter(self.exceptiondesc, RGenOp) self.RGenOp = RGenOp self.current_block = None self.raise_analyzer = hannotator.exceptiontransformer.raise_analyzer Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Thu Feb 21 11:42:17 2008 @@ -163,7 +163,7 @@ red_i = 0 for i, (color, ll_val) in enumerate(zip(argcolors, values)): if color == "green": - greenargs.append(writer.RGenOp.constPrebuiltGlobal(ll_val)) + greenargs.append(rgenop.genconst(ll_val)) else: TYPE = lltype.typeOf(ll_val) kind = rgenop.kindToken(TYPE) From cfbolz at codespeak.net Thu Feb 21 12:19:43 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 21 Feb 2008 12:19:43 +0100 (CET) Subject: [pypy-svn] r51718 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080221111943.F1088168509@codespeak.net> Author: cfbolz Date: Thu Feb 21 12:19:42 2008 New Revision: 51718 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: add a fresh_jitstate method to the interpreter Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Thu Feb 21 12:19:42 2008 @@ -72,6 +72,11 @@ self.bytecode_loop() return self.jitstate + def fresh_jitstate(self, builder): + return rtimeshift.JITState(builder, None, + self.exceptiondesc.null_exc_type_box, + self.exceptiondesc.null_exc_value_box) + def finish_jitstate(self, graphsigtoken): jitstate = self.jitstate exceptiondesc = self.exceptiondesc Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Thu Feb 21 12:19:42 2008 @@ -153,9 +153,8 @@ sigtoken = rgenop.sigToken(self.RESIDUAL_FUNCTYPE) builder, gv_generated, inputargs_gv = rgenop.newgraph(sigtoken, "generated") builder.start_writing() - jitstate = rtimeshift.JITState(builder, None, - writer.exceptiondesc.null_exc_type_box, - writer.exceptiondesc.null_exc_value_box) + jitstate = writer.interpreter.fresh_jitstate(builder) + # build arguments greenargs = [] redargs = [] From cfbolz at codespeak.net Thu Feb 21 14:11:00 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 21 Feb 2008 14:11:00 +0100 (CET) Subject: [pypy-svn] r51719 - pypy/dist/pypy/doc/discussion Message-ID: <20080221131100.0AC1F16850B@codespeak.net> Author: cfbolz Date: Thu Feb 21 14:10:59 2008 New Revision: 51719 Modified: pypy/dist/pypy/doc/discussion/ctypes_todo.txt Log: add a todo item about _fields_ being tuples Modified: pypy/dist/pypy/doc/discussion/ctypes_todo.txt ============================================================================== --- pypy/dist/pypy/doc/discussion/ctypes_todo.txt (original) +++ pypy/dist/pypy/doc/discussion/ctypes_todo.txt Thu Feb 21 14:10:59 2008 @@ -8,6 +8,8 @@ are missing or not comprehensive. Some tests are skipped because I didn't understand the details. + - _fields_ can be tuples too as well as lists + - restype being a function is not working. - there are features, which we don't support like buffer() and From cfbolz at codespeak.net Thu Feb 21 14:32:18 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 21 Feb 2008 14:32:18 +0100 (CET) Subject: [pypy-svn] r51720 - in pypy/dist/pypy/lib: _ctypes app_test/ctypes Message-ID: <20080221133218.5A0A516842C@codespeak.net> Author: cfbolz Date: Thu Feb 21 14:32:18 2008 New Revision: 51720 Modified: pypy/dist/pypy/lib/_ctypes/structure.py pypy/dist/pypy/lib/app_test/ctypes/test_structures.py Log: allow tuples as _fields_ Modified: pypy/dist/pypy/lib/_ctypes/structure.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/structure.py (original) +++ pypy/dist/pypy/lib/_ctypes/structure.py Thu Feb 21 14:32:18 2008 @@ -44,6 +44,8 @@ _CDataMeta.__setattr__(self, name, value) def names_and_fields(_fields_, superclass, zero_offset=False, anon=None): + if isinstance(_fields_, tuple): + _fields_ = list(_fields_) for _, tp in _fields_: if not isinstance(tp, _CDataMeta): raise TypeError("Expected CData subclass, got %s" % (tp,)) Modified: pypy/dist/pypy/lib/app_test/ctypes/test_structures.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_structures.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_structures.py Thu Feb 21 14:32:18 2008 @@ -387,6 +387,16 @@ s.p = None assert s.x == 12345678 + def test_fields_is_a_tuple(self): + class Person(Structure): + _fields_ = (("name", c_char*6), + ("age", c_int)) + + # short enough + p = Person("123456", 6) + assert p.name == "123456" + assert p.age == 6 + class TestRecursiveStructure(BaseCTypesTestChecker): def test_contains_itself(self): class Recursive(Structure): From cfbolz at codespeak.net Thu Feb 21 14:40:06 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 21 Feb 2008 14:40:06 +0100 (CET) Subject: [pypy-svn] r51721 - pypy/dist/pypy/lib/app_test/ctypes Message-ID: <20080221134006.E94C416842C@codespeak.net> Author: cfbolz Date: Thu Feb 21 14:40:06 2008 New Revision: 51721 Modified: pypy/dist/pypy/lib/app_test/ctypes/test_structures.py Log: move the tuple test. wrote another one, which passes but leaks memory :-( Modified: pypy/dist/pypy/lib/app_test/ctypes/test_structures.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_structures.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_structures.py Thu Feb 21 14:40:06 2008 @@ -316,6 +316,7 @@ def get_except(self, func, *args): + # XXX remove this, py.test.raises returns a nice inspectable object try: func(*args) except Exception, detail: @@ -346,6 +347,31 @@ assert "from_address" in dir(type(Structure)) assert "in_dll" in dir(type(Structure)) + def test_fields_is_a_tuple(self): + class Person(Structure): + _fields_ = (("name", c_char*6), + ("age", c_int)) + + # short enough + p = Person("123456", 6) + assert p.name == "123456" + assert p.age == 6 + + def test_subclassing_field_is_a_tuple(self): + py.test.skip("this leaks") + class Person(Structure): + _fields_ = (("name", c_char*6), + ("age", c_int)) + class PersonWithIncome(Person): + _fields_ = [("income", c_int)] + + # short enough + p = PersonWithIncome("123456", 6, 5) + assert p.name == "123456" + assert p.age == 6 + assert p.income == 5 + + class TestPointerMember(BaseCTypesTestChecker): def test_1(self): @@ -387,15 +413,6 @@ s.p = None assert s.x == 12345678 - def test_fields_is_a_tuple(self): - class Person(Structure): - _fields_ = (("name", c_char*6), - ("age", c_int)) - - # short enough - p = Person("123456", 6) - assert p.name == "123456" - assert p.age == 6 class TestRecursiveStructure(BaseCTypesTestChecker): def test_contains_itself(self): From arigo at codespeak.net Thu Feb 21 14:58:44 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 14:58:44 +0100 (CET) Subject: [pypy-svn] r51722 - in pypy/branch/unified-rtti/pypy: rpython rpython/lltypesystem rpython/memory rpython/memory/gctransform translator/c translator/c/src Message-ID: <20080221135844.2908C1683BF@codespeak.net> Author: arigo Date: Thu Feb 21 14:58:42 2008 New Revision: 51722 Modified: pypy/branch/unified-rtti/pypy/rpython/llinterp.py pypy/branch/unified-rtti/pypy/rpython/lltypesystem/llmemory.py pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lloperation.py pypy/branch/unified-rtti/pypy/rpython/memory/gcheader.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/unified-rtti/pypy/translator/c/gc.py pypy/branch/unified-rtti/pypy/translator/c/src/mem.h Log: Progress in porting the refcounting transformer to the new model. Deleted a number of lines in refcounting.py, so I guess it's good. Modified: pypy/branch/unified-rtti/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/llinterp.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/llinterp.py Thu Feb 21 14:58:42 2008 @@ -753,20 +753,12 @@ def op_gc__collect(self): self.heap.collect() - def op_gc_free(self, addr): - # what can you do? - pass - #raise NotImplementedError("gc_free") - def op_gc_fetch_exception(self): raise NotImplementedError("gc_fetch_exception") def op_gc_restore_exception(self, exc): raise NotImplementedError("gc_restore_exception") - def op_gc_deallocate(self, TYPE, addr): - raise NotImplementedError("gc_deallocate") - def op_gc_push_alive_pyobj(self, pyobj): raise NotImplementedError("gc_push_alive_pyobj") Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/llmemory.py Thu Feb 21 14:58:42 2008 @@ -263,6 +263,15 @@ class GCHeaderOffset(AddressOffset): + """Hack: this 'offset' is really 0, but it is used symbolically to + go from a GcStruct instance to its header (by subtraction, as if the + header was before the GcStruct). The GCHeaderOffset internally + holds a reference to a GCHeaderBuilder, which maps between GcStructs + and their headers. The point of this is to *not* have a global + mapping from GcStructs to their headers -- indeed, the same prebuilt + GcStruct instance could be reused by several translations, each with + their own different GCs. + """ def __init__(self, gcheaderbuilder): self.gcheaderbuilder = gcheaderbuilder @@ -306,6 +315,30 @@ assert isinstance(rest[0], GCHeaderOffset) return rest[1]._raw_malloc(rest[2:], zero=zero) +class GCTypeInfoOffset(AddressOffset): + """Hack: this 'offset' is really 0, but it is used symbolically to + go from a Ptr(RuntimeTypeInfo) to the real GC-specific structure + that contains the information. The GCTypeInfoOffset internally + holds a reference to a GCHeaderBuilder, which maps between + RuntimeTypeInfos and their GC-specific equivalents. The point of + this is to *not* have a global mapping -- indeed, the same + RuntimeTypeInfo could be seen in several translations, each with + their own different GCs. + """ + def __init__(self, gcheaderbuilder): + self.gcheaderbuilder = gcheaderbuilder + + def __repr__(self): + return '< GCTypeInfoOffset >' + + def __neg__(self): + raise NotImplementedError("XXX for now, cannot cast from a GC-specific" + " type info to Ptr(RuntimeTypeInfo)") + + def ref(self, rttiptr): + typeinfoptr = self.gcheaderbuilder.typeinfo_from_rtti(rttiptr) + return typeinfoptr + # ____________________________________________________________ def sizeof(TYPE, n=None): Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lloperation.py Thu Feb 21 14:58:42 2008 @@ -387,11 +387,9 @@ # __________ GC operations __________ 'gc__collect': LLOp(canunwindgc=True), - 'gc_free': LLOp(), 'gc_fetch_exception': LLOp(), 'gc_restore_exception': LLOp(), 'gc_runtime_type_info': LLOp(canfold=True), - 'gc_deallocate': LLOp(), 'gc_push_alive_pyobj': LLOp(), 'gc_pop_alive_pyobj': LLOp(), 'gc_reload_possibly_moved': LLOp(), Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gcheader.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gcheader.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gcheader.py Thu Feb 21 14:58:42 2008 @@ -8,11 +8,14 @@ class GCHeaderBuilder(object): - def __init__(self, HDR): + def __init__(self, HDR, TYPEINFO): """NOT_RPYTHON""" self.HDR = HDR + self.TYPEINFO = TYPEINFO self.obj2header = weakref.WeakKeyDictionary() + self.rtti2typeinfo = weakref.WeakKeyDictionary() self.size_gc_header = llmemory.GCHeaderOffset(self) + self.size_gc_typeinfo = llmemory.GCTypeInfoOffset(self) def header_of_object(self, gcptr): # XXX hackhackhack @@ -43,5 +46,16 @@ self.attach_header(gcptr, headerptr) return headerptr + def typeinfo_from_rtti(self, rttiptr): + rttiptr = lltype.cast_pointer(lltype.Ptr(lltype.RuntimeTypeInfo), + rttiptr) + return self.rtti2typeinfo[rttiptr._obj] + + def cast_rtti_to_typeinfo(self, rttiptr): + # this is RPython + addr = llmemory.cast_ptr_to_adr(rttiptr) + addr += self.size_gc_typeinfo + return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.TYPEINFO)) + def _freeze_(self): return True # for reads of size_gc_header Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py Thu Feb 21 14:58:42 2008 @@ -40,11 +40,16 @@ HDR = lltype.Struct("header", ("refcount", lltype.Signed), ("typeptr", RTTIPTR)) + TYPEINFO = lltype.Struct("typeinfo", + ("dealloc", lltype.Ptr(ADDRESS_VOID_FUNC))) + def __init__(self, translator): super(RefcountingGCTransformer, self).__init__(translator, inline=True) - self.gcheaderbuilder = GCHeaderBuilder(self.HDR) + gchdrbuilder = GCHeaderBuilder(self.HDR, self.TYPEINFO) + self.gcheaderbuilder = gchdrbuilder gc_header_offset = self.gcheaderbuilder.size_gc_header self.deallocator_graphs_needing_transforming = [] + self.rtticache = {} # create incref, etc graph @@ -61,9 +66,11 @@ refcount = gcheader.refcount - 1 gcheader.refcount = refcount if refcount == 0: - gcheader.typeptr.xxx(adr) + rtti = gcheader.typeptr + typeinfo = gchdrbuilder.cast_rtti_to_typeinfo(rtti) + typeinfo.dealloc(adr) def ll_no_pointer_dealloc(adr): - llop.gc_free(lltype.Void, adr) + llmemory.raw_free(adr) mh = mallocHelpers() mh.allocate = llmemory.raw_malloc @@ -73,7 +80,14 @@ llmemory.raw_memclear(result, size) result += gc_header_offset return result - def ll_malloc_varsize_no_length(length, size, itemsize): + def ll_malloc_fixedsize_rtti(size, rtti): + size = gc_header_offset + size + result = mh._ll_malloc_fixedsize(size) + llmemory.raw_memclear(result, size) + llmemory.cast_adr_to_ptr(result, HDRPTR).typeptr = rtti + result += gc_header_offset + return result + def ll_malloc_varsize_no_length_rtti(length, size, itemsize, rtti): try: fixsize = gc_header_offset + size varsize = ovfcheck(itemsize * length) @@ -82,34 +96,35 @@ raise MemoryError() result = mh._ll_malloc_fixedsize(tot_size) llmemory.raw_memclear(result, tot_size) + llmemory.cast_adr_to_ptr(result, HDRPTR).typeptr = rtti result += gc_header_offset return result - mh.ll_malloc_varsize_no_length = ll_malloc_varsize_no_length - ll_malloc_varsize = mh.ll_malloc_varsize + def ll_malloc_varsize_rtti(length, size, itemsize, lengthoffset, rtti): + result = ll_malloc_varsize_no_length_rtti(length, size, + itemsize, rtti) + (result + lengthoffset).signed[0] = length + return result if self.translator: self.increfptr = self.inittime_helper( ll_incref, [llmemory.Address], lltype.Void) - self.decref_ptr = self.inittime_helper( - ll_decref, [llmemory.Address, lltype.Ptr(ADDRESS_VOID_FUNC)], - lltype.Void) - self.decref_simple_ptr = self.inittime_helper( - ll_decref_simple, [llmemory.Address], lltype.Void) + self.decrefptr = self.inittime_helper( + ll_decref, [llmemory.Address], lltype.Void) self.no_pointer_dealloc_ptr = self.inittime_helper( ll_no_pointer_dealloc, [llmemory.Address], lltype.Void) self.malloc_fixedsize_ptr = self.inittime_helper( - ll_malloc_fixedsize, [lltype.Signed], llmemory.Address) + ll_malloc_fixedsize_rtti, [lltype.Signed, RTTIPTR], + llmemory.Address) self.malloc_varsize_no_length_ptr = self.inittime_helper( - ll_malloc_varsize_no_length, [lltype.Signed]*3, llmemory.Address) + ll_malloc_varsize_no_length_rtti, [lltype.Signed]*3+[RTTIPTR], + llmemory.Address) self.malloc_varsize_ptr = self.inittime_helper( - ll_malloc_varsize, [lltype.Signed]*4, llmemory.Address) + ll_malloc_varsize_rtti, [lltype.Signed]*4+[RTTIPTR], + llmemory.Address) self.mixlevelannotator.finish() self.mixlevelannotator.backend_optimize() # cache graphs: - self.decref_funcptrs = {} self.static_deallocator_funcptrs = {} - self.dynamic_deallocator_funcptrs = {} - self.queryptr2dynamic_deallocator_funcptr = {} def finish_helpers(self, **kwds): GCTransformer.finish_helpers(self, **kwds) @@ -132,45 +147,34 @@ llops.genop("direct_call", [self.increfptr, v_adr]) def pop_alive_nopyobj(self, var, llops): - PTRTYPE = var.concretetype v_adr = gen_cast(llops, llmemory.Address, var) - - dealloc_fptr = self.dynamic_deallocation_funcptr_for_type(PTRTYPE.TO) - if dealloc_fptr is self.no_pointer_dealloc_ptr.value: - # simple case - llops.genop("direct_call", [self.decref_simple_ptr, v_adr]) - else: - cdealloc_fptr = rmodel.inputconst( - lltype.typeOf(dealloc_fptr), dealloc_fptr) - llops.genop("direct_call", [self.decref_ptr, v_adr, cdealloc_fptr]) + llops.genop("direct_call", [self.decrefptr, v_adr]) def gct_fv_gc_malloc(self, hop, flags, TYPE, c_size): - v_raw = hop.genop("direct_call", [self.malloc_fixedsize_ptr, c_size], + rtti = lltype.getRuntimeTypeInfo(TYPE, self.rtticache) + c_rtti = rmodel.inputconst(RTTIPTR, rtti) + v_raw = hop.genop("direct_call", + [self.malloc_fixedsize_ptr, c_size, c_rtti], resulttype=llmemory.Address) return v_raw def gct_fv_gc_malloc_varsize(self, hop, flags, TYPE, v_length, c_const_size, c_item_size, c_offset_to_length): + rtti = lltype.getRuntimeTypeInfo(TYPE, self.rtticache) + c_rtti = rmodel.inputconst(RTTIPTR, rtti) if c_offset_to_length is None: v_raw = hop.genop("direct_call", [self.malloc_varsize_no_length_ptr, v_length, - c_const_size, c_item_size], + c_const_size, c_item_size, c_rtti], resulttype=llmemory.Address) else: v_raw = hop.genop("direct_call", [self.malloc_varsize_ptr, v_length, - c_const_size, c_item_size, c_offset_to_length], + c_const_size, c_item_size, c_offset_to_length, + c_rtti], resulttype=llmemory.Address) return v_raw - def gct_gc_deallocate(self, hop): - TYPE = hop.spaceop.args[0].value - v_addr = hop.spaceop.args[1] - dealloc_fptr = self.dynamic_deallocation_funcptr_for_type(TYPE) - cdealloc_fptr = rmodel.inputconst( - lltype.typeOf(dealloc_fptr), dealloc_fptr) - hop.genop("direct_call", [cdealloc_fptr, v_addr]) - def consider_constant(self, TYPE, value): if value is not lltype.top_container(value): return @@ -181,18 +185,25 @@ hdr.refcount = sys.maxint // 2 def static_deallocation_funcptr_for_type(self, TYPE): + """The 'static deallocator' for a type is the function that can + free a pointer that we know points exactly to a TYPE structure + (and not to a larger structure that starts with TYPE). This + function is the one that ends up in the 'dealloc' field of + TYPEINFO. + """ if TYPE in self.static_deallocator_funcptrs: return self.static_deallocator_funcptrs[TYPE] #print_call_chain(self) if TYPE._gckind == 'cpy': return # you don't really have an RPython deallocator for PyObjects - rtti = get_rtti(TYPE) - if rtti is not None and hasattr(rtti._obj, 'destructor_funcptr'): - destrptr = rtti._obj.destructor_funcptr + assert TYPE._gckind == 'gc' + + rtti = lltype.getRuntimeTypeInfo(TYPE, self.rtticache) + destrptr = rtti.destructor_funcptr + if destrptr is not None: DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0] else: - destrptr = None DESTR_ARG = None if destrptr is None and not find_gc_ptrs_in_type(TYPE): @@ -206,36 +217,37 @@ def ll_deallocator(addr): exc_instance = llop.gc_fetch_exception(EXC_INSTANCE_TYPE) try: - v = cast_adr_to_ptr(addr, PTR_TYPE) - gcheader = cast_adr_to_ptr(addr - gc_header_offset, HDRPTR) + v = llmemory.cast_adr_to_ptr(addr, PTR_TYPE) + gcheader = llmemory.cast_adr_to_ptr(addr - gc_header_offset, HDRPTR) # refcount is at zero, temporarily bump it to 1: gcheader.refcount = 1 - destr_v = cast_pointer(DESTR_ARG, v) + destr_v = lltype.cast_pointer(DESTR_ARG, v) ll_call_destructor(destrptr, destr_v) refcount = gcheader.refcount - 1 gcheader.refcount = refcount if refcount == 0: %s - llop.%s_free(lltype.Void, addr) + llmemory.raw_free(addr) except: pass llop.gc_restore_exception(lltype.Void, exc_instance) pop_alive(exc_instance) # XXX layering of exceptiontransform versus gcpolicy -""" % (body, TYPE._gckind) +""" % (body,) else: call_del = None body = '\n'.join(_static_deallocator_body_for_type('v', TYPE)) - src = ('def ll_deallocator(addr):\n v = cast_adr_to_ptr(addr, PTR_TYPE)\n' + - body + '\n llop.%s_free(lltype.Void, addr)\n' % (TYPE._gckind,)) + src = ('def ll_deallocator(addr):\n' + + ' v = llmemory.cast_adr_to_ptr(addr, PTR_TYPE)\n' + + body + '\n' + + ' llmemory.raw_free(addr)\n') d = {'pop_alive': LLTransformerOp(self.pop_alive), 'llop': llop, 'lltype': lltype, 'destrptr': destrptr, 'gc_header_offset': self.gcheaderbuilder.size_gc_header, - 'cast_adr_to_ptr': llmemory.cast_adr_to_ptr, - 'cast_pointer': lltype.cast_pointer, + 'llmemory': llmemory, 'PTR_TYPE': lltype.Ptr(TYPE), 'DESTR_ARG': DESTR_ARG, 'EXC_INSTANCE_TYPE': self.translator.rtyper.exceptiondata.lltype_of_exception_value, @@ -248,38 +260,3 @@ for p in find_gc_ptrs_in_type(TYPE): self.static_deallocation_funcptr_for_type(p.TO) return fptr - - def dynamic_deallocation_funcptr_for_type(self, TYPE): - assert TYPE._gckind != 'cpy' - if TYPE in self.dynamic_deallocator_funcptrs: - return self.dynamic_deallocator_funcptrs[TYPE] - #print_call_chain(self) - - rtti = lltype.getRuntimeTypeInfo(TYPE) - if rtti is None: - p = self.static_deallocation_funcptr_for_type(TYPE) - self.dynamic_deallocator_funcptrs[TYPE] = p - return p - - queryptr = rtti._obj.query_funcptr - if queryptr._obj in self.queryptr2dynamic_deallocator_funcptr: - return self.queryptr2dynamic_deallocator_funcptr[queryptr._obj] - - RTTI_PTR = lltype.Ptr(lltype.RuntimeTypeInfo) - QUERY_ARG_TYPE = lltype.typeOf(queryptr).TO.ARGS[0] - gc_header_offset = self.gcheaderbuilder.size_gc_header - HDRPTR = lltype.Ptr(self.HDR) - def ll_dealloc(addr): - # bump refcount to 1 - gcheader = llmemory.cast_adr_to_ptr(addr - gc_header_offset, HDRPTR) - gcheader.refcount = 1 - v = llmemory.cast_adr_to_ptr(addr, QUERY_ARG_TYPE) - rtti = queryptr(v) - gcheader.refcount = 0 - llop.gc_call_rtti_destructor(lltype.Void, rtti, addr) - fptr = self.annotate_helper(ll_dealloc, [llmemory.Address], lltype.Void) - self.dynamic_deallocator_funcptrs[TYPE] = fptr - self.queryptr2dynamic_deallocator_funcptr[queryptr._obj] = fptr - return fptr - - Modified: pypy/branch/unified-rtti/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/gc.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/gc.py Thu Feb 21 14:58:42 2008 @@ -124,10 +124,6 @@ args = [funcgen.expr(v) for v in op.args] line = '%s(%s);' % (args[0], ', '.join(args[1:])) return line - - def OP_GC_FREE(self, funcgen, op): - args = [funcgen.expr(v) for v in op.args] - return 'OP_FREE(%s);' % (args[0], ) def OP_GC__COLLECT(self, funcgen, op): return '' Modified: pypy/branch/unified-rtti/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/src/mem.h (original) +++ pypy/branch/unified-rtti/pypy/translator/c/src/mem.h Thu Feb 21 14:58:42 2008 @@ -69,8 +69,6 @@ /************************************************************/ -#define OP_FREE(p) OP_RAW_FREE(p, do_not_use) - /*------------------------------------------------------------*/ #ifndef COUNT_OP_MALLOCS /*------------------------------------------------------------*/ From arigo at codespeak.net Thu Feb 21 15:26:33 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 15:26:33 +0100 (CET) Subject: [pypy-svn] r51723 - in pypy/branch/unified-rtti/pypy: rpython/lltypesystem rpython/memory rpython/memory/gctransform translator/c translator/c/src Message-ID: <20080221142633.DAC701684DB@codespeak.net> Author: arigo Date: Thu Feb 21 15:26:33 2008 New Revision: 51723 Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py pypy/branch/unified-rtti/pypy/rpython/memory/gcheader.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/unified-rtti/pypy/translator/c/database.py pypy/branch/unified-rtti/pypy/translator/c/gc.py pypy/branch/unified-rtti/pypy/translator/c/node.py pypy/branch/unified-rtti/pypy/translator/c/primitive.py pypy/branch/unified-rtti/pypy/translator/c/src/exception.h Log: First test translating to C passes! Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py Thu Feb 21 15:26:33 2008 @@ -1824,6 +1824,13 @@ return rttiptr getRuntimeTypeInfo._annspecialcase_ = 'specialize:memo' +def getGcTypeForRtti(rttiptr): + assert typeOf(rttiptr) == Ptr(RuntimeTypeInfo) + if not hasattr(rttiptr._obj, '_GCTYPE'): + raise TypeError("rtti object %r is not attached to any type" % ( + rttiptr,)) + return rttiptr._obj._GCTYPE + def _install_rtti(STRUCT, runtime_type_info): if not isinstance(STRUCT, GcStruct): raise TypeError("can only attach a runtime_type_info to a GcStruct") Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gcheader.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gcheader.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gcheader.py Thu Feb 21 15:26:33 2008 @@ -51,6 +51,15 @@ rttiptr) return self.rtti2typeinfo[rttiptr._obj] + def new_typeinfo(self, rttiptr): + rttiptr = lltype.cast_pointer(lltype.Ptr(lltype.RuntimeTypeInfo), + rttiptr) + rtti = rttiptr._obj + assert rtti not in self.rtti2typeinfo + typeinfo = lltype.malloc(self.TYPEINFO, immortal=True) + self.rtti2typeinfo[rtti] = typeinfo + return typeinfo + def cast_rtti_to_typeinfo(self, rttiptr): # this is RPython addr = llmemory.cast_ptr_to_adr(rttiptr) Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py Thu Feb 21 15:26:33 2008 @@ -175,6 +175,23 @@ resulttype=llmemory.Address) return v_raw + def gct_gc_runtime_type_info(self, hop): + # generate inline code equivalent to: + # gcheader = llmemory.cast_adr_to_ptr(adr - gc_header_offset, HDRPTR) + # return gcheader.typeptr + [v_ptr] = hop.spaceop.args + v_adr = hop.genop("cast_ptr_to_adr", [v_ptr], + resulttype=llmemory.Address) + c_gc_header_offset = rmodel.inputconst(lltype.Signed, + self.gcheaderbuilder.size_gc_header) + v_adr = hop.genop("adr_sub", [v_adr, c_gc_header_offset], + resulttype=llmemory.Address) + v_hdr = hop.genop("cast_adr_to_ptr", [v_adr], + resulttype=lltype.Ptr(self.HDR)) + c_typeptr = rmodel.inputconst(lltype.Void, "typeptr") + hop.genop("getfield", [v_hdr, c_typeptr], + resultvar=hop.spaceop.result) + def consider_constant(self, TYPE, value): if value is not lltype.top_container(value): return @@ -260,3 +277,13 @@ for p in find_gc_ptrs_in_type(TYPE): self.static_deallocation_funcptr_for_type(p.TO) return fptr + + def convert_rtti(self, rtti): + rtti = rtti._as_ptr() + try: + return self.gcheaderbuilder.typeinfo_from_rtti(rtti) + except KeyError: + TYPE = lltype.getGcTypeForRtti(rtti) + typeinfo = self.gcheaderbuilder.new_typeinfo(rtti) + typeinfo.dealloc = self.static_deallocation_funcptr_for_type(TYPE) + return typeinfo Modified: pypy/branch/unified-rtti/pypy/translator/c/database.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/database.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/database.py Thu Feb 21 15:26:33 2008 @@ -1,5 +1,5 @@ from pypy.rpython.lltypesystem.lltype import \ - Primitive, Ptr, typeOf, RuntimeTypeInfo, \ + Primitive, Ptr, typeOf, RuntimeTypeInfo, RuntimeTypeInfoType, \ Struct, Array, FuncType, PyObject, Void, \ ContainerType, OpaqueType, FixedSizeArray, _uninitialized from pypy.rpython.lltypesystem import lltype @@ -88,6 +88,9 @@ elif T == WeakRef: REALT = self.gcpolicy.get_real_weakref_type() node = self.gettypedefnode(REALT) + elif T == RuntimeTypeInfo: + REALT = self.gcpolicy.get_real_rtti_type() + node = self.gettypedefnode(REALT) else: raise NoCorrespondingNode("don't know about %r" % (T,)) self.structdefnodes[key] = node @@ -107,7 +110,7 @@ return node.getptrtype() # special-casing because of C typename = self.gettype(T.TO) # who_asks not propagated return typename.replace('@', '*@') - elif isinstance(T, (Struct, Array, _WeakRefType)): + elif isinstance(T, (Struct, Array, _WeakRefType, RuntimeTypeInfoType)): node = self.gettypedefnode(T, varlength=varlength) if who_asks is not None: who_asks.dependencies[node] = True Modified: pypy/branch/unified-rtti/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/gc.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/gc.py Thu Feb 21 15:26:33 2008 @@ -2,8 +2,7 @@ from pypy.translator.c.support import cdecl from pypy.translator.c.node import ContainerNode from pypy.rpython.lltypesystem.lltype import \ - typeOf, Ptr, ContainerType, \ - RuntimeTypeInfo, getRuntimeTypeInfo, top_container + typeOf, Ptr, ContainerType, top_container from pypy.rpython.memory.gctransform import \ refcounting, boehm, framework, stacklessframework, llvmgcroot, asmgcroot from pypy.rpython.lltypesystem import lltype, llmemory @@ -48,14 +47,12 @@ def gc_startup_code(self): return [] - def struct_setup(self, structdefnode, rtti): - return None - - def array_setup(self, arraydefnode): - return None + # for rtti node + def get_real_rtti_type(self): + return self.transformerclass.TYPEINFO - def rtti_type(self): - return '' + def convert_rtti(self, obj): + return self.db.gctransformer.convert_rtti(obj) def OP_GC_PUSH_ALIVE_PYOBJ(self, funcgen, op): expr = funcgen.expr(op.args[0]) @@ -71,11 +68,6 @@ return '' -class RefcountingInfo: - static_deallocator = None - -from pypy.rlib.objectmodel import CDefinedIntSymbolic - class RefcountingGcPolicy(BasicGcPolicy): transformerclass = refcounting.RefcountingGCTransformer @@ -95,29 +87,6 @@ else: return [] - # for structs - - def struct_setup(self, structdefnode, rtti): - if rtti is not None: - transformer = structdefnode.db.gctransformer - fptr = transformer.static_deallocation_funcptr_for_type( - structdefnode.STRUCT) - structdefnode.gcinfo = RefcountingInfo() - structdefnode.gcinfo.static_deallocator = structdefnode.db.get(fptr) - - # for arrays - - def array_setup(self, arraydefnode): - pass - - # for rtti node - - def rtti_type(self): - return 'void (@)(void *)' # void dealloc_xx(struct xx *) - - def rtti_node_factory(self): - return RefcountingRuntimeTypeInfo_OpaqueNode - # zero malloc impl def OP_GC_CALL_RTTI_DESTRUCTOR(self, funcgen, op): @@ -129,49 +98,9 @@ return '' -class RefcountingRuntimeTypeInfo_OpaqueNode(ContainerNode): - nodekind = 'refcnt rtti' - globalcontainer = True - typename = 'void (@)(void *)' - - def __init__(self, db, T, obj): - assert T == RuntimeTypeInfo - assert isinstance(obj.about, GcStruct) - self.db = db - self.T = T - self.obj = obj - defnode = db.gettypedefnode(obj.about) - self.implementationtypename = 'void (@)(void *)' - self.name = defnode.gcinfo.static_deallocator - self.ptrname = '((void (*)(void *)) %s)' % (self.name,) - - def enum_dependencies(self): - return [] - - def implementation(self): - return [] - - - -class BoehmInfo: - finalizer = None - - class BoehmGcPolicy(BasicGcPolicy): transformerclass = boehm.BoehmGCTransformer - def array_setup(self, arraydefnode): - pass - - def struct_setup(self, structdefnode, rtti): - pass - - def rtti_type(self): - return BoehmGcRuntimeTypeInfo_OpaqueNode.typename - - def rtti_node_factory(self): - return BoehmGcRuntimeTypeInfo_OpaqueNode - def gc_libraries(self): if sys.platform == 'win32': return ['gc_pypy'] @@ -210,31 +139,6 @@ nbytes = funcgen.expr(op.args[0]) return 'GC_set_max_heap_size(%s);' % (nbytes,) -class BoehmGcRuntimeTypeInfo_OpaqueNode(ContainerNode): - nodekind = 'boehm rtti' - globalcontainer = True - typename = 'char @' - - def __init__(self, db, T, obj): - assert T == RuntimeTypeInfo - assert isinstance(obj.about, GcStruct) - self.db = db - self.T = T - self.obj = obj - defnode = db.gettypedefnode(obj.about) - self.implementationtypename = self.typename - self.name = self.db.namespace.uniquename('g_rtti_v_'+ defnode.barename) - self.ptrname = '(&%s)' % (self.name,) - - def enum_dependencies(self): - return [] - - def implementation(self): - yield 'char %s /* uninitialized */;' % self.name - -class FrameworkGcRuntimeTypeInfo_OpaqueNode(BoehmGcRuntimeTypeInfo_OpaqueNode): - nodekind = 'framework rtti' - # to get an idea how it looks like with no refcount/gc at all @@ -250,29 +154,6 @@ class FrameworkGcPolicy(BasicGcPolicy): transformerclass = framework.FrameworkGCTransformer - def struct_setup(self, structdefnode, rtti): - if rtti is not None and hasattr(rtti._obj, 'destructor_funcptr'): - destrptr = rtti._obj.destructor_funcptr - # make sure this is seen by the database early, i.e. before - # finish_helpers() on the gctransformer - self.db.get(destrptr) - # the following, on the other hand, will only discover ll_finalizer - # helpers. The get() sees and records a delayed pointer. It is - # still important to see it so that it can be followed as soon as - # the mixlevelannotator resolves it. - gctransf = self.db.gctransformer - fptr = gctransf.finalizer_funcptr_for_type(structdefnode.STRUCT) - self.db.get(fptr) - - def array_setup(self, arraydefnode): - pass - - def rtti_type(self): - return FrameworkGcRuntimeTypeInfo_OpaqueNode.typename - - def rtti_node_factory(self): - return FrameworkGcRuntimeTypeInfo_OpaqueNode - def pre_pre_gc_code(self): yield '#define USING_FRAMEWORK_GC' Modified: pypy/branch/unified-rtti/pypy/translator/c/node.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/node.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/node.py Thu Feb 21 15:26:33 2008 @@ -4,7 +4,7 @@ GcStruct, GcArray, ContainerType, \ parentlink, Ptr, PyObject, Void, OpaqueType, Float, \ RuntimeTypeInfo, getRuntimeTypeInfo, Char, _subarray -from pypy.rpython.lltypesystem import llmemory +from pypy.rpython.lltypesystem import lltype, llmemory from pypy.translator.c.funcgen import FunctionCodeGenerator from pypy.translator.c.external import CExternalFunctionCodeGenerator from pypy.translator.c.support import USESLOTS # set to False if necessary while refactoring @@ -25,15 +25,6 @@ return False # gcheader already in the first field return True -class defaultproperty(object): - def __init__(self, fget): - self.fget = fget - def __get__(self, obj, cls=None): - if obj is None: - return self - else: - return self.fget(obj) - class StructDefNode: typetag = 'struct' @@ -88,23 +79,6 @@ else: typename = db.gettype(T, who_asks=self) self.fields.append((self.c_struct_field_name(name), typename)) - self.gcinfo # force it to be computed - - def computegcinfo(self): - # let the gcpolicy do its own setup - self.gcinfo = None # unless overwritten below - rtti = None - STRUCT = self.STRUCT - if isinstance(STRUCT, GcStruct): - XXX - try: - rtti = getRuntimeTypeInfo(STRUCT) - except ValueError: - pass - if self.varlength == 1: - self.db.gcpolicy.struct_setup(self, rtti) - return self.gcinfo - gcinfo = defaultproperty(computegcinfo) def gettype(self): return '%s %s @' % (self.typetag, self.name) @@ -222,20 +196,11 @@ return # setup() was already called, likely by __init__ db = self.db ARRAY = self.ARRAY - self.gcinfo # force it to be computed if needs_gcheader(ARRAY): for fname, T in db.gcpolicy.array_gcheader_definition(self): self.gcfields.append((fname, db.gettype(T, who_asks=self))) self.itemtypename = db.gettype(ARRAY.OF, who_asks=self) - def computegcinfo(self): - # let the gcpolicy do its own setup - self.gcinfo = None # unless overwritten below - if self.varlength == 1: - self.db.gcpolicy.array_setup(self) - return self.gcinfo - gcinfo = defaultproperty(computegcinfo) - def gettype(self): return '%s %s @' % (self.typetag, self.name) @@ -313,7 +278,6 @@ Implemented directly as a C array instead of a struct with an items field. rffi kind of expects such arrays to be 'bare' C arrays. """ - gcinfo = None name = None forward_decl = None @@ -360,7 +324,6 @@ class FixedSizeArrayDefNode: - gcinfo = None name = None typetag = 'struct' @@ -912,6 +875,13 @@ obj._converted_weakref = container # hack for genllvm :-/ return db.getcontainernode(container, _dont_write_c_code=False) +def rttinode_factory(db, T, obj): + assert isinstance(obj, lltype._rtti) + wrapper = db.gcpolicy.convert_rtti(obj) + container = wrapper._obj + #obj._converted_rtti = container # hack for genllvm :-/ + return db.getcontainernode(container, _dont_write_c_code=False) + ContainerNodeFactory = { Struct: StructNode, @@ -923,4 +893,5 @@ OpaqueType: opaquenode_factory, PyObjectType: PyObjectNode, llmemory._WeakRefType: weakrefnode_factory, + lltype.RuntimeTypeInfoType: rttinode_factory, } Modified: pypy/branch/unified-rtti/pypy/translator/c/primitive.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/primitive.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/primitive.py Thu Feb 21 15:26:33 2008 @@ -7,7 +7,7 @@ from pypy.rpython.lltypesystem.llmemory import Address, \ AddressOffset, ItemOffset, ArrayItemsOffset, FieldOffset, \ CompositeOffset, ArrayLengthOffset, \ - GCHeaderOffset + GCHeaderOffset, GCTypeInfoOffset from pypy.rpython.lltypesystem.llarena import RoundedUpForAllocation from pypy.translator.c.support import cdecl, barebonearray @@ -49,6 +49,8 @@ return '0' elif type(value) == GCHeaderOffset: return '0' + elif type(value) == GCTypeInfoOffset: + return '0' elif type(value) == RoundedUpForAllocation: return 'ROUND_UP_FOR_ALLOCATION(%s)' % ( name_signed(value.basesize, db)) Modified: pypy/branch/unified-rtti/pypy/translator/c/src/exception.h ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/src/exception.h (original) +++ pypy/branch/unified-rtti/pypy/translator/c/src/exception.h Thu Feb 21 15:26:33 2008 @@ -39,7 +39,7 @@ long lineno, const char *functionname) { fprintf(stderr, "%s %s: %s:%ld %s\n", msg, - RPyFetchExceptionType()->ov_name->items, + RPyFetchExceptionType()->ov_name, filename, lineno, functionname); } #endif @@ -98,7 +98,7 @@ PyObject *pycls, *v, *tb; assert(RPyExceptionOccurred()); assert(!PyErr_Occurred()); - clsname = RPyFetchExceptionType()->ov_name->items; + clsname = RPyFetchExceptionType()->ov_name; pycls = PyDict_GetItemString(PyEval_GetBuiltins(), clsname); if (pycls != NULL && PyExceptionClass_Check(pycls) && PyObject_IsSubclass(pycls, PyExc_Exception)) { From arigo at codespeak.net Thu Feb 21 15:30:57 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 15:30:57 +0100 (CET) Subject: [pypy-svn] r51724 - pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test Message-ID: <20080221143057.08F7C1684EA@codespeak.net> Author: arigo Date: Thu Feb 21 15:30:57 2008 New Revision: 51724 Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_llmemory.py Log: Fix test. Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_llmemory.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_llmemory.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_llmemory.py Thu Feb 21 15:30:57 2008 @@ -349,7 +349,7 @@ def test_raw_malloc_gcstruct(): from pypy.rpython.memory import gcheader HDR = lltype.Struct('header', ('a', lltype.Signed)) - builder = gcheader.GCHeaderBuilder(HDR) + builder = gcheader.GCHeaderBuilder(HDR, None) gchdr = builder.size_gc_header S = lltype.GcStruct('S', ('x', lltype.Signed)) @@ -433,7 +433,7 @@ from pypy.rpython.memory.gcheader import GCHeaderBuilder HDR = lltype.Struct('h', ('t', lltype.Signed)) - gh = GCHeaderBuilder(HDR).size_gc_header + gh = GCHeaderBuilder(HDR, None).size_gc_header A = lltype.GcArray(lltype.Signed) adr = raw_malloc(gh+sizeof(A, 10)) From arigo at codespeak.net Thu Feb 21 15:47:46 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 15:47:46 +0100 (CET) Subject: [pypy-svn] r51725 - in pypy/branch/unified-rtti/pypy/rpython/memory: . gctransform Message-ID: <20080221144746.01FEC1684EA@codespeak.net> Author: arigo Date: Thu Feb 21 15:47:44 2008 New Revision: 51725 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gcheader.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py Log: Forgot to set hdr.typeptr on prebuilt objects. Add the ll_assert() that caught this bug. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gcheader.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gcheader.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gcheader.py Thu Feb 21 15:47:44 2008 @@ -1,5 +1,6 @@ import weakref from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rlib.debug import ll_assert # this is global because a header cannot be a header of more than one GcObj @@ -62,6 +63,7 @@ def cast_rtti_to_typeinfo(self, rttiptr): # this is RPython + ll_assert(bool(rttiptr), "NULL rtti pointer") addr = llmemory.cast_ptr_to_adr(rttiptr) addr += self.size_gc_typeinfo return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.TYPEINFO)) Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py Thu Feb 21 15:47:44 2008 @@ -8,6 +8,7 @@ from pypy.rpython import rmodel from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib.debug import ll_assert from pypy.rpython.rbuiltin import gen_cast import sys @@ -69,9 +70,16 @@ rtti = gcheader.typeptr typeinfo = gchdrbuilder.cast_rtti_to_typeinfo(rtti) typeinfo.dealloc(adr) + def ll_no_pointer_dealloc(adr): llmemory.raw_free(adr) + def ll_gc_runtime_type_info(adr): + gcheader = llmemory.cast_adr_to_ptr(adr - gc_header_offset, HDRPTR) + rtti = gcheader.typeptr + ll_assert(bool(rtti), "NULL rtti pointer") + return rtti + mh = mallocHelpers() mh.allocate = llmemory.raw_malloc def ll_malloc_fixedsize(size): @@ -112,6 +120,8 @@ ll_decref, [llmemory.Address], lltype.Void) self.no_pointer_dealloc_ptr = self.inittime_helper( ll_no_pointer_dealloc, [llmemory.Address], lltype.Void) + self.gc_runtime_type_info_ptr = self.inittime_helper( + ll_gc_runtime_type_info, [llmemory.Address], RTTIPTR) self.malloc_fixedsize_ptr = self.inittime_helper( ll_malloc_fixedsize_rtti, [lltype.Signed, RTTIPTR], llmemory.Address) @@ -176,21 +186,14 @@ return v_raw def gct_gc_runtime_type_info(self, hop): - # generate inline code equivalent to: - # gcheader = llmemory.cast_adr_to_ptr(adr - gc_header_offset, HDRPTR) - # return gcheader.typeptr [v_ptr] = hop.spaceop.args v_adr = hop.genop("cast_ptr_to_adr", [v_ptr], resulttype=llmemory.Address) - c_gc_header_offset = rmodel.inputconst(lltype.Signed, - self.gcheaderbuilder.size_gc_header) - v_adr = hop.genop("adr_sub", [v_adr, c_gc_header_offset], - resulttype=llmemory.Address) - v_hdr = hop.genop("cast_adr_to_ptr", [v_adr], - resulttype=lltype.Ptr(self.HDR)) - c_typeptr = rmodel.inputconst(lltype.Void, "typeptr") - hop.genop("getfield", [v_hdr, c_typeptr], - resultvar=hop.spaceop.result) + v_result = hop.spaceop.result + assert v_result.concretetype == RTTIPTR + hop.genop("direct_call", + [self.gc_runtime_type_info_ptr, v_adr], + resultvar = v_result) def consider_constant(self, TYPE, value): if value is not lltype.top_container(value): @@ -200,6 +203,7 @@ if not self.gcheaderbuilder.get_header(p): hdr = self.gcheaderbuilder.new_header(p) hdr.refcount = sys.maxint // 2 + hdr.typeptr = lltype.getRuntimeTypeInfo(TYPE, self.rtticache) def static_deallocation_funcptr_for_type(self, TYPE): """The 'static deallocator' for a type is the function that can From arigo at codespeak.net Thu Feb 21 16:23:57 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 16:23:57 +0100 (CET) Subject: [pypy-svn] r51726 - pypy/branch/unified-rtti/pypy/translator/c Message-ID: <20080221152357.D57A2168469@codespeak.net> Author: arigo Date: Thu Feb 21 16:23:57 2008 New Revision: 51726 Modified: pypy/branch/unified-rtti/pypy/translator/c/genc.py Log: Remove unneeded line. Modified: pypy/branch/unified-rtti/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/genc.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/genc.py Thu Feb 21 16:23:57 2008 @@ -503,7 +503,6 @@ # All declarations # database = self.database - structdeflist = database.getstructdeflist() name = 'structdef.h' fi = self.makefile(name) print >> f, '#include "%s"' % name From arigo at codespeak.net Thu Feb 21 16:24:15 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 16:24:15 +0100 (CET) Subject: [pypy-svn] r51727 - pypy/branch/unified-rtti/pypy/translator/c Message-ID: <20080221152415.0B8AD1684EA@codespeak.net> Author: arigo Date: Thu Feb 21 16:24:15 2008 New Revision: 51727 Modified: pypy/branch/unified-rtti/pypy/translator/c/node.py Log: Forgot dependencies. Modified: pypy/branch/unified-rtti/pypy/translator/c/node.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/node.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/node.py Thu Feb 21 16:24:15 2008 @@ -477,6 +477,9 @@ return self.T._name def enum_dependencies(self): + if needs_gcheader(self.T): + for thing in self.db.gcpolicy.struct_gcheader_initdata(self): + yield thing for name in self.T._names: yield getattr(self.obj, name) @@ -537,7 +540,10 @@ return 'array' def enum_dependencies(self): - return self.obj.items + result = self.obj.items + if needs_gcheader(self.T): + result = result + self.db.gcpolicy.array_gcheader_initdata(self) + return result def getlength(self): return len(self.obj.items) @@ -818,8 +824,6 @@ def opaquenode_factory(db, T, obj): - if T == RuntimeTypeInfo: - return db.gcpolicy.rtti_node_factory()(db, T, obj) if T.hints.get("render_structure", False): return ExtType_OpaqueNode(db, T, obj) raise Exception("don't know about %r" % (T,)) @@ -875,12 +879,24 @@ obj._converted_weakref = container # hack for genllvm :-/ return db.getcontainernode(container, _dont_write_c_code=False) -def rttinode_factory(db, T, obj): - assert isinstance(obj, lltype._rtti) - wrapper = db.gcpolicy.convert_rtti(obj) - container = wrapper._obj - #obj._converted_rtti = container # hack for genllvm :-/ - return db.getcontainernode(container, _dont_write_c_code=False) +class RttiNode(ContainerNode): + nodekind = 'rtti' + + def __init__(self, db, T, obj): + assert isinstance(obj, lltype._rtti) + wrapper = db.gcpolicy.convert_rtti(obj) + self.realobj = wrapper._obj + self.realnode = db.getcontainernode(self.realobj) + ContainerNode.__init__(self, db, T, obj) + + def basename(self): + return self.realnode.basename() + + def enum_dependencies(self): + return self.realnode.enum_dependencies() + + def initializationexpr(self, decoration=''): + return self.realnode.initializationexpr(decoration) ContainerNodeFactory = { @@ -893,5 +909,5 @@ OpaqueType: opaquenode_factory, PyObjectType: PyObjectNode, llmemory._WeakRefType: weakrefnode_factory, - lltype.RuntimeTypeInfoType: rttinode_factory, + lltype.RuntimeTypeInfoType: RttiNode, } From arigo at codespeak.net Thu Feb 21 16:30:46 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 16:30:46 +0100 (CET) Subject: [pypy-svn] r51728 - pypy/branch/unified-rtti/pypy/translator/c Message-ID: <20080221153046.3852C168469@codespeak.net> Author: arigo Date: Thu Feb 21 16:30:45 2008 New Revision: 51728 Modified: pypy/branch/unified-rtti/pypy/translator/c/node.py Log: Fix? Modified: pypy/branch/unified-rtti/pypy/translator/c/node.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/node.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/node.py Thu Feb 21 16:30:45 2008 @@ -886,7 +886,8 @@ assert isinstance(obj, lltype._rtti) wrapper = db.gcpolicy.convert_rtti(obj) self.realobj = wrapper._obj - self.realnode = db.getcontainernode(self.realobj) + self.realnode = db.getcontainernode(self.realobj, + _dont_write_c_code=False) ContainerNode.__init__(self, db, T, obj) def basename(self): From arigo at codespeak.net Thu Feb 21 16:35:02 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 16:35:02 +0100 (CET) Subject: [pypy-svn] r51729 - pypy/branch/unified-rtti/pypy/rpython/lltypesystem Message-ID: <20080221153502.30F93168469@codespeak.net> Author: arigo Date: Thu Feb 21 16:35:02 2008 New Revision: 51729 Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/rvirtualizable.py Log: Fix. Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/rvirtualizable.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/rvirtualizable.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/rvirtualizable.py Thu Feb 21 16:35:02 2008 @@ -34,7 +34,7 @@ rbase = self.rbase accessors = [] if self.top_of_virtualizable_hierarchy: - if len(rbase.allinstancefields) != 1: + if len(rbase.allinstancefields) != 0: raise TyperError("virtulizable class cannot have" " non-virtualizable base class with instance" " fields: %r" % self.classdef) @@ -140,7 +140,7 @@ def setfield(self, vinst, attr, vvalue, llops, force_cast=False, flags={}): - """Write the given attribute (or __class__ for the type) of 'vinst'.""" + """Write the given attribute of 'vinst'.""" if (attr in self.my_redirected_fields and not flags.get('access_directly')): mangled_name, r = self.fields[attr] From arigo at codespeak.net Thu Feb 21 16:42:33 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 16:42:33 +0100 (CET) Subject: [pypy-svn] r51730 - in pypy/branch/unified-rtti/pypy/rpython: lltypesystem memory/gctransform Message-ID: <20080221154233.7DA491684D6@codespeak.net> Author: arigo Date: Thu Feb 21 16:42:32 2008 New Revision: 51730 Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py Log: Fix. Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py Thu Feb 21 16:42:32 2008 @@ -1827,7 +1827,7 @@ def getGcTypeForRtti(rttiptr): assert typeOf(rttiptr) == Ptr(RuntimeTypeInfo) if not hasattr(rttiptr._obj, '_GCTYPE'): - raise TypeError("rtti object %r is not attached to any type" % ( + raise ValueError("rtti object %r is not attached to any type" % ( rttiptr,)) return rttiptr._obj._GCTYPE Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py Thu Feb 21 16:42:32 2008 @@ -287,7 +287,13 @@ try: return self.gcheaderbuilder.typeinfo_from_rtti(rtti) except KeyError: - TYPE = lltype.getGcTypeForRtti(rtti) typeinfo = self.gcheaderbuilder.new_typeinfo(rtti) - typeinfo.dealloc = self.static_deallocation_funcptr_for_type(TYPE) + try: + TYPE = lltype.getGcTypeForRtti(rtti) + except ValueError: + pass # ignore rtti's not attached anywhere, e.g. in the + # vtable of raw-flavored RPython classes + else: + fn = self.static_deallocation_funcptr_for_type(TYPE) + typeinfo.dealloc = fn return typeinfo From arigo at codespeak.net Thu Feb 21 16:44:42 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 16:44:42 +0100 (CET) Subject: [pypy-svn] r51731 - pypy/branch/unified-rtti/pypy/translator/c/src Message-ID: <20080221154442.EFC091684DA@codespeak.net> Author: arigo Date: Thu Feb 21 16:44:38 2008 New Revision: 51731 Modified: pypy/branch/unified-rtti/pypy/translator/c/src/main.h Log: Fix (this is now an rffi.CCHARP directly). Modified: pypy/branch/unified-rtti/pypy/translator/c/src/main.h ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/src/main.h (original) +++ pypy/branch/unified-rtti/pypy/translator/c/src/main.h Thu Feb 21 16:44:38 2008 @@ -39,7 +39,7 @@ /* fish for the exception type, at least */ #ifndef AVR fprintf(stderr, "Fatal RPython error: %s\n", - RPyFetchExceptionType()->ov_name->items); + RPyFetchExceptionType()->ov_name); #endif exitcode = 1; } From arigo at codespeak.net Thu Feb 21 16:53:08 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 16:53:08 +0100 (CET) Subject: [pypy-svn] r51732 - pypy/dist/pypy/rpython/memory/gctransform Message-ID: <20080221155308.556C91684D6@codespeak.net> Author: arigo Date: Thu Feb 21 16:53:05 2008 New Revision: 51732 Modified: pypy/dist/pypy/rpython/memory/gctransform/boehm.py Log: Fix an XXX: remove this run-time check in every Boehm allocation. Modified: pypy/dist/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/boehm.py Thu Feb 21 16:53:05 2008 @@ -14,28 +14,21 @@ atomic_mh = mallocHelpers() atomic_mh.allocate = lambda size: llop.boehm_malloc_atomic(llmemory.Address, size) - def ll_malloc_fixedsize_atomic(size, finalizer): - result = atomic_mh._ll_malloc_fixedsize(size) - if finalizer: # XXX runtime check here is bad? - llop.boehm_register_finalizer(lltype.Void, result, finalizer) - return result + ll_malloc_fixedsize_atomic = atomic_mh._ll_malloc_fixedsize mh = mallocHelpers() mh.allocate = lambda size: llop.boehm_malloc(llmemory.Address, size) - def ll_malloc_fixedsize(size, finalizer): - result = mh._ll_malloc_fixedsize(size) - if finalizer: # XXX runtime check here is bad? - llop.boehm_register_finalizer(lltype.Void, result, finalizer) - return result + ll_malloc_fixedsize = mh._ll_malloc_fixedsize + # XXX, do we need/want an atomic version of this function? ll_malloc_varsize_no_length = mh.ll_malloc_varsize_no_length ll_malloc_varsize = mh.ll_malloc_varsize if self.translator: self.malloc_fixedsize_ptr = self.inittime_helper( - ll_malloc_fixedsize, [lltype.Signed, self.FINALIZER_PTR], llmemory.Address) + ll_malloc_fixedsize, [lltype.Signed], llmemory.Address) self.malloc_fixedsize_atomic_ptr = self.inittime_helper( - ll_malloc_fixedsize_atomic, [lltype.Signed, self.FINALIZER_PTR], llmemory.Address) + ll_malloc_fixedsize_atomic, [lltype.Signed], llmemory.Address) self.malloc_varsize_no_length_ptr = self.inittime_helper( ll_malloc_varsize_no_length, [lltype.Signed]*3, llmemory.Address, inline=False) self.malloc_varsize_ptr = self.inittime_helper( @@ -60,10 +53,13 @@ funcptr = self.malloc_fixedsize_atomic_ptr else: funcptr = self.malloc_fixedsize_ptr - c_finalizer_ptr = Constant(self.finalizer_funcptr_for_type(TYPE), self.FINALIZER_PTR) v_raw = hop.genop("direct_call", - [funcptr, c_size, c_finalizer_ptr], + [funcptr, c_size], resulttype=llmemory.Address) + finalizer_ptr = self.finalizer_funcptr_for_type(TYPE) + if finalizer_ptr: + c_finalizer_ptr = Constant(finalizer_ptr, self.FINALIZER_PTR) + hop.genop("boehm_register_finalizer", [v_raw, c_finalizer_ptr]) return v_raw From arigo at codespeak.net Thu Feb 21 16:53:51 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 16:53:51 +0100 (CET) Subject: [pypy-svn] r51733 - pypy/branch/unified-rtti/pypy/rpython/memory/gctransform Message-ID: <20080221155351.DA2551684D6@codespeak.net> Author: arigo Date: Thu Feb 21 16:53:50 2008 New Revision: 51733 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py Log: Merge r51732 from the trunk. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py Thu Feb 21 16:53:50 2008 @@ -16,28 +16,21 @@ atomic_mh = mallocHelpers() atomic_mh.allocate = lambda size: llop.boehm_malloc_atomic(llmemory.Address, size) - def ll_malloc_fixedsize_atomic(size, finalizer): - result = atomic_mh._ll_malloc_fixedsize(size) - if finalizer: # XXX runtime check here is bad? - llop.boehm_register_finalizer(lltype.Void, result, finalizer) - return result + ll_malloc_fixedsize_atomic = atomic_mh._ll_malloc_fixedsize mh = mallocHelpers() mh.allocate = lambda size: llop.boehm_malloc(llmemory.Address, size) - def ll_malloc_fixedsize(size, finalizer): - result = mh._ll_malloc_fixedsize(size) - if finalizer: # XXX runtime check here is bad? - llop.boehm_register_finalizer(lltype.Void, result, finalizer) - return result + ll_malloc_fixedsize = mh._ll_malloc_fixedsize + # XXX, do we need/want an atomic version of this function? ll_malloc_varsize_no_length = mh.ll_malloc_varsize_no_length ll_malloc_varsize = mh.ll_malloc_varsize if self.translator: self.malloc_fixedsize_ptr = self.inittime_helper( - ll_malloc_fixedsize, [lltype.Signed, self.FINALIZER_PTR], llmemory.Address) + ll_malloc_fixedsize, [lltype.Signed], llmemory.Address) self.malloc_fixedsize_atomic_ptr = self.inittime_helper( - ll_malloc_fixedsize_atomic, [lltype.Signed, self.FINALIZER_PTR], llmemory.Address) + ll_malloc_fixedsize_atomic, [lltype.Signed], llmemory.Address) self.malloc_varsize_no_length_ptr = self.inittime_helper( ll_malloc_varsize_no_length, [lltype.Signed]*3, llmemory.Address, inline=False) self.malloc_varsize_ptr = self.inittime_helper( @@ -62,10 +55,13 @@ funcptr = self.malloc_fixedsize_atomic_ptr else: funcptr = self.malloc_fixedsize_ptr - c_finalizer_ptr = Constant(self.finalizer_funcptr_for_type(TYPE), self.FINALIZER_PTR) v_raw = hop.genop("direct_call", - [funcptr, c_size, c_finalizer_ptr], + [funcptr, c_size], resulttype=llmemory.Address) + finalizer_ptr = self.finalizer_funcptr_for_type(TYPE) + if finalizer_ptr: + c_finalizer_ptr = Constant(finalizer_ptr, self.FINALIZER_PTR) + hop.genop("boehm_register_finalizer", [v_raw, c_finalizer_ptr]) return v_raw From arigo at codespeak.net Thu Feb 21 17:01:28 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 17:01:28 +0100 (CET) Subject: [pypy-svn] r51734 - in pypy/branch/unified-rtti/pypy/rpython/memory/gctransform: . test Message-ID: <20080221160128.0C98E1684EA@codespeak.net> Author: arigo Date: Thu Feb 21 17:01:28 2008 New Revision: 51734 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/test/test_boehm.py Log: Fix the Boehm transformer. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py Thu Feb 21 17:01:28 2008 @@ -9,10 +9,12 @@ class BoehmGCTransformer(GCTransformer): FINALIZER_PTR = lltype.Ptr(lltype.FuncType([llmemory.Address], lltype.Void)) + TYPEINFO = lltype.Struct('typeinfo') # empty def __init__(self, translator, inline=False): super(BoehmGCTransformer, self).__init__(translator, inline=inline) self.finalizer_funcptrs = {} + self.rtticache = {} atomic_mh = mallocHelpers() atomic_mh.allocate = lambda size: llop.boehm_malloc_atomic(llmemory.Address, size) @@ -43,6 +45,10 @@ self.mixlevelannotator.finish() # for now self.mixlevelannotator.backend_optimize() + def convert_rtti(self, rtti): + # no information in the TYPEINFO + return lltype.malloc(self.TYPEINFO, immortal=True) + def push_alive_nopyobj(self, var, llops): pass @@ -84,12 +90,11 @@ if TYPE in self.finalizer_funcptrs: return self.finalizer_funcptrs[TYPE] - rtti = get_rtti(TYPE) - if rtti is not None and hasattr(rtti._obj, 'destructor_funcptr'): - destrptr = rtti._obj.destructor_funcptr + rtti = lltype.getRuntimeTypeInfo(TYPE, self.rtticache) + destrptr = rtti.destructor_funcptr + if destrptr is not None: DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0] else: - destrptr = None DESTR_ARG = None if type_contains_pyobjs(TYPE): Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/test/test_boehm.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/test/test_boehm.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/test/test_boehm.py Thu Feb 21 17:01:28 2008 @@ -40,36 +40,28 @@ assert f is not None def test_boehm_finalizer___del__(): - S = lltype.GcStruct("S", ('x', lltype.Signed)) + pinf = lltype.malloc(lltype.RuntimeTypeInfo, immortal=True) + S = lltype.GcStruct("S", ('x', lltype.Signed), runtime_type_info=pinf) def f(s): s.x = 1 - def type_info_S(p): - return lltype.getRuntimeTypeInfo(S) - qp = lltype.functionptr(lltype.FuncType([lltype.Ptr(S)], - lltype.Ptr(lltype.RuntimeTypeInfo)), - "type_info_S", - _callable=type_info_S) dp = lltype.functionptr(lltype.FuncType([lltype.Ptr(S)], lltype.Void), "destructor_funcptr", _callable=f) - pinf = lltype.attachRuntimeTypeInfo(S, qp, destrptr=dp) + pinf.destructor_funcptr = dp f, t = make_boehm_finalizer(S) assert f is not None def test_boehm_finalizer_nomix___del___and_pyobj(): - S = lltype.GcStruct("S", ('x', lltype.Signed), ('y', lltype.Ptr(lltype.PyObject))) + pinf = lltype.malloc(lltype.RuntimeTypeInfo, immortal=True) + S = lltype.GcStruct("S", ('x', lltype.Signed), + ('y', lltype.Ptr(lltype.PyObject)), + runtime_type_info=pinf) def f(s): s.x = 1 - def type_info_S(p): - return lltype.getRuntimeTypeInfo(S) - qp = lltype.functionptr(lltype.FuncType([lltype.Ptr(S)], - lltype.Ptr(lltype.RuntimeTypeInfo)), - "type_info_S", - _callable=type_info_S) dp = lltype.functionptr(lltype.FuncType([lltype.Ptr(S)], lltype.Void), "destructor_funcptr", _callable=f) - pinf = lltype.attachRuntimeTypeInfo(S, qp, destrptr=dp) + pinf.destructor_funcptr = dp py.test.raises(Exception, "make_boehm_finalizer(S)") From arigo at codespeak.net Thu Feb 21 17:31:38 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 17:31:38 +0100 (CET) Subject: [pypy-svn] r51749 - pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/test Message-ID: <20080221163138.7BD8E1684C1@codespeak.net> Author: arigo Date: Thu Feb 21 17:31:38 2008 New Revision: 51749 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/test/test_refcounting.py Log: Fix tests. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/test/test_refcounting.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/test/test_refcounting.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/test/test_refcounting.py Thu Feb 21 17:31:38 2008 @@ -143,7 +143,7 @@ ops.extend([op for op in block.operations if op.opname != 'same_as']) # XXX assert len(ops) == 1 op = ops[0] - assert op.opname == 'gc_free' + assert op.opname == 'raw_free' def test_deallocator_less_simple(): TPtr = lltype.Ptr(lltype.GcStruct("T", ('a', lltype.Signed))) @@ -157,7 +157,7 @@ ops = getops(dgraph) assert len(ops['direct_call']) == 2 assert len(ops['getfield']) == 2 - assert len(ops['gc_free']) == 1 + assert len(ops['raw_free']) == 1 def test_deallocator_array(): TPtr = lltype.Ptr(lltype.GcStruct("T", ('a', lltype.Signed))) @@ -172,7 +172,7 @@ assert len(ops['getfield']) == 2 assert len(ops['getinteriorfield']) == 2 assert len(ops['getinteriorarraysize']) == 1 - assert len(ops['gc_free']) == 1 + assert len(ops['raw_free']) == 1 def test_deallocator_with_destructor(): S = lltype.GcStruct("S", ('x', lltype.Signed)) From arigo at codespeak.net Thu Feb 21 17:33:35 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 17:33:35 +0100 (CET) Subject: [pypy-svn] r51750 - pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/test Message-ID: <20080221163335.B6B151684C1@codespeak.net> Author: arigo Date: Thu Feb 21 17:33:33 2008 New Revision: 51750 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/test/test_refcounting.py Log: More test fixes. Kill tests about killed code. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/test/test_refcounting.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/test/test_refcounting.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/test/test_refcounting.py Thu Feb 21 17:33:33 2008 @@ -175,96 +175,17 @@ assert len(ops['raw_free']) == 1 def test_deallocator_with_destructor(): - S = lltype.GcStruct("S", ('x', lltype.Signed)) + pinf = lltype.malloc(lltype.RuntimeTypeInfo, immortal=True) + S = lltype.GcStruct("S", ('x', lltype.Signed), runtime_type_info=pinf) def f(s): s.x = 1 - def type_info_S(p): - return lltype.getRuntimeTypeInfo(S) - qp = lltype.functionptr(lltype.FuncType([lltype.Ptr(S)], - lltype.Ptr(lltype.RuntimeTypeInfo)), - "type_info_S", - _callable=type_info_S) dp = lltype.functionptr(lltype.FuncType([lltype.Ptr(S)], lltype.Void), "destructor_funcptr", _callable=f) - pinf = lltype.attachRuntimeTypeInfo(S, qp, destrptr=dp) + pinf.destructor_funcptr = dp graph, t = make_deallocator(S) -def test_caching_dynamic_deallocator(): - S = lltype.GcStruct("S", ('x', lltype.Signed)) - S1 = lltype.GcStruct("S1", ('s', S), ('y', lltype.Signed)) - T = lltype.GcStruct("T", ('x', lltype.Signed)) - def f_S(s): - s.x = 1 - def f_S1(s1): - s1.s.x = 1 - s1.y = 2 - def f_T(s): - s.x = 1 - def type_info_S(p): - return lltype.getRuntimeTypeInfo(S) - def type_info_T(p): - return lltype.getRuntimeTypeInfo(T) - qp = lltype.functionptr(lltype.FuncType([lltype.Ptr(S)], - lltype.Ptr(lltype.RuntimeTypeInfo)), - "type_info_S", - _callable=type_info_S) - dp = lltype.functionptr(lltype.FuncType([lltype.Ptr(S)], - lltype.Void), - "destructor_funcptr", - _callable=f_S) - pinf = lltype.attachRuntimeTypeInfo(S, qp, destrptr=dp) - dp = lltype.functionptr(lltype.FuncType([lltype.Ptr(S)], - lltype.Void), - "destructor_funcptr", - _callable=f_S1) - pinf = lltype.attachRuntimeTypeInfo(S1, qp, destrptr=dp) - qp = lltype.functionptr(lltype.FuncType([lltype.Ptr(T)], - lltype.Ptr(lltype.RuntimeTypeInfo)), - "type_info_S", - _callable=type_info_T) - dp = lltype.functionptr(lltype.FuncType([lltype.Ptr(T)], - lltype.Void), - "destructor_funcptr", - _callable=f_T) - pinf = lltype.attachRuntimeTypeInfo(T, qp, destrptr=dp) - def f(): - pass - t = TranslationContext() - t.buildannotator().build_types(f, []) - t.buildrtyper().specialize() - transformer = RefcountingGCTransformer(t) - p_S = transformer.dynamic_deallocation_funcptr_for_type(S) - p_S1 = transformer.dynamic_deallocation_funcptr_for_type(S1) - p_T = transformer.dynamic_deallocation_funcptr_for_type(T) - assert p_S is not p_T - assert p_S is p_S1 - -def test_dynamic_deallocator(): - class A(object): - pass - class B(A): - pass - def f(x): - a = A() - a.x = 1 - b = B() - b.x = 2 - b.y = 3 - if x: - c = a - else: - c = b - return c.x - t, transformer = rtype_and_transform( - f, [int], RefcountingGCTransformer, check=False) - fgraph = graphof(t, f) - s_instance = t.annotator.bookkeeper.valueoftype(A) - TYPE = t.rtyper.getrepr(s_instance).lowleveltype.TO - p = transformer.dynamic_deallocation_funcptr_for_type(TYPE) - t.rtyper.specialize_more_blocks() - def test_recursive_structure(): F = lltype.GcForwardReference() S = lltype.GcStruct('abc', ('x', lltype.Ptr(F))) From arigo at codespeak.net Thu Feb 21 17:38:50 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 17:38:50 +0100 (CET) Subject: [pypy-svn] r51751 - in pypy/branch/unified-rtti/pypy: rpython/memory/gctransform translator/c Message-ID: <20080221163850.AB1531684DA@codespeak.net> Author: arigo Date: Thu Feb 21 17:38:50 2008 New Revision: 51751 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py pypy/branch/unified-rtti/pypy/translator/c/database.py Log: Refactor GCTransformer to assume that all transformers have a HDR and a TYPEINFO. Move code to the common base. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py Thu Feb 21 17:38:50 2008 @@ -1,4 +1,5 @@ -from pypy.rpython.memory.gctransform.transform import GCTransformer, mallocHelpers +from pypy.rpython.memory.gctransform.transform import GCTransformer +from pypy.rpython.memory.gctransform.transform import mallocHelpers, RTTIPTR from pypy.rpython.memory.gctransform.support import type_contains_pyobjs, \ _static_deallocator_body_for_type, LLTransformerOp, ll_call_destructor from pypy.rpython.lltypesystem import lltype, llmemory @@ -9,12 +10,13 @@ class BoehmGCTransformer(GCTransformer): FINALIZER_PTR = lltype.Ptr(lltype.FuncType([llmemory.Address], lltype.Void)) + # XXX not all objects need a typeptr + HDR = lltype.Struct("header", ("typeptr", RTTIPTR)) TYPEINFO = lltype.Struct('typeinfo') # empty def __init__(self, translator, inline=False): super(BoehmGCTransformer, self).__init__(translator, inline=inline) self.finalizer_funcptrs = {} - self.rtticache = {} atomic_mh = mallocHelpers() atomic_mh.allocate = lambda size: llop.boehm_malloc_atomic(llmemory.Address, size) @@ -45,9 +47,8 @@ self.mixlevelannotator.finish() # for now self.mixlevelannotator.backend_optimize() - def convert_rtti(self, rtti): - # no information in the TYPEINFO - return lltype.malloc(self.TYPEINFO, immortal=True) + def initialize_typeinfo(self, typeinfo, rtti, TYPE): + pass # typeinfo is empty def push_alive_nopyobj(self, var, llops): pass Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py Thu Feb 21 17:38:50 2008 @@ -1,12 +1,12 @@ import py -from pypy.rpython.memory.gctransform.transform import GCTransformer, mallocHelpers +from pypy.rpython.memory.gctransform.transform import GCTransformer +from pypy.rpython.memory.gctransform.transform import mallocHelpers, RTTIPTR from pypy.rpython.memory.gctransform.support import find_gc_ptrs_in_type, \ _static_deallocator_body_for_type, LLTransformerOp, ll_call_destructor from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem.lloperation import llop from pypy.translator.backendopt.support import var_needsgc from pypy.rpython import rmodel -from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rlib.rarithmetic import ovfcheck from pypy.rlib.debug import ll_assert from pypy.rpython.rbuiltin import gen_cast @@ -32,7 +32,6 @@ ## print ' '*i, a, repr(b)[:100-i-len(a)], id(b) ADDRESS_VOID_FUNC = lltype.FuncType([llmemory.Address], lltype.Void) -RTTIPTR = lltype.Ptr(lltype.RuntimeTypeInfo) class RefcountingGCTransformer(GCTransformer): @@ -46,40 +45,30 @@ def __init__(self, translator): super(RefcountingGCTransformer, self).__init__(translator, inline=True) - gchdrbuilder = GCHeaderBuilder(self.HDR, self.TYPEINFO) - self.gcheaderbuilder = gchdrbuilder - gc_header_offset = self.gcheaderbuilder.size_gc_header self.deallocator_graphs_needing_transforming = [] - self.rtticache = {} # create incref, etc graph - memoryError = MemoryError() + gchelpers = self.gchelpers + gc_header_offset = gchelpers.gc_header_offset HDRPTR = lltype.Ptr(self.HDR) def ll_incref(adr): if adr: - gcheader = llmemory.cast_adr_to_ptr(adr - gc_header_offset, HDRPTR) + gcheader = gchelpers.header(adr) gcheader.refcount = gcheader.refcount + 1 def ll_decref(adr): if adr: - gcheader = llmemory.cast_adr_to_ptr(adr - gc_header_offset, HDRPTR) + gcheader = gchelpers.header(adr) refcount = gcheader.refcount - 1 gcheader.refcount = refcount if refcount == 0: - rtti = gcheader.typeptr - typeinfo = gchdrbuilder.cast_rtti_to_typeinfo(rtti) + typeinfo = gchelpers.typeof(adr) typeinfo.dealloc(adr) def ll_no_pointer_dealloc(adr): llmemory.raw_free(adr) - def ll_gc_runtime_type_info(adr): - gcheader = llmemory.cast_adr_to_ptr(adr - gc_header_offset, HDRPTR) - rtti = gcheader.typeptr - ll_assert(bool(rtti), "NULL rtti pointer") - return rtti - mh = mallocHelpers() mh.allocate = llmemory.raw_malloc def ll_malloc_fixedsize(size): @@ -120,8 +109,6 @@ ll_decref, [llmemory.Address], lltype.Void) self.no_pointer_dealloc_ptr = self.inittime_helper( ll_no_pointer_dealloc, [llmemory.Address], lltype.Void) - self.gc_runtime_type_info_ptr = self.inittime_helper( - ll_gc_runtime_type_info, [llmemory.Address], RTTIPTR) self.malloc_fixedsize_ptr = self.inittime_helper( ll_malloc_fixedsize_rtti, [lltype.Signed, RTTIPTR], llmemory.Address) @@ -185,26 +172,6 @@ resulttype=llmemory.Address) return v_raw - def gct_gc_runtime_type_info(self, hop): - [v_ptr] = hop.spaceop.args - v_adr = hop.genop("cast_ptr_to_adr", [v_ptr], - resulttype=llmemory.Address) - v_result = hop.spaceop.result - assert v_result.concretetype == RTTIPTR - hop.genop("direct_call", - [self.gc_runtime_type_info_ptr, v_adr], - resultvar = v_result) - - def consider_constant(self, TYPE, value): - if value is not lltype.top_container(value): - return - if isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)): - p = value._as_ptr() - if not self.gcheaderbuilder.get_header(p): - hdr = self.gcheaderbuilder.new_header(p) - hdr.refcount = sys.maxint // 2 - hdr.typeptr = lltype.getRuntimeTypeInfo(TYPE, self.rtticache) - def static_deallocation_funcptr_for_type(self, TYPE): """The 'static deallocator' for a type is the function that can free a pointer that we know points exactly to a TYPE structure @@ -282,18 +249,9 @@ self.static_deallocation_funcptr_for_type(p.TO) return fptr - def convert_rtti(self, rtti): - rtti = rtti._as_ptr() - try: - return self.gcheaderbuilder.typeinfo_from_rtti(rtti) - except KeyError: - typeinfo = self.gcheaderbuilder.new_typeinfo(rtti) - try: - TYPE = lltype.getGcTypeForRtti(rtti) - except ValueError: - pass # ignore rtti's not attached anywhere, e.g. in the - # vtable of raw-flavored RPython classes - else: - fn = self.static_deallocation_funcptr_for_type(TYPE) - typeinfo.dealloc = fn - return typeinfo + def initialize_constant_header(self, hdr, TYPE, value): + hdr.refcount = sys.maxint // 2 + + def initialize_typeinfo(self, typeinfo, rtti, TYPE): + fn = self.static_deallocation_funcptr_for_type(TYPE) + typeinfo.dealloc = fn Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py Thu Feb 21 17:38:50 2008 @@ -13,6 +13,7 @@ from pypy.annotation import model as annmodel from pypy.rpython import rmodel, annlowlevel from pypy.rpython.memory import gc +from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rpython.memory.gctransform.support import var_ispyobj from pypy.rpython.annlowlevel import MixLevelHelperAnnotator from pypy.rpython.rtyper import LowLevelOpList @@ -23,6 +24,7 @@ from pypy.translator.simplify import join_blocks, cleanup_graph PyObjPtr = lltype.Ptr(lltype.PyObject) +RTTIPTR = lltype.Ptr(lltype.RuntimeTypeInfo) class GcHighLevelOp(object): def __init__(self, gct, op, index, llops): @@ -457,10 +459,51 @@ return mh +def GCHelpers(gcheaderbuilder): + class _GcHelpers(object): + def _freeze_(self): + return True + gh = _GcHelpers() + + gc_header_offset = gh.gc_header_offset = gcheaderbuilder.size_gc_header + HDRPTR = lltype.Ptr(gcheaderbuilder.HDR) + + def header(objaddr): + "Get the header of an object identified by its address." + hdraddr = objaddr - gc_header_offset + return llmemory.cast_adr_to_ptr(hdraddr, HDRPTR) + gh.header = header + + def gc_runtime_type_info(objaddr): + "Implementation of the gc_runtime_type_info operation." + # NB. this assumes that the HDR contains a typeptr field. + # A bit of manual inlining... + hdraddr = objaddr - gc_header_offset + return llmemory.cast_adr_to_ptr(hdraddr, HDRPTR).typeptr + gh.gc_runtime_type_info = gc_runtime_type_info + + def typeof(objaddr): + "Get the typeinfo describing the type of the object at 'objaddr'." + # NB. this assumes that the HDR contains a typeptr field + # A bit of manual inlining... + hdraddr = objaddr - gc_header_offset + rtti = llmemory.cast_adr_to_ptr(hdraddr, HDRPTR).typeptr + return gcheaderbuilder.cast_rtti_to_typeinfo(rtti) + gh.typeof = typeof + + return gh + + class GCTransformer(BaseGCTransformer): def __init__(self, translator, inline=False): super(GCTransformer, self).__init__(translator, inline=inline) + # at the moment, all GC transformers define a HDR structure that + # is added in front of all GC objects, and a TYPEINFO structure + # that works as RuntimeTypeInfo + self.gcheaderbuilder = GCHeaderBuilder(self.HDR, self.TYPEINFO) + self.gchelpers = GCHelpers(self.gcheaderbuilder) + self.rtticache = {} mh = mallocHelpers() mh.allocate = llmemory.raw_malloc @@ -486,6 +529,10 @@ self.stack_malloc_fixedsize_ptr = self.inittime_helper( ll_stack_malloc_fixedsize, [lltype.Signed], llmemory.Address) + self.gc_runtime_type_info_ptr = self.inittime_helper( + self.gchelpers.gc_runtime_type_info, [llmemory.Address], + RTTIPTR) + def gct_malloc(self, hop): TYPE = hop.spaceop.result.concretetype.TO assert not TYPE._is_varsize() @@ -611,3 +658,41 @@ hop.genop('raw_free', [v]) else: assert False, "%s has no support for free with flavor %r" % (self, flavor) + + def gct_gc_runtime_type_info(self, hop): + [v_ptr] = hop.spaceop.args + v_adr = hop.genop("cast_ptr_to_adr", [v_ptr], + resulttype=llmemory.Address) + v_result = hop.spaceop.result + assert v_result.concretetype == RTTIPTR + hop.genop("direct_call", + [self.gc_runtime_type_info_ptr, v_adr], + resultvar = v_result) + + def consider_constant(self, TYPE, value): + if value is not lltype.top_container(value): + return + if isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)): + p = value._as_ptr() + if not self.gcheaderbuilder.get_header(p): + hdr = self.gcheaderbuilder.new_header(p) + hdr.typeptr = lltype.getRuntimeTypeInfo(TYPE, self.rtticache) + self.initialize_constant_header(hdr, TYPE, value) + + def initialize_constant_header(self, hdr, TYPE, value): + pass + + def convert_rtti(self, rtti): + rtti = rtti._as_ptr() + try: + return self.gcheaderbuilder.typeinfo_from_rtti(rtti) + except KeyError: + typeinfo = self.gcheaderbuilder.new_typeinfo(rtti) + try: + TYPE = lltype.getGcTypeForRtti(rtti) + except ValueError: + pass # ignore rtti's not attached anywhere, e.g. in the + # vtable of raw-flavored RPython classes + else: + self.initialize_typeinfo(typeinfo, rtti, TYPE) + return typeinfo Modified: pypy/branch/unified-rtti/pypy/translator/c/database.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/database.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/database.py Thu Feb 21 17:38:50 2008 @@ -153,7 +153,7 @@ except KeyError: T = typeOf(container) if isinstance(T, (lltype.Array, lltype.Struct)): - if hasattr(self.gctransformer, 'consider_constant'): + if self.gctransformer is not None: self.gctransformer.consider_constant(T, container) nodefactory = ContainerNodeFactory[T.__class__] node = nodefactory(self, T, container, **buildkwds) From arigo at codespeak.net Thu Feb 21 17:43:30 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 17:43:30 +0100 (CET) Subject: [pypy-svn] r51752 - pypy/branch/unified-rtti/pypy/rpython/memory/gctransform Message-ID: <20080221164330.533D01684CA@codespeak.net> Author: arigo Date: Thu Feb 21 17:43:29 2008 New Revision: 51752 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py Log: Lost the ll_assert along the way. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py Thu Feb 21 17:43:29 2008 @@ -8,7 +8,6 @@ from pypy.translator.backendopt.support import var_needsgc from pypy.rpython import rmodel from pypy.rlib.rarithmetic import ovfcheck -from pypy.rlib.debug import ll_assert from pypy.rpython.rbuiltin import gen_cast import sys Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py Thu Feb 21 17:43:29 2008 @@ -19,6 +19,7 @@ from pypy.rpython.rtyper import LowLevelOpList from pypy.rpython.rbuiltin import gen_cast from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib.debug import ll_assert import sets, os, sys from pypy.rpython.lltypesystem.lloperation import llop from pypy.translator.simplify import join_blocks, cleanup_graph @@ -479,7 +480,9 @@ # NB. this assumes that the HDR contains a typeptr field. # A bit of manual inlining... hdraddr = objaddr - gc_header_offset - return llmemory.cast_adr_to_ptr(hdraddr, HDRPTR).typeptr + rtti = llmemory.cast_adr_to_ptr(hdraddr, HDRPTR).typeptr + ll_assert(bool(rtti), "NULL rtti") + return rtti gh.gc_runtime_type_info = gc_runtime_type_info def typeof(objaddr): @@ -488,6 +491,7 @@ # A bit of manual inlining... hdraddr = objaddr - gc_header_offset rtti = llmemory.cast_adr_to_ptr(hdraddr, HDRPTR).typeptr + ll_assert(bool(rtti), "NULL rtti") return gcheaderbuilder.cast_rtti_to_typeinfo(rtti) gh.typeof = typeof From arigo at codespeak.net Thu Feb 21 17:49:40 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 17:49:40 +0100 (CET) Subject: [pypy-svn] r51753 - pypy/branch/unified-rtti/pypy/translator/c Message-ID: <20080221164940.5B04B168457@codespeak.net> Author: arigo Date: Thu Feb 21 17:49:39 2008 New Revision: 51753 Modified: pypy/branch/unified-rtti/pypy/translator/c/gc.py Log: Kill code until the Boehm tests pass. :-) Modified: pypy/branch/unified-rtti/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/gc.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/gc.py Thu Feb 21 17:49:39 2008 @@ -15,10 +15,20 @@ self.thread_enabled = thread_enabled def common_gcheader_definition(self, defnode): - return [] + if defnode.db.gctransformer is not None: + HDR = defnode.db.gctransformer.HDR + return [(name, HDR._flds[name]) for name in HDR._names] + else: + return [] def common_gcheader_initdata(self, defnode): - return [] + if defnode.db.gctransformer is not None: + gct = defnode.db.gctransformer + hdr = gct.gcheaderbuilder.header_of_object(top_container(defnode.obj)) + HDR = gct.HDR + return [getattr(hdr, fldname) for fldname in HDR._names] + else: + return [] def struct_gcheader_definition(self, defnode): return self.common_gcheader_definition(defnode) @@ -71,29 +81,6 @@ class RefcountingGcPolicy(BasicGcPolicy): transformerclass = refcounting.RefcountingGCTransformer - def common_gcheader_definition(self, defnode): - if defnode.db.gctransformer is not None: - HDR = defnode.db.gctransformer.HDR - return [(name, HDR._flds[name]) for name in HDR._names] - else: - return [] - - def common_gcheader_initdata(self, defnode): - if defnode.db.gctransformer is not None: - gct = defnode.db.gctransformer - hdr = gct.gcheaderbuilder.header_of_object(top_container(defnode.obj)) - HDR = gct.HDR - return [getattr(hdr, fldname) for fldname in HDR._names] - else: - return [] - - # zero malloc impl - - def OP_GC_CALL_RTTI_DESTRUCTOR(self, funcgen, op): - args = [funcgen.expr(v) for v in op.args] - line = '%s(%s);' % (args[0], ', '.join(args[1:])) - return line - def OP_GC__COLLECT(self, funcgen, op): return '' From arigo at codespeak.net Thu Feb 21 17:59:19 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 17:59:19 +0100 (CET) Subject: [pypy-svn] r51754 - pypy/branch/unified-rtti/pypy/rpython/memory/gctransform Message-ID: <20080221165919.6096B168457@codespeak.net> Author: arigo Date: Thu Feb 21 17:59:18 2008 New Revision: 51754 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py Log: Shuffle in preparation of porting the framework transformer. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py Thu Feb 21 17:59:18 2008 @@ -16,6 +16,7 @@ def __init__(self, translator, inline=False): super(BoehmGCTransformer, self).__init__(translator, inline=inline) + self.newgcheaderbuilder(self.HDR, self.TYPEINFO) self.finalizer_funcptrs = {} atomic_mh = mallocHelpers() Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py Thu Feb 21 17:59:18 2008 @@ -48,6 +48,7 @@ # create incref, etc graph + self.newgcheaderbuilder(self.HDR, self.TYPEINFO) gchelpers = self.gchelpers gc_header_offset = gchelpers.gc_header_offset HDRPTR = lltype.Ptr(self.HDR) Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py Thu Feb 21 17:59:18 2008 @@ -13,7 +13,6 @@ from pypy.annotation import model as annmodel from pypy.rpython import rmodel, annlowlevel from pypy.rpython.memory import gc -from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rpython.memory.gctransform.support import var_ispyobj from pypy.rpython.annlowlevel import MixLevelHelperAnnotator from pypy.rpython.rtyper import LowLevelOpList @@ -502,12 +501,6 @@ def __init__(self, translator, inline=False): super(GCTransformer, self).__init__(translator, inline=inline) - # at the moment, all GC transformers define a HDR structure that - # is added in front of all GC objects, and a TYPEINFO structure - # that works as RuntimeTypeInfo - self.gcheaderbuilder = GCHeaderBuilder(self.HDR, self.TYPEINFO) - self.gchelpers = GCHelpers(self.gcheaderbuilder) - self.rtticache = {} mh = mallocHelpers() mh.allocate = llmemory.raw_malloc @@ -533,6 +526,16 @@ self.stack_malloc_fixedsize_ptr = self.inittime_helper( ll_stack_malloc_fixedsize, [lltype.Signed], llmemory.Address) + def newgcheaderbuilder(self, HDR, TYPEINFO): + from pypy.rpython.memory.gcheader import GCHeaderBuilder + self.setgcheaderbuilder(GCHeaderBuilder(HDR, TYPEINFO)) + + def setgcheaderbuilder(self, gcheaderbuilder): + # at the moment, all GC transformers are based on a GCHeaderBuilder. + self.gcheaderbuilder = gcheaderbuilder + self.gchelpers = GCHelpers(gcheaderbuilder) + self.rtticache = {} + if self.translator: self.gc_runtime_type_info_ptr = self.inittime_helper( self.gchelpers.gc_runtime_type_info, [llmemory.Address], RTTIPTR) From arigo at codespeak.net Thu Feb 21 18:19:35 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 18:19:35 +0100 (CET) Subject: [pypy-svn] r51755 - in pypy/branch/unified-rtti/pypy: rpython/memory rpython/memory/gctransform translator/c Message-ID: <20080221171935.741CE168471@codespeak.net> Author: arigo Date: Thu Feb 21 18:19:33 2008 New Revision: 51755 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py pypy/branch/unified-rtti/pypy/rpython/memory/gctypelayout.py pypy/branch/unified-rtti/pypy/translator/c/gc.py Log: In-progress. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py Thu Feb 21 18:19:33 2008 @@ -5,7 +5,6 @@ from pypy.rpython import rmodel from pypy.rpython.memory import gctypelayout from pypy.rpython.memory.gc import marksweep -from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rlib.rarithmetic import ovfcheck from pypy.rlib.debug import ll_assert from pypy.translator.backendopt import graphanalyze @@ -332,12 +331,6 @@ self.c_const_gc = rmodel.inputconst(r_gc, self.gcdata.gc) self.needs_zero_gc_pointers = GCClass.needs_zero_gc_pointers - HDR = self._gc_HDR = self.gcdata.gc.gcheaderbuilder.HDR - self._gc_fields = fields = [] - for fldname in HDR._names: - FLDTYPE = getattr(HDR, fldname) - fields.append(('_' + fldname, FLDTYPE)) - def build_root_walker(self): return ShadowStackRootWalker(self) @@ -351,14 +344,6 @@ def finalizer_funcptr_for_type(self, TYPE): return self.layoutbuilder.finalizer_funcptr_for_type(TYPE) - def gc_fields(self): - return self._gc_fields - - def gc_field_values_for(self, obj): - hdr = self.gcdata.gc.gcheaderbuilder.header_of_object(obj) - HDR = self._gc_HDR - return [getattr(hdr, fldname) for fldname in HDR._names] - def finish_tables(self): table = self.layoutbuilder.flatten_table() log.info("assigned %s typeids" % (len(table), )) Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctypelayout.py Thu Feb 21 18:19:33 2008 @@ -15,8 +15,9 @@ ADDRESS_VOID_FUNC = lltype.FuncType([llmemory.Address], lltype.Void) FINALIZERTYPE = lltype.Ptr(ADDRESS_VOID_FUNC) - # structure describing the layout of a typeid + # structure describing the layout of a type TYPE_INFO = lltype.Struct("type_info", + ("flags", lltype.Signed), # T_... flags, see below ("finalizer", FINALIZERTYPE), ("fixedsize", lltype.Signed), ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), @@ -26,58 +27,39 @@ ("varofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), ("weakptrofs", lltype.Signed), ) - TYPE_INFO_TABLE = lltype.Array(TYPE_INFO) - def __init__(self, type_info_table): - self.type_info_table = type_info_table - # 'type_info_table' is a list of TYPE_INFO structures when - # running with gcwrapper, or a real TYPE_INFO_TABLE after - # the gctransformer. - - def q_is_varsize(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return (typeid & T_IS_FIXSIZE) == 0 - - def q_has_gcptr_in_varsize(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return (typeid & (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE)) == 0 - - def q_is_gcarrayofgcptr(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return (typeid & - (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE|T_NOT_SIMPLE_GCARRAY)) == 0 - - def q_finalizer(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].finalizer - - def q_offsets_to_gc_pointers(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].ofstoptrs - - def q_fixed_size(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].fixedsize - - def q_varsize_item_sizes(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].varitemsize - - def q_varsize_offset_to_variable_part(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].ofstovar - - def q_varsize_offset_to_length(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].ofstolength - - def q_varsize_offsets_to_gcpointers_in_var_part(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].varofstoptrs - - def q_weakpointer_offset(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].weakptrofs + def q_is_varsize(self, typeinfo): + return (typeinfo.flags & T_IS_VARSIZE) != 0 + + def q_has_gcptr_in_varsize(self, typeinfo): + return (typeinfo.flags & T_HAS_GCPTR_IN_VARSIZE) != 0 + + def q_is_gcarrayofgcptr(self, typeinfo): + return (typeinfo.flags & T_IS_GCARRAYOFGCPTR) != 0 + + def q_finalizer(self, typeinfo): + return typeinfo.finalizer + + def q_offsets_to_gc_pointers(self, typeinfo): + return typeinfo.ofstoptrs + + def q_fixed_size(self, typeinfo): + return typeinfo.fixedsize + + def q_varsize_item_sizes(self, typeinfo): + return typeinfo.varitemsize + + def q_varsize_offset_to_variable_part(self, typeinfo): + return typeinfo.ofstovar + + def q_varsize_offset_to_length(self, typeinfo): + return typeinfo.ofstolength + + def q_varsize_offsets_to_gcpointers_in_var_part(self, typeinfo): + return typeinfo.varofstoptrs + + def q_weakpointer_offset(self, typeinfo): + return typeinfo.weakptrofs def set_query_functions(self, gc): gc.set_query_functions( @@ -93,34 +75,27 @@ self.q_varsize_offsets_to_gcpointers_in_var_part, self.q_weakpointer_offset) -# For the q_xxx functions that return flags, we use bit patterns -# in the typeid instead of entries in the type_info_table. The -# following flag combinations are used (the idea being that it's -# very fast on CPUs to check if all flags in a set are all zero): - -# * if T_IS_FIXSIZE is set, the gc object is not var-sized -# * if T_IS_FIXSIZE and T_NO_GCPTR_IN_VARSIZE are both cleared, -# there are gc ptrs in the var-sized part -# * if T_IS_FIXSIZE, T_NO_GCPTR_IN_VARSIZE and T_NOT_SIMPLE_GCARRAY -# are all cleared, the shape is just like GcArray(gcptr) - -T_IS_FIXSIZE = 0x4 -T_NO_GCPTR_IN_VARSIZE = 0x2 -T_NOT_SIMPLE_GCARRAY = 0x1 - -def get_typeid_bitmask(TYPE): - """Return the bits that we would like to be set or cleared in the type_id - corresponding to TYPE. This returns (mask, expected_value), where - the condition is that 'type_id & mask == expected_value'. +# For the q_xxx functions that return flags, we use bits in the 'flags' +# field. The idea is that at some point, some GCs could copy these bits +# into the header of each object to allow for an indirection-free decoding. +# (Not all combinations of the 3 flags are meaningful; in fact, only 4 are) + +T_IS_VARSIZE = 0x1 +T_HAS_GCPTR_IN_VARSIZE = 0x2 +T_IS_GCARRAYOFGCPTR = 0x4 +T_first_unused_bit = 0x8 + +def get_type_flags(TYPE): + """Compute the 'flags' for the type. """ if not TYPE._is_varsize(): - return (T_IS_FIXSIZE, T_IS_FIXSIZE) # not var-sized + return 0 # not var-sized if (isinstance(TYPE, lltype.GcArray) and isinstance(TYPE.OF, lltype.Ptr) and TYPE.OF.TO._gckind == 'gc'): # a simple GcArray(gcptr) - return (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE|T_NOT_SIMPLE_GCARRAY, 0) + return T_IS_VARSIZE | T_HAS_GCPTR_IN_VARSIZE | T_IS_GCARRAYOFGCPTR if isinstance(TYPE, lltype.Struct): ARRAY = TYPE._flds[TYPE._arrayfld] @@ -129,22 +104,20 @@ assert isinstance(ARRAY, lltype.Array) if ARRAY.OF != lltype.Void and len(offsets_to_gc_pointers(ARRAY.OF)) > 0: # var-sized, with gc pointers in the variable part - return (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE|T_NOT_SIMPLE_GCARRAY, - T_NOT_SIMPLE_GCARRAY) + return T_IS_VARSIZE | T_HAS_GCPTR_IN_VARSIZE else: # var-sized, but no gc pointer in the variable part - return (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE, T_NO_GCPTR_IN_VARSIZE) + return T_IS_VARSIZE def encode_type_shape(builder, info, TYPE): """Encode the shape of the TYPE into the TYPE_INFO structure 'info'.""" offsets = offsets_to_gc_pointers(TYPE) + info.flags = get_type_flags(TYPE) info.ofstoptrs = builder.offsets2table(offsets, TYPE) info.finalizer = builder.make_finalizer_funcptr_for_type(TYPE) info.weakptrofs = weakpointer_offset(TYPE) if not TYPE._is_varsize(): - #info.isvarsize = False - #info.gcptrinvarsize = False info.fixedsize = llarena.round_up_for_allocation( llmemory.sizeof(TYPE)) info.ofstolength = -1 @@ -153,7 +126,6 @@ # varsize ones, the GC must anyway compute the size at run-time # and round up that result. else: - #info.isvarsize = True info.fixedsize = llmemory.sizeof(TYPE, 0) if isinstance(TYPE, lltype.Struct): ARRAY = TYPE._flds[TYPE._arrayfld] @@ -174,20 +146,14 @@ offsets = () info.varofstoptrs = builder.offsets2table(offsets, ARRAY.OF) info.varitemsize = llmemory.sizeof(ARRAY.OF) - #info.gcptrinvarsize = len(offsets) > 0 - #info.gcarrayofgcptr = (isinstance(TYPE, lltype.GcArray) - # and isinstance(TYPE.OF, lltype.Ptr) - # and TYPE.OF.TO._gckind == 'gc') # ____________________________________________________________ class TypeLayoutBuilder(object): - can_add_new_types = True def __init__(self): - self.type_info_list = [None] # don't use typeid 0, helps debugging - self.id_of_type = {} # {LLTYPE: type_id} + self.typeinfos = {} # {LLTYPE: TYPE_INFO} self.seen_roots = {} # the following are lists of addresses of gc pointers living inside the # prebuilt structures. It should list all the locations that could @@ -202,38 +168,17 @@ self.additional_roots_sources = 0 self.finalizer_funcptrs = {} self.offsettable_cache = {} - self.next_typeid_cache = {} - def get_type_id(self, TYPE): + def get_type_info(self, TYPE): try: - return self.id_of_type[TYPE] + return self.typeinfos[TYPE] except KeyError: - assert self.can_add_new_types assert isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)) - # Record the new type_id description as a TYPE_INFO structure. - # It goes into a list for now, which will be turned into a - # TYPE_INFO_TABLE in flatten_table() by the gc transformer. - - # pick the next type_id with the correct bits set or cleared - mask, expected = get_typeid_bitmask(TYPE) - type_id = self.next_typeid_cache.get((mask, expected), 1) - while True: - if type_id == len(self.type_info_list): - self.type_info_list.append(None) - if (self.type_info_list[type_id] is None and - (type_id & mask) == expected): - break # can use this type_id - else: - type_id += 1 # continue searching - self.next_typeid_cache[mask, expected] = type_id + 1 - assert type_id & 0xffff == type_id # make sure it fits into 2 bytes - - # build the TYPE_INFO structure + # Record the new type description as a TYPE_INFO structure. info = lltype.malloc(GCData.TYPE_INFO, immortal=True, zero=True) encode_type_shape(self, info, TYPE) - self.type_info_list[type_id] = info - self.id_of_type[TYPE] = type_id - return type_id + self.typeinfos[TYPE] = info + return info def offsets2table(self, offsets, TYPE): try: @@ -246,21 +191,6 @@ self.offsettable_cache[TYPE] = cachedarray return cachedarray - def flatten_table(self): - self.can_add_new_types = False - self.offsettable_cache = None - table = lltype.malloc(GCData.TYPE_INFO_TABLE, len(self.type_info_list), - immortal=True) - fieldnames = GCData.TYPE_INFO._names - for tableentry, newcontent in zip(table, self.type_info_list): - if newcontent is None: # empty entry - tableentry.weakptrofs = -1 - tableentry.ofstolength = -1 - else: - for name in fieldnames: - setattr(tableentry, name, getattr(newcontent, name)) - return table - def finalizer_funcptr_for_type(self, TYPE): if TYPE in self.finalizer_funcptrs: return self.finalizer_funcptrs[TYPE] @@ -283,10 +213,10 @@ self.seen_roots[id(value)] = True if isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)): - typeid = self.get_type_id(TYPE) + typeinfo = self.get_type_info(TYPE) hdr = gc.gcheaderbuilder.new_header(value) adr = llmemory.cast_ptr_to_adr(hdr) - gc.init_gc_object_immortal(adr, typeid) + gc.init_gc_object_immortal(adr, typeinfo) # The following collects the addresses of all the fields that have # a GC Pointer type, inside the current prebuilt object. All such Modified: pypy/branch/unified-rtti/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/gc.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/gc.py Thu Feb 21 18:19:33 2008 @@ -158,12 +158,6 @@ args = [funcgen.expr(v) for v in op.args] return '%s = %s; /* for moving GCs */' % (args[1], args[0]) - def common_gcheader_definition(self, defnode): - return defnode.db.gctransformer.gc_fields() - - def common_gcheader_initdata(self, defnode): - o = top_container(defnode.obj) - return defnode.db.gctransformer.gc_field_values_for(o) class StacklessFrameworkGcPolicy(FrameworkGcPolicy): transformerclass = stacklessframework.StacklessFrameworkGCTransformer From pypy-svn at codespeak.net Thu Feb 21 18:53:19 2008 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Thu, 21 Feb 2008 18:53:19 +0100 (CET) Subject: [pypy-svn] February 70% OFF Message-ID: <20080221115232.3903.qmail@balzak.customer.top.net.ua> An HTML attachment was scrubbed... URL: From cfbolz at codespeak.net Thu Feb 21 20:48:03 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 21 Feb 2008 20:48:03 +0100 (CET) Subject: [pypy-svn] r51758 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080221194803.CD203168452@codespeak.net> Author: cfbolz Date: Thu Feb 21 20:48:02 2008 New Revision: 51758 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Log: first two portal tests really pass Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py Thu Feb 21 20:48:02 2008 @@ -1,12 +1,298 @@ -# graph transformations for transforming the portal graph(s) +from pypy.jit.hintannotator.model import originalconcretetype +from pypy.jit.timeshifter import rvalue +from pypy.objspace.flow import model as flowmodel +from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.unroll import unrolling_iterable +from pypy.rpython.annlowlevel import llhelper, cachedtype +from pypy.rpython.llinterp import LLInterpreter +from pypy.rpython.lltypesystem import lltype +# graph transformations for transforming the portal graph(s) class PortalRewriter(object): - def __init__(self, hintannotator, rtyper, RGenOp): + def __init__(self, hintannotator, rtyper, RGenOp, codewriter): self.hintannotator = hintannotator self.rtyper = rtyper + self.interpreter = codewriter.interpreter + self.codewriter = codewriter self.RGenOp = RGenOp - def rewrite(self, origportalgraph, view=False): + def rewrite(self, origportalgraph, portalgraph, view=False): self.origportalgraph = origportalgraph + self.portalgraph = portalgraph self.view = view self.readportalgraph = None + self.make_args_specification() + self.PortalState = make_state_class( + self.args_specification, self.RESIDUAL_FUNCTYPE, self.sigtoken, + self.codewriter.all_graphs[self.portalgraph], + self.rtyper) + self.make_state_instance() + self.mutate_origportalgraph() + + def make_args_specification(self): + args_specification = [] + RESTYPE = originalconcretetype( + self.hintannotator.binding(self.portalgraph.getreturnvar())) + ARGS = [] + ORIGARGS = [] + for v in self.portalgraph.getargs(): + ORIGARGS.append(v.concretetype) + binding = self.hintannotator.binding(v) + concretetype = originalconcretetype(binding) + if binding.is_green(): + ORIGARGS.append(concretetype) + arg_spec = "green", None, None + else: + argdesc = self.getportalargdesc(concretetype) + arg_spec = ("red", argdesc.residual_args_collector(), + argdesc.arg_redbox_maker()) + ARGS.extend(argdesc.residual_argtypes()) + args_specification.append(arg_spec) + self.args_specification = args_specification + self.RESIDUAL_FUNCTYPE = lltype.FuncType(ARGS, RESTYPE) + self.PORTAL_FUNCTYPE = lltype.FuncType(ORIGARGS, RESTYPE) + self.sigtoken = self.RGenOp.sigToken(self.RESIDUAL_FUNCTYPE) + + def make_state_instance(self): + state = self.PortalState(self.interpreter) + def portal_entry(*args): + return state.portal_entry(*args) + self.state = state + self.portal_entry = portal_entry + + def mutate_origportalgraph(self): + # XXX + # the following line should really be a call to a mixlevel annotator + # but for now the jit stuff should be run directly to make tests faster + # currently this makes it untranslatable + portal_entry_graph_ptr = llhelper(lltype.Ptr(self.PORTAL_FUNCTYPE), + self.portal_entry) + # the following gives a pdb prompt when portal_entry raises an exception + portal_entry_graph_ptr._obj.__dict__['_debugexc'] = True + # XXX hack hack hack + args = [flowmodel.Constant(portal_entry_graph_ptr, + lltype.Ptr(self.PORTAL_FUNCTYPE))] + args += self.origportalgraph.getargs() + result = flowmodel.Variable() + result.concretetype = self.origportalgraph.getreturnvar().concretetype + block = self.origportalgraph.startblock + block.operations[:] = [ + flowmodel.SpaceOperation("direct_call", args, result)] + block.exitswitch = None + block.exits = [flowmodel.Link([result], self.origportalgraph.returnblock)] + self.origportalgraph.exceptblock = None + + def getportalargdesc(self, lowleveltype): + assert not isinstance(lowleveltype, lltype.ContainerType) + redportargdesccls = RedPortalArgDesc + if isinstance(lowleveltype, lltype.Ptr): + if isinstance(lowleveltype.TO, lltype.Struct): + if lowleveltype.TO._hints.get('virtualizable', False): + redportargdesccls = RedVirtualizableStructPortalArgDesc + else: + redportargdesccls = RedStructPortalArgDesc + return redportargdesccls(lowleveltype, self.RGenOp) + + +def make_state_class(args_specification, RESIDUAL_FUNCTYPE, sigtoken, + portal_jitcode, rtyper): + args_specification = unrolling_iterable(args_specification) + class PortalState(object): + def __init__(self, interpreter): + self.cache = {} + self.graph_compilation_queue = [] + self.interpreter = interpreter + + def compile_more_functions(self): + while self.graph_compilation_queue: + top_jitstate, greenargs, redargs = self.graph_compilation_queue.pop() + builder = top_jitstate.curbuilder + builder.start_writing() + top_jitstate = self.interpreter.run(top_jitstate, + portal_jitcode, + greenargs, redargs) + if top_jitstate is not None: + self.interpreter.finish_jitstate(sigtoken) + builder.end() + builder.show_incremental_progress() + + def make_key(self, *args): + key = () + i = 0 + for color, collect_residual_args, _ in args_specification: + if color == "green": + x = args[i] + if isinstance(lltype.typeOf(x), lltype.Ptr): + x = llmemory.cast_ptr_to_adr(x) + key = key + (x,) + i = i + 1 + return key + + def make_residualargs(self, *args): + residualargs = () + i = 0 + for color, collect_residual_args, _ in args_specification: + if color != "green": + residualargs = residualargs + collect_residual_args(args[i]) + i = i + 1 + return residualargs + + def portal_entry(self, *args): + i = 0 + cache = self.cache + key = self.make_key(*args) + try: + gv_generated = cache[key] + except KeyError: + gv_generated = self.compile(key, *args) + residualargs = self.make_residualargs(*args) + + fn = gv_generated.revealconst(lltype.Ptr(RESIDUAL_FUNCTYPE)) + if not we_are_translated(): + # run the generated code on top of the llinterp for testing + llinterp = LLInterpreter(rtyper) + res = llinterp.eval_graph(fn._obj.graph, residualargs) + return res + else: + return fn(*residualargs) + + + def compile(self, key, *args): + portal_ts_args = () + rgenop = self.interpreter.rgenop + builder, gv_generated, inputargs_gv = rgenop.newgraph(sigtoken, + "generated") + self.cache[key] = gv_generated + top_jitstate = self.interpreter.fresh_jitstate(builder) + greenargs = () + redargs = () + red_i = 0 + for color, _, make_arg_redbox in args_specification: + if color == "green": + llvalue = args[0] + args = args[1:] + greenargs += (rgenop.genconst(llvalue),) + else: + llvalue = args[0] + args = args[1:] + box = make_arg_redbox(top_jitstate, inputargs_gv, red_i) + red_i += make_arg_redbox.consumes + redargs += (box,) + greenargs = list(greenargs) + redargs = list(redargs) + + self.graph_compilation_queue.append((top_jitstate, greenargs, redargs)) + self.compile_more_functions() + return gv_generated + + # debug helpers + def readportal(self, *args): + i = 0 + key = () + for color, _, _ in args_specification: + if color == "green": + x = args[i] + if isinstance(lltype.typeOf(x), lltype.Ptr): + x = llmemory.cast_ptr_to_adr(x) + key = key + (x,) + i = i + 1 + cache = self.cache + try: + gv_generated = cache[key] + except KeyError: + return lltype.nullptr(RESIDUAL_FUNCTYPE) + fn = gv_generated.revealconst(lltype.Ptr(RESIDUAL_FUNCTYPE)) + return fn + + def readallportals(self): + return [gv_gen.revealconst(lltype.Ptr(RESIDUAL_FUNCTYPE)) + for gv_gen in self.cache.values()] + return PortalState + + +class RedPortalArgDesc: + __metaclass__ = cachedtype + + def __init__(self, original_concretetype, RGenOp): + assert original_concretetype is not lltype.Void, ( + "cannot make red boxes for the lltype Void") + self.original_concretetype = original_concretetype + self.RGenOp = RGenOp + self.build_portal_arg_helpers() + + def build_portal_arg_helpers(self): + def collect_residual_args(v): + return (v,) + self.collect_residual_args = collect_residual_args + + TYPE = self.original_concretetype + kind = self.RGenOp.kindToken(TYPE) + boxcls = rvalue.ll_redboxcls(TYPE) + + def make_arg_redbox(jitstate, inputargs_gv, i): + gv_arg = inputargs_gv[i] + box = boxcls(kind, gv_arg) + return box + self.make_arg_redbox = make_arg_redbox + make_arg_redbox.consumes = 1 + + def residual_argtypes(self): + return [self.original_concretetype] + + def residual_args_collector(self): + return self.collect_residual_args + + def arg_redbox_maker(self): + return self.make_arg_redbox + + +class RedVirtualizableStructPortalArgDesc(RedPortalArgDesc): + typedesc = None + _s_c_typedesc = None + + def gettypedesc(self): + if self.typedesc is None: + hrtyper = self.hrtyper + T = self.original_concretetype.TO + self.typedesc = rcontainer.StructTypeDesc(self.RGenOp, T) + return self.typedesc + + def build_portal_arg_helpers(self): + typedesc = self.gettypedesc() + redirected_fielddescs = unrolling_iterable( + typedesc.redirected_fielddescs) + TYPE = self.original_concretetype + kind = self.hrtyper.RGenOp.kindToken(TYPE) + + def make_arg_redbox(jitstate, inputargs_gv, i): + box = typedesc.factory() + jitstate.add_virtualizable(box) + content = box.content + assert isinstance(content, rcontainer.VirtualizableStruct) + content_boxes = content.content_boxes + gv_outside = inputargs_gv[i] + i += 1 + for fieldesc, j in redirected_fielddescs: + content_boxes[j] = fieldesc.makebox(None, inputargs_gv[i]) + i += 1 + content_boxes[-1] = rvalue.PtrRedBox(content_boxes[-1].kind, + gv_outside, + known_nonzero = True) + return box + + self.make_arg_redbox = make_arg_redbox + make_arg_redbox.consumes = len(typedesc.redirected_fielddescs)+1 + + def residual_argtypes(self): + argtypes = [self.original_concretetype] + getredrepr = self.hrtyper.getredrepr + typedesc = self.gettypedesc() + for fielddesc, _ in typedesc.redirected_fielddescs: + FIELDTYPE = fielddesc.RESTYPE + argtypes.append(FIELDTYPE) + return argtypes + + def residual_args_collector(self): + typedesc = self.gettypedesc() + return typedesc.collect_residual_args + Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Thu Feb 21 20:48:02 2008 @@ -45,14 +45,17 @@ # rewire the original portal - rewriter = PortalRewriter(self.hintannotator, self.rtyper, self.RGenOp) + rewriter = PortalRewriter(self.hintannotator, self.rtyper, self.RGenOp, + self.writer) + self.rewriter = rewriter origportalgraph = graphof(t, portal) + portalgraph = graphof(self.hintannotator.translator, portal) rewriter.rewrite(origportalgraph=origportalgraph, + portalgraph=portalgraph, view = conftest.option.view and self.small) if conftest.option.view and self.small: t.view() - self.readportalgraph = rewriter.readportalgraph # Populate the cache if len(self._cache_order) >= 3: @@ -77,13 +80,13 @@ def get_residual_graph(self): llinterp = LLInterpreter(self.rtyper) + portalstate = self.rewriter.state if self.main_is_portal: - residual_graph = llinterp.eval_graph(self.readportalgraph, - self.main_args)._obj.graph + residual_graph = portalstate.readportal(*self.main_args)._obj.graph else: - residual_graphs = llinterp.eval_graph(self.readallportalsgraph, []) - assert residual_graphs.ll_length() == 1 - residual_graph = residual_graphs.ll_getitem_fast(0)._obj.graph + residual_graphs = portalstate.readallportals() + assert len(residual_graphs) == 1 + residual_graph = residual_graphs[0]._obj.graph return residual_graph def count_direct_calls(self): @@ -120,7 +123,6 @@ res = self.timeshift_from_portal(main, evaluate, [4, 7]) assert res == 11 - self.check_insns({"int_add": 1}) def test_main_as_portal(self): def main(x): From cfbolz at codespeak.net Thu Feb 21 21:28:06 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 21 Feb 2008 21:28:06 +0100 (CET) Subject: [pypy-svn] r51759 - pypy/branch/jit-refactoring/pypy/jit/timeshifter/test Message-ID: <20080221202806.56B83168472@codespeak.net> Author: cfbolz Date: Thu Feb 21 21:28:04 2008 New Revision: 51759 Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py Log: kill tests that were already moved over to the rainbow interp Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py Thu Feb 21 21:28:04 2008 @@ -142,52 +142,6 @@ class TestPortal(PortalTest): - def test_simple(self): - - def main(code, x): - return evaluate(code, x) - - def evaluate(y, x): - hint(y, concrete=True) - z = y+x - return z - - res = self.timeshift_from_portal(main, evaluate, [3, 2]) - assert res == 5 - - res = self.timeshift_from_portal(main, evaluate, [3, 5]) - assert res == 8 - - res = self.timeshift_from_portal(main, evaluate, [4, 7]) - assert res == 11 - - def test_main_as_portal(self): - def main(x): - return x - - res = self.timeshift_from_portal(main, main, [42]) - assert res == 42 - - def test_multiple_portal_calls(self): - def ll_function(n): - hint(None, global_merge_point=True) - k = n - if k > 5: - k //= 2 - k = hint(k, promote=True) - k *= 17 - return hint(k, variable=True) - - res = self.timeshift_from_portal(ll_function, ll_function, [4], - policy=P_NOVIRTUAL) - assert res == 68 - self.check_insns(int_floordiv=1, int_mul=0) - - res = self.timeshift_from_portal(ll_function, ll_function, [4], - policy=P_NOVIRTUAL) - assert res == 68 - self.check_insns(int_floordiv=1, int_mul=0) - def test_dfa_compile(self): from pypy.lang.automata.dfa import getautomaton, convertdfa, recognizetable a = getautomaton() From arigo at codespeak.net Thu Feb 21 21:58:12 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 21 Feb 2008 21:58:12 +0100 (CET) Subject: [pypy-svn] r51760 - pypy/dist/pypy/rpython Message-ID: <20080221205812.81026168430@codespeak.net> Author: arigo Date: Thu Feb 21 21:58:10 2008 New Revision: 51760 Modified: pypy/dist/pypy/rpython/annlowlevel.py Log: Expand the comment about the problem of llhelper() with translation. Modified: pypy/dist/pypy/rpython/annlowlevel.py ============================================================================== --- pypy/dist/pypy/rpython/annlowlevel.py (original) +++ pypy/dist/pypy/rpython/annlowlevel.py Thu Feb 21 21:58:10 2008 @@ -342,8 +342,24 @@ def llhelper(F, f): - # implementation for the purpose of direct running only - # XXX need more cleverness to support translation of prebuilt llhelper ptr + """Gives a low-level function pointer of type F which, when called, + invokes the RPython function f(). + """ + # Example - the following code can be either run or translated: + # + # def my_rpython_code(): + # g = llhelper(F, my_other_rpython_function) + # assert typeOf(g) == F + # ... + # g() + # + # however the following doesn't translate (xxx could be fixed with hacks): + # + # prebuilt_g = llhelper(F, f) + # def my_rpython_code(): + # prebuilt_g() + + # the next line is the implementation for the purpose of direct running return lltype.functionptr(F.TO, f.func_name, _callable=f) class LLHelperEntry(extregistry.ExtRegistryEntry): From tverwaes at codespeak.net Thu Feb 21 21:59:14 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Thu, 21 Feb 2008 21:59:14 +0100 (CET) Subject: [pypy-svn] r51761 - in pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk: . test Message-ID: <20080221205914.C180F168430@codespeak.net> Author: tverwaes Date: Thu Feb 21 21:59:13 2008 New Revision: 51761 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/classtable.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/constants.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/objtable.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_interpreter.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_primitives.py Log: adding initial shadows for contexts and making original contexts + tests compatible with new format. Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/classtable.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/classtable.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/classtable.py Thu Feb 21 21:59:13 2008 @@ -75,9 +75,12 @@ create_classtable() def copy_in_globals_classes_known_to_the_vm(): - for name in constants.classes_needed_boot_vm: + for name in constants.classes_in_special_object_table.keys(): name = 'w_' + name - globals()[name] = classtable[name] + try: + globals()[name] = classtable[name] + except KeyError: + globals()[name] = None # ___________________________________________________________________________ # Other classes Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/constants.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/constants.py Thu Feb 21 21:59:13 2008 @@ -112,47 +112,29 @@ SO_FINALIZATION_SEMPAHORE = 41 SO_LARGENEGATIVEINTEGER_CLASS = 42 -classes_needed_boot_vm = [ - "SmallInteger", - "String", - "Array", - "Float", - "MethodContext", - "BlockContext", - "CompiledMethod", - "Character", - "ByteArray", -] - # XXX more missing? classes_in_special_object_table = { - "Bitmap" : SO_BITMAP_CLASS, +# "Bitmap" : SO_BITMAP_CLASS, "SmallInteger" : SO_SMALLINTEGER_CLASS, "String" : SO_STRING_CLASS, "Array" : SO_ARRAY_CLASS, "Float" : SO_FLOAT_CLASS, "MethodContext" : SO_METHODCONTEXT_CLASS, "BlockContext" : SO_BLOCKCONTEXT_CLASS, - "Point" : SO_POINT_CLASS, - "LargePositiveInteger" : SO_LARGEPOSITIVEINTEGER_CLASS, - "Display" : SO_DISPLAY_CLASS, - "Message" : SO_MESSAGE_CLASS, +# "Point" : SO_POINT_CLASS, +# "LargePositiveInteger" : SO_LARGEPOSITIVEINTEGER_CLASS, +# "Display" : SO_DISPLAY_CLASS, +# "Message" : SO_MESSAGE_CLASS, "CompiledMethod" : SO_COMPILEDMETHOD_CLASS, - "Semaphore" : SO_SEMAPHORE_CLASS, +# "Semaphore" : SO_SEMAPHORE_CLASS, "Character" : SO_CHARACTER_CLASS, "ByteArray" : SO_BYTEARRAY_CLASS, - "Process" : SO_PROCESS_CLASS, - "PseudoContext" : SO_PSEUDOCONTEXT_CLASS, - "TranslatedMethod" : SO_TRANSLATEDMETHOD_CLASS, +# "Process" : SO_PROCESS_CLASS, +# "PseudoContext" : SO_PSEUDOCONTEXT_CLASS, +# "TranslatedMethod" : SO_TRANSLATEDMETHOD_CLASS, # "LargeNegativeInteger" : SO_LARGENEGATIVEINTEGER_CLASS, # Not available in mini.image } -objects_needed_boot_vm = [ - "nil", - "true", - "false", -] - objects_in_special_object_table = { "nil": SO_NIL, "true": SO_TRUE, Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py Thu Feb 21 21:59:13 2008 @@ -25,10 +25,10 @@ ONE = objtable.w_one TWO = objtable.w_two - _w_last_active_context = None + _s_last_active_context = None def __init__(self): - self.w_active_context = None + self.s_active_context = None self.cnt = 0 def interpret(self): @@ -42,30 +42,30 @@ return (not objectmodel.we_are_translated()) and option.bc_trace def step(self): - next = self.w_active_context.getNextBytecode() + next = self.s_active_context.getNextBytecode() # we_are_translated returns false on top of CPython and true when # translating the interpreter if not objectmodel.we_are_translated(): bytecodeimpl = BYTECODE_TABLE[next] - if self._w_last_active_context != self.w_active_context: + if self._s_last_active_context != self.s_active_context: cnt = 0 - p = self.w_active_context + p = self.s_active_context # AK make method while p is not None: cnt += 1 - p = p.w_sender + p = p.s_sender() self._last_indent = " " * cnt - self._w_last_active_context = self.w_active_context + self._s_last_active_context = self.s_active_context if self.should_trace(): print "%sStack=%s" % ( self._last_indent, - repr(self.w_active_context.stack),) + repr(self.s_active_context.stack),) print "%sBytecode at %d (%d:%s):" % ( self._last_indent, - self.w_active_context.pc, + self.s_active_context.pc(), next, bytecodeimpl.__name__,) - bytecodeimpl(self.w_active_context, self) + bytecodeimpl(self.s_active_context, self) else: # this is a performance optimization: when translating the # interpreter, the bytecode dispatching is not implemented as a @@ -73,7 +73,7 @@ # below produces the switch (by being unrolled). for code, bytecodeimpl in unrolling_bytecode_table: if code == next: - bytecodeimpl(self.w_active_context, self) + bytecodeimpl(self.s_active_context, self) break @@ -91,7 +91,7 @@ # push bytecodes def pushReceiverVariableBytecode(self, interp): index = self.currentBytecode & 15 - self.push(self.receiver().fetch(index)) + self.push(self.w_receiver().fetch(index)) def pushTemporaryVariableBytecode(self, interp): index = self.currentBytecode & 15 @@ -113,7 +113,7 @@ def storeAndPopReceiverVariableBytecode(self, interp): index = self.currentBytecode & 7 - self.receiver().store(index, self.pop()) + self.w_receiver().store(index, self.pop()) def storeAndPopTemporaryVariableBytecode(self, interp): index = self.currentBytecode & 7 @@ -121,7 +121,7 @@ # push bytecodes def pushReceiverBytecode(self, interp): - self.push(self.receiver()) + self.push(self.w_receiver()) def pushConstantTrueBytecode(self, interp): self.push(interp.TRUE) @@ -163,7 +163,7 @@ def _sendSuperSelector(self, selector, argcount, interp): s_compiledin = self.w_method().compiledin().as_class_get_shadow() - self._sendSelector(selector, argcount, interp, self.receiver(), + self._sendSelector(selector, argcount, interp, self.w_receiver(), s_compiledin.s_superclass) def _sendSelector(self, selector, argcount, interp, @@ -203,33 +203,33 @@ start = len(self.stack) - argcount assert start >= 0 # XXX check in the Blue Book what to do in this case arguments = self.stack[start:] - interp.w_active_context = method.create_frame(receiver, arguments, self) + interp.s_active_context = method.create_frame(receiver, arguments, self) self.pop_n(argcount + 1) - def _return(self, object, interp, w_return_to): + def _return(self, object, interp, s_return_to): # for tests, when returning from the top-level context - if w_return_to is None: + if s_return_to is None: raise ReturnFromTopLevel(object) - w_return_to.push(object) - interp.w_active_context = w_return_to + s_return_to.push(object) + interp.s_active_context = s_return_to def returnReceiver(self, interp): - self._return(self.receiver(), interp, self.w_home.w_sender) + self._return(self.w_receiver(), interp, self.s_home().s_sender()) def returnTrue(self, interp): - self._return(interp.TRUE, interp, self.w_home.w_sender) + self._return(interp.TRUE, interp, self.s_home().s_sender()) def returnFalse(self, interp): - self._return(interp.FALSE, interp, self.w_home.w_sender) + self._return(interp.FALSE, interp, self.s_home().s_sender()) def returnNil(self, interp): - self._return(interp.NIL, interp, self.w_home.w_sender) + self._return(interp.NIL, interp, self.s_home().s_sender()) def returnTopFromMethod(self, interp): - self._return(self.top(), interp, self.w_home.w_sender) + self._return(self.top(), interp, self.s_home().s_sender()) def returnTopFromBlock(self, interp): - self._return(self.top(), interp, self.w_sender) + self._return(self.top(), interp, self.s_sender()) def unknownBytecode(self, interp): raise MissingBytecode("unknownBytecode") @@ -242,7 +242,7 @@ def extendedPushBytecode(self, interp): variableType, variableIndex = self.extendedVariableTypeAndIndex() if variableType == 0: - self.push(self.receiver().fetch(variableIndex)) + self.push(self.w_receiver().fetch(variableIndex)) elif variableType == 1: self.push(self.gettemp(variableIndex)) elif variableType == 2: @@ -257,7 +257,7 @@ def extendedStoreBytecode(self, interp): variableType, variableIndex = self.extendedVariableTypeAndIndex() if variableType == 0: - self.receiver().store(variableIndex, self.top()) + self.w_receiver().store(variableIndex, self.top()) elif variableType == 1: self.settemp(variableIndex, self.top()) elif variableType == 2: @@ -294,7 +294,7 @@ second & 31, interp) elif opType == 2: # pushReceiver - self.push(self.receiver().fetch(third)) + self.push(self.w_receiver().fetch(third)) elif opType == 3: # pushLiteralConstant self.push(self.w_method().getliteral(third)) @@ -304,9 +304,9 @@ assert isinstance(association, model.W_PointersObject) self.push(association.fetch(constants.ASSOCIATION_VALUE_INDEX)) elif opType == 5: - self.receiver().store(third, self.top()) + self.w_receiver().store(third, self.top()) elif opType == 6: - self.receiver().store(third, self.pop()) + self.w_receiver().store(third, self.pop()) elif opType == 7: association = self.w_method().getliteral(third) assert isinstance(association, model.W_PointersObject) @@ -329,7 +329,7 @@ raise MissingBytecode("experimentalBytecode") def jump(self,offset): - self.pc = self.pc + offset + self.store_pc(self.pc() + offset) def jumpConditional(self,bool,position): if self.top() == bool: Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py Thu Feb 21 21:59:13 2008 @@ -196,19 +196,14 @@ def as_semaphore_get_shadow(self): from pypy.lang.smalltalk.shadow import SemaphoreShadow - from pypy.lang.smalltalk import classtable - assert self.getclass() == classtable.classtable["w_Semaphore"] return self.as_special_get_shadow(SemaphoreShadow) def as_linkedlist_get_shadow(self): from pypy.lang.smalltalk.shadow import LinkedListShadow - from pypy.lang.smalltalk import classtable return self.as_special_get_shadow(LinkedListShadow) def as_process_get_shadow(self): from pypy.lang.smalltalk.shadow import ProcessShadow - from pypy.lang.smalltalk import classtable - assert self.getclass() == classtable.classtable["w_Process"] return self.as_special_get_shadow(ProcessShadow) def as_scheduler_get_shadow(self): @@ -219,6 +214,24 @@ from pypy.lang.smalltalk.shadow import AssociationShadow return self.as_special_get_shadow(AssociationShadow) + def as_blockcontext_get_shadow(self): + from pypy.lang.smalltalk.shadow import BlockContextShadow + return self.as_special_get_shadow(BlockContextShadow) + + def as_methodcontext_get_shadow(self): + from pypy.lang.smalltalk.shadow import MethodContextShadow + return self.as_special_get_shadow(MethodContextShadow) + + def as_context_get_shadow(self): + from pypy.lang.smalltalk import classtable + if self.getclass() == classtable.w_MethodContext: + self.as_methodcontext_get_shadow() + elif self.getclass() == classtable.w_BlockContext: + self.as_blockcontext_get_shadow() + else: + # Should not happen... + raise Exception() + def equals(self, other): if not isinstance(other, W_PointersObject): return False @@ -461,30 +474,55 @@ __metaclass__ = extendabletype - def __init__(self, w_home, w_sender): + def __init__(self, s_home, s_sender): self.stack = [] - self.pc = 0 - assert isinstance(w_home, W_MethodContext) - self.w_home = w_home - assert w_sender is None or isinstance(w_sender, W_ContextPart) - self.w_sender = w_sender + self._pc = 0 + #assert isinstance(s_home, W_MethodContext) + self._s_home = s_home + #assert w_sender is None or isinstance(w_sender, W_ContextPart) + self._s_sender = s_sender + + def as_context_get_shadow(self): + # Backward compatibility + return self + + def w_self(self): + # Backward compatibility + return self - def receiver(self): + def pc(self): + return self._pc + + def store_pc(self, pc): + self._pc = pc + + def w_receiver(self): " Return self of the method, or the method that contains the block " - return self.w_home.w_receiver - + return self.s_home().w_receiver() + + def s_home(self): + return self._s_home + + def s_sender(self): + if self._s_sender: + return self._s_sender + + def store_s_sender(self, s_sender): + self._s_sender = s_sender + # ______________________________________________________________________ # Imitate the primitive accessors def fetch(self, index): from pypy.lang.smalltalk import utility, objtable if index == constants.CTXPART_SENDER_INDEX: - if self.w_sender: - return self.w_sender - else: + sender = self.s_sender() + if sender is None: return objtable.w_nil + else: + return sender.w_self() elif index == constants.CTXPART_PC_INDEX: - return utility.wrap_int(self.pc) + return utility.wrap_int(self.pc()) elif index == constants.CTXPART_STACKP_INDEX: return utility.wrap_int(len(self.stack)) @@ -496,25 +534,32 @@ from pypy.lang.smalltalk import utility, objtable if index == constants.CTXPART_SENDER_INDEX: - self.w_sender = w_value + if w_value != objtable.w_nil: + self._s_sender = w_value.as_context_get_shadow() elif index == constants.CTXPART_PC_INDEX: - self.pc = utility.unwrap_int(w_value) + self._pc = utility.unwrap_int(w_value) elif index == constants.CTXPART_STACKP_INDEX: - self.stack = [objtable.w_nil] * utility.unwrap_int(w_value) + size = utility.unwrap_int(w_value) + size = size - self.stackstart() + self.stack = [objtable.w_nil] * size else: # Invalid! raise IndexError + def stackstart(self): + return self.w_method().argsize + self.w_method().tempsize + constants.MTHDCTX_TEMP_FRAME_START + # ______________________________________________________________________ # Method that contains the bytecode for this method/block context def w_method(self): - return self.w_home._w_method + return self.s_home().w_method() def getbytecode(self): - bytecode = self.w_method().bytes[self.pc] + pc = self.pc() + bytecode = self.w_method().bytes[pc] currentBytecode = ord(bytecode) - self.pc = self.pc + 1 + self.store_pc(pc + 1) return currentBytecode def getNextBytecode(self): @@ -527,10 +572,10 @@ # Are always fetched relative to the home method context. def gettemp(self, index): - return self.w_home.temps[index] + return self.s_home().gettemp(index) def settemp(self, index, w_value): - self.w_home.temps[index] = w_value + self.s_home().settemp(index, w_value) # ______________________________________________________________________ # Stack Manipulation @@ -569,10 +614,13 @@ class W_BlockContext(W_ContextPart): - def __init__(self, w_home, w_sender, argcnt, initialip): - W_ContextPart.__init__(self, w_home, w_sender) + def __init__(self, s_home, s_sender, argcnt, initialip): + W_ContextPart.__init__(self, s_home, s_sender) self.argcnt = argcnt - self.initialip = initialip + self._initialip = initialip + + def initialip(self): + return self._initialip def expected_argument_count(self): return self.argcnt @@ -580,6 +628,10 @@ def getclass(self): from pypy.lang.smalltalk.classtable import w_BlockContext return w_BlockContext + + def as_blockcontext_get_shadow(self): + # Backwards compatibility + return self def fetch(self, index): from pypy.lang.smalltalk import utility @@ -588,7 +640,7 @@ elif index == constants.BLKCTX_INITIAL_IP_INDEX: return utility.wrap_int(self.initialip) elif index == constants.BLKCTX_HOME_INDEX: - return self.w_home + return self.s_home() elif index >= constants.BLKCTX_TEMP_FRAME_START: stack_index = len(self.stack) - index - 1 return self.stack[stack_index] @@ -604,8 +656,7 @@ elif index == constants.BLKCTX_INITIAL_IP_INDEX: self.pc = utility.unwrap_int(value) elif index == constants.BLKCTX_HOME_INDEX: - assert isinstance(value, W_MethodContext) - self.w_home = value + self._s_home = value.as_methodcontext_get_shadow() elif index >= constants.BLKCTX_TEMP_FRAME_START: stack_index = len(self.stack) - index - 1 self.stack[stack_index] = value @@ -614,23 +665,33 @@ class W_MethodContext(W_ContextPart): def __init__(self, w_method, w_receiver, - arguments, w_sender=None): - W_ContextPart.__init__(self, self, w_sender) + arguments, s_sender=None): + W_ContextPart.__init__(self, self, s_sender) self._w_method = w_method - self.w_receiver = w_receiver + self._w_receiver = w_receiver self.temps = arguments + [w_nil] * w_method.tempsize + def as_methodcontext_get_shadow(self): + # Backwards compatibility + return self + def getclass(self): from pypy.lang.smalltalk.classtable import w_MethodContext return w_MethodContext + def w_receiver(self): + return self._w_receiver + + def store_w_receiver(self, w_receiver): + self._w_receiver = w_receiver + def fetch(self, index): if index == constants.MTHDCTX_METHOD: return self.w_method() elif index == constants.MTHDCTX_RECEIVER_MAP: # what is this thing? return w_nil elif index == constants.MTHDCTX_RECEIVER: - return self.w_receiver + return self._w_receiver elif index >= constants.MTHDCTX_TEMP_FRAME_START: # First set of indices are temporary variables: offset = index - constants.MTHDCTX_TEMP_FRAME_START @@ -650,7 +711,7 @@ elif index == constants.MTHDCTX_RECEIVER_MAP: # what is this thing? pass elif index == constants.MTHDCTX_RECEIVER: - self.w_receiver = w_object + self._w_receiver = w_object elif index >= constants.MTHDCTX_TEMP_FRAME_START: # First set of indices are temporary variables: offset = index - constants.MTHDCTX_TEMP_FRAME_START @@ -664,6 +725,15 @@ else: W_ContextPart.store(self, index, w_object) + def gettemp(self, idx): + return self.temps[idx] + + def settemp(self, idx, w_value): + self.temps[idx] = w_value + + def w_method(self): + return self._w_method + # Use black magic to create w_nil without running the constructor, # thus allowing it to be used even in the constructor of its own Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/objtable.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/objtable.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/objtable.py Thu Feb 21 21:59:13 2008 @@ -30,5 +30,8 @@ objtable = {} -for name in constants.objects_needed_boot_vm: - objtable["w_" + name] = globals()["w_" + name] +for name in constants.objects_in_special_object_table: + try: + objtable["w_" + name] = globals()["w_" + name] + except KeyError, e: + objtable["w_" + name ] = None Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py Thu Feb 21 21:59:13 2008 @@ -68,7 +68,7 @@ w_result = func(interp, argument_count_m1) if not no_result: assert w_result is not None - interp.w_active_context.push(w_result) + interp.s_active_context.push(w_result) return w_result else: len_unwrap_spec = len(unwrap_spec) @@ -77,7 +77,7 @@ unrolling_unwrap_spec = unrolling_iterable(enumerate(unwrap_spec)) def wrapped(interp, argument_count_m1): argument_count = argument_count_m1 + 1 # to account for the rcvr - frame = interp.w_active_context + frame = interp.s_active_context assert argument_count == len_unwrap_spec if len(frame.stack) < len_unwrap_spec: raise PrimitiveFailedError() @@ -104,7 +104,7 @@ frame.pop_n(len_unwrap_spec) # only if no exception occurs! if not no_result: assert w_result is not None - interp.w_active_context.push(w_result) + interp.s_active_context.push(w_result) wrapped.func_name = "wrap_prim_" + name prim_table[code] = wrapped prim_table_implemented_only.append((code, wrapped)) @@ -611,7 +611,7 @@ @expose_primitive(PRIMITIVE_BLOCK_COPY, unwrap_spec=[object, int]) def func(interp, w_context, argcnt): - frame = interp.w_active_context + frame = interp.s_active_context # From B.B.: If receiver is a MethodContext, then it becomes # the new BlockContext's home context. Otherwise, the home @@ -619,20 +619,20 @@ # Note that in our impl, MethodContext.w_home == self if not isinstance(w_context, model.W_ContextPart): raise PrimitiveFailedError() - w_method_context = w_context.w_home + s_method_context = w_context.as_context_get_shadow().s_home() # The block bytecodes are stored inline: so we skip past the # byteodes to invoke this primitive to find them (hence +2) - initialip = frame.pc + 2 - w_new_context = model.W_BlockContext( - w_method_context, None, argcnt, initialip) - return w_new_context + initialip = frame.pc() + 2 + s_new_context = model.W_BlockContext( + s_method_context, None, argcnt, initialip) + return s_new_context.w_self() -def finalize_block_ctx(interp, w_block_ctx, frame): +def finalize_block_ctx(interp, s_block_ctx, frame): # Set some fields - w_block_ctx.pc = w_block_ctx.initialip - w_block_ctx.w_sender = frame - interp.w_active_context = w_block_ctx + s_block_ctx.store_pc(s_block_ctx.initialip()) + s_block_ctx.store_s_sender(frame) + interp.s_active_context = s_block_ctx @expose_primitive(PRIMITIVE_VALUE, no_result=True) def func(interp, argument_count): @@ -642,31 +642,31 @@ # Rcvr | Arg 0 | Arg1 | Arg 2 # - frame = interp.w_active_context + frame = interp.s_active_context # Validate that we have a block on the stack and that it received # the proper number of arguments: w_block_ctx = frame.peek(argument_count) - if not isinstance(w_block_ctx, model.W_BlockContext): + if w_block_ctx.getclass() != classtable.w_BlockContext: raise PrimitiveFailedError() - exp_arg_cnt = w_block_ctx.expected_argument_count() + s_block_ctx = w_block_ctx.as_blockcontext_get_shadow() + exp_arg_cnt = s_block_ctx.expected_argument_count() if argument_count != exp_arg_cnt: # exp_arg_cnt doesn't count self raise PrimitiveFailedError() # Initialize the block stack with the arguments that were # pushed. Also pop the receiver. block_args = frame.pop_and_return_n(exp_arg_cnt) - w_block_ctx.push_all(block_args) + s_block_ctx.push_all(block_args) frame.pop() - finalize_block_ctx(interp, w_block_ctx, frame) + finalize_block_ctx(interp, s_block_ctx, frame) @expose_primitive(PRIMITIVE_VALUE_WITH_ARGS, unwrap_spec=[object, object], no_result=True) def func(interp, w_block_ctx, w_args): - if not isinstance(w_block_ctx, model.W_BlockContext): - raise PrimitiveFailedError() - exp_arg_cnt = w_block_ctx.expected_argument_count() + s_block_ctx = w_block_ctx.as_blockcontext_get_shadow() + exp_arg_cnt = s_block_ctx.expected_argument_count() # Check that our arguments have pointers format and the right size: if w_args.getclass() != classtable.w_Array: @@ -676,12 +676,12 @@ # Push all the items from the array for i in range(exp_arg_cnt): - w_block_ctx.push(w_args.fetchvarpointer(i)) + s_block_ctx.push(w_args.fetchvarpointer(i)) # XXX Check original logic. Image does not test this anyway # because falls back to value + internal implementation - finalize_block_ctx(interp, w_block_ctx, interp.w_active_context) + finalize_block_ctx(interp, s_block_ctx, interp.s_active_context) @expose_primitive(PRIMITIVE_PERFORM) def func(interp, argcount): @@ -694,11 +694,11 @@ w_method = w_rcvr.shadow_of_my_class().lookup(sel) assert w_method - w_frame = w_method.create_frame(w_rcvr, + s_frame = w_method.create_frame(w_rcvr, [w_args.fetch(i) for i in range(w_args.size())]) - w_frame.w_sender = interp.w_active_context - interp.w_active_context = w_frame + s_frame.store_s_sender(interp.s_active_context) + interp.s_active_context = s_frame @expose_primitive(PRIMITIVE_SIGNAL, unwrap_spec=[object]) Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py Thu Feb 21 21:59:13 2008 @@ -9,6 +9,9 @@ """XXX This should get called whenever the base Smalltalk object changes.""" + def w_self(self): + return self._w_self + # ____________________________________________________________ POINTERS = 0 @@ -29,7 +32,7 @@ (i.e. used as the class of another Smalltalk object). """ def __init__(self, w_self): - self.w_self = w_self + self._w_self = w_self self.invalidate() def invalidate(self): @@ -47,7 +50,7 @@ "Update the ClassShadow with data from the w_self class." - w_self = self.w_self + w_self = self._w_self # read and painfully decode the format classformat = utility.unwrap_int( w_self.fetch(constants.CLASS_FORMAT_INDEX)) @@ -126,7 +129,7 @@ def new(self, extrasize=0): from pypy.lang.smalltalk import classtable - w_cls = self.w_self + w_cls = self._w_self if w_cls == classtable.w_BlockContext: return model.W_BlockContext(None, None, 0, 0) @@ -212,23 +215,23 @@ "NOT_RPYTHON" # this is only for testing. assert isinstance(method, model.W_CompiledMethod) self.methoddict[selector] = method - method.w_compiledin = self.w_self + method.w_compiledin = self._w_self class LinkedListShadow(AbstractShadow): def __init__(self, w_self): - self.w_self = w_self + self._w_self = w_self def firstlink(self): - return self.w_self.at0(constants.FIRST_LINK_INDEX) + return self._w_self.at0(constants.FIRST_LINK_INDEX) def store_firstlink(self, w_object): - return self.w_self.atput0(constants.FIRST_LINK_INDEX, w_object) + return self._w_self.atput0(constants.FIRST_LINK_INDEX, w_object) def lastlink(self): - return self.w_self.at0(constants.LAST_LINK_INDEX) + return self._w_self.at0(constants.LAST_LINK_INDEX) def store_lastlink(self, w_object): - return self.w_self.atput0(constants.LAST_LINK_INDEX, w_object) + return self._w_self.atput0(constants.LAST_LINK_INDEX, w_object) def is_empty_list(self): from pypy.lang.smalltalk import objtable @@ -259,14 +262,14 @@ """A shadow for Smalltalk objects that are semaphores """ def __init__(self, w_self): - self.w_self = w_self + self._w_self = w_self def put_to_sleep(self, s_process): priority = s_process.priority() s_scheduler = self.scheduler() w_process_lists = s_scheduler.process_lists() w_process_list = w_process_lists.at0(priority) - w_process_list.as_linkedlist_get_shadow().add_last_link(s_process.w_self) + w_process_list.as_linkedlist_get_shadow().add_last_link(s_process._w_self) s_process.store_my_list(w_process_list) def transfer_to(self, s_process, interp): @@ -274,9 +277,9 @@ s_scheduler = self.scheduler() s_old_process = s_scheduler.active_process() s_scheduler.store_active_process(s_process) - s_old_process.store_suspended_context(interp.w_active_context) - interp.w_active_context = s_process.suspended_context() - s_process.store_suspended_context(objtable.w_nil) + s_old_process.store_w_suspended_context(interp.s_active_context.w_self()) + interp.s_active_context = s_process.s_suspended_context() + s_process.store_w_suspended_context(objtable.w_nil) #reclaimableContextCount := 0 def scheduler(self): @@ -300,62 +303,202 @@ def synchronous_signal(self, interp): print "SYNCHRONOUS SIGNAL" if self.is_empty_list(): - w_value = self.w_self.at0(constants.EXCESS_SIGNALS_INDEX) + w_value = self._w_self.at0(constants.EXCESS_SIGNALS_INDEX) w_value = utility.wrap_int(utility.unwrap_int(w_value) + 1) - self.w_self.atput0(constants.EXCESS_SIGNALS_INDEX, w_value) + self._w_self.atput0(constants.EXCESS_SIGNALS_INDEX, w_value) else: self.resume(self.remove_first_link_of_list(), interp) class LinkShadow(AbstractShadow): def __init__(self, w_self): - self.w_self = self + self._w_self = self def next(self): - return self.w_self.at0(constants.NEXT_LINK_INDEX) + return self._w_self.at0(constants.NEXT_LINK_INDEX) def store_next(self, w_object): - self.w_self.atput0(constants.NEXT_LINK_INDEX, w_object) + self._w_self.atput0(constants.NEXT_LINK_INDEX, w_object) class ProcessShadow(LinkShadow): """A shadow for Smalltalk objects that are processes """ def __init__(self, w_self): - self.w_self = w_self + self._w_self = w_self def priority(self): - return utility.unwrap_int(self.w_self.at0(constants.PROCESS_PRIORITY_INDEX)) + return utility.unwrap_int(self._w_self.at0(constants.PROCESS_PRIORITY_INDEX)) def my_list(self): - return self.w_self.at0(constants.PROCESS_MY_LIST_INDEX) + return self._w_self.at0(constants.PROCESS_MY_LIST_INDEX) def store_my_list(self, w_object): - self.w_self.atput0(constants.PROCESS_MY_LIST_INDEX, w_object) + self._w_self.atput0(constants.PROCESS_MY_LIST_INDEX, w_object) - def suspended_context(self): - return self.w_self.at0(constants.PROCESS_SUSPENDED_CONTEXT_INDEX) + def s_suspended_context(self): + return self._w_self.fetch(constants.PROCESS_SUSPENDED_CONTEXT_INDEX).as_context_get_shadow() - def store_suspended_context(self, w_object): - self.w_self.atput0(constants.PROCESS_SUSPENDED_CONTEXT_INDEX, w_object) + def store_w_suspended_context(self, w_object): + self._w_self.atput0(constants.PROCESS_SUSPENDED_CONTEXT_INDEX, w_object) class AssociationShadow(AbstractShadow): def __init__(self, w_self): - self.w_self = w_self + self._w_self = w_self def key(self): - return self.w_self.at0(constants.ASSOCIATION_KEY_INDEX) + return self._w_self.at0(constants.ASSOCIATION_KEY_INDEX) def value(self): - return self.w_self.at0(constants.ASSOCIATION_VALUE_INDEX) + return self._w_self.at0(constants.ASSOCIATION_VALUE_INDEX) class SchedulerShadow(AbstractShadow): def __init__(self, w_self): - self.w_self = w_self + self._w_self = w_self def active_process(self): - return self.w_self.at0(constants.SCHEDULER_ACTIVE_PROCESS_INDEX).as_process_get_shadow() + return self._w_self.at0(constants.SCHEDULER_ACTIVE_PROCESS_INDEX).as_process_get_shadow() def store_active_process(self, w_object): - self.w_self.atput0(constants.SCHEDULER_ACTIVE_PROCESS_INDEX, w_object) + self._w_self.atput0(constants.SCHEDULER_ACTIVE_PROCESS_INDEX, w_object) def process_lists(self): - return self.w_self.at0(constants.SCHEDULER_PROCESS_LISTS_INDEX) + return self._w_self.at0(constants.SCHEDULER_PROCESS_LISTS_INDEX) + +class ContextPartShadow(AbstractShadow): + + #__metaclass__ = extendabletype + + def __init__(self, w_self): + self._w_self = w_self + + def s_home(self): + raise NotImplementedError() + + def w_receiver(self): + " Return self of the method, or the method that contains the block " + return self.s_home().w_receiver() + + def s_sender(self): + return self._w_self.fetch(constants.CTXPART_SENDER_INDEX).as_context_get_shadow() + + def store_s_sender(self, s_sender): + self._w_self.store(constants.CTXPART_SENDER_INDEX, s_sender._w_self()) + + def pc(self): + return utility.unwrap_int(self._w_self.fetch(constants.CTXPART_PC_INDEX)) + + def store_pc(self, newpc): + self._w_self.store(constants.CTXPART_PC_INDEX, utility.wrap_int(newpc)) + + def stackpointer(self): + return utility.unwrap_int(self._w_self.fetch(constants.CTXPART_SENDER_STACKP_INDEX)) + + def store_stackpointer(self, pointer): + self._w_self.store(constants.CTXPART_SENDER_STACKP_INDEX, + utility.wrap_int(pointer)) + + # ______________________________________________________________________ + # Method that contains the bytecode for this method/block context + + def w_method(self): + return self.s_home().w_method() + + def getbytecode(self): + pc = self.pc() + bytecode = self.w_method().bytes[pc] + currentBytecode = ord(bytecode) + self.store_pc(pc + 1) + return currentBytecode + + def getNextBytecode(self): + self.currentBytecode = self.getbytecode() + return self.currentBytecode + + # ______________________________________________________________________ + # Temporary Variables + # + # Are always fetched relative to the home method context. + + def gettemp(self, index): + return self.s_home().gettemp(index) + + def settemp(self, index, w_value): + self.s_home().settemp(index, w_value) + + # ______________________________________________________________________ + # Stack Manipulation + def pop(self): + idx = self.stackpointer() + w_v = self._w_self.fetch(idx) + self.store_stackpointer(idx - 1) + return w_v + + def push(self, w_v): + idx = self.stackpointer() + 1 + self._w_self.store(idx, w_v) + self.store_stackpointer(idx) + + def push_all(self, lst): + for x in lst: + self.push(x) + + def top(self): + return self.peek(0) + + def peek(self, idx): + return self._w_self.fetch(self.stackpointer()-idx) + + def pop_n(self, n): + assert n >= 0 + assert n > (self.stackpointer() - self.stackstart()) + self.store_stackpointer(self.stackpointer() - n) + + def pop_and_return_n(self, n): + self.pop_n(n) + start = self.stackpointer() + 1 + return [self._w_self.fetch(i) for i in range(start, start+n)] + +class BlockContextShadow(ContextPartShadow): + + def __init__(self, w_self): + self._w_self = w_self + + def expected_argument_count(self): + return utility.unwrap_int(self._w_self.fetch(constants.BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX)) + + def store_expected_argument_count(self, argc): + return self._w_self.store(constants.BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX, + utility.wrap_int(argc)) + + def initialip(self): + return utility.unwrap_int(self._w_self.fetch(constants.BLKCTX_INITIAL_IP_INDEX)) + + def s_home(self): + return self._w_self.fetch(constants.BLKCTX_HOME_INDEX).as_methodcontext_get_shadow() + + def stackstart(self): + return constants.BLKCTX_TEMP_FRAME_START + +class MethodContextShadow(ContextPartShadow): + def __init__(self, w_self): + self._w_self = w_self + + def w_method(self): + return self._w_self.fetch(constants.MTHDCTX_METHOD) + + def w_receiver(self): + return self._w_self.fetch(constants.MTHDCTX_RECEIVER) + + def store_w_receiver(self, w_receiver): + self._w_self.store(constants.MTHDCTX_RECEIVER, w_receiver) + + def gettemp(self, index): + return self._w_self.fetch(constants.MTHDCTX_TEMP_FRAME_START + index) + + def settemp(self, index, w_value): + self._w_self.store(constants.MTHDCTX_TEMP_FRAME_START + index, w_value) + + def s_home(self): + return self + + def stackstart(self): + return self.w_method().argsize + self.w_method().tempsize + constants.MTHDCTX_TEMP_FRAME_START Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py Thu Feb 21 21:59:13 2008 @@ -123,25 +123,15 @@ for chunk in self.chunks.itervalues(): chunk.g_object.init_w_object() - def class_indxs(self): - from pypy.lang.smalltalk import constants - return map(lambda key: (key, constants.classes_in_special_object_table[key]), - constants.classes_needed_boot_vm) - - def object_indxs(self): - from pypy.lang.smalltalk import constants - return map(lambda key: (key, constants.objects_in_special_object_table[key]), - constants.objects_needed_boot_vm) - def assign_prebuilt_constants(self): from pypy.lang.smalltalk import classtable, constants, objtable # assign w_objects for objects that are already in classtable - for name, so_index in self.class_indxs(): + for name, so_index in constants.classes_in_special_object_table.items(): # w_object = getattr(classtable, "w_" + name) w_object = classtable.classtable["w_" + name] self.special_object(so_index).w_object = w_object # assign w_objects for objects that are already in objtable - for name, so_index in self.object_indxs(): + for name, so_index in constants.objects_in_special_object_table.items(): # w_object = getattr(objtable, "w_" + name) w_object = objtable.objtable["w_" + name] self.special_object(so_index).w_object = w_object @@ -211,14 +201,8 @@ self.special_objects = [g_object.w_object for g_object in reader.chunks[reader.specialobjectspointer] .g_object.pointers] - # After loading we know of more classes than before. Copy back. - for name, so_index in constants.classes_in_special_object_table.items(): - classtable.classtable["w_" + name] = self.special(so_index) - # After loading we know of more objects than before. Copy back. self.objects = [chunk.g_object.w_object for chunk in reader.chunklist] - for name, so_index in constants.objects_in_special_object_table.items(): - objtable.objtable["w_" + name] = self.special(so_index) def special(self, index): return self.special_objects[index] Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_interpreter.py Thu Feb 21 21:59:13 2008 @@ -74,7 +74,7 @@ w_method.tempsize=1 w_frame = w_method.create_frame(receiver, ["foo", "bar"]) interp = interpreter.Interpreter() - interp.w_active_context = w_frame + interp.s_active_context = w_frame.as_methodcontext_get_shadow() return interp def test_create_frame(): @@ -83,7 +83,7 @@ w_method.argsize=2 w_method.tempsize=1 w_frame = w_method.create_frame("receiver", ["foo", "bar"]) - assert w_frame.w_receiver == "receiver" + assert w_frame.w_receiver() == "receiver" assert w_frame.gettemp(0) == "foo" assert w_frame.gettemp(1) == "bar" assert w_frame.gettemp(2) is objtable.w_nil @@ -95,7 +95,7 @@ def test_push_pop(): interp = new_interpreter("") - frame = interp.w_active_context + frame = interp.s_active_context frame.push(12) frame.push(34) frame.push(56) @@ -116,7 +116,7 @@ def test_pushReceiverBytecode(): interp = new_interpreter(pushReceiverBytecode) interp.step() - assert interp.w_active_context.top() == interp.w_active_context.w_receiver + assert interp.s_active_context.top() == interp.s_active_context.w_receiver() def test_pushReceiverVariableBytecode(bytecode = (pushReceiverVariableBytecode(0) + pushReceiverVariableBytecode(1) + @@ -129,27 +129,27 @@ interp.step() interp.step() interp.step() - assert interp.w_active_context.stack == ["egg", "bar", "baz"] + assert interp.s_active_context.stack == ["egg", "bar", "baz"] def test_pushTemporaryVariableBytecode(bytecode=(pushTemporaryVariableBytecode(0) + pushTemporaryVariableBytecode(1) + pushTemporaryVariableBytecode(2))): interp = new_interpreter(bytecode) - interp.w_active_context.settemp(2, "temp") + interp.s_active_context.settemp(2, "temp") interp.step() interp.step() interp.step() - assert interp.w_active_context.stack == ["foo", "bar", "temp"] + assert interp.s_active_context.stack == ["foo", "bar", "temp"] def test_pushLiteralConstantBytecode(bytecode=pushLiteralConstantBytecode(0) + pushLiteralConstantBytecode(1) + pushLiteralConstantBytecode(2)): interp = new_interpreter(bytecode) - interp.w_active_context.w_method().literals = fakeliterals("a", "b", "c") + interp.s_active_context.w_method().literals = fakeliterals("a", "b", "c") interp.step() interp.step() interp.step() - assert interp.w_active_context.stack == [fakesymbol("a"), + assert interp.s_active_context.stack == [fakesymbol("a"), fakesymbol("b"), fakesymbol("c")] @@ -158,9 +158,9 @@ w_association.store(0, "mykey") w_association.store(1, "myvalue") interp = new_interpreter(bytecode) - interp.w_active_context.w_method().literals = fakeliterals(w_association) + interp.s_active_context.w_method().literals = fakeliterals(w_association) interp.step() - assert interp.w_active_context.stack == ["myvalue"] + assert interp.s_active_context.stack == ["myvalue"] def test_storeAndPopReceiverVariableBytecode(bytecode=storeAndPopReceiverVariableBytecode, popped=True): @@ -168,13 +168,13 @@ for index in range(8): w_object = shadow.new() interp = new_interpreter(pushConstantTrueBytecode + bytecode(index)) - interp.w_active_context.w_receiver = w_object + interp.s_active_context.store_w_receiver(w_object) interp.step() interp.step() if popped: - assert interp.w_active_context.stack == [] + assert interp.s_active_context.stack == [] else: - assert interp.w_active_context.stack == [interp.TRUE] + assert interp.s_active_context.stack == [interp.TRUE] for test_index in range(8): if test_index == index: @@ -185,167 +185,167 @@ def test_storeAndPopTemporaryVariableBytecode(bytecode=storeAndPopTemporaryVariableBytecode): for index in range(8): interp = new_interpreter(pushConstantTrueBytecode + bytecode(index)) - interp.w_active_context.temps = [None] * 8 + interp.s_active_context.temps = [None] * 8 interp.step() interp.step() - assert interp.w_active_context.stack == [] + assert interp.s_active_context.stack == [] for test_index in range(8): if test_index == index: - assert interp.w_active_context.temps[test_index] == interp.TRUE + assert interp.s_active_context.temps[test_index] == interp.TRUE else: - assert interp.w_active_context.temps[test_index] == None + assert interp.s_active_context.temps[test_index] == None def test_pushConstantTrueBytecode(): interp = new_interpreter(pushConstantTrueBytecode) interp.step() - assert interp.w_active_context.pop() == interp.TRUE - assert interp.w_active_context.stack == [] + assert interp.s_active_context.pop() == interp.TRUE + assert interp.s_active_context.stack == [] def test_pushConstantFalseBytecode(): interp = new_interpreter(pushConstantFalseBytecode) interp.step() - assert interp.w_active_context.pop() == interp.FALSE - assert interp.w_active_context.stack == [] + assert interp.s_active_context.pop() == interp.FALSE + assert interp.s_active_context.stack == [] def test_pushConstantNilBytecode(): interp = new_interpreter(pushConstantNilBytecode) interp.step() - assert interp.w_active_context.pop() == interp.NIL - assert interp.w_active_context.stack == [] + assert interp.s_active_context.pop() == interp.NIL + assert interp.s_active_context.stack == [] def test_pushConstantMinusOneBytecode(): interp = new_interpreter(pushConstantMinusOneBytecode) interp.step() - assert interp.w_active_context.pop() == interp.MINUS_ONE - assert interp.w_active_context.stack == [] + assert interp.s_active_context.pop() == interp.MINUS_ONE + assert interp.s_active_context.stack == [] def test_pushConstantZeroBytecode(): interp = new_interpreter(pushConstantZeroBytecode) interp.step() - assert interp.w_active_context.pop() == interp.ZERO - assert interp.w_active_context.stack == [] + assert interp.s_active_context.pop() == interp.ZERO + assert interp.s_active_context.stack == [] def test_pushConstantOneBytecode(): interp = new_interpreter(pushConstantOneBytecode) interp.step() - assert interp.w_active_context.pop() == interp.ONE - assert interp.w_active_context.stack == [] + assert interp.s_active_context.pop() == interp.ONE + assert interp.s_active_context.stack == [] def test_pushConstantTwoBytecode(): interp = new_interpreter(pushConstantTwoBytecode) interp.step() - assert interp.w_active_context.pop() == interp.TWO - assert interp.w_active_context.stack == [] + assert interp.s_active_context.pop() == interp.TWO + assert interp.s_active_context.stack == [] def test_pushActiveContextBytecode(): interp = new_interpreter(pushActiveContextBytecode) interp.step() - assert interp.w_active_context.pop() == interp.w_active_context - assert interp.w_active_context.stack == [] + assert interp.s_active_context.pop() == interp.s_active_context.w_self() + assert interp.s_active_context.stack == [] def test_duplicateTopBytecode(): interp = new_interpreter(pushConstantZeroBytecode + duplicateTopBytecode) interp.step() interp.step() - assert interp.w_active_context.stack == [interp.ZERO, interp.ZERO] + assert interp.s_active_context.stack == [interp.ZERO, interp.ZERO] def test_bytecodePrimBitAnd(): interp = new_interpreter(pushConstantOneBytecode + pushConstantTwoBytecode + bytecodePrimBitAnd) interp.step() interp.step() interp.step() - assert interp.w_active_context.pop().value == 0 - assert interp.w_active_context.stack == [] + assert interp.s_active_context.pop().value == 0 + assert interp.s_active_context.stack == [] def test_bytecodePrimBitOr(): interp = new_interpreter(pushConstantOneBytecode + pushConstantTwoBytecode + bytecodePrimBitOr) interp.step() interp.step() interp.step() - assert interp.w_active_context.pop().value == 3 - assert interp.w_active_context.stack == [] + assert interp.s_active_context.pop().value == 3 + assert interp.s_active_context.stack == [] def test_bytecodePrimBitShift(): interp = new_interpreter(pushConstantOneBytecode + pushConstantTwoBytecode + bytecodePrimBitShift) interp.step() interp.step() interp.step() - assert interp.w_active_context.pop().value == 4 - assert interp.w_active_context.stack == [] + assert interp.s_active_context.pop().value == 4 + assert interp.s_active_context.stack == [] def test_bytecodePrimClass(): interp = new_interpreter(pushConstantOneBytecode + bytecodePrimClass) interp.step() interp.step() - assert interp.w_active_context.pop() == classtable.w_SmallInteger - assert interp.w_active_context.stack == [] + assert interp.s_active_context.pop() == classtable.w_SmallInteger + assert interp.s_active_context.stack == [] def test_bytecodePrimSubtract(): interp = new_interpreter(pushConstantOneBytecode + pushConstantTwoBytecode + bytecodePrimSubtract) interp.step() interp.step() interp.step() - assert interp.w_active_context.pop().value == -1 - assert interp.w_active_context.stack == [] + assert interp.s_active_context.pop().value == -1 + assert interp.s_active_context.stack == [] def test_bytecodePrimMultiply(): interp = new_interpreter(pushConstantMinusOneBytecode + pushConstantTwoBytecode + bytecodePrimMultiply) interp.step() interp.step() interp.step() - assert interp.w_active_context.pop().value == -2 - assert interp.w_active_context.stack == [] + assert interp.s_active_context.pop().value == -2 + assert interp.s_active_context.stack == [] def test_bytecodePrimDivide(): interp = new_interpreter(pushConstantTwoBytecode + pushConstantMinusOneBytecode + bytecodePrimDivide) interp.step() interp.step() interp.step() - assert interp.w_active_context.pop().value == -2 - assert interp.w_active_context.stack == [] + assert interp.s_active_context.pop().value == -2 + assert interp.s_active_context.stack == [] def test_bytecodePrimDiv(): interp = new_interpreter(pushConstantTwoBytecode + pushConstantMinusOneBytecode + bytecodePrimDiv) interp.step() interp.step() interp.step() - assert interp.w_active_context.pop().value == -2 - assert interp.w_active_context.stack == [] + assert interp.s_active_context.pop().value == -2 + assert interp.s_active_context.stack == [] def test_bytecodePrimMod(): interp = new_interpreter(pushConstantTwoBytecode + pushConstantMinusOneBytecode + bytecodePrimMod) interp.step() interp.step() interp.step() - assert interp.w_active_context.pop().value == 0 - assert interp.w_active_context.stack == [] + assert interp.s_active_context.pop().value == 0 + assert interp.s_active_context.stack == [] def test_bytecodePrimEquivalent(): interp = new_interpreter(pushConstantTwoBytecode + pushConstantMinusOneBytecode + bytecodePrimEquivalent) interp.step() interp.step() interp.step() - assert interp.w_active_context.pop() == interpreter.Interpreter.FALSE - assert interp.w_active_context.stack == [] + assert interp.s_active_context.pop() == interpreter.Interpreter.FALSE + assert interp.s_active_context.stack == [] interp = new_interpreter(pushConstantOneBytecode + pushConstantOneBytecode + bytecodePrimEquivalent) interp.step() interp.step() interp.step() - assert interp.w_active_context.pop() == interpreter.Interpreter.TRUE - assert interp.w_active_context.stack == [] + assert interp.s_active_context.pop() == interpreter.Interpreter.TRUE + assert interp.s_active_context.stack == [] def test_bytecodePrimNew(): w_fakeclassclass = mockclass(10, name='fakeclassclass') w_fakeclass = mockclass(1, name='fakeclass', varsized=False, w_metaclass=w_fakeclassclass) interp = new_interpreter(bytecodePrimNew) - interp.w_active_context.push(w_fakeclass) + interp.s_active_context.push(w_fakeclass) run_with_faked_methods( [[w_fakeclassclass, primitives.NEW, 0, "new"]], interp.step) - w_fakeinst = interp.w_active_context.pop() - assert interp.w_active_context.stack == [] + w_fakeinst = interp.s_active_context.pop() + assert interp.s_active_context.stack == [] assert w_fakeinst.getclass() == w_fakeclass assert w_fakeinst.size() == 1 @@ -354,13 +354,13 @@ w_fakeclass = mockclass(1, name='fakeclass', varsized=True, w_metaclass=w_fakeclassclass) interp = new_interpreter(bytecodePrimNewWithArg) - interp.w_active_context.push(w_fakeclass) - interp.w_active_context.push(interpreter.Interpreter.TWO) + interp.s_active_context.push(w_fakeclass) + interp.s_active_context.push(interpreter.Interpreter.TWO) run_with_faked_methods( [[w_fakeclassclass, primitives.NEW_WITH_ARG, 1, "new:"]], interp.step) - w_fakeinst = interp.w_active_context.pop() - assert interp.w_active_context.stack == [] + w_fakeinst = interp.s_active_context.pop() + assert interp.s_active_context.stack == [] assert w_fakeinst.getclass() == w_fakeclass assert w_fakeinst.size() == 3 @@ -368,12 +368,12 @@ w_fakeclass = mockclass(2, name='fakeclass', varsized=True) w_fakeinst = w_fakeclass.as_class_get_shadow().new(5) interp = new_interpreter(bytecodePrimSize) - interp.w_active_context.push(w_fakeinst) + interp.s_active_context.push(w_fakeinst) run_with_faked_methods( [[w_fakeclass, primitives.SIZE, 0, "size"]], interp.step) - assert interp.w_active_context.pop().value == 5 - assert interp.w_active_context.stack == [] + assert interp.s_active_context.pop().value == 5 + assert interp.s_active_context.stack == [] # w_class - the class from which the method is going to be called # (and on which it is going to be installed) @@ -390,19 +390,19 @@ w_method.bytes = pushConstantOneBytecode + bytecode shadow.installmethod("foo", w_method) interp = new_interpreter(bytecodes) - interp.w_active_context.w_method().literals = fakeliterals("foo") - interp.w_active_context.push(w_object) - callerContext = interp.w_active_context - interp.step() - assert interp.w_active_context.w_sender == callerContext - assert interp.w_active_context.stack == [] - assert interp.w_active_context.w_receiver == w_object - assert interp.w_active_context.w_method() == shadow.methoddict["foo"] + interp.s_active_context.w_method().literals = fakeliterals("foo") + interp.s_active_context.push(w_object) + callerContext = interp.s_active_context + interp.step() + assert interp.s_active_context.s_sender() == callerContext + assert interp.s_active_context.stack == [] + assert interp.s_active_context.w_receiver() == w_object + assert interp.s_active_context.w_method() == shadow.methoddict["foo"] assert callerContext.stack == [] interp.step() interp.step() - assert interp.w_active_context == callerContext - assert interp.w_active_context.stack == [result] + assert interp.s_active_context == callerContext + assert interp.s_active_context.stack == [result] def test_sendLiteralSelectorBytecode(): w_class = mockclass(0) @@ -420,9 +420,9 @@ shadow.installmethod("fib:", method) w_object = shadow.new() interp = new_interpreter(sendLiteralSelectorBytecode(16) + returnTopFromMethod) - interp.w_active_context.w_method().literals = fakeliterals("fib:") - interp.w_active_context.push(w_object) - interp.w_active_context.push(wrap_int(8)) + interp.s_active_context.w_method().literals = fakeliterals("fib:") + interp.s_active_context.push(w_object) + interp.s_active_context.push(wrap_int(8)) result = interp.interpret() assert unwrap_int(result) == 34 @@ -430,14 +430,14 @@ def test(): interp = new_interpreter(sendLiteralSelectorBytecode(1 + 16)) - interp.w_active_context.w_method().literals = fakeliterals("foo", "sub") - interp.w_active_context.push(wrap_int(50)) - interp.w_active_context.push(wrap_int(8)) - callerContext = interp.w_active_context - interp.step() - assert interp.w_active_context is callerContext - assert len(interp.w_active_context.stack) == 1 - w_result = interp.w_active_context.pop() + interp.s_active_context.w_method().literals = fakeliterals("foo", "sub") + interp.s_active_context.push(wrap_int(50)) + interp.s_active_context.push(wrap_int(8)) + callerContext = interp.s_active_context + interp.step() + assert interp.s_active_context is callerContext + assert len(interp.s_active_context.stack) == 1 + w_result = interp.s_active_context.pop() assert unwrap_int(w_result) == 42 run_with_faked_methods( @@ -447,58 +447,58 @@ def test_longJumpIfTrue(): interp = new_interpreter(longJumpIfTrue(0) + chr(15) + longJumpIfTrue(0) + chr(15)) - interp.w_active_context.push(interp.FALSE) - pc = interp.w_active_context.pc + 2 + interp.s_active_context.push(interp.FALSE) + pc = interp.s_active_context.pc() + 2 interp.step() - assert interp.w_active_context.pc == pc - interp.w_active_context.push(interp.TRUE) - pc = interp.w_active_context.pc + 2 + assert interp.s_active_context.pc() == pc + interp.s_active_context.push(interp.TRUE) + pc = interp.s_active_context.pc() + 2 interp.step() - assert interp.w_active_context.pc == pc + 15 + assert interp.s_active_context.pc() == pc + 15 def test_longJumpIfFalse(): interp = new_interpreter(pushConstantTrueBytecode + longJumpIfFalse(0) + chr(15) + pushConstantFalseBytecode + longJumpIfFalse(0) + chr(15)) interp.step() - pc = interp.w_active_context.pc + 2 + pc = interp.s_active_context.pc() + 2 interp.step() - assert interp.w_active_context.pc == pc + assert interp.s_active_context.pc() == pc interp.step() - pc = interp.w_active_context.pc + 2 + pc = interp.s_active_context.pc() + 2 interp.step() - assert interp.w_active_context.pc == pc + 15 + assert interp.s_active_context.pc() == pc + 15 def test_longUnconditionalJump(): interp = new_interpreter(longUnconditionalJump(4) + chr(15)) - pc = interp.w_active_context.pc + 2 + pc = interp.s_active_context.pc() + 2 interp.step() - assert interp.w_active_context.pc == pc + 15 + assert interp.s_active_context.pc() == pc + 15 def test_shortUnconditionalJump(): interp = new_interpreter(chr(145)) - pc = interp.w_active_context.pc + 1 + pc = interp.s_active_context.pc() + 1 interp.step() - assert interp.w_active_context.pc == pc + 2 + assert interp.s_active_context.pc() == pc + 2 def test_shortConditionalJump(): interp = new_interpreter(pushConstantTrueBytecode + shortConditionalJump(3) + pushConstantFalseBytecode + shortConditionalJump(3)) interp.step() - pc = interp.w_active_context.pc + 1 + pc = interp.s_active_context.pc() + 1 interp.step() - assert interp.w_active_context.pc == pc + assert interp.s_active_context.pc() == pc interp.step() - pc = interp.w_active_context.pc + 1 + pc = interp.s_active_context.pc() + 1 interp.step() - assert interp.w_active_context.pc == pc + 4 + assert interp.s_active_context.pc() == pc + 4 def test_popStackBytecode(): interp = new_interpreter(pushConstantTrueBytecode + popStackBytecode) interp.step() - assert interp.w_active_context.stack == [interp.TRUE] + assert interp.s_active_context.stack == [interp.TRUE] interp.step() - assert interp.w_active_context.stack == [] + assert interp.s_active_context.stack == [] def test_extendedPushBytecode(): test_pushReceiverVariableBytecode(extendedPushBytecode + chr((0<<6) + 0) + @@ -520,7 +520,7 @@ w_association.store(0, "mykey") w_association.store(1, "myvalue") interp = new_interpreter(pushConstantOneBytecode + bytecode) - interp.w_active_context.w_method().literals = fakeliterals(w_association) + interp.s_active_context.w_method().literals = fakeliterals(w_association) interp.step() interp.step() assert w_association.fetch(1) == interp.ONE @@ -545,13 +545,13 @@ shadow.installmethod("+", w_method) w_object = shadow.new() - interp.w_active_context.push(w_object) - interp.w_active_context.push(interp.ONE) + interp.s_active_context.push(w_object) + interp.s_active_context.push(interp.ONE) interp.step() - assert interp.w_active_context.w_method() == shadow.methoddict["+"] - assert interp.w_active_context.w_receiver is w_object - assert interp.w_active_context.gettemp(0) == interp.ONE - assert interp.w_active_context.stack == [] + assert interp.s_active_context.w_method() == shadow.methoddict["+"] + assert interp.s_active_context.w_receiver() is w_object + assert interp.s_active_context.gettemp(0) == interp.ONE + assert interp.s_active_context.stack == [] def test_bytecodePrimBool(): interp = new_interpreter(bytecodePrimLessThan + @@ -561,10 +561,10 @@ bytecodePrimEqual + bytecodePrimNotEqual) for i in range(6): - interp.w_active_context.push(interp.ONE) - interp.w_active_context.push(interp.TWO) + interp.s_active_context.push(interp.ONE) + interp.s_active_context.push(interp.TWO) interp.step() - assert interp.w_active_context.stack == [interp.TRUE, interp.FALSE, + assert interp.s_active_context.stack == [interp.TRUE, interp.FALSE, interp.TRUE, interp.FALSE, interp.FALSE, interp.TRUE] @@ -593,18 +593,18 @@ meth1.literals = fakeliterals("foo") meth2.literals = fakeliterals("foo") interp = new_interpreter(bytecodes) - interp.w_active_context.w_method().literals = fakeliterals("foo") - interp.w_active_context.push(w_object) + interp.s_active_context.w_method().literals = fakeliterals("foo") + interp.s_active_context.push(w_object) interp.step() for w_specificclass in [w_super, w_supersuper]: - callerContext = interp.w_active_context + callerContext = interp.s_active_context interp.step() interp.step() - assert interp.w_active_context.w_sender == callerContext - assert interp.w_active_context.stack == [] - assert interp.w_active_context.w_receiver == w_object + assert interp.s_active_context.s_sender() == callerContext + assert interp.s_active_context.stack == [] + assert interp.s_active_context.w_receiver() == w_object meth = w_specificclass.as_class_get_shadow().methoddict["foo"] - assert interp.w_active_context.w_method() == meth + assert interp.s_active_context.w_method() == meth assert callerContext.stack == [] def test_secondExtendedSendBytecode(): @@ -639,7 +639,7 @@ def interpret_bc(bcodes, literals, receiver=objtable.w_nil): bcode = "".join([chr(x) for x in bcodes]) interp = new_interpreter(bcode, receiver=receiver) - interp.w_active_context.w_method().literals = literals + interp.s_active_context.w_method().literals = literals return interp.interpret() # tests: bytecodePrimValue & bytecodePrimValueWithArg Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py Thu Feb 21 21:59:13 2008 @@ -200,15 +200,15 @@ w_method = s_class.lookup("abs") assert w_method - w_frame = w_method.create_frame(w_object, []) - interp.w_active_context = w_frame + s_frame = w_method.create_frame(w_object, []) + interp.s_active_context = s_frame print w_method while True: try: interp.step() - print interp.w_active_context.stack + print interp.s_active_context.stack except interpreter.ReturnFromTopLevel, e: assert e.object.value == abs(int) return @@ -258,12 +258,12 @@ s_class = w_receiver.shadow_of_my_class() w_method = s_class.lookup(selector) assert w_method - w_frame = w_method.create_frame(w_receiver, list(arguments_w)) - interp.w_active_context = w_frame + s_frame = w_method.create_frame(w_receiver, list(arguments_w)) + interp.s_active_context = s_frame while True: try: interp.step() - #print interp.w_active_context.stack + #print interp.s_active_context.stack except interpreter.ReturnFromTopLevel, e: return e.object.value Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_primitives.py Thu Feb 21 21:59:13 2008 @@ -25,24 +25,24 @@ mapped_stack = [wrap(x) for x in stack] frame = MockFrame(mapped_stack) interp = interpreter.Interpreter() - interp.w_active_context = frame + interp.s_active_context = frame return (interp, len(stack)) def prim(code, stack): interp, argument_count = mock(stack) prim_table[code](interp, argument_count-1) - res = interp.w_active_context.pop() - assert not len(interp.w_active_context.stack) # check args are consumed + res = interp.s_active_context.pop() + assert not len(interp.s_active_context.stack) # check args are consumed return res def prim_fails(code, stack): interp, argument_count = mock(stack) - orig_stack = list(interp.w_active_context.stack) + orig_stack = list(interp.s_active_context.stack) try: prim_table[code](interp, argument_count-1) py.test.fail("Expected PrimitiveFailedError") except PrimitiveFailedError: - assert interp.w_active_context.stack == orig_stack + assert interp.s_active_context.stack == orig_stack # smallinteger tests def test_small_int_add(): From tverwaes at codespeak.net Fri Feb 22 00:32:32 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Fri, 22 Feb 2008 00:32:32 +0100 (CET) Subject: [pypy-svn] r51763 - in pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk: . test Message-ID: <20080221233232.9C1AF1684EE@codespeak.net> Author: tverwaes Date: Fri Feb 22 00:32:32 2008 New Revision: 51763 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py Log: added some tests; make context-stack work as described in blue book Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py Fri Feb 22 00:32:32 2008 @@ -194,6 +194,10 @@ shadow.check_for_updates() return shadow + def as_link_get_shadow(self): + from pypy.lang.smalltalk.shadow import LinkShadow + return self.as_special_get_shadow(LinkShadow) + def as_semaphore_get_shadow(self): from pypy.lang.smalltalk.shadow import SemaphoreShadow return self.as_special_get_shadow(SemaphoreShadow) @@ -225,9 +229,9 @@ def as_context_get_shadow(self): from pypy.lang.smalltalk import classtable if self.getclass() == classtable.w_MethodContext: - self.as_methodcontext_get_shadow() + return self.as_methodcontext_get_shadow() elif self.getclass() == classtable.w_BlockContext: - self.as_blockcontext_get_shadow() + return self.as_blockcontext_get_shadow() else: # Should not happen... raise Exception() @@ -524,7 +528,7 @@ elif index == constants.CTXPART_PC_INDEX: return utility.wrap_int(self.pc()) elif index == constants.CTXPART_STACKP_INDEX: - return utility.wrap_int(len(self.stack)) + return utility.wrap_int(len(self.stack)+self.stackstart()) # Invalid! raise IndexError Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py Fri Feb 22 00:32:32 2008 @@ -222,13 +222,13 @@ self._w_self = w_self def firstlink(self): - return self._w_self.at0(constants.FIRST_LINK_INDEX) + return self._w_self.fetch(constants.FIRST_LINK_INDEX) def store_firstlink(self, w_object): return self._w_self.atput0(constants.FIRST_LINK_INDEX, w_object) def lastlink(self): - return self._w_self.at0(constants.LAST_LINK_INDEX) + return self._w_self.fetch(constants.LAST_LINK_INDEX) def store_lastlink(self, w_object): return self._w_self.atput0(constants.LAST_LINK_INDEX, w_object) @@ -266,9 +266,9 @@ def put_to_sleep(self, s_process): priority = s_process.priority() - s_scheduler = self.scheduler() + s_scheduler = self.s_scheduler() w_process_lists = s_scheduler.process_lists() - w_process_list = w_process_lists.at0(priority) + w_process_list = w_process_lists.fetch(priority) w_process_list.as_linkedlist_get_shadow().add_last_link(s_process._w_self) s_process.store_my_list(w_process_list) @@ -282,7 +282,7 @@ s_process.store_w_suspended_context(objtable.w_nil) #reclaimableContextCount := 0 - def scheduler(self): + def s_scheduler(self): from pypy.lang.smalltalk import objtable w_association = objtable.objtable["w_schedulerassociationpointer"] w_scheduler = w_association.as_association_get_shadow().value() @@ -290,7 +290,7 @@ def resume(self, w_process, interp): s_process = w_process.as_process_get_shadow() - s_scheduler = self.scheduler() + s_scheduler = self.s_scheduler() s_active_process = s_scheduler.active_process() active_priority = s_active_process.priority() new_priority = s_process.priority() @@ -303,7 +303,7 @@ def synchronous_signal(self, interp): print "SYNCHRONOUS SIGNAL" if self.is_empty_list(): - w_value = self._w_self.at0(constants.EXCESS_SIGNALS_INDEX) + w_value = self._w_self.fetch(constants.EXCESS_SIGNALS_INDEX) w_value = utility.wrap_int(utility.unwrap_int(w_value) + 1) self._w_self.atput0(constants.EXCESS_SIGNALS_INDEX, w_value) else: @@ -311,10 +311,10 @@ class LinkShadow(AbstractShadow): def __init__(self, w_self): - self._w_self = self + self._w_self = w_self def next(self): - return self._w_self.at0(constants.NEXT_LINK_INDEX) + return self._w_self.fetch(constants.NEXT_LINK_INDEX) def store_next(self, w_object): self._w_self.atput0(constants.NEXT_LINK_INDEX, w_object) @@ -326,10 +326,10 @@ self._w_self = w_self def priority(self): - return utility.unwrap_int(self._w_self.at0(constants.PROCESS_PRIORITY_INDEX)) + return utility.unwrap_int(self._w_self.fetch(constants.PROCESS_PRIORITY_INDEX)) def my_list(self): - return self._w_self.at0(constants.PROCESS_MY_LIST_INDEX) + return self._w_self.fetch(constants.PROCESS_MY_LIST_INDEX) def store_my_list(self, w_object): self._w_self.atput0(constants.PROCESS_MY_LIST_INDEX, w_object) @@ -345,23 +345,23 @@ self._w_self = w_self def key(self): - return self._w_self.at0(constants.ASSOCIATION_KEY_INDEX) + return self._w_self.fetch(constants.ASSOCIATION_KEY_INDEX) def value(self): - return self._w_self.at0(constants.ASSOCIATION_VALUE_INDEX) + return self._w_self.fetch(constants.ASSOCIATION_VALUE_INDEX) class SchedulerShadow(AbstractShadow): def __init__(self, w_self): self._w_self = w_self def active_process(self): - return self._w_self.at0(constants.SCHEDULER_ACTIVE_PROCESS_INDEX).as_process_get_shadow() + return self._w_self.fetch(constants.SCHEDULER_ACTIVE_PROCESS_INDEX).as_process_get_shadow() def store_active_process(self, w_object): self._w_self.atput0(constants.SCHEDULER_ACTIVE_PROCESS_INDEX, w_object) def process_lists(self): - return self._w_self.at0(constants.SCHEDULER_PROCESS_LISTS_INDEX) + return self._w_self.fetch(constants.SCHEDULER_PROCESS_LISTS_INDEX) class ContextPartShadow(AbstractShadow): @@ -378,10 +378,16 @@ return self.s_home().w_receiver() def s_sender(self): - return self._w_self.fetch(constants.CTXPART_SENDER_INDEX).as_context_get_shadow() + # XXX XXX + from pypy.lang.smalltalk import objtable + w_sender = self.w_self().fetch(constants.CTXPART_SENDER_INDEX) + if w_sender == objtable.w_nil: + return None + else: + return w_sender.as_context_get_shadow() def store_s_sender(self, s_sender): - self._w_self.store(constants.CTXPART_SENDER_INDEX, s_sender._w_self()) + self._w_self.store(constants.CTXPART_SENDER_INDEX, s_sender.w_self()) def pc(self): return utility.unwrap_int(self._w_self.fetch(constants.CTXPART_PC_INDEX)) @@ -390,10 +396,10 @@ self._w_self.store(constants.CTXPART_PC_INDEX, utility.wrap_int(newpc)) def stackpointer(self): - return utility.unwrap_int(self._w_self.fetch(constants.CTXPART_SENDER_STACKP_INDEX)) + return utility.unwrap_int(self._w_self.fetch(constants.CTXPART_STACKP_INDEX)) def store_stackpointer(self, pointer): - self._w_self.store(constants.CTXPART_SENDER_STACKP_INDEX, + self._w_self.store(constants.CTXPART_STACKP_INDEX, utility.wrap_int(pointer)) # ______________________________________________________________________ @@ -428,13 +434,13 @@ # Stack Manipulation def pop(self): idx = self.stackpointer() - w_v = self._w_self.fetch(idx) + w_v = self.w_self().fetch(idx) self.store_stackpointer(idx - 1) return w_v def push(self, w_v): idx = self.stackpointer() + 1 - self._w_self.store(idx, w_v) + self.w_self().store(idx, w_v) self.store_stackpointer(idx) def push_all(self, lst): @@ -445,17 +451,17 @@ return self.peek(0) def peek(self, idx): - return self._w_self.fetch(self.stackpointer()-idx) + return self.w_self().fetch(self.stackpointer()-idx) def pop_n(self, n): assert n >= 0 - assert n > (self.stackpointer() - self.stackstart()) + assert n <= self.stackpointer() + 1 self.store_stackpointer(self.stackpointer() - n) def pop_and_return_n(self, n): self.pop_n(n) start = self.stackpointer() + 1 - return [self._w_self.fetch(i) for i in range(start, start+n)] + return [self.w_self().fetch(i) for i in range(start, start+n)] class BlockContextShadow(ContextPartShadow): Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py Fri Feb 22 00:32:32 2008 @@ -203,6 +203,9 @@ .g_object.pointers] self.objects = [chunk.g_object.w_object for chunk in reader.chunklist] + from pypy.lang.smalltalk import constants, objtable + for name, idx in constants.objects_in_special_object_table.items(): + objtable.objtable["w_" + name] = self.objects[idx] def special(self, index): return self.special_objects[index] Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py Fri Feb 22 00:32:32 2008 @@ -226,6 +226,11 @@ w_false = image.special(constants.SO_FALSE) assert w_false is objtable.w_false +def test_scheduler(): + from pypy.lang.smalltalk.shadow import SemaphoreShadow + s_semaphore = SemaphoreShadow(None) + s_scheduler = s_semaphore.s_scheduler() + def test_compile_method(): # py.test.skip("Not working yet.") sourcecode = """fib Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py Fri Feb 22 00:32:32 2008 @@ -69,3 +69,75 @@ w_class = build_smalltalk_class("Demo", 0x90, methods=methods) classshadow = w_class.as_class_get_shadow() assert classshadow.methoddict == methods + +def test_link(): + w_object = model.W_PointersObject(None, 1) + w_object.store(constants.NEXT_LINK_INDEX, 'foo') + assert w_object.as_link_get_shadow().next() == 'foo' + +def method(tempsize=3,argsize=2, bytes="abcde"): + w_m = model.W_CompiledMethod() + w_m.bytes = bytes + w_m.tempsize = tempsize + w_m.argsize = argsize + return w_m + +def methodcontext(w_sender=objtable.w_nil, pc=1, stackpointer=0, stacksize=5, + method=method()): + stackstart = method.argsize + method.tempsize + 7 + w_object = model.W_PointersObject(classtable.w_MethodContext, stackstart+stacksize) + w_object.store(constants.CTXPART_SENDER_INDEX, w_sender) + w_object.store(constants.CTXPART_PC_INDEX, utility.wrap_int(pc)) + w_object.store(constants.CTXPART_STACKP_INDEX, utility.wrap_int(stackstart+stackpointer)) + w_object.store(constants.MTHDCTX_METHOD, method) + # XXX + w_object.store(constants.MTHDCTX_RECEIVER_MAP, '???') + w_object.store(constants.MTHDCTX_RECEIVER, 'receiver') + w_object.store(constants.MTHDCTX_TEMP_FRAME_START, + utility.wrap_int(constants.MTHDCTX_TEMP_FRAME_START)) + return w_object + +def test_methodcontext(): + w_m = method() + w_object = methodcontext(stackpointer=2, method=w_m) + w_object2 = methodcontext(w_sender=w_object) + s_object = w_object.as_methodcontext_get_shadow() + s_object2 = w_object2.as_methodcontext_get_shadow() + assert w_object2.fetch(constants.CTXPART_SENDER_INDEX) == w_object + assert s_object.w_self() == w_object + assert s_object2.w_self() == w_object2 + assert s_object.s_sender() == None + assert s_object2.s_sender() == s_object + assert s_object.w_receiver() == 'receiver' + s_object2.settemp(0, 'a') + s_object2.settemp(1, 'b') + assert s_object2.gettemp(0) == 'a' + assert s_object2.gettemp(1) == 'b' + assert s_object.w_method() == w_m + idx = s_object.stackstart() + w_object.store(idx + 1, 'f') + w_object.store(idx + 2, 'g') + w_object.store(idx + 3, 'h') + assert s_object.top() == 'h' + s_object.push('i') + assert s_object.top() == 'i' + assert s_object.peek(1) == 'h' + assert s_object.pop() == 'i' + assert s_object.pop_and_return_n(2) == ['g', 'h'] + assert s_object.pop() == 'f' + assert s_object.stackpointer() == s_object.stackstart() + +def test_process(priority=utility.wrap_int(3)): + w_context = methodcontext() + w_object = model.W_PointersObject(None, 4) + w_object.store(constants.NEXT_LINK_INDEX, 'foo') + w_object.store(constants.PROCESS_SUSPENDED_CONTEXT_INDEX, w_context) + w_object.store(constants.PROCESS_PRIORITY_INDEX, priority) + w_object.store(constants.PROCESS_MY_LIST_INDEX, 'mli') + s_object = w_object.as_process_get_shadow() + assert s_object.next() == 'foo' + assert s_object.priority() == 3 + assert s_object.my_list() == 'mli' + assert s_object.s_suspended_context() == w_context.as_context_get_shadow() + + From tverwaes at codespeak.net Fri Feb 22 00:44:05 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Fri, 22 Feb 2008 00:44:05 +0100 (CET) Subject: [pypy-svn] r51764 - pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test Message-ID: <20080221234405.B75B2168447@codespeak.net> Author: tverwaes Date: Fri Feb 22 00:44:04 2008 New Revision: 51764 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py Log: added extra shadow tests Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py Fri Feb 22 00:44:04 2008 @@ -127,17 +127,36 @@ assert s_object.pop() == 'f' assert s_object.stackpointer() == s_object.stackstart() -def test_process(priority=utility.wrap_int(3)): - w_context = methodcontext() +def process(w_context=methodcontext(), priority=utility.wrap_int(3)): w_object = model.W_PointersObject(None, 4) w_object.store(constants.NEXT_LINK_INDEX, 'foo') w_object.store(constants.PROCESS_SUSPENDED_CONTEXT_INDEX, w_context) w_object.store(constants.PROCESS_PRIORITY_INDEX, priority) w_object.store(constants.PROCESS_MY_LIST_INDEX, 'mli') + return w_object + +def test_process(): + w_context = methodcontext() + w_object = process(w_context) s_object = w_object.as_process_get_shadow() assert s_object.next() == 'foo' assert s_object.priority() == 3 assert s_object.my_list() == 'mli' assert s_object.s_suspended_context() == w_context.as_context_get_shadow() +def test_association(): + w_object = model.W_PointersObject(None, 2) + w_object.store(constants.ASSOCIATION_KEY_INDEX, 'key') + w_object.store(constants.ASSOCIATION_VALUE_INDEX, 'value') + s_object = w_object.as_association_get_shadow() + assert s_object.key() == 'key' + assert s_object.value() == 'value' +def test_scheduler(): + w_process = process() + w_object = model.W_PointersObject(None, 2) + w_object.store(constants.SCHEDULER_ACTIVE_PROCESS_INDEX, w_process) + w_object.store(constants.SCHEDULER_PROCESS_LISTS_INDEX, 'pl') + s_object = w_object.as_scheduler_get_shadow() + assert s_object.active_process() == w_process.as_process_get_shadow() + assert s_object.process_lists() == 'pl' From tverwaes at codespeak.net Fri Feb 22 01:18:40 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Fri, 22 Feb 2008 01:18:40 +0100 (CET) Subject: [pypy-svn] r51765 - in pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk: . test Message-ID: <20080222001840.C8943168450@codespeak.net> Author: tverwaes Date: Fri Feb 22 01:18:38 2008 New Revision: 51765 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/classtable.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/constants.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py Log: adding tests + making sure Process and Semaphore are known to the VM, fixing getting correct overloaded shadows Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/classtable.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/classtable.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/classtable.py Fri Feb 22 01:18:38 2008 @@ -111,8 +111,8 @@ define_cls("w_SmallInteger", "w_Integer") define_cls("w_Float", "w_Number", format=shadow.BYTES) define_cls("w_Collection", "w_Object") -define_cls("w_SequencableCollection", "w_Collection") -define_cls("w_ArrayedCollection", "w_SequencableCollection") +define_cls("w_SequenceableCollection", "w_Collection") +define_cls("w_ArrayedCollection", "w_SequenceableCollection") define_cls("w_Array", "w_ArrayedCollection", varsized=True) define_cls("w_String", "w_ArrayedCollection", format=shadow.BYTES) define_cls("w_UndefinedObject", "w_Object") @@ -122,9 +122,12 @@ define_cls("w_ByteArray", "w_ArrayedCollection", format=shadow.BYTES) define_cls("w_MethodDict", "w_Object", instvarsize=2, varsized=True) define_cls("w_CompiledMethod", "w_ByteArray", format=shadow.COMPILED_METHOD) -define_cls("w_MethodContext", "w_Object") define_cls("w_ContextPart", "w_Object") define_cls("w_MethodContext", "w_ContextPart") +define_cls("w_Link", "w_Object") +define_cls("w_Process", "w_Link") +define_cls("w_LinkedList", "w_SequenceableCollection") +define_cls("w_Semaphore", "w_LinkedList") define_cls("w_BlockContext", "w_ContextPart", instvarsize=constants.BLKCTX_TEMP_FRAME_START) Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/constants.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/constants.py Fri Feb 22 01:18:38 2008 @@ -15,43 +15,53 @@ CLASS_FORMAT_INDEX = 2 CLASS_NAME_INDEX = 6 # in the mini.image, at least -NEXT_LINK_INDEX = 0 # " +# Link +NEXT_LINK_INDEX = 0 -PROCESS_SUSPENDED_CONTEXT_INDEX = 1 # " -PROCESS_PRIORITY_INDEX = 2 # " -PROCESS_MY_LIST_INDEX = 3 # " - -FIRST_LINK_INDEX = 0 # " -LAST_LINK_INDEX = 1 # " -EXCESS_SIGNALS_INDEX = 2 # " - -SCHEDULER_PROCESS_LISTS_INDEX = 0 # " -SCHEDULER_ACTIVE_PROCESS_INDEX = 1 # " +# Process < Link +PROCESS_SUSPENDED_CONTEXT_INDEX = 1 +PROCESS_PRIORITY_INDEX = 2 +PROCESS_MY_LIST_INDEX = 3 + +# LinkedList +FIRST_LINK_INDEX = 0 +LAST_LINK_INDEX = 1 + +# Semaphore < LinkedList +EXCESS_SIGNALS_INDEX = 2 + +# Scheduler +SCHEDULER_PROCESS_LISTS_INDEX = 0 +SCHEDULER_ACTIVE_PROCESS_INDEX = 1 +# MethodDict METHODDICT_TALLY_INDEX = 0 METHODDICT_VALUES_INDEX = 1 METHODDICT_NAMES_INDEX = 2 +# Message MESSAGE_SELECTOR_INDEX = 0 MESSAGE_ARGUMENTS_INDEX = 1 MESSAGE_LOOKUP_CLASS_INDEX = 2 +# Association ASSOCIATION_KEY_INDEX = 0 ASSOCIATION_VALUE_INDEX = 1 +# ContextPart CTXPART_SENDER_INDEX = 0 CTXPART_PC_INDEX = 1 CTXPART_STACKP_INDEX = 2 METHOD_HEADER_INDEX = 0 -# Extends CTXPART_* +# BlockContext < ContextPart BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX = 3 BLKCTX_INITIAL_IP_INDEX = 4 BLKCTX_HOME_INDEX = 5 BLKCTX_TEMP_FRAME_START = 6 -# Extends CTXPART_* +# MethodContext < ContextPart MTHDCTX_METHOD = 3 MTHDCTX_RECEIVER_MAP = 4 MTHDCTX_RECEIVER = 5 @@ -126,10 +136,10 @@ # "Display" : SO_DISPLAY_CLASS, # "Message" : SO_MESSAGE_CLASS, "CompiledMethod" : SO_COMPILEDMETHOD_CLASS, -# "Semaphore" : SO_SEMAPHORE_CLASS, + "Semaphore" : SO_SEMAPHORE_CLASS, "Character" : SO_CHARACTER_CLASS, "ByteArray" : SO_BYTEARRAY_CLASS, -# "Process" : SO_PROCESS_CLASS, + "Process" : SO_PROCESS_CLASS, # "PseudoContext" : SO_PSEUDOCONTEXT_CLASS, # "TranslatedMethod" : SO_TRANSLATEDMETHOD_CLASS, # "LargeNegativeInteger" : SO_LARGENEGATIVEINTEGER_CLASS, # Not available in mini.image Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py Fri Feb 22 01:18:38 2008 @@ -181,6 +181,8 @@ return (W_AbstractObjectWithClassReference.invariant(self) and isinstance(self._vars, list)) + # XXX XXX + # Need to find better way of handling overloading of shadows!!! def as_special_get_shadow(self, TheClass): shadow = self._shadow if shadow is None: @@ -195,6 +197,9 @@ return shadow def as_link_get_shadow(self): + from pypy.lang.smalltalk import classtable + if self.getclass() == classtable.w_Process: + return self.as_process_get_shadow() from pypy.lang.smalltalk.shadow import LinkShadow return self.as_special_get_shadow(LinkShadow) @@ -203,6 +208,9 @@ return self.as_special_get_shadow(SemaphoreShadow) def as_linkedlist_get_shadow(self): + from pypy.lang.smalltalk import classtable + if self.getclass() == classtable.w_Semaphore: + return self.as_semaphore_get_shadow() from pypy.lang.smalltalk.shadow import LinkedListShadow return self.as_special_get_shadow(LinkedListShadow) Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py Fri Feb 22 01:18:38 2008 @@ -221,40 +221,40 @@ def __init__(self, w_self): self._w_self = w_self - def firstlink(self): + def w_firstlink(self): return self._w_self.fetch(constants.FIRST_LINK_INDEX) - def store_firstlink(self, w_object): + def store_w_firstlink(self, w_object): return self._w_self.atput0(constants.FIRST_LINK_INDEX, w_object) - def lastlink(self): + def w_lastlink(self): return self._w_self.fetch(constants.LAST_LINK_INDEX) - def store_lastlink(self, w_object): + def store_w_lastlink(self, w_object): return self._w_self.atput0(constants.LAST_LINK_INDEX, w_object) def is_empty_list(self): from pypy.lang.smalltalk import objtable - return self.firstlink() == objtable.w_nil + return self.w_firstlink() == objtable.w_nil def add_last_link(self, w_object): if self.is_empty_list(): - self.store_firstlink(w_object) + self.store_w_firstlink(w_object) else: - self.lastlink().store_next(w_object) + self.w_lastlink().as_link_get_shadow().store_next(w_object) # XXX Slang version stores list in process here... - self.store_lastlink(w_object) + self.store_w_lastlink(w_object) def remove_first_link_of_list(self): from pypy.lang.smalltalk import objtable - first = self.firstlink() - last = self.lastlink() + first = self.w_firstlink() + last = self.w_lastlink() if first == last: - self.store_firstlink(objtable.w_nil) - self.store_lastlink(objtable.w_nil) + self.store_w_firstlink(objtable.w_nil) + self.store_w_lastlink(objtable.w_nil) else: next = first.as_process_get_shadow().next() - self.store_firstlink(next) + self.store_w_firstlink(next) first.as_process_get_shadow().store_next(objtable.w_nil) return first @@ -275,8 +275,8 @@ def transfer_to(self, s_process, interp): from pypy.lang.smalltalk import objtable s_scheduler = self.scheduler() - s_old_process = s_scheduler.active_process() - s_scheduler.store_active_process(s_process) + s_old_process = s_scheduler.s_active_process() + s_scheduler.store_w_active_process(s_process.w_self()) s_old_process.store_w_suspended_context(interp.s_active_context.w_self()) interp.s_active_context = s_process.s_suspended_context() s_process.store_w_suspended_context(objtable.w_nil) @@ -291,7 +291,7 @@ def resume(self, w_process, interp): s_process = w_process.as_process_get_shadow() s_scheduler = self.s_scheduler() - s_active_process = s_scheduler.active_process() + s_active_process = s_scheduler.s_active_process() active_priority = s_active_process.priority() new_priority = s_process.priority() if new_priority > active_priority: @@ -354,11 +354,11 @@ def __init__(self, w_self): self._w_self = w_self - def active_process(self): + def s_active_process(self): return self._w_self.fetch(constants.SCHEDULER_ACTIVE_PROCESS_INDEX).as_process_get_shadow() - def store_active_process(self, w_object): - self._w_self.atput0(constants.SCHEDULER_ACTIVE_PROCESS_INDEX, w_object) + def store_w_active_process(self, w_object): + self._w_self.store(constants.SCHEDULER_ACTIVE_PROCESS_INDEX, w_object) def process_lists(self): return self._w_self.fetch(constants.SCHEDULER_PROCESS_LISTS_INDEX) Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py Fri Feb 22 01:18:38 2008 @@ -205,7 +205,7 @@ self.objects = [chunk.g_object.w_object for chunk in reader.chunklist] from pypy.lang.smalltalk import constants, objtable for name, idx in constants.objects_in_special_object_table.items(): - objtable.objtable["w_" + name] = self.objects[idx] + objtable.objtable["w_" + name] = self.special_objects[idx] def special(self, index): return self.special_objects[index] Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py Fri Feb 22 01:18:38 2008 @@ -70,9 +70,13 @@ classshadow = w_class.as_class_get_shadow() assert classshadow.methoddict == methods -def test_link(): +def link(w_next='foo'): w_object = model.W_PointersObject(None, 1) - w_object.store(constants.NEXT_LINK_INDEX, 'foo') + w_object.store(constants.NEXT_LINK_INDEX, w_next) + return w_object + +def test_link(): + w_object = link() assert w_object.as_link_get_shadow().next() == 'foo' def method(tempsize=3,argsize=2, bytes="abcde"): @@ -158,5 +162,39 @@ w_object.store(constants.SCHEDULER_ACTIVE_PROCESS_INDEX, w_process) w_object.store(constants.SCHEDULER_PROCESS_LISTS_INDEX, 'pl') s_object = w_object.as_scheduler_get_shadow() - assert s_object.active_process() == w_process.as_process_get_shadow() + assert s_object.s_active_process() == w_process.as_process_get_shadow() + assert s_object.process_lists() == 'pl' + w_process2 = process() + s_object.store_w_active_process(w_process2) assert s_object.process_lists() == 'pl' + assert s_object.s_active_process() != w_process.as_process_get_shadow() + assert s_object.s_active_process() == w_process2.as_process_get_shadow() + +def test_linkedlist(): + w_object = model.W_PointersObject(None,2) + w_last = link(objtable.w_nil) + w_lb1 = link(w_last) + w_lb2 = link(w_lb1) + w_lb3 = link(w_lb2) + w_lb4 = link(w_lb3) + w_first = link(w_lb4) + w_object.store(constants.FIRST_LINK_INDEX, w_first) + w_object.store(constants.LAST_LINK_INDEX, w_last) + s_object = w_object.as_linkedlist_get_shadow() + assert w_first == s_object.w_firstlink() + assert w_last == s_object.w_lastlink() + assert s_object.remove_first_link_of_list() == w_first + assert s_object.remove_first_link_of_list() == w_lb4 + assert s_object.remove_first_link_of_list() == w_lb3 + assert not s_object.is_empty_list() + assert s_object.remove_first_link_of_list() == w_lb2 + assert s_object.remove_first_link_of_list() == w_lb1 + assert s_object.remove_first_link_of_list() == w_last + assert s_object.is_empty_list() + s_object.add_last_link(w_first) + assert s_object.w_firstlink() == w_first + assert s_object.w_lastlink() == w_first + s_object.add_last_link(w_last) + assert s_object.w_firstlink() == w_first + assert s_object.w_lastlink() == w_last + From tverwaes at codespeak.net Fri Feb 22 02:21:50 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Fri, 22 Feb 2008 02:21:50 +0100 (CET) Subject: [pypy-svn] r51766 - in pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk: . test Message-ID: <20080222012150.D501B168512@codespeak.net> Author: tverwaes Date: Fri Feb 22 02:21:49 2008 New Revision: 51766 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py Log: making shadows and oldstyle methodcontexts more polymorph Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py Fri Feb 22 02:21:49 2008 @@ -2,6 +2,7 @@ from pypy.lang.smalltalk import model, constants, primitives from pypy.lang.smalltalk import objtable from pypy.lang.smalltalk.model import W_ContextPart +from pypy.lang.smalltalk.shadow import ContextPartShadow from pypy.lang.smalltalk.conftest import option from pypy.rlib import objectmodel, unroll @@ -106,10 +107,9 @@ # which is an object with two named vars, and fetches the second # named var (the value). index = self.currentBytecode & 31 - association = self.w_method().getliteral(index) - assert isinstance(association, model.W_PointersObject) - assert association.size() == 2 - self.push(association.fetch(constants.ASSOCIATION_VALUE_INDEX)) + w_association = self.w_method().getliteral(index) + s_association = w_association.as_association_get_shadow() + self.push(s_association.value()) def storeAndPopReceiverVariableBytecode(self, interp): index = self.currentBytecode & 7 @@ -200,11 +200,9 @@ if interp.should_trace(): print "PRIMITIVE FAILED: %d %s" % (method.primitive, selector,) pass # ignore this error and fall back to the Smalltalk version - start = len(self.stack) - argcount - assert start >= 0 # XXX check in the Blue Book what to do in this case - arguments = self.stack[start:] + arguments = self.pop_and_return_n(argcount) interp.s_active_context = method.create_frame(receiver, arguments, self) - self.pop_n(argcount + 1) + self.pop() def _return(self, object, interp, s_return_to): # for tests, when returning from the top-level context @@ -248,9 +246,9 @@ elif variableType == 2: self.push(self.w_method().getliteral(variableIndex)) elif variableType == 3: - association = self.w_method().getliteral(variableIndex) - assert isinstance(association, model.W_PointersObject) - self.push(association.fetch(constants.ASSOCIATION_VALUE_INDEX)) + w_association = self.w_method().getliteral(variableIndex) + s_association = w_association.as_association_get_shadow() + self.push(s_association.value()) else: assert 0 @@ -263,9 +261,9 @@ elif variableType == 2: raise IllegalStoreError elif variableType == 3: - association = self.w_method().getliteral(variableIndex) - assert isinstance(association, model.W_PointersObject) - association.store(constants.ASSOCIATION_VALUE_INDEX, self.top()) + w_association = self.w_method().getliteral(variableIndex) + s_association = w_association.as_association_get_shadow() + s_association.store_value(self.top()) def extendedStoreAndPopBytecode(self, interp): self.extendedStoreBytecode(interp) @@ -300,17 +298,17 @@ self.push(self.w_method().getliteral(third)) elif opType == 4: # pushLiteralVariable - association = self.w_method().getliteral(third) - assert isinstance(association, model.W_PointersObject) - self.push(association.fetch(constants.ASSOCIATION_VALUE_INDEX)) + w_association = self.w_method().getliteral(third) + s_association = w_association.as_association_get_shadow() + self.push(s_association.value()) elif opType == 5: self.w_receiver().store(third, self.top()) elif opType == 6: self.w_receiver().store(third, self.pop()) elif opType == 7: - association = self.w_method().getliteral(third) - assert isinstance(association, model.W_PointersObject) - association.store(constants.ASSOCIATION_VALUE_INDEX, self.top()) + w_association = self.w_method().getliteral(third) + s_association = w_association.as_association_get_shadow() + s_association.store_value(self.top()) def singleExtendedSuperBytecode(self, interp): selector, argcount = self.getExtendedSelectorArgcount() Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py Fri Feb 22 02:21:49 2008 @@ -364,9 +364,9 @@ # (Blue book, p 607) All CompiledMethods that contain extended-super bytecodes have the clain which they are found as their last literal variable. # Last of the literals is an association with compiledin # as a class - association = self.literals[-1] - assert isinstance(association, W_PointersObject) - self.w_compiledin = association.fetch(constants.ASSOCIATION_VALUE_INDEX) + w_association = self.literals[-1] + s_association = w_association.as_association_get_shadow() + self.w_compiledin = s_association.value() return self.w_compiledin def getclass(self): @@ -451,6 +451,16 @@ else: self.literals[index0-1] = w_value + def fetchbyte(self, index1): + index0 = index1 - 1 + index0 -= self.headersize() + index0 -= self.getliteralsize() + assert index0 < len(self.bytes) + return self.bytes[index0] + + def store(self, index0, w_v): + self.atput0(index0, w_v) + def at0(self, index0): from pypy.lang.smalltalk import utility # XXX Not tested @@ -459,11 +469,7 @@ self.literalat0(index0) else: index0 = index0 - self.getliteralsize() - if index0 > len(self.bytes): - print "ERROR" - print self.getliteralsize() - print len(self.bytes) - print self.bytes + assert index0 < len(self.bytes) return utility.wrap_int(ord(self.bytes[index0])) def atput0(self, index0, w_value): @@ -522,41 +528,43 @@ def store_s_sender(self, s_sender): self._s_sender = s_sender + def stackpointer(self): + return len(self.stack) + self.stackstart() # ______________________________________________________________________ # Imitate the primitive accessors - def fetch(self, index): - from pypy.lang.smalltalk import utility, objtable - if index == constants.CTXPART_SENDER_INDEX: - sender = self.s_sender() - if sender is None: - return objtable.w_nil - else: - return sender.w_self() - elif index == constants.CTXPART_PC_INDEX: - return utility.wrap_int(self.pc()) - elif index == constants.CTXPART_STACKP_INDEX: - return utility.wrap_int(len(self.stack)+self.stackstart()) - - # Invalid! - raise IndexError - - def store(self, index, w_value): - # XXX Untested code... + #def fetch(self, index): + # from pypy.lang.smalltalk import utility, objtable + # if index == constants.CTXPART_SENDER_INDEX: + # sender = self.s_sender() + # if sender is None: + # return objtable.w_nil + # else: + # return sender.w_self() + # elif index == constants.CTXPART_PC_INDEX: + # return utility.wrap_int(self.pc()) + # elif index == constants.CTXPART_STACKP_INDEX: + # return utility.wrap_int(self.stackpointer()) + # + # # Invalid! + # raise IndexError - from pypy.lang.smalltalk import utility, objtable - if index == constants.CTXPART_SENDER_INDEX: - if w_value != objtable.w_nil: - self._s_sender = w_value.as_context_get_shadow() - elif index == constants.CTXPART_PC_INDEX: - self._pc = utility.unwrap_int(w_value) - elif index == constants.CTXPART_STACKP_INDEX: - size = utility.unwrap_int(w_value) - size = size - self.stackstart() - self.stack = [objtable.w_nil] * size - else: - # Invalid! - raise IndexError + #def store(self, index, w_value): + # # XXX Untested code... + # + # from pypy.lang.smalltalk import utility, objtable + # if index == constants.CTXPART_SENDER_INDEX: + # if w_value != objtable.w_nil: + # self._s_sender = w_value.as_context_get_shadow() + # elif index == constants.CTXPART_PC_INDEX: + # self._pc = utility.unwrap_int(w_value) + # elif index == constants.CTXPART_STACKP_INDEX: + # size = utility.unwrap_int(w_value) + # size = size - self.stackstart() + # self.stack = [objtable.w_nil] * size + # else: + # # Invalid! + # raise IndexError def stackstart(self): return self.w_method().argsize + self.w_method().tempsize + constants.MTHDCTX_TEMP_FRAME_START Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py Fri Feb 22 02:21:49 2008 @@ -79,7 +79,7 @@ argument_count = argument_count_m1 + 1 # to account for the rcvr frame = interp.s_active_context assert argument_count == len_unwrap_spec - if len(frame.stack) < len_unwrap_spec: + if frame.stackpointer() - frame.stackstart() < len_unwrap_spec: raise PrimitiveFailedError() args = () for i, spec in unrolling_unwrap_spec: Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py Fri Feb 22 02:21:49 2008 @@ -1,5 +1,6 @@ import weakref from pypy.lang.smalltalk import model, constants, utility, error +from pypy.tool.pairtype import extendabletype class AbstractShadow(object): """A shadow is an optional extra bit of information that @@ -50,7 +51,7 @@ "Update the ClassShadow with data from the w_self class." - w_self = self._w_self + w_self = self.w_self() # read and painfully decode the format classformat = utility.unwrap_int( w_self.fetch(constants.CLASS_FORMAT_INDEX)) @@ -129,7 +130,7 @@ def new(self, extrasize=0): from pypy.lang.smalltalk import classtable - w_cls = self._w_self + w_cls = self.w_self() if w_cls == classtable.w_BlockContext: return model.W_BlockContext(None, None, 0, 0) @@ -215,23 +216,23 @@ "NOT_RPYTHON" # this is only for testing. assert isinstance(method, model.W_CompiledMethod) self.methoddict[selector] = method - method.w_compiledin = self._w_self + method.w_compiledin = self.w_self() class LinkedListShadow(AbstractShadow): def __init__(self, w_self): self._w_self = w_self def w_firstlink(self): - return self._w_self.fetch(constants.FIRST_LINK_INDEX) + return self.w_self().fetch(constants.FIRST_LINK_INDEX) def store_w_firstlink(self, w_object): - return self._w_self.atput0(constants.FIRST_LINK_INDEX, w_object) + return self.w_self().store(constants.FIRST_LINK_INDEX, w_object) def w_lastlink(self): - return self._w_self.fetch(constants.LAST_LINK_INDEX) + return self.w_self().fetch(constants.LAST_LINK_INDEX) def store_w_lastlink(self, w_object): - return self._w_self.atput0(constants.LAST_LINK_INDEX, w_object) + return self.w_self().store(constants.LAST_LINK_INDEX, w_object) def is_empty_list(self): from pypy.lang.smalltalk import objtable @@ -269,7 +270,7 @@ s_scheduler = self.s_scheduler() w_process_lists = s_scheduler.process_lists() w_process_list = w_process_lists.fetch(priority) - w_process_list.as_linkedlist_get_shadow().add_last_link(s_process._w_self) + w_process_list.as_linkedlist_get_shadow().add_last_link(s_process.w_self()) s_process.store_my_list(w_process_list) def transfer_to(self, s_process, interp): @@ -301,11 +302,10 @@ self.put_to_sleep(s_process) def synchronous_signal(self, interp): - print "SYNCHRONOUS SIGNAL" if self.is_empty_list(): - w_value = self._w_self.fetch(constants.EXCESS_SIGNALS_INDEX) + w_value = self.w_self().fetch(constants.EXCESS_SIGNALS_INDEX) w_value = utility.wrap_int(utility.unwrap_int(w_value) + 1) - self._w_self.atput0(constants.EXCESS_SIGNALS_INDEX, w_value) + self.w_self().store(constants.EXCESS_SIGNALS_INDEX, w_value) else: self.resume(self.remove_first_link_of_list(), interp) @@ -314,10 +314,10 @@ self._w_self = w_self def next(self): - return self._w_self.fetch(constants.NEXT_LINK_INDEX) + return self.w_self().fetch(constants.NEXT_LINK_INDEX) def store_next(self, w_object): - self._w_self.atput0(constants.NEXT_LINK_INDEX, w_object) + self.w_self().store(constants.NEXT_LINK_INDEX, w_object) class ProcessShadow(LinkShadow): """A shadow for Smalltalk objects that are processes @@ -326,44 +326,51 @@ self._w_self = w_self def priority(self): - return utility.unwrap_int(self._w_self.fetch(constants.PROCESS_PRIORITY_INDEX)) + return utility.unwrap_int(self.w_self().fetch(constants.PROCESS_PRIORITY_INDEX)) def my_list(self): - return self._w_self.fetch(constants.PROCESS_MY_LIST_INDEX) + return self.w_self().fetch(constants.PROCESS_MY_LIST_INDEX) def store_my_list(self, w_object): - self._w_self.atput0(constants.PROCESS_MY_LIST_INDEX, w_object) + self.w_self().store(constants.PROCESS_MY_LIST_INDEX, w_object) def s_suspended_context(self): - return self._w_self.fetch(constants.PROCESS_SUSPENDED_CONTEXT_INDEX).as_context_get_shadow() + return self.w_self().fetch(constants.PROCESS_SUSPENDED_CONTEXT_INDEX).as_context_get_shadow() def store_w_suspended_context(self, w_object): - self._w_self.atput0(constants.PROCESS_SUSPENDED_CONTEXT_INDEX, w_object) + self.w_self().store(constants.PROCESS_SUSPENDED_CONTEXT_INDEX, w_object) class AssociationShadow(AbstractShadow): def __init__(self, w_self): self._w_self = w_self def key(self): - return self._w_self.fetch(constants.ASSOCIATION_KEY_INDEX) + return self.w_self().fetch(constants.ASSOCIATION_KEY_INDEX) def value(self): - return self._w_self.fetch(constants.ASSOCIATION_VALUE_INDEX) + return self.w_self().fetch(constants.ASSOCIATION_VALUE_INDEX) + + def store_value(self, w_value): + self.w_self().store(constants.ASSOCIATION_VALUE_INDEX, w_value) class SchedulerShadow(AbstractShadow): def __init__(self, w_self): self._w_self = w_self def s_active_process(self): - return self._w_self.fetch(constants.SCHEDULER_ACTIVE_PROCESS_INDEX).as_process_get_shadow() + return self.w_self().fetch(constants.SCHEDULER_ACTIVE_PROCESS_INDEX).as_process_get_shadow() def store_w_active_process(self, w_object): - self._w_self.store(constants.SCHEDULER_ACTIVE_PROCESS_INDEX, w_object) + self.w_self().store(constants.SCHEDULER_ACTIVE_PROCESS_INDEX, w_object) def process_lists(self): - return self._w_self.fetch(constants.SCHEDULER_PROCESS_LISTS_INDEX) + return self.w_self().fetch(constants.SCHEDULER_PROCESS_LISTS_INDEX) -class ContextPartShadow(AbstractShadow): +# XXX This should be changed once everything works using contextshadows +# current hack to make the class extension-system for bytecode lookup happy... +# Should be AbstractShadow +# XXX XXX +class ContextPartShadow(model.W_ContextPart): #__metaclass__ = extendabletype @@ -373,6 +380,14 @@ def s_home(self): raise NotImplementedError() + # XXX XXX Remove function when fixing superclass to AbstractShadow + def w_self(self): + return self._w_self + + # XXX XXX Remove function when fixing superclass to AbstractShadow + def invalidate(self): + pass + def w_receiver(self): " Return self of the method, or the method that contains the block " return self.s_home().w_receiver() @@ -387,19 +402,19 @@ return w_sender.as_context_get_shadow() def store_s_sender(self, s_sender): - self._w_self.store(constants.CTXPART_SENDER_INDEX, s_sender.w_self()) + self.w_self().store(constants.CTXPART_SENDER_INDEX, s_sender.w_self()) def pc(self): - return utility.unwrap_int(self._w_self.fetch(constants.CTXPART_PC_INDEX)) + return utility.unwrap_int(self.w_self().fetch(constants.CTXPART_PC_INDEX)) def store_pc(self, newpc): - self._w_self.store(constants.CTXPART_PC_INDEX, utility.wrap_int(newpc)) + self.w_self().store(constants.CTXPART_PC_INDEX, utility.wrap_int(newpc)) def stackpointer(self): - return utility.unwrap_int(self._w_self.fetch(constants.CTXPART_STACKP_INDEX)) + return utility.unwrap_int(self.w_self().fetch(constants.CTXPART_STACKP_INDEX)) def store_stackpointer(self, pointer): - self._w_self.store(constants.CTXPART_STACKP_INDEX, + self.w_self().store(constants.CTXPART_STACKP_INDEX, utility.wrap_int(pointer)) # ______________________________________________________________________ @@ -410,7 +425,7 @@ def getbytecode(self): pc = self.pc() - bytecode = self.w_method().bytes[pc] + bytecode = self.w_method().fetchbyte(pc) currentBytecode = ord(bytecode) self.store_pc(pc + 1) return currentBytecode @@ -469,17 +484,17 @@ self._w_self = w_self def expected_argument_count(self): - return utility.unwrap_int(self._w_self.fetch(constants.BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX)) + return utility.unwrap_int(self.w_self().fetch(constants.BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX)) def store_expected_argument_count(self, argc): - return self._w_self.store(constants.BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX, + return self.w_self().store(constants.BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX, utility.wrap_int(argc)) def initialip(self): - return utility.unwrap_int(self._w_self.fetch(constants.BLKCTX_INITIAL_IP_INDEX)) + return utility.unwrap_int(self.w_self().fetch(constants.BLKCTX_INITIAL_IP_INDEX)) def s_home(self): - return self._w_self.fetch(constants.BLKCTX_HOME_INDEX).as_methodcontext_get_shadow() + return self.w_self().fetch(constants.BLKCTX_HOME_INDEX).as_methodcontext_get_shadow() def stackstart(self): return constants.BLKCTX_TEMP_FRAME_START @@ -489,19 +504,19 @@ self._w_self = w_self def w_method(self): - return self._w_self.fetch(constants.MTHDCTX_METHOD) + return self.w_self().fetch(constants.MTHDCTX_METHOD) def w_receiver(self): - return self._w_self.fetch(constants.MTHDCTX_RECEIVER) + return self.w_self().fetch(constants.MTHDCTX_RECEIVER) def store_w_receiver(self, w_receiver): - self._w_self.store(constants.MTHDCTX_RECEIVER, w_receiver) + self.w_self().store(constants.MTHDCTX_RECEIVER, w_receiver) def gettemp(self, index): - return self._w_self.fetch(constants.MTHDCTX_TEMP_FRAME_START + index) + return self.w_self().fetch(constants.MTHDCTX_TEMP_FRAME_START + index) def settemp(self, index, w_value): - self._w_self.store(constants.MTHDCTX_TEMP_FRAME_START + index, w_value) + self.w_self().store(constants.MTHDCTX_TEMP_FRAME_START + index, w_value) def s_home(self): return self Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py Fri Feb 22 02:21:49 2008 @@ -226,10 +226,16 @@ w_false = image.special(constants.SO_FALSE) assert w_false is objtable.w_false -def test_scheduler(): +def test_runimage(): from pypy.lang.smalltalk.shadow import SemaphoreShadow s_semaphore = SemaphoreShadow(None) s_scheduler = s_semaphore.s_scheduler() + s_ap = s_scheduler.s_active_process() + s_ctx = s_ap.s_suspended_context() + s_ap.store_w_suspended_context(objtable.w_nil) + interp = interpreter.Interpreter() + interp.s_active_context = s_ctx + interp.interpret() def test_compile_method(): # py.test.skip("Not working yet.") From tverwaes at codespeak.net Fri Feb 22 04:28:45 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Fri, 22 Feb 2008 04:28:45 +0100 (CET) Subject: [pypy-svn] r51767 - in pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk: . test Message-ID: <20080222032845.2A776168510@codespeak.net> Author: tverwaes Date: Fri Feb 22 04:28:42 2008 New Revision: 51767 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_interpreter.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_primitives.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py Log: getting all the numbers and tests right. added a test which restarts the suspended thread of the image Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py Fri Feb 22 04:28:42 2008 @@ -61,7 +61,7 @@ print "%sStack=%s" % ( self._last_indent, - repr(self.s_active_context.stack),) + repr(self.s_active_context.stack()),) print "%sBytecode at %d (%d:%s):" % ( self._last_indent, self.s_active_context.pc(), @@ -171,7 +171,7 @@ if interp.should_trace(): print "%sSending selector %r to %r with: %r" % ( interp._last_indent, selector, receiver, - [self.stack[i-argcount] for i in range(argcount)]) + [self.peek(argcount-1-i) for i in range(argcount)]) pass assert argcount >= 0 method = receiverclassshadow.lookup(selector) Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py Fri Feb 22 04:28:42 2008 @@ -406,7 +406,7 @@ self.primitive is not None) def size(self): - return self.getliteralsize() + len(self.bytes) + self.headersize() + return self.headersize() + self.getliteralsize() + len(self.bytes) def getliteralsize(self): return self.literalsize * constants.BYTES_PER_WORD @@ -453,7 +453,6 @@ def fetchbyte(self, index1): index0 = index1 - 1 - index0 -= self.headersize() index0 -= self.getliteralsize() assert index0 < len(self.bytes) return self.bytes[index0] @@ -493,7 +492,7 @@ __metaclass__ = extendabletype def __init__(self, s_home, s_sender): - self.stack = [] + self._stack = [] self._pc = 0 #assert isinstance(s_home, W_MethodContext) self._s_home = s_home @@ -511,6 +510,9 @@ def pc(self): return self._pc + def stack(self): + return self._stack + def store_pc(self, pc): self._pc = pc @@ -529,45 +531,44 @@ self._s_sender = s_sender def stackpointer(self): - return len(self.stack) + self.stackstart() + return len(self.stack()) + self.stackstart() - 1 # ______________________________________________________________________ # Imitate the primitive accessors - #def fetch(self, index): - # from pypy.lang.smalltalk import utility, objtable - # if index == constants.CTXPART_SENDER_INDEX: - # sender = self.s_sender() - # if sender is None: - # return objtable.w_nil - # else: - # return sender.w_self() - # elif index == constants.CTXPART_PC_INDEX: - # return utility.wrap_int(self.pc()) - # elif index == constants.CTXPART_STACKP_INDEX: - # return utility.wrap_int(self.stackpointer()) - # - # # Invalid! - # raise IndexError + def fetch(self, index): + from pypy.lang.smalltalk import utility, objtable + if index == constants.CTXPART_SENDER_INDEX: + sender = self.s_sender() + if sender is None: + return objtable.w_nil + else: + return sender.w_self() + elif index == constants.CTXPART_PC_INDEX: + return utility.wrap_int(self.pc()) + elif index == constants.CTXPART_STACKP_INDEX: + return utility.wrap_int(self.stackpointer()) + + # Invalid! + raise IndexError - #def store(self, index, w_value): - # # XXX Untested code... - # - # from pypy.lang.smalltalk import utility, objtable - # if index == constants.CTXPART_SENDER_INDEX: - # if w_value != objtable.w_nil: - # self._s_sender = w_value.as_context_get_shadow() - # elif index == constants.CTXPART_PC_INDEX: - # self._pc = utility.unwrap_int(w_value) - # elif index == constants.CTXPART_STACKP_INDEX: - # size = utility.unwrap_int(w_value) - # size = size - self.stackstart() - # self.stack = [objtable.w_nil] * size - # else: - # # Invalid! - # raise IndexError + def store(self, index, w_value): + # XXX Untested code... + from pypy.lang.smalltalk import utility, objtable + if index == constants.CTXPART_SENDER_INDEX: + if w_value != objtable.w_nil: + self._s_sender = w_value.as_context_get_shadow() + elif index == constants.CTXPART_PC_INDEX: + self._pc = utility.unwrap_int(w_value) + elif index == constants.CTXPART_STACKP_INDEX: + size = utility.unwrap_int(w_value) + size = 1 + size - self.stackstart() + self._stack = [objtable.w_nil] * size + else: + # Invalid! + raise IndexError def stackstart(self): - return self.w_method().argsize + self.w_method().tempsize + constants.MTHDCTX_TEMP_FRAME_START + return constants.MTHDCTX_TEMP_FRAME_START # ______________________________________________________________________ # Method that contains the bytecode for this method/block context @@ -601,35 +602,35 @@ # Stack Manipulation def pop(self): - return self.stack.pop() + return self.stack().pop() def push(self, w_v): assert w_v - self.stack.append(w_v) + self.stack().append(w_v) def push_all(self, lst): " Equivalent to 'for x in lst: self.push(x)' where x is a lst " assert None not in lst - self.stack += lst + self._stack += lst def top(self): return self.peek(0) def peek(self, idx): - return self.stack[-(idx+1)] + return self.stack()[-(idx+1)] def pop_n(self, n): assert n >= 0 - start = len(self.stack) - n + start = len(self.stack()) - n assert start >= 0 # XXX what if this fails? - del self.stack[start:] + del self.stack()[start:] def pop_and_return_n(self, n): assert n >= 0 - start = len(self.stack) - n + start = len(self.stack()) - n assert start >= 0 # XXX what if this fails? - res = self.stack[start:] - del self.stack[start:] + res = self.stack()[start:] + del self.stack()[start:] return res class W_BlockContext(W_ContextPart): @@ -662,8 +663,8 @@ elif index == constants.BLKCTX_HOME_INDEX: return self.s_home() elif index >= constants.BLKCTX_TEMP_FRAME_START: - stack_index = len(self.stack) - index - 1 - return self.stack[stack_index] + stack_index = len(self.stack()) - index - 1 + return self.stack()[stack_index] else: return W_ContextPart.fetch(self, index) @@ -678,11 +679,14 @@ elif index == constants.BLKCTX_HOME_INDEX: self._s_home = value.as_methodcontext_get_shadow() elif index >= constants.BLKCTX_TEMP_FRAME_START: - stack_index = len(self.stack) - index - 1 - self.stack[stack_index] = value + stack_index = len(self.stack()) - index - 1 + self.stack()[stack_index] = value else: W_ContextPart.store(self, index, value) + def stackstart(self): + return constants.BLKCTX_TEMP_FRAME_START + class W_MethodContext(W_ContextPart): def __init__(self, w_method, w_receiver, arguments, s_sender=None): @@ -720,8 +724,8 @@ # After that comes the stack: offset -= len(self.temps) - stack_index = len(self.stack) - offset - 1 - return self.stack[stack_index] + stack_index = len(self.stack()) - offset - 1 + return self.stack()[stack_index] else: return W_ContextPart.fetch(self, index) @@ -740,8 +744,8 @@ # After that comes the stack: offset -= len(self.temps) - stack_index = len(self.stack) - offset - 1 - self.stack[stack_index] = w_object + stack_index = len(self.stack()) - offset - 1 + self.stack()[stack_index] = w_object else: W_ContextPart.store(self, index, w_object) Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py Fri Feb 22 04:28:42 2008 @@ -79,12 +79,12 @@ argument_count = argument_count_m1 + 1 # to account for the rcvr frame = interp.s_active_context assert argument_count == len_unwrap_spec - if frame.stackpointer() - frame.stackstart() < len_unwrap_spec: + if frame.stackpointer() - frame.stackstart() + 1 < len_unwrap_spec: raise PrimitiveFailedError() args = () for i, spec in unrolling_unwrap_spec: - index = -len_unwrap_spec + i - w_arg = frame.stack[index] + index = len_unwrap_spec - 1 - i + w_arg = frame.peek(index) if spec is int: args += (utility.unwrap_int(w_arg), ) elif spec is index1_0: Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py Fri Feb 22 04:28:42 2008 @@ -473,6 +473,9 @@ assert n <= self.stackpointer() + 1 self.store_stackpointer(self.stackpointer() - n) + def stack(self): + return [self.w_self().fetch(i) for i in range(self.stackstart(), self.stackpointer() + 1)] + def pop_and_return_n(self, n): self.pop_n(n) start = self.stackpointer() + 1 @@ -522,4 +525,4 @@ return self def stackstart(self): - return self.w_method().argsize + self.w_method().tempsize + constants.MTHDCTX_TEMP_FRAME_START + return constants.MTHDCTX_TEMP_FRAME_START Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_interpreter.py Fri Feb 22 04:28:42 2008 @@ -129,7 +129,7 @@ interp.step() interp.step() interp.step() - assert interp.s_active_context.stack == ["egg", "bar", "baz"] + assert interp.s_active_context.stack() == ["egg", "bar", "baz"] def test_pushTemporaryVariableBytecode(bytecode=(pushTemporaryVariableBytecode(0) + pushTemporaryVariableBytecode(1) + @@ -139,7 +139,7 @@ interp.step() interp.step() interp.step() - assert interp.s_active_context.stack == ["foo", "bar", "temp"] + assert interp.s_active_context.stack() == ["foo", "bar", "temp"] def test_pushLiteralConstantBytecode(bytecode=pushLiteralConstantBytecode(0) + pushLiteralConstantBytecode(1) + @@ -149,7 +149,7 @@ interp.step() interp.step() interp.step() - assert interp.s_active_context.stack == [fakesymbol("a"), + assert interp.s_active_context.stack() == [fakesymbol("a"), fakesymbol("b"), fakesymbol("c")] @@ -160,7 +160,7 @@ interp = new_interpreter(bytecode) interp.s_active_context.w_method().literals = fakeliterals(w_association) interp.step() - assert interp.s_active_context.stack == ["myvalue"] + assert interp.s_active_context.stack() == ["myvalue"] def test_storeAndPopReceiverVariableBytecode(bytecode=storeAndPopReceiverVariableBytecode, popped=True): @@ -172,9 +172,9 @@ interp.step() interp.step() if popped: - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] else: - assert interp.s_active_context.stack == [interp.TRUE] + assert interp.s_active_context.stack() == [interp.TRUE] for test_index in range(8): if test_index == index: @@ -188,7 +188,7 @@ interp.s_active_context.temps = [None] * 8 interp.step() interp.step() - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] for test_index in range(8): if test_index == index: assert interp.s_active_context.temps[test_index] == interp.TRUE @@ -199,55 +199,55 @@ interp = new_interpreter(pushConstantTrueBytecode) interp.step() assert interp.s_active_context.pop() == interp.TRUE - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] def test_pushConstantFalseBytecode(): interp = new_interpreter(pushConstantFalseBytecode) interp.step() assert interp.s_active_context.pop() == interp.FALSE - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] def test_pushConstantNilBytecode(): interp = new_interpreter(pushConstantNilBytecode) interp.step() assert interp.s_active_context.pop() == interp.NIL - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] def test_pushConstantMinusOneBytecode(): interp = new_interpreter(pushConstantMinusOneBytecode) interp.step() assert interp.s_active_context.pop() == interp.MINUS_ONE - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] def test_pushConstantZeroBytecode(): interp = new_interpreter(pushConstantZeroBytecode) interp.step() assert interp.s_active_context.pop() == interp.ZERO - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] def test_pushConstantOneBytecode(): interp = new_interpreter(pushConstantOneBytecode) interp.step() assert interp.s_active_context.pop() == interp.ONE - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] def test_pushConstantTwoBytecode(): interp = new_interpreter(pushConstantTwoBytecode) interp.step() assert interp.s_active_context.pop() == interp.TWO - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] def test_pushActiveContextBytecode(): interp = new_interpreter(pushActiveContextBytecode) interp.step() assert interp.s_active_context.pop() == interp.s_active_context.w_self() - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] def test_duplicateTopBytecode(): interp = new_interpreter(pushConstantZeroBytecode + duplicateTopBytecode) interp.step() interp.step() - assert interp.s_active_context.stack == [interp.ZERO, interp.ZERO] + assert interp.s_active_context.stack() == [interp.ZERO, interp.ZERO] def test_bytecodePrimBitAnd(): interp = new_interpreter(pushConstantOneBytecode + pushConstantTwoBytecode + bytecodePrimBitAnd) @@ -255,7 +255,7 @@ interp.step() interp.step() assert interp.s_active_context.pop().value == 0 - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] def test_bytecodePrimBitOr(): interp = new_interpreter(pushConstantOneBytecode + pushConstantTwoBytecode + bytecodePrimBitOr) @@ -263,7 +263,7 @@ interp.step() interp.step() assert interp.s_active_context.pop().value == 3 - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] def test_bytecodePrimBitShift(): interp = new_interpreter(pushConstantOneBytecode + pushConstantTwoBytecode + bytecodePrimBitShift) @@ -271,14 +271,14 @@ interp.step() interp.step() assert interp.s_active_context.pop().value == 4 - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] def test_bytecodePrimClass(): interp = new_interpreter(pushConstantOneBytecode + bytecodePrimClass) interp.step() interp.step() assert interp.s_active_context.pop() == classtable.w_SmallInteger - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] def test_bytecodePrimSubtract(): interp = new_interpreter(pushConstantOneBytecode + pushConstantTwoBytecode + bytecodePrimSubtract) @@ -286,7 +286,7 @@ interp.step() interp.step() assert interp.s_active_context.pop().value == -1 - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] def test_bytecodePrimMultiply(): interp = new_interpreter(pushConstantMinusOneBytecode + pushConstantTwoBytecode + bytecodePrimMultiply) @@ -294,7 +294,7 @@ interp.step() interp.step() assert interp.s_active_context.pop().value == -2 - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] def test_bytecodePrimDivide(): interp = new_interpreter(pushConstantTwoBytecode + pushConstantMinusOneBytecode + bytecodePrimDivide) @@ -302,7 +302,7 @@ interp.step() interp.step() assert interp.s_active_context.pop().value == -2 - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] def test_bytecodePrimDiv(): interp = new_interpreter(pushConstantTwoBytecode + pushConstantMinusOneBytecode + bytecodePrimDiv) @@ -310,7 +310,7 @@ interp.step() interp.step() assert interp.s_active_context.pop().value == -2 - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] def test_bytecodePrimMod(): interp = new_interpreter(pushConstantTwoBytecode + pushConstantMinusOneBytecode + bytecodePrimMod) @@ -318,7 +318,7 @@ interp.step() interp.step() assert interp.s_active_context.pop().value == 0 - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] def test_bytecodePrimEquivalent(): interp = new_interpreter(pushConstantTwoBytecode + pushConstantMinusOneBytecode + bytecodePrimEquivalent) @@ -326,14 +326,14 @@ interp.step() interp.step() assert interp.s_active_context.pop() == interpreter.Interpreter.FALSE - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] interp = new_interpreter(pushConstantOneBytecode + pushConstantOneBytecode + bytecodePrimEquivalent) interp.step() interp.step() interp.step() assert interp.s_active_context.pop() == interpreter.Interpreter.TRUE - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] def test_bytecodePrimNew(): w_fakeclassclass = mockclass(10, name='fakeclassclass') @@ -345,7 +345,7 @@ [[w_fakeclassclass, primitives.NEW, 0, "new"]], interp.step) w_fakeinst = interp.s_active_context.pop() - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] assert w_fakeinst.getclass() == w_fakeclass assert w_fakeinst.size() == 1 @@ -360,7 +360,7 @@ [[w_fakeclassclass, primitives.NEW_WITH_ARG, 1, "new:"]], interp.step) w_fakeinst = interp.s_active_context.pop() - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] assert w_fakeinst.getclass() == w_fakeclass assert w_fakeinst.size() == 3 @@ -373,7 +373,7 @@ [[w_fakeclass, primitives.SIZE, 0, "size"]], interp.step) assert interp.s_active_context.pop().value == 5 - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] # w_class - the class from which the method is going to be called # (and on which it is going to be installed) @@ -395,14 +395,14 @@ callerContext = interp.s_active_context interp.step() assert interp.s_active_context.s_sender() == callerContext - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] assert interp.s_active_context.w_receiver() == w_object assert interp.s_active_context.w_method() == shadow.methoddict["foo"] - assert callerContext.stack == [] + assert callerContext.stack() == [] interp.step() interp.step() assert interp.s_active_context == callerContext - assert interp.s_active_context.stack == [result] + assert interp.s_active_context.stack() == [result] def test_sendLiteralSelectorBytecode(): w_class = mockclass(0) @@ -436,7 +436,7 @@ callerContext = interp.s_active_context interp.step() assert interp.s_active_context is callerContext - assert len(interp.s_active_context.stack) == 1 + assert len(interp.s_active_context.stack()) == 1 w_result = interp.s_active_context.pop() assert unwrap_int(w_result) == 42 @@ -496,9 +496,9 @@ interp = new_interpreter(pushConstantTrueBytecode + popStackBytecode) interp.step() - assert interp.s_active_context.stack == [interp.TRUE] + assert interp.s_active_context.stack() == [interp.TRUE] interp.step() - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] def test_extendedPushBytecode(): test_pushReceiverVariableBytecode(extendedPushBytecode + chr((0<<6) + 0) + @@ -551,7 +551,7 @@ assert interp.s_active_context.w_method() == shadow.methoddict["+"] assert interp.s_active_context.w_receiver() is w_object assert interp.s_active_context.gettemp(0) == interp.ONE - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] def test_bytecodePrimBool(): interp = new_interpreter(bytecodePrimLessThan + @@ -564,7 +564,7 @@ interp.s_active_context.push(interp.ONE) interp.s_active_context.push(interp.TWO) interp.step() - assert interp.s_active_context.stack == [interp.TRUE, interp.FALSE, + assert interp.s_active_context.stack() == [interp.TRUE, interp.FALSE, interp.TRUE, interp.FALSE, interp.FALSE, interp.TRUE] @@ -601,11 +601,11 @@ interp.step() interp.step() assert interp.s_active_context.s_sender() == callerContext - assert interp.s_active_context.stack == [] + assert interp.s_active_context.stack() == [] assert interp.s_active_context.w_receiver() == w_object meth = w_specificclass.as_class_get_shadow().methoddict["foo"] assert interp.s_active_context.w_method() == meth - assert callerContext.stack == [] + assert callerContext.stack() == [] def test_secondExtendedSendBytecode(): w_class = mockclass(0) Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py Fri Feb 22 04:28:42 2008 @@ -227,6 +227,7 @@ assert w_false is objtable.w_false def test_runimage(): + #py.test.skip("This method actually runs an image. Fails since no graphical primitives yet") from pypy.lang.smalltalk.shadow import SemaphoreShadow s_semaphore = SemaphoreShadow(None) s_scheduler = s_semaphore.s_scheduler() @@ -238,7 +239,7 @@ interp.interpret() def test_compile_method(): - # py.test.skip("Not working yet.") + py.test.skip("Not working yet.") sourcecode = """fib ^self < 2 ifTrue: [ 1 ] Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_primitives.py Fri Feb 22 04:28:42 2008 @@ -11,7 +11,7 @@ class MockFrame(model.W_MethodContext): def __init__(self, stack): - self.stack = stack + self._stack = stack def wrap(x): if isinstance(x, int): return utility.wrap_int(x) @@ -32,17 +32,17 @@ interp, argument_count = mock(stack) prim_table[code](interp, argument_count-1) res = interp.s_active_context.pop() - assert not len(interp.s_active_context.stack) # check args are consumed + assert not len(interp.s_active_context.stack()) # check args are consumed return res def prim_fails(code, stack): interp, argument_count = mock(stack) - orig_stack = list(interp.s_active_context.stack) + orig_stack = list(interp.s_active_context.stack()) try: prim_table[code](interp, argument_count-1) py.test.fail("Expected PrimitiveFailedError") except PrimitiveFailedError: - assert interp.s_active_context.stack == orig_stack + assert interp.s_active_context.stack() == orig_stack # smallinteger tests def test_small_int_add(): Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py Fri Feb 22 04:28:42 2008 @@ -88,7 +88,7 @@ def methodcontext(w_sender=objtable.w_nil, pc=1, stackpointer=0, stacksize=5, method=method()): - stackstart = method.argsize + method.tempsize + 7 + stackstart = 7 # (len notation, not idx notation) w_object = model.W_PointersObject(classtable.w_MethodContext, stackstart+stacksize) w_object.store(constants.CTXPART_SENDER_INDEX, w_sender) w_object.store(constants.CTXPART_PC_INDEX, utility.wrap_int(pc)) From tverwaes at codespeak.net Fri Feb 22 04:53:22 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Fri, 22 Feb 2008 04:53:22 +0100 (CET) Subject: [pypy-svn] r51768 - pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test Message-ID: <20080222035322.8209416850F@codespeak.net> Author: tverwaes Date: Fri Feb 22 04:53:19 2008 New Revision: 51768 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py Log: adding stack-test + weirdness comment Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py Fri Feb 22 04:53:19 2008 @@ -97,8 +97,16 @@ # XXX w_object.store(constants.MTHDCTX_RECEIVER_MAP, '???') w_object.store(constants.MTHDCTX_RECEIVER, 'receiver') - w_object.store(constants.MTHDCTX_TEMP_FRAME_START, - utility.wrap_int(constants.MTHDCTX_TEMP_FRAME_START)) + + # XXX Might want to check the realness of the next assumption, + # XXX made by hooking into the suspended thread of the image. + # XXX it seems the only possibility, and using this assumption + # XXX it actually runs... + # Weirdly enough, undependant from the size of the tempsize and + # argsize, the stackpointer can point to anything starting from + # the temp_frame_start. That's why stacks always print all elements + # including possible "temps or args" + w_object.store(constants.MTHDCTX_TEMP_FRAME_START, 'el') return w_object def test_methodcontext(): @@ -123,6 +131,7 @@ w_object.store(idx + 2, 'g') w_object.store(idx + 3, 'h') assert s_object.top() == 'h' + assert s_object.stack() == ['el', 'f', 'g', 'h' ] s_object.push('i') assert s_object.top() == 'i' assert s_object.peek(1) == 'h' @@ -197,4 +206,3 @@ s_object.add_last_link(w_last) assert s_object.w_firstlink() == w_first assert s_object.w_lastlink() == w_last - From arigo at codespeak.net Fri Feb 22 11:40:23 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 22 Feb 2008 11:40:23 +0100 (CET) Subject: [pypy-svn] r51780 - pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test Message-ID: <20080222104023.0DF33168469@codespeak.net> Author: arigo Date: Fri Feb 22 11:40:22 2008 New Revision: 51780 Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_llmemory.py Log: Support for 'adr|flag' and 'adr&~flag' to set and clear flags in the lower-order bits of an aligned address. Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_llmemory.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_llmemory.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_llmemory.py Fri Feb 22 11:40:22 2008 @@ -570,3 +570,41 @@ adr += gcarrayofptr_itemsoffset + 2 * gcarrayofptr_singleitemoffset adr = adr.address[0] # => s2[1][2] assert (adr + FieldOffset(S1, 'x')).signed[0] == -33 + +def test_flags_in_low_bits(): + S = lltype.GcStruct('S', ('x', lltype.Signed)) + s = lltype.malloc(S) + a = cast_ptr_to_adr(s) + assert cast_adr_to_int(a) & 3 == 0 + assert cast_adr_to_int(a | 0) & 3 == 0 + assert cast_adr_to_int(a | 1) & 3 == 1 + assert cast_adr_to_int(a | 2) & 3 == 2 + assert cast_adr_to_int(a | 3) & 3 == 3 + assert cast_adr_to_int(a & ~0) & 3 == 0 + assert cast_adr_to_int(a & ~1) & 3 == 0 + assert cast_adr_to_int(a & ~2) & 3 == 0 + assert cast_adr_to_int(a & ~3) & 3 == 0 + assert cast_adr_to_int((a | 1) & ~0) & 3 == 1 + assert cast_adr_to_int((a | 1) & ~1) & 3 == 0 + assert cast_adr_to_int((a | 1) & ~2) & 3 == 1 + assert cast_adr_to_int((a | 1) & ~3) & 3 == 0 + assert cast_adr_to_int((a | 2) & ~0) & 3 == 2 + assert cast_adr_to_int((a | 2) & ~1) & 3 == 2 + assert cast_adr_to_int((a | 2) & ~2) & 3 == 0 + assert cast_adr_to_int((a | 2) & ~3) & 3 == 0 + assert cast_adr_to_int((a | 3) & ~0) & 3 == 3 + assert cast_adr_to_int((a | 3) & ~1) & 3 == 2 + assert cast_adr_to_int((a | 3) & ~2) & 3 == 1 + assert cast_adr_to_int((a | 3) & ~3) & 3 == 0 + a |= 1 + assert a != cast_ptr_to_adr(s) + py.test.raises(ValueError, cast_adr_to_ptr, a, lltype.Ptr(S)) + a |= 2 + assert a != cast_ptr_to_adr(s) + py.test.raises(ValueError, cast_adr_to_ptr, a, lltype.Ptr(S)) + a &= ~1 + assert a != cast_ptr_to_adr(s) + py.test.raises(ValueError, cast_adr_to_ptr, a, lltype.Ptr(S)) + a &= ~2 + assert a == cast_ptr_to_adr(s) + assert cast_adr_to_ptr(a, lltype.Ptr(S)) == s From arigo at codespeak.net Fri Feb 22 11:47:42 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 22 Feb 2008 11:47:42 +0100 (CET) Subject: [pypy-svn] r51781 - pypy/branch/unified-rtti/pypy/rpython/lltypesystem Message-ID: <20080222104742.2239F168469@codespeak.net> Author: arigo Date: Fri Feb 22 11:47:38 2008 New Revision: 51781 Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/llmemory.py Log: Forgot this in the previous check-in. Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/llmemory.py Fri Feb 22 11:47:38 2008 @@ -439,6 +439,11 @@ def __ge__(self, other): return not (self < other) + def __or__(self, other): + return _or_flags(self, 0, other) + def __and__(self, other): + return _and_flags(self, 0, other) + def ref(self): if not self: raise NullAddressError @@ -458,6 +463,75 @@ # ____________________________________________________________ +class fakeaddresswithflags(object): + MASK = 3 + + def __init__(self, adr, flags): + assert isinstance(adr, fakeaddress) + assert flags != 0 + if adr._cast_to_int() & self.MASK != 0: + raise ValueError("'addr | flag': the base address seems to be at " + "a non-aligned memory location") + self.adr = adr + self.flags = flags + + def __repr__(self): + return '<%r | 0x%x>' % (self.adr, self.flags) + + def __eq__(self, other): + if isinstance(other, fakeaddresswithflags): + return self.adr == other.adr and self.flags == other.flags + if isinstance(other, fakeaddress): + return False + return NotImplemented + + def __ne__(self, other): + if isinstance(other, fakeaddresswithflags): + return not (self.adr == other.adr and self.flags == other.flags) + if isinstance(other, fakeaddress): + return True + return NotImplemented + + def __or__(self, other): + return _or_flags(self.adr, self.flags, other) + + def __and__(self, other): + return _and_flags(self.adr, self.flags, other) + + def _cast_to_ptr(self, EXPECTED_TYPE): + raise ValueError("cannot cast %r") + + def _cast_to_int(self): + value = self.adr._cast_to_int() + assert value & self.MASK == 0 + return value | self.flags + +def _or_flags(adr, flags1, flags2): + if not isinstance(flags2, int): + return NotImplemented + if not (0 <= flags2 <= fakeaddresswithflags.MASK): + raise ValueError("'addr | flag': can only set the lower bits, " + "got flag=0x%x" % (flags2,)) + flags = flags1 | flags2 + if flags == 0: + return adr + else: + return fakeaddresswithflags(adr, flags) + +def _and_flags(adr, flags1, flags2): + if not isinstance(flags2, int): + return NotImplemented + if not (0 <= ~flags2 <= fakeaddresswithflags.MASK): + raise ValueError("'addr & flag': can only clear the lower bits, " + "got flag=0x%x" % (flags2,)) + flags = flags1 & flags2 + if flags == 0: + return adr + else: + return fakeaddresswithflags(adr, flags) + +# ____________________________________________________________ + class NullAddressError(Exception): pass @@ -550,6 +624,7 @@ fakeaddress.char = property(_char_fakeaccessor) fakeaddress.address = property(_address_fakeaccessor) fakeaddress._TYPE = Address +fakeaddresswithflags._TYPE = Address # the obtained address will not keep the object alive. e.g. if the object is # only reachable through an address, it might get collected From arigo at codespeak.net Fri Feb 22 11:48:01 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 22 Feb 2008 11:48:01 +0100 (CET) Subject: [pypy-svn] r51782 - in pypy/branch/unified-rtti/pypy: annotation rpython rpython/lltypesystem rpython/test Message-ID: <20080222104801.A93301684EE@codespeak.net> Author: arigo Date: Fri Feb 22 11:47:59 2008 New Revision: 51782 Modified: pypy/branch/unified-rtti/pypy/annotation/binaryop.py pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lloperation.py pypy/branch/unified-rtti/pypy/rpython/lltypesystem/opimpl.py pypy/branch/unified-rtti/pypy/rpython/raddress.py pypy/branch/unified-rtti/pypy/rpython/test/test_rptr.py Log: Translation support for adr|int, adr&~int. Modified: pypy/branch/unified-rtti/pypy/annotation/binaryop.py ============================================================================== --- pypy/branch/unified-rtti/pypy/annotation/binaryop.py (original) +++ pypy/branch/unified-rtti/pypy/annotation/binaryop.py Fri Feb 22 11:47:59 2008 @@ -1019,6 +1019,12 @@ def sub((s_addr, s_int)): return SomeAddress(is_null=False) + def or_((s_addr, s_int)): + return SomeAddress() + + def and_((s_addr, s_int)): + return SomeAddress() + class __extend__(pairtype(SomeAddress, SomeImpossibleValue)): # need to override this specifically to hide the 'raise UnionError' # of pairtype(SomeAddress, SomeObject). Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lloperation.py Fri Feb 22 11:47:59 2008 @@ -374,6 +374,8 @@ 'adr_ne': LLOp(canfold=True), 'adr_gt': LLOp(canfold=True), 'adr_ge': LLOp(canfold=True), + 'adr_or': LLOp(canfold=True), # to add flags in bits 0,1 + 'adr_and': LLOp(canfold=True), # to remove these flags 'adr_call': LLOp(canraise=(Exception,)), 'cast_ptr_to_adr': LLOp(sideeffects=False), 'cast_adr_to_ptr': LLOp(canfold=True), Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/opimpl.py Fri Feb 22 11:47:59 2008 @@ -368,6 +368,16 @@ assert lltype.typeOf(offset) is lltype.Signed return addr - offset +def op_adr_or(addr, flags): + checkadr(addr) + assert lltype.typeOf(flags) is lltype.Signed + return addr | flags + +def op_adr_and(addr, flags): + checkadr(addr) + assert lltype.typeOf(flags) is lltype.Signed + return addr & flags + def op_adr_delta(addr1, addr2): checkadr(addr1) checkadr(addr2) Modified: pypy/branch/unified-rtti/pypy/rpython/raddress.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/raddress.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/raddress.py Fri Feb 22 11:47:59 2008 @@ -99,6 +99,22 @@ return NotImplemented rtype_inplace_sub = rtype_sub + def rtype_or_((r_addr, r_int), hop): + if r_int.lowleveltype == lltype.Signed: + v_addr, v_flags = hop.inputargs(Address, lltype.Signed) + return hop.genop('adr_or', [v_addr, v_flags], resulttype=Address) + + return NotImplemented + rtype_inplace_or = rtype_or_ + + def rtype_and_((r_addr, r_int), hop): + if r_int.lowleveltype == lltype.Signed: + v_addr, v_flags = hop.inputargs(Address, lltype.Signed) + return hop.genop('adr_and', [v_addr, v_flags], resulttype=Address) + + return NotImplemented + rtype_inplace_and = rtype_and_ + class __extend__(pairtype(AddressRepr, AddressRepr)): Modified: pypy/branch/unified-rtti/pypy/rpython/test/test_rptr.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/test/test_rptr.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/test/test_rptr.py Fri Feb 22 11:47:59 2008 @@ -162,6 +162,43 @@ res = interpret(fn, [5]) assert res is True +def test_flags_in_low_bits(): + S = GcStruct('S', ('x', Signed)) + def fn(): + s = malloc(S) + a = llmemory.cast_ptr_to_adr(s) + assert llmemory.cast_adr_to_int(a) & 3 == 0 + assert llmemory.cast_adr_to_int(a | 0) & 3 == 0 + assert llmemory.cast_adr_to_int(a | 1) & 3 == 1 + assert llmemory.cast_adr_to_int(a | 2) & 3 == 2 + assert llmemory.cast_adr_to_int(a | 3) & 3 == 3 + assert llmemory.cast_adr_to_int(a & ~0) & 3 == 0 + assert llmemory.cast_adr_to_int(a & ~1) & 3 == 0 + assert llmemory.cast_adr_to_int(a & ~2) & 3 == 0 + assert llmemory.cast_adr_to_int(a & ~3) & 3 == 0 + assert llmemory.cast_adr_to_int((a | 1) & ~0) & 3 == 1 + assert llmemory.cast_adr_to_int((a | 1) & ~1) & 3 == 0 + assert llmemory.cast_adr_to_int((a | 1) & ~2) & 3 == 1 + assert llmemory.cast_adr_to_int((a | 1) & ~3) & 3 == 0 + assert llmemory.cast_adr_to_int((a | 2) & ~0) & 3 == 2 + assert llmemory.cast_adr_to_int((a | 2) & ~1) & 3 == 2 + assert llmemory.cast_adr_to_int((a | 2) & ~2) & 3 == 0 + assert llmemory.cast_adr_to_int((a | 2) & ~3) & 3 == 0 + assert llmemory.cast_adr_to_int((a | 3) & ~0) & 3 == 3 + assert llmemory.cast_adr_to_int((a | 3) & ~1) & 3 == 2 + assert llmemory.cast_adr_to_int((a | 3) & ~2) & 3 == 1 + assert llmemory.cast_adr_to_int((a | 3) & ~3) & 3 == 0 + a |= 1 + assert a != llmemory.cast_ptr_to_adr(s) + a |= 2 + assert a != llmemory.cast_ptr_to_adr(s) + a &= ~1 + assert a != llmemory.cast_ptr_to_adr(s) + a &= ~2 + assert a == llmemory.cast_ptr_to_adr(s) + assert llmemory.cast_adr_to_ptr(a, Ptr(S)) == s + interpret(fn, []) + def test_flavored_malloc(): T = GcStruct('T', ('y', Signed)) def fn(n): From arigo at codespeak.net Fri Feb 22 11:54:09 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 22 Feb 2008 11:54:09 +0100 (CET) Subject: [pypy-svn] r51783 - in pypy/branch/unified-rtti/pypy/translator/c: src test Message-ID: <20080222105409.280FE168452@codespeak.net> Author: arigo Date: Fri Feb 22 11:54:08 2008 New Revision: 51783 Modified: pypy/branch/unified-rtti/pypy/translator/c/src/address.h pypy/branch/unified-rtti/pypy/translator/c/test/test_lladdresses.py Log: translation to C of adr|int, adr&int Modified: pypy/branch/unified-rtti/pypy/translator/c/src/address.h ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/src/address.h (original) +++ pypy/branch/unified-rtti/pypy/translator/c/src/address.h Fri Feb 22 11:54:08 2008 @@ -18,3 +18,7 @@ #define OP_CAST_ADR_TO_INT(x, r) r = ((long)x) #define OP_CAST_INT_TO_ADR(x, r) r = ((void *)(x)) + +/* XXX assumes that addresses fit in a long */ +#define OP_ADR_OR(x,y,r) r = (char *)((long)(x) | (y)) +#define OP_ADR_AND(x,y,r) r = (char *)((long)(x) & (y)) Modified: pypy/branch/unified-rtti/pypy/translator/c/test/test_lladdresses.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/test/test_lladdresses.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/test/test_lladdresses.py Fri Feb 22 11:54:08 2008 @@ -60,6 +60,44 @@ res = fc(10, "c") assert res == "c" +def test_flags_in_low_bits(): + S = lltype.GcStruct('S', ('x', lltype.Signed)) + def fn(): + s = lltype.malloc(S) + a = cast_ptr_to_adr(s) + assert cast_adr_to_int(a) & 3 == 0 + assert cast_adr_to_int(a | 0) & 3 == 0 + assert cast_adr_to_int(a | 1) & 3 == 1 + assert cast_adr_to_int(a | 2) & 3 == 2 + assert cast_adr_to_int(a | 3) & 3 == 3 + assert cast_adr_to_int(a & ~0) & 3 == 0 + assert cast_adr_to_int(a & ~1) & 3 == 0 + assert cast_adr_to_int(a & ~2) & 3 == 0 + assert cast_adr_to_int(a & ~3) & 3 == 0 + assert cast_adr_to_int((a | 1) & ~0) & 3 == 1 + assert cast_adr_to_int((a | 1) & ~1) & 3 == 0 + assert cast_adr_to_int((a | 1) & ~2) & 3 == 1 + assert cast_adr_to_int((a | 1) & ~3) & 3 == 0 + assert cast_adr_to_int((a | 2) & ~0) & 3 == 2 + assert cast_adr_to_int((a | 2) & ~1) & 3 == 2 + assert cast_adr_to_int((a | 2) & ~2) & 3 == 0 + assert cast_adr_to_int((a | 2) & ~3) & 3 == 0 + assert cast_adr_to_int((a | 3) & ~0) & 3 == 3 + assert cast_adr_to_int((a | 3) & ~1) & 3 == 2 + assert cast_adr_to_int((a | 3) & ~2) & 3 == 1 + assert cast_adr_to_int((a | 3) & ~3) & 3 == 0 + a |= 1 + assert a != cast_ptr_to_adr(s) + a |= 2 + assert a != cast_ptr_to_adr(s) + a &= ~1 + assert a != cast_ptr_to_adr(s) + a &= ~2 + assert a == cast_ptr_to_adr(s) + assert cast_adr_to_ptr(a, lltype.Ptr(S)) == s + fc = compile(fn, []) + fc() + def test_raw_memcopy(): def f(): addr = raw_malloc(100) From arigo at codespeak.net Fri Feb 22 12:31:09 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 22 Feb 2008 12:31:09 +0100 (CET) Subject: [pypy-svn] r51784 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20080222113109.6CC0D16850D@codespeak.net> Author: arigo Date: Fri Feb 22 12:31:09 2008 New Revision: 51784 Modified: pypy/dist/pypy/rpython/memory/gc/marksweep.py Log: Kill kill major code duplication. Modified: pypy/dist/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/dist/pypy/rpython/memory/gc/marksweep.py Fri Feb 22 12:31:09 2008 @@ -71,10 +71,20 @@ self.collect_in_progress = False self.prev_collect_end_time = 0.0 + def maybe_collect(self): + if self.bytes_malloced > self.bytes_malloced_threshold: + self.collect() + + def write_malloc_statistics(self, typeid, size, result, varsize): + pass + + def write_free_statistics(self, typeid, result): + pass + def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, contains_weakptr=False): - if can_collect and self.bytes_malloced > self.bytes_malloced_threshold: - self.collect() + if can_collect: + self.maybe_collect() size_gc_header = self.gcheaderbuilder.size_gc_header try: tot_size = size_gc_header + size @@ -101,13 +111,14 @@ result += size_gc_header #llop.debug_print(lltype.Void, 'malloc typeid', typeid, # '->', llmemory.cast_adr_to_int(result)) + self.write_malloc_statistics(typeid, tot_size, result, False) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) malloc_fixedsize._dont_inline_ = True def malloc_fixedsize_clear(self, typeid, size, can_collect, has_finalizer=False, contains_weakptr=False): - if can_collect and self.bytes_malloced > self.bytes_malloced_threshold: - self.collect() + if can_collect: + self.maybe_collect() size_gc_header = self.gcheaderbuilder.size_gc_header try: tot_size = size_gc_header + size @@ -135,13 +146,14 @@ result += size_gc_header #llop.debug_print(lltype.Void, 'malloc typeid', typeid, # '->', llmemory.cast_adr_to_int(result)) + self.write_malloc_statistics(typeid, tot_size, result, False) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) malloc_fixedsize_clear._dont_inline_ = True def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, can_collect, has_finalizer=False): - if can_collect and self.bytes_malloced > self.bytes_malloced_threshold: - self.collect() + if can_collect: + self.maybe_collect() size_gc_header = self.gcheaderbuilder.size_gc_header try: fixsize = size_gc_header + size @@ -170,14 +182,15 @@ #llop.debug_print(lltype.Void, 'malloc_varsize length', length, # 'typeid', typeid, # '->', llmemory.cast_adr_to_int(result)) + self.write_malloc_statistics(typeid, tot_size, result, True) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) malloc_varsize._dont_inline_ = True def malloc_varsize_clear(self, typeid, length, size, itemsize, offset_to_length, can_collect, has_finalizer=False): - if can_collect and self.bytes_malloced > self.bytes_malloced_threshold: - self.collect() + if can_collect: + self.maybe_collect() size_gc_header = self.gcheaderbuilder.size_gc_header try: fixsize = size_gc_header + size @@ -207,6 +220,7 @@ #llop.debug_print(lltype.Void, 'malloc_varsize length', length, # 'typeid', typeid, # '->', llmemory.cast_adr_to_int(result)) + self.write_malloc_statistics(typeid, tot_size, result, True) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) malloc_varsize_clear._dont_inline_ = True @@ -307,6 +321,9 @@ surviving = hdr curr_heap_size += estimate else: + gc_info = llmemory.cast_ptr_to_adr(hdr) + weakref_obj = gc_info + size_gc_header + self.write_free_statistics(typeid, weakref_obj) freed_size += estimate raw_free(addr) hdr = next @@ -338,6 +355,9 @@ ppnext += llmemory.offsetof(self.HDR, 'next') curr_heap_size += estimate else: + gc_info = llmemory.cast_ptr_to_adr(hdr) + obj = gc_info + size_gc_header + self.write_free_statistics(typeid, obj) freed_size += estimate raw_free(addr) hdr = next @@ -682,6 +702,11 @@ MarkSweepGC.__init__(self, chunk_size, start_heap_size) self.count_mallocs = 0 + def maybe_collect(self): + self.count_mallocs += 1 + if self.count_mallocs > self.COLLECT_EVERY: + self.collect() + def write_malloc_statistics(self, typeid, size, result, varsize): if varsize: what = "malloc_varsize" @@ -692,397 +717,6 @@ def write_free_statistics(self, typeid, result): llop.debug_print(lltype.Void, "free", typeid, " ", result) - - #XXX XXX XXX XXX XXX XXX XXX XXX - # the next methods are nearly copies of the MarkSweepGC methods - # not clear how that can be improved - - def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, - contains_weakptr=False): - self.count_mallocs += 1 - if can_collect and self.count_mallocs > self.COLLECT_EVERY: - self.collect() - size_gc_header = self.gcheaderbuilder.size_gc_header - try: - tot_size = size_gc_header + size - usage = raw_malloc_usage(tot_size) - bytes_malloced = ovfcheck(self.bytes_malloced+usage) - ovfcheck(self.heap_usage + bytes_malloced) - except OverflowError: - raise memoryError - result = raw_malloc(tot_size) - if not result: - raise memoryError - hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) - hdr.typeid = typeid << 1 - if has_finalizer: - hdr.next = self.malloced_objects_with_finalizer - self.malloced_objects_with_finalizer = hdr - elif contains_weakptr: - hdr.next = self.objects_with_weak_pointers - self.objects_with_weak_pointers = hdr - else: - hdr.next = self.malloced_objects - self.malloced_objects = hdr - self.bytes_malloced = bytes_malloced - result += size_gc_header - #llop.debug_print(lltype.Void, 'malloc typeid', typeid, - # '->', llmemory.cast_adr_to_int(result)) - self.write_malloc_statistics(typeid, tot_size, result, False) - return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) - - def malloc_fixedsize_clear(self, typeid, size, can_collect, - has_finalizer=False, contains_weakptr=False): - self.count_mallocs += 1 - if can_collect and self.count_mallocs > self.COLLECT_EVERY: - self.collect() - size_gc_header = self.gcheaderbuilder.size_gc_header - try: - tot_size = size_gc_header + size - usage = raw_malloc_usage(tot_size) - bytes_malloced = ovfcheck(self.bytes_malloced+usage) - ovfcheck(self.heap_usage + bytes_malloced) - except OverflowError: - raise memoryError - result = raw_malloc(tot_size) - if not result: - raise memoryError - raw_memclear(result, tot_size) - hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) - hdr.typeid = typeid << 1 - if has_finalizer: - hdr.next = self.malloced_objects_with_finalizer - self.malloced_objects_with_finalizer = hdr - elif contains_weakptr: - hdr.next = self.objects_with_weak_pointers - self.objects_with_weak_pointers = hdr - else: - hdr.next = self.malloced_objects - self.malloced_objects = hdr - self.bytes_malloced = bytes_malloced - result += size_gc_header - #llop.debug_print(lltype.Void, 'malloc typeid', typeid, - # '->', llmemory.cast_adr_to_int(result)) - self.write_malloc_statistics(typeid, tot_size, result, False) - return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) - - def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, - can_collect, has_finalizer=False): - self.count_mallocs += 1 - if can_collect and self.count_mallocs > self.COLLECT_EVERY: - self.collect() - size_gc_header = self.gcheaderbuilder.size_gc_header - try: - fixsize = size_gc_header + size - varsize = ovfcheck(itemsize * length) - tot_size = ovfcheck(fixsize + varsize) - usage = raw_malloc_usage(tot_size) - bytes_malloced = ovfcheck(self.bytes_malloced+usage) - ovfcheck(self.heap_usage + bytes_malloced) - except OverflowError: - raise memoryError - result = raw_malloc(tot_size) - if not result: - raise memoryError - (result + size_gc_header + offset_to_length).signed[0] = length - hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) - hdr.typeid = typeid << 1 - if has_finalizer: - hdr.next = self.malloced_objects_with_finalizer - self.malloced_objects_with_finalizer = hdr - else: - hdr.next = self.malloced_objects - self.malloced_objects = hdr - self.bytes_malloced = bytes_malloced - - result += size_gc_header - #llop.debug_print(lltype.Void, 'malloc_varsize length', length, - # 'typeid', typeid, - # '->', llmemory.cast_adr_to_int(result)) - self.write_malloc_statistics(typeid, tot_size, result, True) - return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) - - def malloc_varsize_clear(self, typeid, length, size, itemsize, - offset_to_length, can_collect, - has_finalizer=False): - self.count_mallocs += 1 - if can_collect and self.count_mallocs > self.COLLECT_EVERY: - self.collect() - size_gc_header = self.gcheaderbuilder.size_gc_header - try: - fixsize = size_gc_header + size - varsize = ovfcheck(itemsize * length) - tot_size = ovfcheck(fixsize + varsize) - usage = raw_malloc_usage(tot_size) - bytes_malloced = ovfcheck(self.bytes_malloced+usage) - ovfcheck(self.heap_usage + bytes_malloced) - except OverflowError: - raise memoryError - result = raw_malloc(tot_size) - if not result: - raise memoryError - raw_memclear(result, tot_size) - (result + size_gc_header + offset_to_length).signed[0] = length - hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) - hdr.typeid = typeid << 1 - if has_finalizer: - hdr.next = self.malloced_objects_with_finalizer - self.malloced_objects_with_finalizer = hdr - else: - hdr.next = self.malloced_objects - self.malloced_objects = hdr - self.bytes_malloced = bytes_malloced - - result += size_gc_header - #llop.debug_print(lltype.Void, 'malloc_varsize length', length, - # 'typeid', typeid, - # '->', llmemory.cast_adr_to_int(result)) - self.write_malloc_statistics(typeid, tot_size, result, True) - return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) - def collect(self): - # 1. mark from the roots, and also the objects that objects-with-del - # point to (using the list of malloced_objects_with_finalizer) - # 2. walk the list of objects-without-del and free the ones not marked - # 3. walk the list of objects-with-del and for the ones not marked: - # call __del__, move the object to the list of object-without-del - import time - from pypy.rpython.lltypesystem.lloperation import llop self.count_mallocs = 0 - start_time = time.time() - self.collect_in_progress = True - size_gc_header = self.gcheaderbuilder.size_gc_header -## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, -## size_gc_header) - - # push the roots on the mark stack - objects = self.AddressStack() # mark stack - self._mark_stack = objects - self.root_walker.walk_roots( - MarkSweepGC._mark_root, # stack roots - MarkSweepGC._mark_root, # static in prebuilt non-gc structures - MarkSweepGC._mark_root) # static in prebuilt gc objects - - # from this point onwards, no more mallocs should be possible - old_malloced = self.bytes_malloced - self.bytes_malloced = 0 - curr_heap_size = 0 - freed_size = 0 - - # mark objects reachable by objects with a finalizer, but not those - # themselves. add their size to curr_heap_size, since they always - # survive the collection - hdr = self.malloced_objects_with_finalizer - while hdr: - next = hdr.next - typeid = hdr.typeid >> 1 - gc_info = llmemory.cast_ptr_to_adr(hdr) - obj = gc_info + size_gc_header - if not hdr.typeid & 1: - self.add_reachable_to_stack(obj, objects) - addr = llmemory.cast_ptr_to_adr(hdr) - size = self.fixed_size(typeid) - if self.is_varsize(typeid): - length = (obj + self.varsize_offset_to_length(typeid)).signed[0] - size += self.varsize_item_sizes(typeid) * length - estimate = raw_malloc_usage(size_gc_header + size) - curr_heap_size += estimate - hdr = next - - # mark thinks on the mark stack and put their descendants onto the - # stack until the stack is empty - while objects.non_empty(): #mark - curr = objects.pop() - gc_info = curr - size_gc_header - hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - if hdr.typeid & 1: - continue - self.add_reachable_to_stack(curr, objects) - hdr.typeid = hdr.typeid | 1 - objects.delete() - # also mark self.curpool - if self.curpool: - gc_info = llmemory.cast_ptr_to_adr(self.curpool) - size_gc_header - hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - hdr.typeid = hdr.typeid | 1 - # go through the list of objects containing weak pointers - # and kill the links if they go to dead objects - # if the object itself is not marked, free it - hdr = self.objects_with_weak_pointers - surviving = lltype.nullptr(self.HDR) - while hdr: - typeid = hdr.typeid >> 1 - next = hdr.next - addr = llmemory.cast_ptr_to_adr(hdr) - size = self.fixed_size(typeid) - estimate = raw_malloc_usage(size_gc_header + size) - if hdr.typeid & 1: - typeid = hdr.typeid >> 1 - offset = self.weakpointer_offset(typeid) - hdr.typeid = hdr.typeid & (~1) - gc_info = llmemory.cast_ptr_to_adr(hdr) - weakref_obj = gc_info + size_gc_header - pointing_to = (weakref_obj + offset).address[0] - if pointing_to: - gc_info_pointing_to = pointing_to - size_gc_header - hdr_pointing_to = llmemory.cast_adr_to_ptr( - gc_info_pointing_to, self.HDRPTR) - # pointed to object will die - # XXX what to do if the object has a finalizer which resurrects - # the object? - if not hdr_pointing_to.typeid & 1: - (weakref_obj + offset).address[0] = NULL - hdr.next = surviving - surviving = hdr - curr_heap_size += estimate - else: - gc_info = llmemory.cast_ptr_to_adr(hdr) - weakref_obj = gc_info + size_gc_header - self.write_free_statistics(typeid, weakref_obj) - freed_size += estimate - raw_free(addr) - hdr = next - self.objects_with_weak_pointers = surviving - # sweep: delete objects without del if they are not marked - # unmark objects without del that are marked - firstpoolnode = lltype.malloc(self.POOLNODE, flavor='raw') - firstpoolnode.linkedlist = self.malloced_objects - firstpoolnode.nextnode = self.poolnodes - prevpoolnode = lltype.nullptr(self.POOLNODE) - poolnode = firstpoolnode - while poolnode: #sweep - ppnext = llmemory.cast_ptr_to_adr(poolnode) - ppnext += llmemory.offsetof(self.POOLNODE, 'linkedlist') - hdr = poolnode.linkedlist - while hdr: #sweep - typeid = hdr.typeid >> 1 - next = hdr.next - addr = llmemory.cast_ptr_to_adr(hdr) - size = self.fixed_size(typeid) - if self.is_varsize(typeid): - length = (addr + size_gc_header + self.varsize_offset_to_length(typeid)).signed[0] - size += self.varsize_item_sizes(typeid) * length - estimate = raw_malloc_usage(size_gc_header + size) - if hdr.typeid & 1: - hdr.typeid = hdr.typeid & (~1) - ppnext.address[0] = addr - ppnext = llmemory.cast_ptr_to_adr(hdr) - ppnext += llmemory.offsetof(self.HDR, 'next') - curr_heap_size += estimate - else: - gc_info = llmemory.cast_ptr_to_adr(hdr) - obj = gc_info + size_gc_header - self.write_free_statistics(typeid, obj) - freed_size += estimate - raw_free(addr) - hdr = next - ppnext.address[0] = llmemory.NULL - next = poolnode.nextnode - if not poolnode.linkedlist and prevpoolnode: - # completely empty node - prevpoolnode.nextnode = next - lltype.free(poolnode, flavor='raw') - else: - prevpoolnode = poolnode - poolnode = next - self.malloced_objects = firstpoolnode.linkedlist - self.poolnodes = firstpoolnode.nextnode - lltype.free(firstpoolnode, flavor='raw') - #llop.debug_view(lltype.Void, self.malloced_objects, self.malloced_objects_with_finalizer, size_gc_header) - - end_time = time.time() - compute_time = start_time - self.prev_collect_end_time - collect_time = end_time - start_time - - garbage_collected = old_malloced - (curr_heap_size - self.heap_usage) - - if (collect_time * curr_heap_size > - 0.02 * garbage_collected * compute_time): - self.bytes_malloced_threshold += self.bytes_malloced_threshold / 2 - if (collect_time * curr_heap_size < - 0.005 * garbage_collected * compute_time): - self.bytes_malloced_threshold /= 2 - - # Use atleast as much memory as current live objects. - if curr_heap_size > self.bytes_malloced_threshold: - self.bytes_malloced_threshold = curr_heap_size - - # Cap at 1/4 GB - self.bytes_malloced_threshold = min(self.bytes_malloced_threshold, - 256 * 1024 * 1024) - self.total_collection_time += collect_time - self.prev_collect_end_time = end_time - if DEBUG_PRINT: - llop.debug_print(lltype.Void, - " malloced since previous collection:", - old_malloced, "bytes") - llop.debug_print(lltype.Void, - " heap usage at start of collection: ", - self.heap_usage + old_malloced, "bytes") - llop.debug_print(lltype.Void, - " freed: ", - freed_size, "bytes") - llop.debug_print(lltype.Void, - " new heap usage: ", - curr_heap_size, "bytes") - llop.debug_print(lltype.Void, - " total time spent collecting: ", - self.total_collection_time, "seconds") - llop.debug_print(lltype.Void, - " collecting time: ", - collect_time) - llop.debug_print(lltype.Void, - " computing time: ", - collect_time) - llop.debug_print(lltype.Void, - " new threshold: ", - self.bytes_malloced_threshold) -## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, -## size_gc_header) - assert self.heap_usage + old_malloced == curr_heap_size + freed_size - - self.heap_usage = curr_heap_size - hdr = self.malloced_objects_with_finalizer - self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR) - last = lltype.nullptr(self.HDR) - while hdr: - next = hdr.next - if hdr.typeid & 1: - hdr.next = lltype.nullptr(self.HDR) - if not self.malloced_objects_with_finalizer: - self.malloced_objects_with_finalizer = hdr - else: - last.next = hdr - hdr.typeid = hdr.typeid & (~1) - last = hdr - else: - obj = llmemory.cast_ptr_to_adr(hdr) + size_gc_header - finalizer = self.getfinalizer(hdr.typeid >> 1) - # make malloced_objects_with_finalizer consistent - # for the sake of a possible collection caused by finalizer - if not self.malloced_objects_with_finalizer: - self.malloced_objects_with_finalizer = next - else: - last.next = next - hdr.next = self.malloced_objects - self.malloced_objects = hdr - #llop.debug_view(lltype.Void, self.malloced_objects, self.malloced_objects_with_finalizer, size_gc_header) - finalizer(obj) - if not self.collect_in_progress: # another collection was caused? - llop.debug_print(lltype.Void, "outer collect interrupted " - "by recursive collect") - return - if not last: - if self.malloced_objects_with_finalizer == next: - self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR) - else: - # now it gets annoying: finalizer caused a malloc of something - # with a finalizer - last = self.malloced_objects_with_finalizer - while last.next != next: - last = last.next - last.next = lltype.nullptr(self.HDR) - else: - last.next = lltype.nullptr(self.HDR) - hdr = next - self.collect_in_progress = False + MarkSweepGC.collect(self) From arigo at codespeak.net Fri Feb 22 12:34:35 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 22 Feb 2008 12:34:35 +0100 (CET) Subject: [pypy-svn] r51785 - pypy/branch/unified-rtti/pypy/rpython/memory/gc Message-ID: <20080222113435.668D016851A@codespeak.net> Author: arigo Date: Fri Feb 22 12:34:34 2008 New Revision: 51785 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py Log: Merge 51784 from trunk. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py Fri Feb 22 12:34:34 2008 @@ -71,10 +71,20 @@ self.collect_in_progress = False self.prev_collect_end_time = 0.0 + def maybe_collect(self): + if self.bytes_malloced > self.bytes_malloced_threshold: + self.collect() + + def write_malloc_statistics(self, typeid, size, result, varsize): + pass + + def write_free_statistics(self, typeid, result): + pass + def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, contains_weakptr=False): - if can_collect and self.bytes_malloced > self.bytes_malloced_threshold: - self.collect() + if can_collect: + self.maybe_collect() size_gc_header = self.gcheaderbuilder.size_gc_header try: tot_size = size_gc_header + size @@ -101,13 +111,14 @@ result += size_gc_header #llop.debug_print(lltype.Void, 'malloc typeid', typeid, # '->', llmemory.cast_adr_to_int(result)) + self.write_malloc_statistics(typeid, tot_size, result, False) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) malloc_fixedsize._dont_inline_ = True def malloc_fixedsize_clear(self, typeid, size, can_collect, has_finalizer=False, contains_weakptr=False): - if can_collect and self.bytes_malloced > self.bytes_malloced_threshold: - self.collect() + if can_collect: + self.maybe_collect() size_gc_header = self.gcheaderbuilder.size_gc_header try: tot_size = size_gc_header + size @@ -135,13 +146,14 @@ result += size_gc_header #llop.debug_print(lltype.Void, 'malloc typeid', typeid, # '->', llmemory.cast_adr_to_int(result)) + self.write_malloc_statistics(typeid, tot_size, result, False) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) malloc_fixedsize_clear._dont_inline_ = True def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, can_collect, has_finalizer=False): - if can_collect and self.bytes_malloced > self.bytes_malloced_threshold: - self.collect() + if can_collect: + self.maybe_collect() size_gc_header = self.gcheaderbuilder.size_gc_header try: fixsize = size_gc_header + size @@ -170,14 +182,15 @@ #llop.debug_print(lltype.Void, 'malloc_varsize length', length, # 'typeid', typeid, # '->', llmemory.cast_adr_to_int(result)) + self.write_malloc_statistics(typeid, tot_size, result, True) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) malloc_varsize._dont_inline_ = True def malloc_varsize_clear(self, typeid, length, size, itemsize, offset_to_length, can_collect, has_finalizer=False): - if can_collect and self.bytes_malloced > self.bytes_malloced_threshold: - self.collect() + if can_collect: + self.maybe_collect() size_gc_header = self.gcheaderbuilder.size_gc_header try: fixsize = size_gc_header + size @@ -207,6 +220,7 @@ #llop.debug_print(lltype.Void, 'malloc_varsize length', length, # 'typeid', typeid, # '->', llmemory.cast_adr_to_int(result)) + self.write_malloc_statistics(typeid, tot_size, result, True) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) malloc_varsize_clear._dont_inline_ = True @@ -307,6 +321,9 @@ surviving = hdr curr_heap_size += estimate else: + gc_info = llmemory.cast_ptr_to_adr(hdr) + weakref_obj = gc_info + size_gc_header + self.write_free_statistics(typeid, weakref_obj) freed_size += estimate raw_free(addr) hdr = next @@ -338,6 +355,9 @@ ppnext += llmemory.offsetof(self.HDR, 'next') curr_heap_size += estimate else: + gc_info = llmemory.cast_ptr_to_adr(hdr) + obj = gc_info + size_gc_header + self.write_free_statistics(typeid, obj) freed_size += estimate raw_free(addr) hdr = next @@ -852,6 +872,11 @@ MarkSweepGC.__init__(self, chunk_size, start_heap_size) self.count_mallocs = 0 + def maybe_collect(self): + self.count_mallocs += 1 + if self.count_mallocs > self.COLLECT_EVERY: + self.collect() + def write_malloc_statistics(self, typeid, size, result, varsize): if varsize: what = "malloc_varsize" @@ -862,397 +887,6 @@ def write_free_statistics(self, typeid, result): llop.debug_print(lltype.Void, "free", typeid, " ", result) - - #XXX XXX XXX XXX XXX XXX XXX XXX - # the next methods are nearly copies of the MarkSweepGC methods - # not clear how that can be improved - - def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, - contains_weakptr=False): - self.count_mallocs += 1 - if can_collect and self.count_mallocs > self.COLLECT_EVERY: - self.collect() - size_gc_header = self.gcheaderbuilder.size_gc_header - try: - tot_size = size_gc_header + size - usage = raw_malloc_usage(tot_size) - bytes_malloced = ovfcheck(self.bytes_malloced+usage) - ovfcheck(self.heap_usage + bytes_malloced) - except OverflowError: - raise memoryError - result = raw_malloc(tot_size) - if not result: - raise memoryError - hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) - hdr.typeid = typeid << 1 - if has_finalizer: - hdr.next = self.malloced_objects_with_finalizer - self.malloced_objects_with_finalizer = hdr - elif contains_weakptr: - hdr.next = self.objects_with_weak_pointers - self.objects_with_weak_pointers = hdr - else: - hdr.next = self.malloced_objects - self.malloced_objects = hdr - self.bytes_malloced = bytes_malloced - result += size_gc_header - #llop.debug_print(lltype.Void, 'malloc typeid', typeid, - # '->', llmemory.cast_adr_to_int(result)) - self.write_malloc_statistics(typeid, tot_size, result, False) - return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) - - def malloc_fixedsize_clear(self, typeid, size, can_collect, - has_finalizer=False, contains_weakptr=False): - self.count_mallocs += 1 - if can_collect and self.count_mallocs > self.COLLECT_EVERY: - self.collect() - size_gc_header = self.gcheaderbuilder.size_gc_header - try: - tot_size = size_gc_header + size - usage = raw_malloc_usage(tot_size) - bytes_malloced = ovfcheck(self.bytes_malloced+usage) - ovfcheck(self.heap_usage + bytes_malloced) - except OverflowError: - raise memoryError - result = raw_malloc(tot_size) - if not result: - raise memoryError - raw_memclear(result, tot_size) - hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) - hdr.typeid = typeid << 1 - if has_finalizer: - hdr.next = self.malloced_objects_with_finalizer - self.malloced_objects_with_finalizer = hdr - elif contains_weakptr: - hdr.next = self.objects_with_weak_pointers - self.objects_with_weak_pointers = hdr - else: - hdr.next = self.malloced_objects - self.malloced_objects = hdr - self.bytes_malloced = bytes_malloced - result += size_gc_header - #llop.debug_print(lltype.Void, 'malloc typeid', typeid, - # '->', llmemory.cast_adr_to_int(result)) - self.write_malloc_statistics(typeid, tot_size, result, False) - return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) - - def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, - can_collect, has_finalizer=False): - self.count_mallocs += 1 - if can_collect and self.count_mallocs > self.COLLECT_EVERY: - self.collect() - size_gc_header = self.gcheaderbuilder.size_gc_header - try: - fixsize = size_gc_header + size - varsize = ovfcheck(itemsize * length) - tot_size = ovfcheck(fixsize + varsize) - usage = raw_malloc_usage(tot_size) - bytes_malloced = ovfcheck(self.bytes_malloced+usage) - ovfcheck(self.heap_usage + bytes_malloced) - except OverflowError: - raise memoryError - result = raw_malloc(tot_size) - if not result: - raise memoryError - (result + size_gc_header + offset_to_length).signed[0] = length - hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) - hdr.typeid = typeid << 1 - if has_finalizer: - hdr.next = self.malloced_objects_with_finalizer - self.malloced_objects_with_finalizer = hdr - else: - hdr.next = self.malloced_objects - self.malloced_objects = hdr - self.bytes_malloced = bytes_malloced - - result += size_gc_header - #llop.debug_print(lltype.Void, 'malloc_varsize length', length, - # 'typeid', typeid, - # '->', llmemory.cast_adr_to_int(result)) - self.write_malloc_statistics(typeid, tot_size, result, True) - return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) - - def malloc_varsize_clear(self, typeid, length, size, itemsize, - offset_to_length, can_collect, - has_finalizer=False): - self.count_mallocs += 1 - if can_collect and self.count_mallocs > self.COLLECT_EVERY: - self.collect() - size_gc_header = self.gcheaderbuilder.size_gc_header - try: - fixsize = size_gc_header + size - varsize = ovfcheck(itemsize * length) - tot_size = ovfcheck(fixsize + varsize) - usage = raw_malloc_usage(tot_size) - bytes_malloced = ovfcheck(self.bytes_malloced+usage) - ovfcheck(self.heap_usage + bytes_malloced) - except OverflowError: - raise memoryError - result = raw_malloc(tot_size) - if not result: - raise memoryError - raw_memclear(result, tot_size) - (result + size_gc_header + offset_to_length).signed[0] = length - hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) - hdr.typeid = typeid << 1 - if has_finalizer: - hdr.next = self.malloced_objects_with_finalizer - self.malloced_objects_with_finalizer = hdr - else: - hdr.next = self.malloced_objects - self.malloced_objects = hdr - self.bytes_malloced = bytes_malloced - - result += size_gc_header - #llop.debug_print(lltype.Void, 'malloc_varsize length', length, - # 'typeid', typeid, - # '->', llmemory.cast_adr_to_int(result)) - self.write_malloc_statistics(typeid, tot_size, result, True) - return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) - def collect(self): - # 1. mark from the roots, and also the objects that objects-with-del - # point to (using the list of malloced_objects_with_finalizer) - # 2. walk the list of objects-without-del and free the ones not marked - # 3. walk the list of objects-with-del and for the ones not marked: - # call __del__, move the object to the list of object-without-del - import time - from pypy.rpython.lltypesystem.lloperation import llop self.count_mallocs = 0 - start_time = time.time() - self.collect_in_progress = True - size_gc_header = self.gcheaderbuilder.size_gc_header -## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, -## size_gc_header) - - # push the roots on the mark stack - objects = self.AddressStack() # mark stack - self._mark_stack = objects - self.root_walker.walk_roots( - MarkSweepGC._mark_root, # stack roots - MarkSweepGC._mark_root, # static in prebuilt non-gc structures - MarkSweepGC._mark_root) # static in prebuilt gc objects - - # from this point onwards, no more mallocs should be possible - old_malloced = self.bytes_malloced - self.bytes_malloced = 0 - curr_heap_size = 0 - freed_size = 0 - - # mark objects reachable by objects with a finalizer, but not those - # themselves. add their size to curr_heap_size, since they always - # survive the collection - hdr = self.malloced_objects_with_finalizer - while hdr: - next = hdr.next - typeid = hdr.typeid >> 1 - gc_info = llmemory.cast_ptr_to_adr(hdr) - obj = gc_info + size_gc_header - if not hdr.typeid & 1: - self.add_reachable_to_stack(obj, objects) - addr = llmemory.cast_ptr_to_adr(hdr) - size = self.fixed_size(typeid) - if self.is_varsize(typeid): - length = (obj + self.varsize_offset_to_length(typeid)).signed[0] - size += self.varsize_item_sizes(typeid) * length - estimate = raw_malloc_usage(size_gc_header + size) - curr_heap_size += estimate - hdr = next - - # mark thinks on the mark stack and put their descendants onto the - # stack until the stack is empty - while objects.non_empty(): #mark - curr = objects.pop() - gc_info = curr - size_gc_header - hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - if hdr.typeid & 1: - continue - self.add_reachable_to_stack(curr, objects) - hdr.typeid = hdr.typeid | 1 - objects.delete() - # also mark self.curpool - if self.curpool: - gc_info = llmemory.cast_ptr_to_adr(self.curpool) - size_gc_header - hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - hdr.typeid = hdr.typeid | 1 - # go through the list of objects containing weak pointers - # and kill the links if they go to dead objects - # if the object itself is not marked, free it - hdr = self.objects_with_weak_pointers - surviving = lltype.nullptr(self.HDR) - while hdr: - typeid = hdr.typeid >> 1 - next = hdr.next - addr = llmemory.cast_ptr_to_adr(hdr) - size = self.fixed_size(typeid) - estimate = raw_malloc_usage(size_gc_header + size) - if hdr.typeid & 1: - typeid = hdr.typeid >> 1 - offset = self.weakpointer_offset(typeid) - hdr.typeid = hdr.typeid & (~1) - gc_info = llmemory.cast_ptr_to_adr(hdr) - weakref_obj = gc_info + size_gc_header - pointing_to = (weakref_obj + offset).address[0] - if pointing_to: - gc_info_pointing_to = pointing_to - size_gc_header - hdr_pointing_to = llmemory.cast_adr_to_ptr( - gc_info_pointing_to, self.HDRPTR) - # pointed to object will die - # XXX what to do if the object has a finalizer which resurrects - # the object? - if not hdr_pointing_to.typeid & 1: - (weakref_obj + offset).address[0] = NULL - hdr.next = surviving - surviving = hdr - curr_heap_size += estimate - else: - gc_info = llmemory.cast_ptr_to_adr(hdr) - weakref_obj = gc_info + size_gc_header - self.write_free_statistics(typeid, weakref_obj) - freed_size += estimate - raw_free(addr) - hdr = next - self.objects_with_weak_pointers = surviving - # sweep: delete objects without del if they are not marked - # unmark objects without del that are marked - firstpoolnode = lltype.malloc(self.POOLNODE, flavor='raw') - firstpoolnode.linkedlist = self.malloced_objects - firstpoolnode.nextnode = self.poolnodes - prevpoolnode = lltype.nullptr(self.POOLNODE) - poolnode = firstpoolnode - while poolnode: #sweep - ppnext = llmemory.cast_ptr_to_adr(poolnode) - ppnext += llmemory.offsetof(self.POOLNODE, 'linkedlist') - hdr = poolnode.linkedlist - while hdr: #sweep - typeid = hdr.typeid >> 1 - next = hdr.next - addr = llmemory.cast_ptr_to_adr(hdr) - size = self.fixed_size(typeid) - if self.is_varsize(typeid): - length = (addr + size_gc_header + self.varsize_offset_to_length(typeid)).signed[0] - size += self.varsize_item_sizes(typeid) * length - estimate = raw_malloc_usage(size_gc_header + size) - if hdr.typeid & 1: - hdr.typeid = hdr.typeid & (~1) - ppnext.address[0] = addr - ppnext = llmemory.cast_ptr_to_adr(hdr) - ppnext += llmemory.offsetof(self.HDR, 'next') - curr_heap_size += estimate - else: - gc_info = llmemory.cast_ptr_to_adr(hdr) - obj = gc_info + size_gc_header - self.write_free_statistics(typeid, obj) - freed_size += estimate - raw_free(addr) - hdr = next - ppnext.address[0] = llmemory.NULL - next = poolnode.nextnode - if not poolnode.linkedlist and prevpoolnode: - # completely empty node - prevpoolnode.nextnode = next - lltype.free(poolnode, flavor='raw') - else: - prevpoolnode = poolnode - poolnode = next - self.malloced_objects = firstpoolnode.linkedlist - self.poolnodes = firstpoolnode.nextnode - lltype.free(firstpoolnode, flavor='raw') - #llop.debug_view(lltype.Void, self.malloced_objects, self.malloced_objects_with_finalizer, size_gc_header) - - end_time = time.time() - compute_time = start_time - self.prev_collect_end_time - collect_time = end_time - start_time - - garbage_collected = old_malloced - (curr_heap_size - self.heap_usage) - - if (collect_time * curr_heap_size > - 0.02 * garbage_collected * compute_time): - self.bytes_malloced_threshold += self.bytes_malloced_threshold / 2 - if (collect_time * curr_heap_size < - 0.005 * garbage_collected * compute_time): - self.bytes_malloced_threshold /= 2 - - # Use atleast as much memory as current live objects. - if curr_heap_size > self.bytes_malloced_threshold: - self.bytes_malloced_threshold = curr_heap_size - - # Cap at 1/4 GB - self.bytes_malloced_threshold = min(self.bytes_malloced_threshold, - 256 * 1024 * 1024) - self.total_collection_time += collect_time - self.prev_collect_end_time = end_time - if DEBUG_PRINT: - llop.debug_print(lltype.Void, - " malloced since previous collection:", - old_malloced, "bytes") - llop.debug_print(lltype.Void, - " heap usage at start of collection: ", - self.heap_usage + old_malloced, "bytes") - llop.debug_print(lltype.Void, - " freed: ", - freed_size, "bytes") - llop.debug_print(lltype.Void, - " new heap usage: ", - curr_heap_size, "bytes") - llop.debug_print(lltype.Void, - " total time spent collecting: ", - self.total_collection_time, "seconds") - llop.debug_print(lltype.Void, - " collecting time: ", - collect_time) - llop.debug_print(lltype.Void, - " computing time: ", - collect_time) - llop.debug_print(lltype.Void, - " new threshold: ", - self.bytes_malloced_threshold) -## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, -## size_gc_header) - assert self.heap_usage + old_malloced == curr_heap_size + freed_size - - self.heap_usage = curr_heap_size - hdr = self.malloced_objects_with_finalizer - self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR) - last = lltype.nullptr(self.HDR) - while hdr: - next = hdr.next - if hdr.typeid & 1: - hdr.next = lltype.nullptr(self.HDR) - if not self.malloced_objects_with_finalizer: - self.malloced_objects_with_finalizer = hdr - else: - last.next = hdr - hdr.typeid = hdr.typeid & (~1) - last = hdr - else: - obj = llmemory.cast_ptr_to_adr(hdr) + size_gc_header - finalizer = self.getfinalizer(hdr.typeid >> 1) - # make malloced_objects_with_finalizer consistent - # for the sake of a possible collection caused by finalizer - if not self.malloced_objects_with_finalizer: - self.malloced_objects_with_finalizer = next - else: - last.next = next - hdr.next = self.malloced_objects - self.malloced_objects = hdr - #llop.debug_view(lltype.Void, self.malloced_objects, self.malloced_objects_with_finalizer, size_gc_header) - finalizer(obj) - if not self.collect_in_progress: # another collection was caused? - llop.debug_print(lltype.Void, "outer collect interrupted " - "by recursive collect") - return - if not last: - if self.malloced_objects_with_finalizer == next: - self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR) - else: - # now it gets annoying: finalizer caused a malloc of something - # with a finalizer - last = self.malloced_objects_with_finalizer - while last.next != next: - last = last.next - last.next = lltype.nullptr(self.HDR) - else: - last.next = lltype.nullptr(self.HDR) - hdr = next - self.collect_in_progress = False + MarkSweepGC.collect(self) From arigo at codespeak.net Fri Feb 22 12:41:13 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 22 Feb 2008 12:41:13 +0100 (CET) Subject: [pypy-svn] r51786 - in pypy/branch/unified-rtti/pypy/rpython: lltypesystem memory/gc memory/gctransform memory/test Message-ID: <20080222114113.452F216851A@codespeak.net> Author: arigo Date: Fri Feb 22 12:41:12 2008 New Revision: 51786 Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lloperation.py pypy/branch/unified-rtti/pypy/rpython/memory/gc/base.py pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py pypy/branch/unified-rtti/pypy/rpython/memory/test/test_transformed_gc.py Log: Merge r51632 from the trunk (kill x_become). Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lloperation.py Fri Feb 22 12:41:12 2008 @@ -403,10 +403,6 @@ 'gc_x_clone': LLOp(canraise=(MemoryError, RuntimeError), canunwindgc=True), 'gc_x_size_header': LLOp(), - # this one is even more experimental; only implemented with the - # Mark&Sweep GC, and likely only useful when combined with - # stackless: - 'gc_x_become': LLOp(canraise=(RuntimeError,), canunwindgc=True), # for llvm.gcroot() support. can change at any time 'llvm_frameaddress': LLOp(sideeffects=False), Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gc/base.py Fri Feb 22 12:41:12 2008 @@ -111,9 +111,6 @@ def x_clone(self, clonedata): raise RuntimeError("no support for x_clone in the GC") - def x_become(self, target_addr, source_addr): - raise RuntimeError("no support for x_become in the GC") - def trace(self, obj, callback, arg): """Enumerate the locations inside the given obj that can contain GC pointers. For each such location, callback(pointer, arg) is Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py Fri Feb 22 12:41:12 2008 @@ -693,176 +693,6 @@ # reinstall the pool that was current at the beginning of x_clone() clonedata.pool = self.x_swap_pool(curpool) - def add_reachable_to_stack2(self, obj, objects): - size_gc_header = self.gcheaderbuilder.size_gc_header - gc_info = obj - size_gc_header - hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - if hdr.typeid & 1: - return - self.trace(obj, self._add_reachable_and_rename, objects) - - def _add_reachable_and_rename(self, pointer, objects): - obj = pointer.address[0] - if obj: - if obj == self.x_become_target_addr: - obj = pointer.address[0] = self.x_become_source_addr - objects.append(obj) - - def x_become(self, target_addr, source_addr): - # 1. mark from the roots, and also the objects that objects-with-del - # point to (using the list of malloced_objects_with_finalizer) - # 2. walk the list of objects-without-del and free the ones not marked - # 3. walk the list of objects-with-del and for the ones not marked: - # call __del__, move the object to the list of object-without-del - import time - from pypy.rpython.lltypesystem.lloperation import llop - if DEBUG_PRINT: - llop.debug_print(lltype.Void, 'collecting...') - start_time = time.time() - size_gc_header = self.gcheaderbuilder.size_gc_header -## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, -## size_gc_header) - - # push the roots on the mark stack - objects = self.AddressStack() # mark stack - self._mark_stack = objects - # the last sweep did not clear the mark bit of static roots, - # since they are not in the malloced_objects list - self.root_walker.walk_roots( - MarkSweepGC._mark_root_and_clear_bit, # stack roots - MarkSweepGC._mark_root_and_clear_bit, # static in prebuilt non-gc - MarkSweepGC._mark_root_and_clear_bit) # static in prebuilt gc - - # from this point onwards, no more mallocs should be possible - old_malloced = self.bytes_malloced - self.bytes_malloced = 0 - curr_heap_size = 0 - freed_size = 0 - - self.x_become_target_addr = target_addr - self.x_become_source_addr = source_addr - - # mark objects reachable by objects with a finalizer, but not those - # themselves. add their size to curr_heap_size, since they always - # survive the collection - hdr = self.malloced_objects_with_finalizer - while hdr: - next = hdr.next - typeid = hdr.typeid >> 1 - gc_info = llmemory.cast_ptr_to_adr(hdr) - obj = gc_info + size_gc_header - self.add_reachable_to_stack2(obj, objects) - addr = llmemory.cast_ptr_to_adr(hdr) - size = self.fixed_size(typeid) - if self.is_varsize(typeid): - length = (obj + self.varsize_offset_to_length(typeid)).signed[0] - size += self.varsize_item_sizes(typeid) * length - estimate = raw_malloc_usage(size_gc_header + size) - curr_heap_size += estimate - hdr = next - - # mark thinks on the mark stack and put their descendants onto the - # stack until the stack is empty - while objects.non_empty(): #mark - curr = objects.pop() - self.add_reachable_to_stack2(curr, objects) - gc_info = curr - size_gc_header - hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - if hdr.typeid & 1: - continue - hdr.typeid = hdr.typeid | 1 - objects.delete() - # also mark self.curpool - if self.curpool: - gc_info = llmemory.cast_ptr_to_adr(self.curpool) - size_gc_header - hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - hdr.typeid = hdr.typeid | 1 - - # sweep: delete objects without del if they are not marked - # unmark objects without del that are marked - firstpoolnode = lltype.malloc(self.POOLNODE, flavor='raw') - firstpoolnode.linkedlist = self.malloced_objects - firstpoolnode.nextnode = self.poolnodes - prevpoolnode = lltype.nullptr(self.POOLNODE) - poolnode = firstpoolnode - while poolnode: #sweep - ppnext = llmemory.cast_ptr_to_adr(poolnode) - ppnext += llmemory.offsetof(self.POOLNODE, 'linkedlist') - hdr = poolnode.linkedlist - while hdr: #sweep - typeid = hdr.typeid >> 1 - next = hdr.next - addr = llmemory.cast_ptr_to_adr(hdr) - size = self.fixed_size(typeid) - if self.is_varsize(typeid): - length = (addr + size_gc_header + self.varsize_offset_to_length(typeid)).signed[0] - size += self.varsize_item_sizes(typeid) * length - estimate = raw_malloc_usage(size_gc_header + size) - if hdr.typeid & 1: - hdr.typeid = hdr.typeid & (~1) - ppnext.address[0] = addr - ppnext = llmemory.cast_ptr_to_adr(hdr) - ppnext += llmemory.offsetof(self.HDR, 'next') - curr_heap_size += estimate - else: - freed_size += estimate - raw_free(addr) - hdr = next - ppnext.address[0] = llmemory.NULL - next = poolnode.nextnode - if not poolnode.linkedlist and prevpoolnode: - # completely empty node - prevpoolnode.nextnode = next - lltype.free(poolnode, flavor='raw') - else: - prevpoolnode = poolnode - poolnode = next - self.malloced_objects = firstpoolnode.linkedlist - self.poolnodes = firstpoolnode.nextnode - lltype.free(firstpoolnode, flavor='raw') - - if curr_heap_size > self.bytes_malloced_threshold: - self.bytes_malloced_threshold = curr_heap_size - end_time = time.time() - self.total_collection_time += end_time - start_time - if DEBUG_PRINT: - llop.debug_print(lltype.Void, - " malloced since previous collection:", - old_malloced, "bytes") - llop.debug_print(lltype.Void, - " heap usage at start of collection: ", - self.heap_usage + old_malloced, "bytes") - llop.debug_print(lltype.Void, - " freed: ", - freed_size, "bytes") - llop.debug_print(lltype.Void, - " new heap usage: ", - curr_heap_size, "bytes") - llop.debug_print(lltype.Void, - " total time spent collecting: ", - self.total_collection_time, "seconds") -## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, -## size_gc_header) - assert self.heap_usage + old_malloced == curr_heap_size + freed_size - - # call finalizers if needed - self.heap_usage = curr_heap_size - hdr = self.malloced_objects_with_finalizer - self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR) - while hdr: - next = hdr.next - if hdr.typeid & 1: - hdr.next = self.malloced_objects_with_finalizer - self.malloced_objects_with_finalizer = hdr - hdr.typeid = hdr.typeid & (~1) - else: - obj = llmemory.cast_ptr_to_adr(hdr) + size_gc_header - finalizer = self.getfinalizer(hdr.typeid >> 1) - finalizer(obj) - hdr.next = self.malloced_objects - self.malloced_objects = hdr - hdr = next - class PrintingMarkSweepGC(MarkSweepGC): _alloc_flavor_ = "raw" Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py Fri Feb 22 12:41:12 2008 @@ -22,7 +22,7 @@ class CollectAnalyzer(graphanalyze.GraphAnalyzer): def operation_is_true(self, op): - if op.opname in ('gc__collect', 'gc_x_become'): + if op.opname == 'gc__collect': return True if op.opname in ('malloc', 'malloc_varsize'): flags = op.args[1].value @@ -315,11 +315,6 @@ annmodel.s_None, minimal_transform = False) - self.x_become_ptr = getfn( - GCClass.x_become.im_func, - [s_gc, annmodel.SomeAddress(), annmodel.SomeAddress()], - annmodel.s_None) - annhelper.finish() # at this point, annotate all mix-level helpers annhelper.backend_optimize() @@ -534,15 +529,6 @@ [c_result], resultvar=op.result) - def gct_gc_x_become(self, hop): - op = hop.spaceop - [v_target, v_source] = op.args - livevars = self.push_roots(hop) - hop.genop("direct_call", - [self.x_become_ptr, self.c_const_gc, v_target, v_source], - resultvar=op.result) - self.pop_roots(hop, livevars) - def gct_zero_gc_pointers_inside(self, hop): if self.needs_zero_gc_pointers: v_ob = hop.spaceop.args[0] Modified: pypy/branch/unified-rtti/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/test/test_transformed_gc.py Fri Feb 22 12:41:12 2008 @@ -729,26 +729,6 @@ def test_instances(self): py.test.skip("fails for a stupid reasons") - def test_x_become(self): - from pypy.rlib import objectmodel - S = lltype.GcStruct("S", ('x', lltype.Signed)) - def f(): - x = lltype.malloc(S) - x.x = 10 - y = lltype.malloc(S) - y.x = 20 - z = x - llop.gc_x_become(lltype.Void, - llmemory.cast_ptr_to_adr(x), - llmemory.cast_ptr_to_adr(y)) - # keep 'y' alive until the x_become() is finished, because in - # theory it could go away as soon as only its address is present - objectmodel.keepalive_until_here(y) - return z.x - run = self.runner(f) - res = run([]) - assert res == 20 - class TestPrintingGC(GenericGCTests): gcname = "statistics" From arigo at codespeak.net Fri Feb 22 13:06:10 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 22 Feb 2008 13:06:10 +0100 (CET) Subject: [pypy-svn] r51787 - in pypy/branch/unified-rtti/pypy/rpython/memory: . gc gctransform Message-ID: <20080222120610.9C40E16851F@codespeak.net> Author: arigo Date: Fri Feb 22 13:06:08 2008 New Revision: 51787 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/base.py pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py pypy/branch/unified-rtti/pypy/rpython/memory/gcheader.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py pypy/branch/unified-rtti/pypy/rpython/memory/gctypelayout.py pypy/branch/unified-rtti/pypy/rpython/memory/gcwrapper.py pypy/branch/unified-rtti/pypy/rpython/memory/support.py Log: In-progress: some MarkSweep tests pass. Many other tests broke :-) Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gc/base.py Fri Feb 22 13:06:08 2008 @@ -1,4 +1,5 @@ from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.memory import gcheader, gctypelayout from pypy.rlib.debug import ll_assert class GCBase(object): @@ -8,6 +9,10 @@ needs_zero_gc_pointers = True prebuilt_gc_objects_are_static_roots = True + def __init__(self): + TYPE_INFO = gctypelayout.GCData.TYPE_INFO + self.gcheaderbuilder = gcheader.GCHeaderBuilder(self.HDR, TYPE_INFO) + def set_query_functions(self, is_varsize, has_gcptr_in_varsize, is_gcarrayofgcptr, getfinalizer, Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py Fri Feb 22 13:06:08 2008 @@ -1,13 +1,13 @@ from pypy.rpython.lltypesystem.llmemory import raw_malloc, raw_free from pypy.rpython.lltypesystem.llmemory import raw_memcopy, raw_memclear from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage -from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE +from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE, RTTIPTR from pypy.rpython.memory.support import get_address_stack -from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rlib.objectmodel import free_non_gc_object from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib.debug import ll_assert from pypy.rpython.memory.gc.base import GCBase @@ -19,6 +19,24 @@ ('pool', X_POOL_PTR)) X_CLONE_PTR = lltype.Ptr(X_CLONE) +def ll_getnext(hdr): + "Return the 'next' header by reading the 'next_and_flags' fields" + next = hdr.next_and_flags & ~1 + return llmemory.cast_adr_to_ptr(next, MarkSweepGC.HDRPTR) + +def ll_setnext_clear(hdr, next): + hdr.next_and_flags = llmemory.cast_ptr_to_adr(next) + +def ll_ismarked(hdr): + return (llmemory.cast_adr_to_int(hdr.next_and_flags) & 1) != 0 + +def ll_setmark(hdr): + hdr.next_and_flags |= 1 + +def ll_clearmark(hdr): + hdr.next_and_flags &= ~1 + + DEBUG_PRINT = False memoryError = MemoryError() class MarkSweepGC(GCBase): @@ -28,18 +46,24 @@ HDRPTR = lltype.Ptr(HDR) # need to maintain a linked list of malloced objects, since we used the # systems allocator and can't walk the heap - HDR.become(lltype.Struct('header', ('typeid', lltype.Signed), - ('next', HDRPTR))) + HDR.become(lltype.Struct('header', ('typeptr', RTTIPTR), + ('next_and_flags', llmemory.Address), + adtmeths={'getnext': ll_getnext, + 'setnext_clear': ll_setnext_clear, + 'ismarked': ll_ismarked, + 'setmark': ll_setmark, + 'clearmark': ll_clearmark})) POOL = lltype.GcStruct('gc_pool') POOLPTR = lltype.Ptr(POOL) POOLNODE = lltype.ForwardReference() POOLNODEPTR = lltype.Ptr(POOLNODE) - POOLNODE.become(lltype.Struct('gc_pool_node', ('linkedlist', HDRPTR), + POOLNODE.become(lltype.Struct('gc_pool_node', ('linkedlisthdr', HDR), ('nextnode', POOLNODEPTR))) def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, start_heap_size=4096): + GCBase.__init__(self) self.heap_usage = 0 # at the end of the latest collection self.bytes_malloced = 0 # since the latest collection self.bytes_malloced_threshold = start_heap_size @@ -50,7 +74,6 @@ # these are usually only the small bits of memory that make a # weakref object self.objects_with_weak_pointers = lltype.nullptr(self.HDR) - self.gcheaderbuilder = GCHeaderBuilder(self.HDR) # pools, for x_swap_pool(): # 'curpool' is the current pool, lazily allocated (i.e. NULL means # the current POOL object is not yet malloc'ed). POOL objects are @@ -97,15 +120,15 @@ if not result: raise memoryError hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) - hdr.typeid = typeid << 1 + hdr.typeptr = typeid if has_finalizer: - hdr.next = self.malloced_objects_with_finalizer + hdr.setnext_clear(self.malloced_objects_with_finalizer) self.malloced_objects_with_finalizer = hdr elif contains_weakptr: - hdr.next = self.objects_with_weak_pointers + hdr.setnext_clear(self.objects_with_weak_pointers) self.objects_with_weak_pointers = hdr else: - hdr.next = self.malloced_objects + hdr.setnext_clear(self.malloced_objects) self.malloced_objects = hdr self.bytes_malloced = bytes_malloced result += size_gc_header @@ -132,15 +155,15 @@ raise memoryError raw_memclear(result, tot_size) hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) - hdr.typeid = typeid << 1 + hdr.typeptr = typeid if has_finalizer: - hdr.next = self.malloced_objects_with_finalizer + hdr.setnext_clear(self.malloced_objects_with_finalizer) self.malloced_objects_with_finalizer = hdr elif contains_weakptr: - hdr.next = self.objects_with_weak_pointers + hdr.setnext_clear(self.objects_with_weak_pointers) self.objects_with_weak_pointers = hdr else: - hdr.next = self.malloced_objects + hdr.setnext_clear(self.malloced_objects) self.malloced_objects = hdr self.bytes_malloced = bytes_malloced result += size_gc_header @@ -169,12 +192,12 @@ raise memoryError (result + size_gc_header + offset_to_length).signed[0] = length hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) - hdr.typeid = typeid << 1 + hdr.typeptr = typeid if has_finalizer: - hdr.next = self.malloced_objects_with_finalizer + hdr.setnext_clear(self.malloced_objects_with_finalizer) self.malloced_objects_with_finalizer = hdr else: - hdr.next = self.malloced_objects + hdr.setnext_clear(self.malloced_objects) self.malloced_objects = hdr self.bytes_malloced = bytes_malloced @@ -207,12 +230,12 @@ raw_memclear(result, tot_size) (result + size_gc_header + offset_to_length).signed[0] = length hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) - hdr.typeid = typeid << 1 + hdr.typeptr = typeid if has_finalizer: - hdr.next = self.malloced_objects_with_finalizer + hdr.setnext_clear(self.malloced_objects_with_finalizer) self.malloced_objects_with_finalizer = hdr else: - hdr.next = self.malloced_objects + hdr.setnext_clear(self.malloced_objects) self.malloced_objects = hdr self.bytes_malloced = bytes_malloced @@ -259,20 +282,19 @@ # survive the collection hdr = self.malloced_objects_with_finalizer while hdr: - next = hdr.next - typeid = hdr.typeid >> 1 gc_info = llmemory.cast_ptr_to_adr(hdr) obj = gc_info + size_gc_header - if not hdr.typeid & 1: + if not hdr.ismarked(): self.add_reachable_to_stack(obj, objects) addr = llmemory.cast_ptr_to_adr(hdr) + typeid = hdr.typeptr size = self.fixed_size(typeid) if self.is_varsize(typeid): length = (obj + self.varsize_offset_to_length(typeid)).signed[0] size += self.varsize_item_sizes(typeid) * length estimate = raw_malloc_usage(size_gc_header + size) curr_heap_size += estimate - hdr = next + hdr = hdr.getnext() # mark thinks on the mark stack and put their descendants onto the # stack until the stack is empty @@ -280,31 +302,29 @@ curr = objects.pop() gc_info = curr - size_gc_header hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - if hdr.typeid & 1: + if hdr.ismarked(): continue self.add_reachable_to_stack(curr, objects) - hdr.typeid = hdr.typeid | 1 + hdr.setmark() objects.delete() # also mark self.curpool if self.curpool: gc_info = llmemory.cast_ptr_to_adr(self.curpool) - size_gc_header hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - hdr.typeid = hdr.typeid | 1 + hdr.setmark() # go through the list of objects containing weak pointers # and kill the links if they go to dead objects # if the object itself is not marked, free it hdr = self.objects_with_weak_pointers surviving = lltype.nullptr(self.HDR) while hdr: - typeid = hdr.typeid >> 1 - next = hdr.next + typeid = hdr.typeptr + next = hdr.getnext() addr = llmemory.cast_ptr_to_adr(hdr) size = self.fixed_size(typeid) estimate = raw_malloc_usage(size_gc_header + size) - if hdr.typeid & 1: - typeid = hdr.typeid >> 1 + if hdr.ismarked(): offset = self.weakpointer_offset(typeid) - hdr.typeid = hdr.typeid & (~1) gc_info = llmemory.cast_ptr_to_adr(hdr) weakref_obj = gc_info + size_gc_header pointing_to = (weakref_obj + offset).address[0] @@ -315,9 +335,9 @@ # pointed to object will die # XXX what to do if the object has a finalizer which resurrects # the object? - if not hdr_pointing_to.typeid & 1: + if not hdr_pointing_to.ismarked(): (weakref_obj + offset).address[0] = NULL - hdr.next = surviving + hdr.setnext_clear(surviving) # this also clears the mark surviving = hdr curr_heap_size += estimate else: @@ -331,28 +351,27 @@ # sweep: delete objects without del if they are not marked # unmark objects without del that are marked firstpoolnode = lltype.malloc(self.POOLNODE, flavor='raw') - firstpoolnode.linkedlist = self.malloced_objects + firstpoolnode.linkedlisthdr.setnext_clear(self.malloced_objects) firstpoolnode.nextnode = self.poolnodes prevpoolnode = lltype.nullptr(self.POOLNODE) poolnode = firstpoolnode while poolnode: #sweep - ppnext = llmemory.cast_ptr_to_adr(poolnode) - ppnext += llmemory.offsetof(self.POOLNODE, 'linkedlist') - hdr = poolnode.linkedlist + previous = poolnode.linkedlisthdr + hdr = previous.getnext() while hdr: #sweep - typeid = hdr.typeid >> 1 - next = hdr.next + typeid = hdr.typeptr + next = hdr.getnext() addr = llmemory.cast_ptr_to_adr(hdr) size = self.fixed_size(typeid) if self.is_varsize(typeid): length = (addr + size_gc_header + self.varsize_offset_to_length(typeid)).signed[0] size += self.varsize_item_sizes(typeid) * length estimate = raw_malloc_usage(size_gc_header + size) - if hdr.typeid & 1: - hdr.typeid = hdr.typeid & (~1) - ppnext.address[0] = addr - ppnext = llmemory.cast_ptr_to_adr(hdr) - ppnext += llmemory.offsetof(self.HDR, 'next') + if hdr.ismarked(): + # hdr.clearmark() -- done automatically by the + # call to setnext_clear() that will follow + previous.setnext_clear(hdr) + previous = hdr curr_heap_size += estimate else: gc_info = llmemory.cast_ptr_to_adr(hdr) @@ -361,16 +380,16 @@ freed_size += estimate raw_free(addr) hdr = next - ppnext.address[0] = llmemory.NULL + previous.setnext_clear(lltype.nullptr(self.HDR)) next = poolnode.nextnode - if not poolnode.linkedlist and prevpoolnode: + if not poolnode.linkedlisthdr.getnext() and prevpoolnode: # completely empty node prevpoolnode.nextnode = next lltype.free(poolnode, flavor='raw') else: prevpoolnode = poolnode poolnode = next - self.malloced_objects = firstpoolnode.linkedlist + self.malloced_objects = firstpoolnode.linkedlisthdr.getnext() self.poolnodes = firstpoolnode.nextnode lltype.free(firstpoolnode, flavor='raw') #llop.debug_view(lltype.Void, self.malloced_objects, self.malloced_objects_with_finalizer, size_gc_header) @@ -431,25 +450,24 @@ self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR) last = lltype.nullptr(self.HDR) while hdr: - next = hdr.next - if hdr.typeid & 1: - hdr.next = lltype.nullptr(self.HDR) + next = hdr.getnext() + if hdr.ismarked(): + hdr.setnext_clear(lltype.nullptr(self.HDR)) # also clears mark if not self.malloced_objects_with_finalizer: self.malloced_objects_with_finalizer = hdr else: - last.next = hdr - hdr.typeid = hdr.typeid & (~1) + last.setnext_clear(hdr) last = hdr else: obj = llmemory.cast_ptr_to_adr(hdr) + size_gc_header - finalizer = self.getfinalizer(hdr.typeid >> 1) + finalizer = self.getfinalizer(hdr.typeptr) # make malloced_objects_with_finalizer consistent # for the sake of a possible collection caused by finalizer if not self.malloced_objects_with_finalizer: self.malloced_objects_with_finalizer = next else: - last.next = next - hdr.next = self.malloced_objects + last.setnext_clear(next) + hdr.setnext_clear(self.malloced_objects) self.malloced_objects = hdr #llop.debug_view(lltype.Void, self.malloced_objects, self.malloced_objects_with_finalizer, size_gc_header) finalizer(obj) @@ -464,11 +482,11 @@ # now it gets annoying: finalizer caused a malloc of something # with a finalizer last = self.malloced_objects_with_finalizer - while last.next != next: - last = last.next - last.next = lltype.nullptr(self.HDR) + while last.getnext() != next: + last = last.getnext() + last.setnext_clear(lltype.nullptr(self.HDR)) else: - last.next = lltype.nullptr(self.HDR) + last.setnext_clear(lltype.nullptr(self.HDR)) hdr = next self.collect_in_progress = False @@ -482,7 +500,7 @@ size_gc_header = self.gcheaderbuilder.size_gc_header gc_info = gcobjectaddr - size_gc_header hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - hdr.typeid = hdr.typeid & (~1) + hdr.clearmark() STAT_HEAP_USAGE = 0 STAT_BYTES_MALLOCED = 1 @@ -492,7 +510,7 @@ size_gc_header = self.gcheaderbuilder.size_gc_header gc_info = obj - size_gc_header hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - return hdr.typeid >> 1 + return hdr.typeptr def add_reachable_to_stack(self, obj, objects): self.trace(obj, self._add_reachable, objects) @@ -511,15 +529,10 @@ return self.bytes_malloced return -1 - def init_gc_object(self, addr, typeid): - hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) - hdr.typeid = typeid << 1 - - def init_gc_object_immortal(self, addr, typeid, flags=0): + def init_gc_object_immortal(self, hdr, typeid): # prebuilt gc structures always have the mark bit set - # ignore flags - hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) - hdr.typeid = (typeid << 1) | 1 + hdr.typeptr = typeid + hdr.setmark() # experimental support for thread cloning def x_swap_pool(self, newpool): @@ -542,7 +555,7 @@ hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) # put this new POOL object in the poolnodes list node = lltype.malloc(self.POOLNODE, flavor="raw") - node.linkedlist = hdr + node.linkedlisthdr.setnext_clear(hdr) node.nextnode = self.poolnodes self.poolnodes = node else: @@ -550,7 +563,7 @@ addr = llmemory.cast_ptr_to_adr(oldpool) addr -= size_gc_header hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) - hdr.next = self.malloced_objects + hdr.setnext_clear(self.malloced_objects) newpool = lltype.cast_opaque_ptr(self.POOLPTR, newpool) if newpool: @@ -558,11 +571,11 @@ addr = llmemory.cast_ptr_to_adr(newpool) addr -= size_gc_header hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) - self.malloced_objects = hdr.next + self.malloced_objects = hdr.getnext() # invariant: now that objects in the hdr.next list are accessible # through self.malloced_objects, make sure they are not accessible # via poolnodes (which has a node pointing to newpool): - hdr.next = lltype.nullptr(self.HDR) + hdr.setnext_clear(lltype.nullptr(self.HDR)) else: # start a fresh new linked list self.malloced_objects = lltype.nullptr(self.HDR) @@ -575,7 +588,6 @@ # in the specified pool. A new pool is built to contain the # copies, and the 'gcobjectptr' and 'pool' fields of clonedata # are adjusted to refer to the result. - CURPOOL_FLAG = sys.maxint // 2 + 1 # install a new pool into which all the mallocs go curpool = self.x_swap_pool(lltype.nullptr(X_POOL)) @@ -589,11 +601,13 @@ addr -= size_gc_header hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) - hdr = hdr.next # skip the POOL object itself + hdr = hdr.getnext() # skip the POOL object itself while hdr: - next = hdr.next - hdr.typeid |= CURPOOL_FLAG # mark all objects from malloced_list - hdr.next = lltype.nullptr(self.HDR) # abused to point to the copy + next = hdr.getnext() + # 'hdr.next' is abused to point to the copy + ll_assert(not hdr.ismarked(), "x_clone: object already marked") + hdr.setnext_clear(lltype.nullptr(self.HDR)) + hdr.setmark() # mark all objects from malloced_list oldobjects.append(llmemory.cast_ptr_to_adr(hdr)) hdr = next @@ -609,12 +623,11 @@ continue # pointer is NULL oldhdr = llmemory.cast_adr_to_ptr(oldobj_addr - size_gc_header, self.HDRPTR) - typeid = oldhdr.typeid - if not (typeid & CURPOOL_FLAG): + if not oldhdr.ismarked(): continue # ignore objects that were not in the malloced_list - newhdr = oldhdr.next # abused to point to the copy + newhdr = oldhdr.getnext() # abused to point to the copy if not newhdr: - typeid = (typeid & ~CURPOOL_FLAG) >> 1 + typeid = hdr.typeptr size = self.fixed_size(typeid) # XXX! collect() at the beginning if the free heap is low if self.is_varsize(typeid): @@ -640,11 +653,11 @@ newhdr_addr = newobj_addr - size_gc_header newhdr = llmemory.cast_adr_to_ptr(newhdr_addr, self.HDRPTR) - saved_id = newhdr.typeid # XXX hack needed for genc - saved_next = newhdr.next # where size_gc_header == 0 + saved_tid = newhdr.typeptr # XXX hack needed for genc + saved_next = newhdr.next_and_flags # where size_gc_header == 0 raw_memcopy(oldobj_addr, newobj_addr, size) - newhdr.typeid = saved_id - newhdr.next = saved_next + newhdr.typeptr = saved_tid + newhdr.next_and_flags = saved_next offsets = self.offsets_to_gc_pointers(typeid) i = 0 @@ -669,7 +682,8 @@ j += 1 i += 1 - oldhdr.next = newhdr + oldhdr.setnext_clear(newhdr) + oldhdr.setmark() newobj_addr = llmemory.cast_ptr_to_adr(newhdr) + size_gc_header gcptr_addr.address[0] = newobj_addr stack.delete() @@ -678,8 +692,7 @@ next = lltype.nullptr(self.HDR) while oldobjects.non_empty(): hdr = llmemory.cast_adr_to_ptr(oldobjects.pop(), self.HDRPTR) - hdr.typeid &= ~CURPOOL_FLAG # reset the flag - hdr.next = next + hdr.setnext_clear(next) # this also resets the mark next = hdr oldobjects.delete() @@ -687,7 +700,7 @@ addr = llmemory.cast_ptr_to_adr(oldpool) addr -= size_gc_header hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) - assert hdr.next == next + ll_assert(hdr.getnext() == next, "bad .next at the end of x_clone") # build the new pool object collecting the new objects, and # reinstall the pool that was current at the beginning of x_clone() Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gcheader.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gcheader.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gcheader.py Fri Feb 22 13:06:08 2008 @@ -17,6 +17,7 @@ self.rtti2typeinfo = weakref.WeakKeyDictionary() self.size_gc_header = llmemory.GCHeaderOffset(self) self.size_gc_typeinfo = llmemory.GCTypeInfoOffset(self) + self.rtticache = {} def header_of_object(self, gcptr): # XXX hackhackhack @@ -68,5 +69,8 @@ addr += self.size_gc_typeinfo return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.TYPEINFO)) + def getRtti(self, TYPE): + return lltype.getRuntimeTypeInfo(TYPE, self.rtticache) + def _freeze_(self): return True # for reads of size_gc_header Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py Fri Feb 22 13:06:08 2008 @@ -148,7 +148,7 @@ llops.genop("direct_call", [self.decrefptr, v_adr]) def gct_fv_gc_malloc(self, hop, flags, TYPE, c_size): - rtti = lltype.getRuntimeTypeInfo(TYPE, self.rtticache) + rtti = self.gcheaderbuilder.getTypeInfo(TYPE) c_rtti = rmodel.inputconst(RTTIPTR, rtti) v_raw = hop.genop("direct_call", [self.malloc_fixedsize_ptr, c_size, c_rtti], @@ -157,7 +157,7 @@ def gct_fv_gc_malloc_varsize(self, hop, flags, TYPE, v_length, c_const_size, c_item_size, c_offset_to_length): - rtti = lltype.getRuntimeTypeInfo(TYPE, self.rtticache) + rtti = self.gcheaderbuilder.getTypeInfo(TYPE) c_rtti = rmodel.inputconst(RTTIPTR, rtti) if c_offset_to_length is None: v_raw = hop.genop("direct_call", @@ -187,7 +187,7 @@ return # you don't really have an RPython deallocator for PyObjects assert TYPE._gckind == 'gc' - rtti = lltype.getRuntimeTypeInfo(TYPE, self.rtticache) + rtti = self.gcheaderbuilder.getTypeInfo(TYPE) destrptr = rtti.destructor_funcptr if destrptr is not None: DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0] Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py Fri Feb 22 13:06:08 2008 @@ -534,7 +534,6 @@ # at the moment, all GC transformers are based on a GCHeaderBuilder. self.gcheaderbuilder = gcheaderbuilder self.gchelpers = GCHelpers(gcheaderbuilder) - self.rtticache = {} if self.translator: self.gc_runtime_type_info_ptr = self.inittime_helper( self.gchelpers.gc_runtime_type_info, [llmemory.Address], @@ -683,7 +682,7 @@ p = value._as_ptr() if not self.gcheaderbuilder.get_header(p): hdr = self.gcheaderbuilder.new_header(p) - hdr.typeptr = lltype.getRuntimeTypeInfo(TYPE, self.rtticache) + hdr.typeptr = self.gcheaderbuilder.getTypeInfo(TYPE) self.initialize_constant_header(hdr, TYPE, value) def initialize_constant_header(self, hdr, TYPE, value): Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctypelayout.py Fri Feb 22 13:06:08 2008 @@ -28,37 +28,52 @@ ("weakptrofs", lltype.Signed), ) - def q_is_varsize(self, typeinfo): + def __init__(self, gcheaderbuilder): + self.gcheaderbuilder = gcheaderbuilder + self.cast_rtti_to_typeinfo = gcheaderbuilder.cast_rtti_to_typeinfo + + def q_is_varsize(self, typeid): + typeinfo = self.cast_rtti_to_typeinfo(typeid) return (typeinfo.flags & T_IS_VARSIZE) != 0 - def q_has_gcptr_in_varsize(self, typeinfo): + def q_has_gcptr_in_varsize(self, typeid): + typeinfo = self.cast_rtti_to_typeinfo(typeid) return (typeinfo.flags & T_HAS_GCPTR_IN_VARSIZE) != 0 - def q_is_gcarrayofgcptr(self, typeinfo): + def q_is_gcarrayofgcptr(self, typeid): + typeinfo = self.cast_rtti_to_typeinfo(typeid) return (typeinfo.flags & T_IS_GCARRAYOFGCPTR) != 0 - def q_finalizer(self, typeinfo): + def q_finalizer(self, typeid): + typeinfo = self.cast_rtti_to_typeinfo(typeid) return typeinfo.finalizer - def q_offsets_to_gc_pointers(self, typeinfo): + def q_offsets_to_gc_pointers(self, typeid): + typeinfo = self.cast_rtti_to_typeinfo(typeid) return typeinfo.ofstoptrs - def q_fixed_size(self, typeinfo): + def q_fixed_size(self, typeid): + typeinfo = self.cast_rtti_to_typeinfo(typeid) return typeinfo.fixedsize - def q_varsize_item_sizes(self, typeinfo): + def q_varsize_item_sizes(self, typeid): + typeinfo = self.cast_rtti_to_typeinfo(typeid) return typeinfo.varitemsize - def q_varsize_offset_to_variable_part(self, typeinfo): + def q_varsize_offset_to_variable_part(self, typeid): + typeinfo = self.cast_rtti_to_typeinfo(typeid) return typeinfo.ofstovar - def q_varsize_offset_to_length(self, typeinfo): + def q_varsize_offset_to_length(self, typeid): + typeinfo = self.cast_rtti_to_typeinfo(typeid) return typeinfo.ofstolength - def q_varsize_offsets_to_gcpointers_in_var_part(self, typeinfo): + def q_varsize_offsets_to_gcpointers_in_var_part(self, typeid): + typeinfo = self.cast_rtti_to_typeinfo(typeid) return typeinfo.varofstoptrs - def q_weakpointer_offset(self, typeinfo): + def q_weakpointer_offset(self, typeid): + typeinfo = self.cast_rtti_to_typeinfo(typeid) return typeinfo.weakptrofs def set_query_functions(self, gc): @@ -152,8 +167,8 @@ class TypeLayoutBuilder(object): - def __init__(self): - self.typeinfos = {} # {LLTYPE: TYPE_INFO} + def __init__(self, gcheaderbuilder): + self.id_of_type = {} # {LLTYPE: fully-initialized-RTTIPTR} self.seen_roots = {} # the following are lists of addresses of gc pointers living inside the # prebuilt structures. It should list all the locations that could @@ -168,17 +183,25 @@ self.additional_roots_sources = 0 self.finalizer_funcptrs = {} self.offsettable_cache = {} + self.gcheaderbuilder = gcheaderbuilder + assert gcheaderbuilder.TYPEINFO == GCData.TYPE_INFO - def get_type_info(self, TYPE): + def get_type_id(self, TYPE): + """The 'typeid' of a TYPE is an opaque RTTIPTR; this is the + same pointer as the 'type info', but the latter is not opaque. + """ try: - return self.typeinfos[TYPE] + return self.id_of_type[TYPE] except KeyError: - assert isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)) # Record the new type description as a TYPE_INFO structure. - info = lltype.malloc(GCData.TYPE_INFO, immortal=True, zero=True) - encode_type_shape(self, info, TYPE) - self.typeinfos[TYPE] = info - return info + rtti = self.gcheaderbuilder.getRtti(TYPE) + try: + self.gcheaderbuilder.typeinfo_from_rtti(rtti) + except KeyError: + typeinfo = self.gcheaderbuilder.new_typeinfo(rtti) + encode_type_shape(self, typeinfo, TYPE) + self.id_of_type[TYPE] = rtti + return rtti def offsets2table(self, offsets, TYPE): try: @@ -203,7 +226,7 @@ return lltype.nullptr(GCData.ADDRESS_VOID_FUNC) def initialize_gc_query_function(self, gc): - return GCData(self.type_info_list).set_query_functions(gc) + return GCData(self.gcheaderbuilder).set_query_functions(gc) def consider_constant(self, TYPE, value, gc): if value is not lltype.top_container(value): @@ -213,10 +236,9 @@ self.seen_roots[id(value)] = True if isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)): - typeinfo = self.get_type_info(TYPE) + typeid = self.get_type_id(TYPE) hdr = gc.gcheaderbuilder.new_header(value) - adr = llmemory.cast_ptr_to_adr(hdr) - gc.init_gc_object_immortal(adr, typeinfo) + gc.init_gc_object_immortal(hdr, typeid) # The following collects the addresses of all the fields that have # a GC Pointer type, inside the current prebuilt object. All such Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gcwrapper.py Fri Feb 22 13:06:08 2008 @@ -15,7 +15,8 @@ self.gc.setup() def prepare_graphs(self, flowgraphs): - layoutbuilder = DirectRunLayoutBuilder(self.llinterp) + layoutbuilder = DirectRunLayoutBuilder(self.gc.gcheaderbuilder, + self.llinterp) self.get_type_id = layoutbuilder.get_type_id layoutbuilder.initialize_gc_query_function(self.gc) @@ -85,8 +86,8 @@ # we have to be lazy in reading the llinterp variable containing # the 'obj' pointer, because the gc.malloc() call below could # move it around - type_id = self.get_type_id(gctypelayout.WEAKREF) - addr = self.gc.malloc(type_id, None, zero=False) + type_info = self.get_type_info(gctypelayout.WEAKREF) + addr = self.gc.malloc(type_info, None, zero=False) result = llmemory.cast_adr_to_ptr(addr, gctypelayout.WEAKREFPTR) result.weakptr = llmemory.cast_ptr_to_adr(objgetter()) return llmemory.cast_ptr_to_weakrefptr(result) @@ -132,16 +133,16 @@ class DirectRunLayoutBuilder(gctypelayout.TypeLayoutBuilder): - def __init__(self, llinterp): + def __init__(self, gcheaderbuilder, llinterp): self.llinterp = llinterp - super(DirectRunLayoutBuilder, self).__init__() + super(DirectRunLayoutBuilder, self).__init__(gcheaderbuilder) def make_finalizer_funcptr_for_type(self, TYPE): - from pypy.rpython.memory.gctransform.support import get_rtti, \ + from pypy.rpython.memory.gctransform.support import \ type_contains_pyobjs - rtti = get_rtti(TYPE) - if rtti is not None and hasattr(rtti._obj, 'destructor_funcptr'): - destrptr = rtti._obj.destructor_funcptr + rtti = self.gcheaderbuilder.getRtti(TYPE) + destrptr = rtti.destructor_funcptr + if destrptr is not None: DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0] destrgraph = destrptr._obj.graph else: Modified: pypy/branch/unified-rtti/pypy/rpython/memory/support.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/support.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/support.py Fri Feb 22 13:06:08 2008 @@ -2,6 +2,7 @@ from pypy.rlib.objectmodel import free_non_gc_object, we_are_translated from pypy.rlib.debug import ll_assert +RTTIPTR = lltype.Ptr(lltype.RuntimeTypeInfo) DEFAULT_CHUNK_SIZE = 1019 From arigo at codespeak.net Fri Feb 22 13:37:22 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 22 Feb 2008 13:37:22 +0100 (CET) Subject: [pypy-svn] r51788 - in pypy/branch/unified-rtti/pypy: annotation rpython translator/c translator/c/src translator/c/test Message-ID: <20080222123722.9DADE168438@codespeak.net> Author: arigo Date: Fri Feb 22 13:37:21 2008 New Revision: 51788 Modified: pypy/branch/unified-rtti/pypy/annotation/bookkeeper.py pypy/branch/unified-rtti/pypy/rpython/raddress.py pypy/branch/unified-rtti/pypy/translator/c/primitive.py pypy/branch/unified-rtti/pypy/translator/c/src/address.h pypy/branch/unified-rtti/pypy/translator/c/test/test_lladdresses.py Log: Support prebuilt fakeaddresswithflags. Modified: pypy/branch/unified-rtti/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/unified-rtti/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/unified-rtti/pypy/annotation/bookkeeper.py Fri Feb 22 13:37:21 2008 @@ -407,7 +407,8 @@ result = entry.compute_annotation_bk(self) elif isinstance(x, lltype._ptr): result = SomePtr(lltype.typeOf(x)) - elif isinstance(x, llmemory.fakeaddress): + elif isinstance(x, (llmemory.fakeaddress, + llmemory.fakeaddresswithflags)): result = SomeAddress(is_null=not x) elif isinstance(x, ootype._static_meth): result = SomeOOStaticMeth(ootype.typeOf(x)) Modified: pypy/branch/unified-rtti/pypy/rpython/raddress.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/raddress.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/raddress.py Fri Feb 22 13:37:21 2008 @@ -2,7 +2,7 @@ from pypy.tool.pairtype import pairtype from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem.llmemory import NULL, Address, \ - cast_adr_to_int, fakeaddress + cast_adr_to_int, fakeaddress, fakeaddresswithflags from pypy.rpython.rmodel import Repr, IntegerRepr from pypy.rpython.rptr import PtrRepr from pypy.rpython.lltypesystem import lltype @@ -28,7 +28,8 @@ def convert_const(self, value): # note that llarena.fakearenaaddress is not supported as a constant # in graphs - assert type(value) is fakeaddress + assert (type(value) is fakeaddress or + type(value) is fakeaddresswithflags) return value def ll_str(self, a): Modified: pypy/branch/unified-rtti/pypy/translator/c/primitive.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/primitive.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/primitive.py Fri Feb 22 13:37:21 2008 @@ -7,7 +7,8 @@ from pypy.rpython.lltypesystem.llmemory import Address, \ AddressOffset, ItemOffset, ArrayItemsOffset, FieldOffset, \ CompositeOffset, ArrayLengthOffset, \ - GCHeaderOffset, GCTypeInfoOffset + GCHeaderOffset, GCTypeInfoOffset, \ + fakeaddress, fakeaddresswithflags from pypy.rpython.lltypesystem.llarena import RoundedUpForAllocation from pypy.translator.c.support import cdecl, barebonearray @@ -126,7 +127,13 @@ def name_address(value, db): if value: - return db.get(value.ref()) + if type(value) is fakeaddress: + return db.get(value.ref()) + elif type(value) is fakeaddresswithflags: + return 'ADR_OR(%s, %d)' % (name_address(value.adr, db), + value.flags) + else: + raise TypeError("not supported: name_address(%r)" % (value,)) else: return 'NULL' Modified: pypy/branch/unified-rtti/pypy/translator/c/src/address.h ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/src/address.h (original) +++ pypy/branch/unified-rtti/pypy/translator/c/src/address.h Fri Feb 22 13:37:21 2008 @@ -20,5 +20,8 @@ #define OP_CAST_INT_TO_ADR(x, r) r = ((void *)(x)) /* XXX assumes that addresses fit in a long */ -#define OP_ADR_OR(x,y,r) r = (char *)((long)(x) | (y)) -#define OP_ADR_AND(x,y,r) r = (char *)((long)(x) & (y)) +#define ADR_OR(x,y) ((char *)((long)(x) | (y))) +#define ADR_AND(x,y) ((char *)((long)(x) & (y))) + +#define OP_ADR_OR(x,y,r) r = ADR_OR(x,y) +#define OP_ADR_AND(x,y,r) r = ADR_AND(x,y) Modified: pypy/branch/unified-rtti/pypy/translator/c/test/test_lladdresses.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/test/test_lladdresses.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/test/test_lladdresses.py Fri Feb 22 13:37:21 2008 @@ -62,9 +62,17 @@ def test_flags_in_low_bits(): S = lltype.GcStruct('S', ('x', lltype.Signed)) - def fn(): + s1 = lltype.malloc(S) + s1.x = 42 + a1 = cast_ptr_to_adr(s1) | 1 + def fn(flag): s = lltype.malloc(S) + s.x = 6161 a = cast_ptr_to_adr(s) + if flag: + a2 = a1 + else: + a2 = a | 1 assert cast_adr_to_int(a) & 3 == 0 assert cast_adr_to_int(a | 0) & 3 == 0 assert cast_adr_to_int(a | 1) & 3 == 1 @@ -95,8 +103,15 @@ a &= ~2 assert a == cast_ptr_to_adr(s) assert cast_adr_to_ptr(a, lltype.Ptr(S)) == s - fc = compile(fn, []) - fc() + s1bis = cast_adr_to_ptr(a1 & ~1, lltype.Ptr(S)) + assert s1bis.x == 42 + ster = cast_adr_to_ptr(a2 & ~1, lltype.Ptr(S)) + return ster.x + fc = compile(fn, [int]) + res = fc(1) + assert res == 42 + res = fc(0) + assert res == 6161 def test_raw_memcopy(): def f(): From arigo at codespeak.net Fri Feb 22 13:48:24 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 22 Feb 2008 13:48:24 +0100 (CET) Subject: [pypy-svn] r51789 - pypy/branch/unified-rtti/pypy/rpython/memory/gc Message-ID: <20080222124824.8E9501684CA@codespeak.net> Author: arigo Date: Fri Feb 22 13:48:22 2008 New Revision: 51789 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py Log: Fix mark collision. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py Fri Feb 22 13:48:22 2008 @@ -21,7 +21,7 @@ def ll_getnext(hdr): "Return the 'next' header by reading the 'next_and_flags' fields" - next = hdr.next_and_flags & ~1 + next = hdr.next_and_flags & ~3 return llmemory.cast_adr_to_ptr(next, MarkSweepGC.HDRPTR) def ll_setnext_clear(hdr, next): @@ -36,6 +36,15 @@ def ll_clearmark(hdr): hdr.next_and_flags &= ~1 +def ll_ismarked2(hdr): + return (llmemory.cast_adr_to_int(hdr.next_and_flags) & 2) != 0 + +def ll_setmark2(hdr): + hdr.next_and_flags |= 2 + +def ll_clearmark2(hdr): + hdr.next_and_flags &= ~2 + DEBUG_PRINT = False memoryError = MemoryError() @@ -52,7 +61,10 @@ 'setnext_clear': ll_setnext_clear, 'ismarked': ll_ismarked, 'setmark': ll_setmark, - 'clearmark': ll_clearmark})) + 'clearmark': ll_clearmark, + 'ismarked2': ll_ismarked2, + 'setmark2': ll_setmark2, + 'clearmark2': ll_clearmark2})) POOL = lltype.GcStruct('gc_pool') POOLPTR = lltype.Ptr(POOL) @@ -605,9 +617,9 @@ while hdr: next = hdr.getnext() # 'hdr.next' is abused to point to the copy - ll_assert(not hdr.ismarked(), "x_clone: object already marked") + ll_assert(not hdr.ismarked2(), "x_clone: object already marked") hdr.setnext_clear(lltype.nullptr(self.HDR)) - hdr.setmark() # mark all objects from malloced_list + hdr.setmark2() # mark all objects from malloced_list oldobjects.append(llmemory.cast_ptr_to_adr(hdr)) hdr = next @@ -623,7 +635,7 @@ continue # pointer is NULL oldhdr = llmemory.cast_adr_to_ptr(oldobj_addr - size_gc_header, self.HDRPTR) - if not oldhdr.ismarked(): + if not oldhdr.ismarked2(): continue # ignore objects that were not in the malloced_list newhdr = oldhdr.getnext() # abused to point to the copy if not newhdr: @@ -683,7 +695,7 @@ i += 1 oldhdr.setnext_clear(newhdr) - oldhdr.setmark() + oldhdr.setmark2() newobj_addr = llmemory.cast_ptr_to_adr(newhdr) + size_gc_header gcptr_addr.address[0] = newobj_addr stack.delete() @@ -692,7 +704,7 @@ next = lltype.nullptr(self.HDR) while oldobjects.non_empty(): hdr = llmemory.cast_adr_to_ptr(oldobjects.pop(), self.HDRPTR) - hdr.setnext_clear(next) # this also resets the mark + hdr.setnext_clear(next) # this also resets the mark2 next = hdr oldobjects.delete() From arigo at codespeak.net Fri Feb 22 13:49:06 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 22 Feb 2008 13:49:06 +0100 (CET) Subject: [pypy-svn] r51790 - in pypy/branch/unified-rtti/pypy: rpython/memory rpython/memory/gctransform translator/c Message-ID: <20080222124906.2C5FE1684F9@codespeak.net> Author: arigo Date: Fri Feb 22 13:49:05 2008 New Revision: 51790 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py pypy/branch/unified-rtti/pypy/rpython/memory/gctypelayout.py pypy/branch/unified-rtti/pypy/rpython/memory/gcwrapper.py pypy/branch/unified-rtti/pypy/translator/c/gc.py Log: Progress on the framework gc transformer. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py Fri Feb 22 13:49:05 2008 @@ -92,7 +92,7 @@ if TYPE in self.finalizer_funcptrs: return self.finalizer_funcptrs[TYPE] - rtti = lltype.getRuntimeTypeInfo(TYPE, self.rtticache) + rtti = self.gcheaderbuilder.getRtti(TYPE) destrptr = rtti.destructor_funcptr if destrptr is not None: DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0] Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py Fri Feb 22 13:49:05 2008 @@ -1,4 +1,4 @@ -from pypy.rpython.memory.gctransform.transform import GCTransformer +from pypy.rpython.memory.gctransform.transform import GCTransformer, RTTIPTR from pypy.rpython.memory.gctransform.support import find_gc_ptrs_in_type, \ ll_call_destructor, type_contains_pyobjs, var_ispyobj from pypy.rpython.lltypesystem import lltype, llmemory @@ -107,14 +107,13 @@ # for regular translation: pick the GC from the config GCClass, GC_PARAMS = choose_gc_from_config(translator.config) + gc = GCClass(**GC_PARAMS) + self.setgcheaderbuilder(gc.gcheaderbuilder) self.layoutbuilder = TransformerLayoutBuilder(self) self.get_type_id = self.layoutbuilder.get_type_id - # set up dummy a table, to be overwritten with the real one in finish() - type_info_table = lltype._ptr( - lltype.Ptr(gctypelayout.GCData.TYPE_INFO_TABLE), - "delayed!type_info_table", solid=True) - gcdata = gctypelayout.GCData(type_info_table) + gcdata = gctypelayout.GCData(self.gcheaderbuilder) + gcdata.gc = gc # initialize the following two fields with a random non-NULL address, # to make the annotator happy. The fields are patched in finish() @@ -128,7 +127,6 @@ self.gcdata = gcdata self.malloc_fnptr_cache = {} - gcdata.gc = GCClass(**GC_PARAMS) root_walker = self.build_root_walker() gcdata.set_query_functions(gcdata.gc) gcdata.gc.set_root_walker(root_walker) @@ -190,7 +188,7 @@ malloc_fixedsize_clear_meth = GCClass.malloc_fixedsize_clear.im_func self.malloc_fixedsize_clear_ptr = getfn( malloc_fixedsize_clear_meth, - [s_gc, annmodel.SomeInteger(nonneg=True), + [s_gc, annmodel.SomePtr(RTTIPTR), annmodel.SomeInteger(nonneg=True), annmodel.SomeBool(), annmodel.SomeBool(), annmodel.SomeBool()], s_gcref, @@ -199,7 +197,7 @@ malloc_fixedsize_meth = GCClass.malloc_fixedsize.im_func self.malloc_fixedsize_ptr = getfn( malloc_fixedsize_meth, - [s_gc, annmodel.SomeInteger(nonneg=True), + [s_gc, annmodel.SomePtr(RTTIPTR), annmodel.SomeInteger(nonneg=True), annmodel.SomeBool(), annmodel.SomeBool(), annmodel.SomeBool()], s_gcref, @@ -207,13 +205,10 @@ else: malloc_fixedsize_meth = None self.malloc_fixedsize_ptr = self.malloc_fixedsize_clear_ptr -## self.malloc_varsize_ptr = getfn( -## GCClass.malloc_varsize.im_func, -## [s_gc] + [annmodel.SomeInteger(nonneg=True) for i in range(5)] -## + [annmodel.SomeBool(), annmodel.SomeBool()], s_gcref) self.malloc_varsize_clear_ptr = getfn( GCClass.malloc_varsize_clear.im_func, - [s_gc] + [annmodel.SomeInteger(nonneg=True) for i in range(5)] + [s_gc, annmodel.SomePtr(RTTIPTR)] + + [annmodel.SomeInteger(nonneg=True) for i in range(4)] + [annmodel.SomeBool(), annmodel.SomeBool()], s_gcref) self.collect_ptr = getfn(GCClass.collect.im_func, [s_gc], annmodel.s_None) @@ -256,7 +251,7 @@ s_True = annmodel.SomeBool(); s_True .const = True self.malloc_varsize_clear_fast_ptr = getfn( malloc_varsize_clear_fast, - [s_gc, annmodel.SomeInteger(nonneg=True), + [s_gc, annmodel.SomePtr(RTTIPTR), annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True), @@ -340,7 +335,7 @@ return self.layoutbuilder.finalizer_funcptr_for_type(TYPE) def finish_tables(self): - table = self.layoutbuilder.flatten_table() + table = self.layoutbuilder.id_of_type log.info("assigned %s typeids" % (len(table), )) log.info("added %s push/pop stack root instructions" % ( self.num_pushs, )) @@ -348,13 +343,6 @@ log.info("inserted %s write barrier calls" % ( self.write_barrier_calls, )) - # replace the type_info_table pointer in gcdata -- at this point, - # the database is in principle complete, so it has already seen - # the delayed pointer. We need to force it to consider the new - # array now. - - self.gcdata.type_info_table._become(table) - # XXX because we call inputconst already in replace_malloc, we can't # modify the instance, we have to modify the 'rtyped instance' # instead. horrors. is there a better way? @@ -388,13 +376,15 @@ def write_typeid_list(self): """write out the list of type ids together with some info""" from pypy.tool.udir import udir + # for debugging, we put unique numbers in typeinfo.flags + lines = [] + for TYPE, typeid in self.layoutbuilder.id_of_type.iteritems(): + typeinfo = self.gcheaderbuilder.cast_rtti_to_typeinfo(typeid) + lines.append("%6d %s\n" % (typeinfo.flags, TYPE)) + lines.sort() # XXX not ideal since it is not per compilation, but per run f = udir.join("typeids.txt").open("w") - all = [(typeid, TYPE) - for TYPE, typeid in self.layoutbuilder.id_of_type.iteritems()] - all.sort() - for typeid, TYPE in all: - f.write("%s %s\n" % (typeid, TYPE)) + f.writelines(lines) f.close() def transform_graph(self, graph): @@ -424,8 +414,8 @@ assert PTRTYPE.TO == TYPE type_id = self.get_type_id(TYPE) - c_type_id = rmodel.inputconst(lltype.Signed, type_id) - info = self.layoutbuilder.type_info_list[type_id] + c_type_id = rmodel.inputconst(RTTIPTR, type_id) + info = self.gcheaderbuilder.cast_rtti_to_typeinfo(type_id) c_size = rmodel.inputconst(lltype.Signed, info.fixedsize) has_finalizer = bool(self.finalizer_funcptr_for_type(TYPE)) c_has_finalizer = rmodel.inputconst(lltype.Bool, has_finalizer) @@ -475,8 +465,8 @@ assert PTRTYPE.TO == TYPE type_id = self.get_type_id(TYPE) - c_type_id = rmodel.inputconst(lltype.Signed, type_id) - info = self.layoutbuilder.type_info_list[type_id] + c_type_id = rmodel.inputconst(RTTIPTR, type_id) + info = self.gcheaderbuilder.cast_rtti_to_typeinfo(type_id) c_size = rmodel.inputconst(lltype.Signed, info.fixedsize) has_finalizer = bool(self.finalizer_funcptr_for_type(TYPE)) assert not has_finalizer @@ -540,8 +530,8 @@ type_id = self.get_type_id(WEAKREF) - c_type_id = rmodel.inputconst(lltype.Signed, type_id) - info = self.layoutbuilder.type_info_list[type_id] + c_type_id = rmodel.inputconst(RTTIPTR, type_id) + info = self.gcheaderbuilder.cast_rtti_to_typeinfo(type_id) c_size = rmodel.inputconst(lltype.Signed, info.fixedsize) malloc_ptr = self.malloc_fixedsize_ptr c_has_finalizer = rmodel.inputconst(lltype.Bool, False) @@ -682,17 +672,17 @@ class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder): def __init__(self, transformer): - super(TransformerLayoutBuilder, self).__init__() + self.gcheaderbuilder = transformer.gcheaderbuilder + super(TransformerLayoutBuilder, self).__init__(self.gcheaderbuilder) self.transformer = transformer self.offsettable_cache = {} def make_finalizer_funcptr_for_type(self, TYPE): - rtti = get_rtti(TYPE) - if rtti is not None and hasattr(rtti._obj, 'destructor_funcptr'): - destrptr = rtti._obj.destructor_funcptr + rtti = self.gcheaderbuilder.getRtti(TYPE) + destrptr = rtti.destructor_funcptr + if destrptr is not None: DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0] else: - destrptr = None DESTR_ARG = None assert not type_contains_pyobjs(TYPE), "not implemented" Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py Fri Feb 22 13:49:05 2008 @@ -148,7 +148,7 @@ llops.genop("direct_call", [self.decrefptr, v_adr]) def gct_fv_gc_malloc(self, hop, flags, TYPE, c_size): - rtti = self.gcheaderbuilder.getTypeInfo(TYPE) + rtti = self.gcheaderbuilder.getRtti(TYPE) c_rtti = rmodel.inputconst(RTTIPTR, rtti) v_raw = hop.genop("direct_call", [self.malloc_fixedsize_ptr, c_size, c_rtti], @@ -157,7 +157,7 @@ def gct_fv_gc_malloc_varsize(self, hop, flags, TYPE, v_length, c_const_size, c_item_size, c_offset_to_length): - rtti = self.gcheaderbuilder.getTypeInfo(TYPE) + rtti = self.gcheaderbuilder.getRtti(TYPE) c_rtti = rmodel.inputconst(RTTIPTR, rtti) if c_offset_to_length is None: v_raw = hop.genop("direct_call", @@ -187,7 +187,7 @@ return # you don't really have an RPython deallocator for PyObjects assert TYPE._gckind == 'gc' - rtti = self.gcheaderbuilder.getTypeInfo(TYPE) + rtti = self.gcheaderbuilder.getRtti(TYPE) destrptr = rtti.destructor_funcptr if destrptr is not None: DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0] Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py Fri Feb 22 13:49:05 2008 @@ -682,7 +682,7 @@ p = value._as_ptr() if not self.gcheaderbuilder.get_header(p): hdr = self.gcheaderbuilder.new_header(p) - hdr.typeptr = self.gcheaderbuilder.getTypeInfo(TYPE) + hdr.typeptr = self.gcheaderbuilder.getRtti(TYPE) self.initialize_constant_header(hdr, TYPE, value) def initialize_constant_header(self, hdr, TYPE, value): Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctypelayout.py Fri Feb 22 13:49:05 2008 @@ -129,6 +129,10 @@ """Encode the shape of the TYPE into the TYPE_INFO structure 'info'.""" offsets = offsets_to_gc_pointers(TYPE) info.flags = get_type_flags(TYPE) + # for debugging, we also encode a unique non-zero identifier in the .flags + builder.idcount += 1 + info.flags += T_first_unused_bit * builder.idcount + info.ofstoptrs = builder.offsets2table(offsets, TYPE) info.finalizer = builder.make_finalizer_funcptr_for_type(TYPE) info.weakptrofs = weakpointer_offset(TYPE) @@ -169,6 +173,7 @@ def __init__(self, gcheaderbuilder): self.id_of_type = {} # {LLTYPE: fully-initialized-RTTIPTR} + self.idcount = 0 self.seen_roots = {} # the following are lists of addresses of gc pointers living inside the # prebuilt structures. It should list all the locations that could Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gcwrapper.py Fri Feb 22 13:49:05 2008 @@ -86,8 +86,8 @@ # we have to be lazy in reading the llinterp variable containing # the 'obj' pointer, because the gc.malloc() call below could # move it around - type_info = self.get_type_info(gctypelayout.WEAKREF) - addr = self.gc.malloc(type_info, None, zero=False) + typeid = self.get_type_id(gctypelayout.WEAKREF) + addr = self.gc.malloc(typeid, None, zero=False) result = llmemory.cast_adr_to_ptr(addr, gctypelayout.WEAKREFPTR) result.weakptr = llmemory.cast_ptr_to_adr(objgetter()) return llmemory.cast_ptr_to_weakrefptr(result) Modified: pypy/branch/unified-rtti/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/gc.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/gc.py Fri Feb 22 13:49:05 2008 @@ -16,7 +16,7 @@ def common_gcheader_definition(self, defnode): if defnode.db.gctransformer is not None: - HDR = defnode.db.gctransformer.HDR + HDR = defnode.db.gctransformer.gcheaderbuilder.HDR return [(name, HDR._flds[name]) for name in HDR._names] else: return [] @@ -25,7 +25,7 @@ if defnode.db.gctransformer is not None: gct = defnode.db.gctransformer hdr = gct.gcheaderbuilder.header_of_object(top_container(defnode.obj)) - HDR = gct.HDR + HDR = gct.gcheaderbuilder.HDR return [getattr(hdr, fldname) for fldname in HDR._names] else: return [] @@ -59,7 +59,7 @@ # for rtti node def get_real_rtti_type(self): - return self.transformerclass.TYPEINFO + return self.db.gctransformer.gcheaderbuilder.TYPEINFO def convert_rtti(self, obj): return self.db.gctransformer.convert_rtti(obj) From pedronis at codespeak.net Fri Feb 22 15:57:23 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 22 Feb 2008 15:57:23 +0100 (CET) Subject: [pypy-svn] r51791 - in pypy/dist/pypy: lib/_ctypes lib/app_test/ctypes module/_rawffi module/_rawffi/test rlib rlib/test Message-ID: <20080222145723.7124E168471@codespeak.net> Author: pedronis Date: Fri Feb 22 15:57:21 2008 New Revision: 51791 Modified: pypy/dist/pypy/lib/_ctypes/structure.py pypy/dist/pypy/lib/app_test/ctypes/test_as_parameter.py pypy/dist/pypy/lib/app_test/ctypes/test_functions.py pypy/dist/pypy/module/_rawffi/interp_rawffi.py pypy/dist/pypy/module/_rawffi/test/test__rawffi.py pypy/dist/pypy/rlib/libffi.py pypy/dist/pypy/rlib/test/test_libffi.py Log: Passing structure as values to functions. _ffiletter is probably not anymore the right name for this. This leaks ffitypes at the moment, given that for some ABIs we really need the completely filled ffitype we may want to approach this by having gettypecode return something that contains a full ffitype for the structure(/array) type, instead of the current tuple. The type would own its ffitype then. Modified: pypy/dist/pypy/lib/_ctypes/structure.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/structure.py (original) +++ pypy/dist/pypy/lib/_ctypes/structure.py Fri Feb 22 15:57:21 2008 @@ -106,6 +106,7 @@ typedict.get('_anonymous_', None)) res._ffistruct = _rawffi.Structure(rawfields) res._ffishape = res._ffistruct.gettypecode() + res._ffiletter = res._ffishape def __init__(self, *args, **kwds): if not hasattr(self, '_ffistruct'): @@ -158,7 +159,6 @@ class Structure(_CData): __metaclass__ = StructureMeta - _ffiletter = 'P' _needs_free = False def _subarray(self, fieldtype, name): @@ -196,7 +196,7 @@ return fieldtype._CData_output(suba, self, offset) def _get_buffer_for_param(self): - return CArgObject(self._buffer.byptr()) + return self def _get_buffer_value(self): return self._buffer.buffer Modified: pypy/dist/pypy/lib/app_test/ctypes/test_as_parameter.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_as_parameter.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_as_parameter.py Fri Feb 22 15:57:21 2008 @@ -144,7 +144,6 @@ assert 13577625587 == int(f(self.wrap(1000000000000), self.wrap(cb))) def test_byval(self): - py.test.skip("Structure by value") # without prototype ptin = POINT(1, 2) ptout = POINT() Modified: pypy/dist/pypy/lib/app_test/ctypes/test_functions.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_functions.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_functions.py Fri Feb 22 15:57:21 2008 @@ -310,7 +310,6 @@ raises(ValueError, c_int.in_dll, dll, "_xxx_yyy") def test_byval(self): - py.test.skip("Structure by value") # without prototype ptin = POINT(1, 2) ptout = POINT() Modified: pypy/dist/pypy/module/_rawffi/interp_rawffi.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/interp_rawffi.py (original) +++ pypy/dist/pypy/module/_rawffi/interp_rawffi.py Fri Feb 22 15:57:21 2008 @@ -86,6 +86,14 @@ raise OperationError(space.w_ValueError, space.wrap( "Unknown type letter %s" % (key,))) +def unpack_typecode(space, w_typecode): + if space.is_true(space.isinstance(w_typecode, space.w_str)): + letter = space.str_w(w_typecode) + return letter2tp(space, letter) + else: + w_size, w_align = space.unpacktuple(w_typecode, expected_length=2) + return ('V', space.int_w(w_size), space.int_w(w_align)) # value object + def _get_type_(space, key): try: return TYPEMAP[key] @@ -100,6 +108,13 @@ self.w_cache = space.newdict() self.space = space + def get_arg_type(self, letter, argsize, argalignment): + space = self.space + if letter == 'V': # xxx leaks + return make_struct_ffitype(argsize, argalignment) + else: + return _get_type_(space, letter) + def get_type(self, key): space = self.space return _get_type_(space, key) @@ -125,8 +140,10 @@ else: raise argtypes_w = space.unpackiterable(w_argtypes) - argtypes = [space.str_w(w_arg) for w_arg in argtypes_w] - ffi_argtypes = [self.get_type(arg) for arg in argtypes] + argtypes = [unpack_typecode(space, w_arg) for w_arg in argtypes_w] + ffi_argtypes = [self.get_arg_type(letter, argsize, argalignment) + for letter, argsize, argalignment + in argtypes] try: ptr = self.cdll.getrawpointer(name, ffi_argtypes, ffi_restype) w_funcptr = W_FuncPtr(space, ptr, argtypes, restype) @@ -180,14 +197,6 @@ w_exception = space.getattr(w_mod, space.wrap("SegfaultException")) return OperationError(w_exception, space.wrap(reason)) -def unpack_typecode(space, w_typecode): - if space.is_true(space.isinstance(w_typecode, space.w_str)): - letter = space.str_w(w_typecode) - return letter2tp(space, letter) - else: - w_size, w_align = space.unpacktuple(w_typecode, expected_length=2) - return ('?', space.int_w(w_size), space.int_w(w_align)) - class W_DataInstance(Wrappable): def __init__(self, space, size, address=r_uint(0)): @@ -308,6 +317,7 @@ def call(self, space, args_w): from pypy.module._rawffi.array import W_ArrayInstance + from pypy.module._rawffi.structure import W_StructureInstance argnum = len(args_w) if argnum != len(self.argtypes): msg = "Wrong number of argument: expected %d, got %d" % ( @@ -315,20 +325,31 @@ raise OperationError(space.w_TypeError, space.wrap(msg)) args_ll = [] for i in range(argnum): - argtype = self.argtypes[i] + argtype_letter, argtype_size, argtype_alignment = self.argtypes[i] w_arg = args_w[i] - arg = space.interp_w(W_ArrayInstance, w_arg) - if arg.length != 1: - msg = ("Argument %d should be an array of length 1, " - "got length %d" % (i+1, arg.length)) - raise OperationError(space.w_TypeError, space.wrap(msg)) - letter = arg.shape.itemtp[0] - if letter != argtype: - if not (argtype in TYPEMAP_PTR_LETTERS and - letter in TYPEMAP_PTR_LETTERS): - msg = "Argument %d should be typecode %s, got %s" % ( - i+1, argtype, letter) + if argtype_letter == 'V': # by value object + arg = space.interp_w(W_StructureInstance, w_arg) + if (arg.shape.size != argtype_size or + arg.shape.alignment != argtype_alignment): + msg = ("Argument %d should be a structure of size %d and " + "alignment %d, " + "got instead size %d and alignment %d" % + (i+1, argtype_size, argtype_alignment, + arg.shape.size, arg.shape.alignment)) + raise OperationError(space.w_TypeError, space.wrap(msg)) + else: + arg = space.interp_w(W_ArrayInstance, w_arg) + if arg.length != 1: + msg = ("Argument %d should be an array of length 1, " + "got length %d" % (i+1, arg.length)) raise OperationError(space.w_TypeError, space.wrap(msg)) + letter = arg.shape.itemtp[0] + if letter != argtype_letter: + if not (argtype_letter in TYPEMAP_PTR_LETTERS and + letter in TYPEMAP_PTR_LETTERS): + msg = "Argument %d should be typecode %s, got %s" % ( + i+1, argtype_letter, letter) + raise OperationError(space.w_TypeError, space.wrap(msg)) args_ll.append(arg.ll_buffer) # XXX we could avoid the intermediate list args_ll if self.resarray is not None: Modified: pypy/dist/pypy/module/_rawffi/test/test__rawffi.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/test/test__rawffi.py (original) +++ pypy/dist/pypy/module/_rawffi/test/test__rawffi.py Fri Feb 22 15:57:21 2008 @@ -112,6 +112,15 @@ return callback(); } + struct x_y { + long x; + long y; + }; + + long sum_x_y(struct x_y s) { + return s.x + s.y; + } + ''')) return compile_c_module([c_file], 'x', ExternalCompilationInfo()) prepare_c_example = staticmethod(prepare_c_example) @@ -498,7 +507,7 @@ s = struct.calcsize("i") assert (repr(_rawffi.Array('i')) == "<_rawffi.Array 'i' (%d, %d)>" % (s, s)) - assert repr(_rawffi.Array((18, 2))) == "<_rawffi.Array '?' (18, 2)>" + assert repr(_rawffi.Array((18, 2))) == "<_rawffi.Array 'V' (18, 2)>" assert (repr(_rawffi.Structure([('x', 'i'), ('yz', 'i')])) == "<_rawffi.Structure 'x' 'yz' (%d, %d)>" % (2*s, s)) @@ -612,6 +621,18 @@ raises(_rawffi.SegfaultException, a.__getitem__, 3) raises(_rawffi.SegfaultException, a.__setitem__, 3, 3) + def test_struct_byvalue(self): + import _rawffi + X_Y = _rawffi.Structure([('x', 'l'), ('y', 'l')]) + x_y = X_Y() + lib = _rawffi.CDLL(self.lib_name) + sum_x_y = lib.ptr('sum_x_y', [X_Y.gettypecode()], 'l') + x_y.x = 200 + x_y.y = 220 + res = sum_x_y(x_y) + assert res[0] == 420 + x_y.free() + class AppTestAutoFree: def setup_class(cls): Modified: pypy/dist/pypy/rlib/libffi.py ============================================================================== --- pypy/dist/pypy/rlib/libffi.py (original) +++ pypy/dist/pypy/rlib/libffi.py Fri Feb 22 15:57:21 2008 @@ -40,6 +40,8 @@ FFI_BAD_TYPEDEF = rffi_platform.ConstantInteger('FFI_BAD_TYPEDEF') FFI_DEFAULT_ABI = rffi_platform.ConstantInteger('FFI_DEFAULT_ABI') + FFI_TYPE_STRUCT = rffi_platform.ConstantInteger('FFI_TYPE_STRUCT') + size_t = rffi_platform.SimpleType("size_t", rffi.ULONG) ffi_type = rffi_platform.Struct('ffi_type', [('size', rffi.ULONG), @@ -113,6 +115,7 @@ FFI_OK = cConfig.FFI_OK FFI_BAD_TYPEDEF = cConfig.FFI_BAD_TYPEDEF FFI_DEFAULT_ABI = rffi.cast(rffi.USHORT, cConfig.FFI_DEFAULT_ABI) +FFI_TYPE_STRUCT = rffi.cast(rffi.USHORT, cConfig.FFI_TYPE_STRUCT) FFI_CIFP = rffi.COpaquePtr('ffi_cif', compilation_info=CConfig. _compilation_info_) @@ -161,6 +164,14 @@ # XXX rffi.cast here... return res +def make_struct_ffitype(size, aligment): + tp = lltype.malloc(FFI_TYPE_P.TO, flavor='raw') + tp.c_type = FFI_TYPE_STRUCT + tp.c_size = rffi.cast(rffi.SIZE_T, size) + tp.c_alignment = rffi.cast(rffi.USHORT, aligment) + tp.c_elements = lltype.nullptr(FFI_TYPE_PP.TO) + return tp + def cast_type_to_ffitype(tp): """ This function returns ffi representation of rpython type tp """ Modified: pypy/dist/pypy/rlib/test/test_libffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_libffi.py (original) +++ pypy/dist/pypy/rlib/test/test_libffi.py Fri Feb 22 15:57:21 2008 @@ -175,3 +175,59 @@ del pow del libm assert not ALLOCATED + + def test_make_struct_fftiype(self): + tp = make_struct_ffitype(6, 2) + assert tp.c_type == FFI_TYPE_STRUCT + assert tp.c_size == 6 + assert tp.c_alignment == 2 + lltype.free(tp, flavor='raw') + + def test_struct_by_val(self): + from pypy.translator.tool.cbuild import compile_c_module, \ + ExternalCompilationInfo + from pypy.tool.udir import udir + + c_file = udir.ensure("test_libffi", dir=1).join("xlib.c") + c_file.write(py.code.Source(''' + #include + #include + + struct x_y { + long x; + long y; + }; + + long sum_x_y(struct x_y s) { + return s.x + s.y; + } + + long sum_x_y_p(struct x_y *p) { + return p->x + p->y; + } + + ''')) + lib_name = compile_c_module([c_file], 'x', ExternalCompilationInfo()) + + lib = CDLL(lib_name) + + size = ffi_type_slong.c_size*2 + alignment = ffi_type_slong.c_alignment + tp = make_struct_ffitype(size, alignment) + + sum_x_y = lib.getrawpointer('sum_x_y', [tp], ffi_type_slong) + + buffer = lltype.malloc(rffi.LONGP.TO, 3, flavor='raw') + buffer[0] = 200 + buffer[1] = 220 + buffer[2] = 666 + sum_x_y.call([rffi.cast(rffi.VOIDP, buffer)], + rffi.cast(rffi.VOIDP, rffi.ptradd(buffer, 2))) + assert buffer[2] == 420 + + lltype.free(buffer, flavor='raw') + del sum_x_y + lltype.free(tp, flavor='raw') + del lib + + assert not ALLOCATED From tverwaes at codespeak.net Fri Feb 22 16:14:25 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Fri, 22 Feb 2008 16:14:25 +0100 (CET) Subject: [pypy-svn] r51794 - in pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk: . test Message-ID: <20080222151425.0C07C1684F6@codespeak.net> Author: tverwaes Date: Fri Feb 22 16:14:24 2008 New Revision: 51794 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py Log: keeping shadows up to date Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py Fri Feb 22 16:14:24 2008 @@ -183,17 +183,22 @@ # XXX XXX # Need to find better way of handling overloading of shadows!!! + def setshadow(self, shadow): + self._shadow = shadow + def as_special_get_shadow(self, TheClass): shadow = self._shadow if shadow is None: - self._shadow = shadow = TheClass(self) - assert isinstance(shadow, TheClass) + shadow = TheClass(self) + elif not isinstance(shadow, TheClass): + shadow.invalidate() + shadow = TheClass(self) + shadow.check_for_updates() return shadow def as_class_get_shadow(self): from pypy.lang.smalltalk.shadow import ClassShadow shadow = self.as_special_get_shadow(ClassShadow) - shadow.check_for_updates() return shadow def as_link_get_shadow(self): Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py Fri Feb 22 16:14:24 2008 @@ -6,13 +6,28 @@ """A shadow is an optional extra bit of information that can be attached at run-time to any Smalltalk object. """ + def __init__(self, w_self): + self._w_self = w_self + self.invalidate() + def invalidate(self): """XXX This should get called whenever the base Smalltalk object changes.""" + self.invalid = True def w_self(self): return self._w_self + def check_for_updates(self): + if self.invalid: + self.w_self().setshadow(self) + self.update_shadow() + + # XXX XXX Remove function when fixing superclass to AbstractShadow + def update_shadow(self): + pass + + # ____________________________________________________________ POINTERS = 0 @@ -33,18 +48,13 @@ (i.e. used as the class of another Smalltalk object). """ def __init__(self, w_self): - self._w_self = w_self - self.invalidate() + AbstractShadow.__init__(self, w_self) def invalidate(self): + AbstractShadow.invalidate(self) self.methoddict = {} self.s_superclass = None # the ClassShadow of the super class self.name = None - self.invalid = True - - def check_for_updates(self): - if self.invalid: - self.update_shadow() def update_shadow(self): from pypy.lang.smalltalk import objtable @@ -220,7 +230,7 @@ class LinkedListShadow(AbstractShadow): def __init__(self, w_self): - self._w_self = w_self + AbstractShadow.__init__(self, w_self) def w_firstlink(self): return self.w_self().fetch(constants.FIRST_LINK_INDEX) @@ -263,7 +273,7 @@ """A shadow for Smalltalk objects that are semaphores """ def __init__(self, w_self): - self._w_self = w_self + LinkedListShadow.__init__(self, w_self) def put_to_sleep(self, s_process): priority = s_process.priority() @@ -311,7 +321,7 @@ class LinkShadow(AbstractShadow): def __init__(self, w_self): - self._w_self = w_self + AbstractShadow.__init__(self, w_self) def next(self): return self.w_self().fetch(constants.NEXT_LINK_INDEX) @@ -323,7 +333,7 @@ """A shadow for Smalltalk objects that are processes """ def __init__(self, w_self): - self._w_self = w_self + LinkShadow.__init__(self, w_self) def priority(self): return utility.unwrap_int(self.w_self().fetch(constants.PROCESS_PRIORITY_INDEX)) @@ -342,7 +352,7 @@ class AssociationShadow(AbstractShadow): def __init__(self, w_self): - self._w_self = w_self + AbstractShadow.__init__(self, w_self) def key(self): return self.w_self().fetch(constants.ASSOCIATION_KEY_INDEX) @@ -355,7 +365,7 @@ class SchedulerShadow(AbstractShadow): def __init__(self, w_self): - self._w_self = w_self + AbstractShadow.__init__(self, w_self) def s_active_process(self): return self.w_self().fetch(constants.SCHEDULER_ACTIVE_PROCESS_INDEX).as_process_get_shadow() @@ -376,6 +386,7 @@ def __init__(self, w_self): self._w_self = w_self + self.invalidate() def s_home(self): raise NotImplementedError() @@ -386,6 +397,16 @@ # XXX XXX Remove function when fixing superclass to AbstractShadow def invalidate(self): + self.invalid = True + + # XXX XXX Remove function when fixing superclass to AbstractShadow + def check_for_updates(self): + if self.invalid: + self.w_self().setshadow(self) + self.update_shadow() + + # XXX XXX Remove function when fixing superclass to AbstractShadow + def update_shadow(self): pass def w_receiver(self): @@ -484,7 +505,7 @@ class BlockContextShadow(ContextPartShadow): def __init__(self, w_self): - self._w_self = w_self + ContextPartShadow.__init__(self, w_self) def expected_argument_count(self): return utility.unwrap_int(self.w_self().fetch(constants.BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX)) @@ -504,7 +525,7 @@ class MethodContextShadow(ContextPartShadow): def __init__(self, w_self): - self._w_self = w_self + ContextPartShadow.__init__(self, w_self) def w_method(self): return self.w_self().fetch(constants.MTHDCTX_METHOD) Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py Fri Feb 22 16:14:24 2008 @@ -24,7 +24,6 @@ w_selector = utility.wrap_string(selector) w_methoddict.store(constants.METHODDICT_NAMES_INDEX+pos, w_selector) w_array.store(pos, w_compiledmethod) - #print w_methoddict._vars return w_methoddict def build_smalltalk_class(name, format, w_superclass=w_Object, From pedronis at codespeak.net Fri Feb 22 16:17:53 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 22 Feb 2008 16:17:53 +0100 (CET) Subject: [pypy-svn] r51796 - pypy/dist/pypy/lib/_ctypes Message-ID: <20080222151753.86A2316843E@codespeak.net> Author: pedronis Date: Fri Feb 22 16:17:53 2008 New Revision: 51796 Modified: pypy/dist/pypy/lib/_ctypes/array.py pypy/dist/pypy/lib/_ctypes/function.py pypy/dist/pypy/lib/_ctypes/pointer.py pypy/dist/pypy/lib/_ctypes/primitive.py pypy/dist/pypy/lib/_ctypes/structure.py pypy/dist/pypy/lib/_ctypes/union.py Log: introduce _ffiargshape, use this for arguments instead of _ffiletter, this is really a (size,align) tuple for structures now. _ffiletter should go away once we have added support for functions returning structs. Modified: pypy/dist/pypy/lib/_ctypes/array.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/array.py (original) +++ pypy/dist/pypy/lib/_ctypes/array.py Fri Feb 22 16:17:53 2008 @@ -125,7 +125,7 @@ class Array(_CData): __metaclass__ = ArrayMeta - _ffiletter = 'P' + _ffiargshape = _ffiletter = 'P' _needs_free = False def __init__(self, *args): Modified: pypy/dist/pypy/lib/_ctypes/function.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/function.py (original) +++ pypy/dist/pypy/lib/_ctypes/function.py Fri Feb 22 16:17:53 2008 @@ -17,7 +17,7 @@ _argtypes_ = None _restype_ = None - _ffiletter = 'P' + _ffiargshape = _ffiletter = 'P' _ffishape = 'P' _needs_free = False @@ -47,7 +47,7 @@ # XXX finish this one, we need to be able to jump there somehow elif callable(argument): self.callable = argument - argtypes = [arg._ffiletter for arg in self._argtypes_] + argtypes = [arg._ffiargshape for arg in self._argtypes_] restype = self._restype_._ffiletter self._ptr = _rawffi.CallbackPtr(argument, argtypes, restype) self._needs_free = True @@ -90,8 +90,8 @@ if restype is None: import ctypes restype = ctypes.c_int - argletters = [arg._ffiletter for arg in argtypes] - return self.dll._handle.ptr(self.name, argletters, restype._ffiletter) + argshapes = [arg._ffiargshape for arg in argtypes] + return self.dll._handle.ptr(self.name, argshapes, restype._ffiletter) def _guess_argtypes(self, args): from _ctypes import _CData Modified: pypy/dist/pypy/lib/_ctypes/pointer.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/pointer.py (original) +++ pypy/dist/pypy/lib/_ctypes/pointer.py Fri Feb 22 16:17:53 2008 @@ -13,6 +13,7 @@ size = _rawffi.sizeof('P'), align = _rawffi.alignment('P'), length = 1, + _ffiargshape = 'P', _ffiletter = 'P', _ffishape = 'P', ) Modified: pypy/dist/pypy/lib/_ctypes/primitive.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/primitive.py (original) +++ pypy/dist/pypy/lib/_ctypes/primitive.py Fri Feb 22 16:17:53 2008 @@ -56,6 +56,7 @@ ffiarray = _rawffi.Array(tp) result = type.__new__(self, name, bases, dct) result._ffiletter = tp + result._ffiargshape = tp result._ffishape = tp result._ffiarray = ffiarray if tp == 'z': Modified: pypy/dist/pypy/lib/_ctypes/structure.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/structure.py (original) +++ pypy/dist/pypy/lib/_ctypes/structure.py Fri Feb 22 16:17:53 2008 @@ -13,7 +13,6 @@ alignment = 1 pos = [] for fieldname, ctype in fields: - letter = ctype._ffiletter fieldsize = ctypes.sizeof(ctype) fieldalignment = ctypes.alignment(ctype) size = round_up(size, fieldalignment) @@ -39,7 +38,7 @@ self.__dict__.get('_anonymous_', None)) self._ffistruct = _rawffi.Structure(rawfields) _CDataMeta.__setattr__(self, '_fields_', value) - self._ffishape = self._ffistruct.gettypecode() + self._ffiargshape = self._ffishape = self._ffistruct.gettypecode() return _CDataMeta.__setattr__(self, name, value) @@ -106,7 +105,7 @@ typedict.get('_anonymous_', None)) res._ffistruct = _rawffi.Structure(rawfields) res._ffishape = res._ffistruct.gettypecode() - res._ffiletter = res._ffishape + res._ffiargshape = res._ffishape def __init__(self, *args, **kwds): if not hasattr(self, '_ffistruct'): Modified: pypy/dist/pypy/lib/_ctypes/union.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/union.py (original) +++ pypy/dist/pypy/lib/_ctypes/union.py Fri Feb 22 16:17:53 2008 @@ -15,6 +15,7 @@ typedict.get('_anonymous_', None)) res._ffishape = (res._sizeofinstances(), res._alignmentofinstances()) + res._ffiargshape = res._ffishape # we need to create an array of size one for each # of our elements res._ffiarrays = {} @@ -58,14 +59,13 @@ for name, field in self._fieldtypes.iteritems(): self._ffiarrays[name] = _rawffi.Array(field.ctype._ffishape) _CDataMeta.__setattr__(self, '_fields_', value) - self._ffishape = (self._sizeofinstances(), - self._alignmentofinstances()) + self._ffiargshape = self._ffishape = (self._sizeofinstances(), + self._alignmentofinstances()) return _CDataMeta.__setattr__(self, name, value) class Union(_CData): __metaclass__ = UnionMeta - _ffiletter = 'P' _needs_free = False def __getattr__(self, name): From arigo at codespeak.net Fri Feb 22 16:18:02 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 22 Feb 2008 16:18:02 +0100 (CET) Subject: [pypy-svn] r51797 - pypy/branch/unified-rtti/pypy/rpython/memory/gctransform Message-ID: <20080222151802.E7D1B1684F6@codespeak.net> Author: arigo Date: Fri Feb 22 16:18:01 2008 New Revision: 51797 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py Log: Explain why this implementation is not there. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py Fri Feb 22 16:18:01 2008 @@ -327,6 +327,10 @@ def consider_constant(self, TYPE, value): self.layoutbuilder.consider_constant(TYPE, value, self.gcdata.gc) + def initialize_typeinfo(self, typeinfo, rtti, TYPE): + raise Exception("for now, the layoutbuilder should have found " + "all possible GC types - got %r" % (TYPE,)) + #def get_type_id(self, TYPE): # this method is attached to the instance and redirects to # layoutbuilder.get_type_id(). From arigo at codespeak.net Fri Feb 22 16:18:28 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 22 Feb 2008 16:18:28 +0100 (CET) Subject: [pypy-svn] r51798 - pypy/branch/unified-rtti/pypy/rpython/memory/gc Message-ID: <20080222151828.E787E16843E@codespeak.net> Author: arigo Date: Fri Feb 22 16:18:28 2008 New Revision: 51798 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/generation.py Log: Merge r51633 from the trunk. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gc/generation.py Fri Feb 22 16:18:28 2008 @@ -11,11 +11,11 @@ # in the nursery. It is initially set on all prebuilt and old objects, # and gets cleared by the write_barrier() when we write in them a # pointer to a young object. -GCFLAG_NO_YOUNG_PTRS = SemiSpaceGC.first_unused_gcflag << 1 +GCFLAG_NO_YOUNG_PTRS = SemiSpaceGC.first_unused_gcflag << 0 # The following flag is set for static roots which are not on the list # of static roots yet, but will appear with write barrier -GCFLAG_NO_HEAP_PTRS = SemiSpaceGC.first_unused_gcflag << 2 +GCFLAG_NO_HEAP_PTRS = SemiSpaceGC.first_unused_gcflag << 1 DEBUG_PRINT = False @@ -29,7 +29,7 @@ inline_simple_malloc_varsize = True needs_write_barrier = True prebuilt_gc_objects_are_static_roots = False - first_unused_gcflag = SemiSpaceGC.first_unused_gcflag << 3 + first_unused_gcflag = SemiSpaceGC.first_unused_gcflag << 2 def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, nursery_size=128, From arigo at codespeak.net Fri Feb 22 16:19:03 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 22 Feb 2008 16:19:03 +0100 (CET) Subject: [pypy-svn] r51799 - in pypy/branch/unified-rtti/pypy/rpython: lltypesystem lltypesystem/test memory memory/gc Message-ID: <20080222151903.D08FE16843E@codespeak.net> Author: arigo Date: Fri Feb 22 16:19:03 2008 New Revision: 51799 Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/llarena.py pypy/branch/unified-rtti/pypy/rpython/lltypesystem/llmemory.py pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_llarena.py pypy/branch/unified-rtti/pypy/rpython/memory/gc/generation.py pypy/branch/unified-rtti/pypy/rpython/memory/gc/semispace.py pypy/branch/unified-rtti/pypy/rpython/memory/gcheader.py pypy/branch/unified-rtti/pypy/rpython/memory/support.py Log: Merge r51675 from the trunk. Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/llarena.py Fri Feb 22 16:19:03 2008 @@ -1,4 +1,4 @@ -import array +import array, weakref from pypy.rpython.lltypesystem import lltype, llmemory # An "arena" is a large area of memory which can hold a number of @@ -15,6 +15,7 @@ class Arena(object): object_arena_location = {} # {container: (arena, offset)} + old_object_arena_location = weakref.WeakKeyDictionary() def __init__(self, nbytes, zero): self.nbytes = nbytes @@ -29,7 +30,7 @@ if size is None: stop = self.nbytes else: - stop = start + size + stop = start + llmemory.raw_malloc_usage(size) assert 0 <= start <= stop <= self.nbytes for offset, ptr in self.objectptrs.items(): size = self.objectsizes[offset] @@ -79,9 +80,7 @@ addr2 = size._raw_malloc([], zero=zero) pattern = 'X' + 'x'*(bytes-1) self.usagemap[offset:offset+bytes] = array.array('c', pattern) - self.objectptrs[offset] = addr2.ptr - self.objectsizes[offset] = bytes - Arena.object_arena_location[addr2.ptr._obj] = self, offset + self.setobject(addr2, offset, bytes) # common case: 'size' starts with a GCHeaderOffset. In this case # we can also remember that the real object starts after the header. while isinstance(size, RoundedUpForAllocation): @@ -91,12 +90,17 @@ objaddr = addr2 + size.offsets[0] hdrbytes = llmemory.raw_malloc_usage(size.offsets[0]) objoffset = offset + hdrbytes - assert objoffset not in self.objectptrs - self.objectptrs[objoffset] = objaddr.ptr - self.objectsizes[objoffset] = bytes - hdrbytes - Arena.object_arena_location[objaddr.ptr._obj] = self, objoffset + self.setobject(objaddr, objoffset, bytes - hdrbytes) return addr2 + def setobject(self, objaddr, offset, bytes): + assert offset not in self.objectptrs + self.objectptrs[offset] = objaddr.ptr + self.objectsizes[offset] = bytes + container = objaddr.ptr._obj + Arena.object_arena_location[container] = self, offset + Arena.old_object_arena_location[container] = self, offset + class fakearenaaddress(llmemory.fakeaddress): def __init__(self, arena, offset): @@ -148,6 +152,7 @@ return True def compare_with_fakeaddr(self, other): + other = other._fixup() if not other: return None, None obj = other.ptr._obj @@ -205,6 +210,30 @@ return self.arena._getid() + self.offset +def _getfakearenaaddress(addr): + """Logic to handle test_replace_object_with_stub().""" + if isinstance(addr, fakearenaaddress): + return addr + else: + assert isinstance(addr, llmemory.fakeaddress) + assert addr, "NULL address" + # it must be possible to use the address of an already-freed + # arena object + obj = addr.ptr._getobj(check=False) + return _oldobj_to_address(obj) + +def _oldobj_to_address(obj): + obj = obj._normalizedcontainer(check=False) + try: + arena, offset = Arena.old_object_arena_location[obj] + except KeyError: + if obj._was_freed(): + msg = "taking address of %r, but it was freed" + else: + msg = "taking address of %r, but it is not in an arena" + raise RuntimeError(msg % (obj,)) + return arena.getaddr(offset) + class RoundedUpForAllocation(llmemory.AddressOffset): """A size that is rounded up in order to preserve alignment of objects following it. For arenas containing heterogenous objects. @@ -247,7 +276,7 @@ """Free all objects in the arena, which can then be reused. The arena is filled with zeroes if 'zero' is True. This can also be used on a subrange of the arena.""" - assert isinstance(arena_addr, fakearenaaddress) + arena_addr = _getfakearenaaddress(arena_addr) arena_addr.arena.reset(zero, arena_addr.offset, size) def arena_reserve(addr, size, check_alignment=True): @@ -256,7 +285,7 @@ overlap. The size must be symbolic; in non-translated version this is used to know what type of lltype object to allocate.""" from pypy.rpython.memory.lltypelayout import memory_alignment - assert isinstance(addr, fakearenaaddress) + addr = _getfakearenaaddress(addr) if check_alignment and (addr.offset & (memory_alignment-1)) != 0: raise ArenaError("object at offset %d would not be correctly aligned" % (addr.offset,)) Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/llmemory.py Fri Feb 22 16:19:03 2008 @@ -372,7 +372,9 @@ # NOTE: the 'ptr' in the addresses must be normalized. # Use cast_ptr_to_adr() instead of directly fakeaddress() if unsure. def __init__(self, ptr): - self.ptr = ptr or None # null ptr => None + if ptr is not None and ptr._obj0 is None: + ptr = None # null ptr => None + self.ptr = ptr def __repr__(self): if self.ptr is None: @@ -407,8 +409,8 @@ def __eq__(self, other): if isinstance(other, fakeaddress): - obj1 = self.ptr - obj2 = other.ptr + obj1 = self._fixup().ptr + obj2 = other._fixup().ptr if obj1 is not None: obj1 = obj1._obj if obj2 is not None: obj2 = obj2._obj return obj1 == obj2 @@ -450,8 +452,9 @@ return self.ptr def _cast_to_ptr(self, EXPECTED_TYPE): - if self: - return cast_any_ptr(EXPECTED_TYPE, self.ptr) + addr = self._fixup() + if addr: + return cast_any_ptr(EXPECTED_TYPE, addr.ptr) else: return lltype.nullptr(EXPECTED_TYPE.TO) @@ -461,6 +464,14 @@ else: return 0 + def _fixup(self): + if self.ptr is not None and self.ptr._was_freed(): + # hack to support llarena.test_replace_object_with_stub() + from pypy.rpython.lltypesystem import llarena + return llarena._getfakearenaaddress(self) + else: + return self + # ____________________________________________________________ class fakeaddresswithflags(object): Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/lltype.py Fri Feb 22 16:19:03 2008 @@ -956,7 +956,7 @@ self._set_solid(solid) self._set_obj0(obj0) - def _getobj(self): + def _getobj(self, check=True): obj = self._obj0 if obj is not None: if self._weak: @@ -965,12 +965,17 @@ raise RuntimeError("accessing already garbage collected %r" % (self._T,)) if isinstance(obj, _container): - obj._check() + if check: + obj._check() elif isinstance(obj, str) and obj.startswith("delayed!"): raise DelayedPointer return obj _obj = property(_getobj) + def _was_freed(self): + return (self._obj0 is not None and + self._getobj(check=False)._was_freed()) + def __getattr__(self, field_name): # ! can only return basic or ptr ! if isinstance(self._T, Struct): if field_name in self._T._flds: @@ -1171,6 +1176,10 @@ from pypy.rpython.lltypesystem import llmemory if isinstance(self._T, FuncType): return llmemory.fakeaddress(self) + elif self._was_freed(): + # hack to support llarena.test_replace_object_with_stub() + from pypy.rpython.lltypesystem import llarena + return llarena._oldobj_to_address(self._getobj(check=False)) elif isinstance(self._obj, _subarray): return llmemory.fakeaddress(self) ## # return an address built as an offset in the whole array @@ -1185,8 +1194,8 @@ def _as_ptr(self): return self - def _as_obj(self): - return self._obj + def _as_obj(self, check=True): + return self._getobj(check=check) def _expose(self, offset, val): """XXX A nice docstring here""" @@ -1251,13 +1260,13 @@ class _container(object): __slots__ = () - def _parentstructure(self): + def _parentstructure(self, check=True): return None def _check(self): pass def _as_ptr(self): return _ptr(Ptr(self._TYPE), self, True) - def _as_obj(self): + def _as_obj(self, check=True): return self def _normalizedcontainer(self): return self @@ -1288,7 +1297,16 @@ self._storage = None def _was_freed(self): - return self._storage is None + if self._storage is None: + return True + if self._wrparent is None: + return False + parent = self._wrparent() + if parent is None: + raise RuntimeError("accessing sub%s %r,\n" + "but already garbage collected parent %r" + % (self._kind, self, self._parent_type)) + return parent._was_freed() def _setparentstructure(self, parent, parentindex): self._wrparent = weakref.ref(parent) @@ -1300,14 +1318,15 @@ # keep strong reference to parent, we share the same allocation self._keepparent = parent - def _parentstructure(self): + def _parentstructure(self, check=True): if self._wrparent is not None: parent = self._wrparent() if parent is None: raise RuntimeError("accessing sub%s %r,\n" "but already garbage collected parent %r" % (self._kind, self, self._parent_type)) - parent._check() + if check: + parent._check() return parent return None @@ -1316,14 +1335,15 @@ raise RuntimeError("accessing freed %r" % self._TYPE) self._parentstructure() - def _normalizedcontainer(self): + def _normalizedcontainer(self, check=True): # if we are the first inlined substructure of a structure, # return the whole (larger) structure instead container = self while True: - parent, index = parentlink(container) + parent = container._parentstructure(check=check) if parent is None: break + index = container._parent_index T = typeOf(parent) if (not isinstance(T, Struct) or T._first_struct()[0] != index or isinstance(T, FixedSizeArray)): @@ -1518,7 +1538,7 @@ def __repr__(self): return '<_subarray at %r in %r>' % (self._parent_index, - self._parentstructure()) + self._parentstructure(check=False)) def getlength(self): assert isinstance(self._TYPE, FixedSizeArray) Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_llarena.py Fri Feb 22 16:19:03 2008 @@ -165,6 +165,8 @@ plist.append(llmemory.cast_adr_to_ptr(b, SPTR)) # clear blist[1] and blist[2] but not blist[0] nor blist[3] arena_reset(blist[1], llmemory.raw_malloc_usage(precomputed_size)*2, False) + py.test.raises(RuntimeError, "plist[1].x") # marked as freed + py.test.raises(RuntimeError, "plist[2].x") # marked as freed # re-reserve object at index 1 and 2 blist[1] = reserve(1) blist[2] = reserve(2) @@ -173,6 +175,10 @@ assert plist[3].x == 103 py.test.raises(RuntimeError, "plist[1].x") # marked as freed py.test.raises(RuntimeError, "plist[2].x") # marked as freed + # but we can still cast the old ptrs to addresses, which compare equal + # to the new ones we gotq + assert llmemory.cast_ptr_to_adr(plist[1]) == blist[1] + assert llmemory.cast_ptr_to_adr(plist[2]) == blist[2] # check via addresses assert (blist[0] + llmemory.offsetof(SX, 'x')).signed[0] == 100 assert (blist[3] + llmemory.offsetof(SX, 'x')).signed[0] == 103 @@ -204,6 +210,45 @@ assert llmemory.cast_adr_to_int(a) == llmemory.cast_adr_to_int(a1) assert llmemory.cast_adr_to_int(a+1) == llmemory.cast_adr_to_int(a1) + 1 +def test_replace_object_with_stub(): + from pypy.rpython.memory.gcheader import GCHeaderBuilder + HDR = lltype.Struct('HDR', ('x', lltype.Signed)) + S = lltype.GcStruct('S', ('y', lltype.Signed), ('z', lltype.Signed)) + STUB = lltype.GcStruct('STUB', ('t', lltype.Char)) + gcheaderbuilder = GCHeaderBuilder(HDR) + size_gc_header = gcheaderbuilder.size_gc_header + + a = arena_malloc(50, True) + hdraddr = a + 12 + arena_reserve(hdraddr, size_gc_header + llmemory.sizeof(S)) + hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR)) + hdr.x = 42 + obj = llmemory.cast_adr_to_ptr(hdraddr + size_gc_header, lltype.Ptr(S)) + obj.y = -5 + obj.z = -6 + + hdraddr = llmemory.cast_ptr_to_adr(obj) - size_gc_header + arena_reset(hdraddr, size_gc_header + llmemory.sizeof(S), False) + arena_reserve(hdraddr, size_gc_header + llmemory.sizeof(STUB)) + + # check that it possible to reach the newly reserved HDR+STUB + # via the header of the old 'obj' pointer, both via the existing + # 'hdraddr': + hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR)) + hdr.x = 46 + stub = llmemory.cast_adr_to_ptr(hdraddr + size_gc_header, lltype.Ptr(STUB)) + stub.t = '!' + + # and via a (now-invalid) pointer to the old 'obj': (this is needed + # because during a garbage collection there are still pointers to + # the old 'obj' around to be fixed) + hdraddr = llmemory.cast_ptr_to_adr(obj) - size_gc_header + hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR)) + assert hdr.x == 46 + stub = llmemory.cast_adr_to_ptr(hdraddr + size_gc_header, + lltype.Ptr(STUB)) + assert stub.t == '!' + def test_llinterpreted(): from pypy.rpython.test.test_llinterp import interpret Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gc/generation.py Fri Feb 22 16:19:03 2008 @@ -22,8 +22,8 @@ class GenerationGC(SemiSpaceGC): """A basic generational GC: it's a SemiSpaceGC with an additional nursery for young objects. A write barrier is used to ensure that - old objects that contain pointers to young objects are in a linked - list, chained to each other via their 'forw' header field. + old objects that contain pointers to young objects are recorded in + a list. """ inline_simple_malloc = True inline_simple_malloc_varsize = True @@ -44,19 +44,16 @@ self.initial_nursery_size = nursery_size self.auto_nursery_size = auto_nursery_size self.min_nursery_size = min_nursery_size - - def setup(self): - SemiSpaceGC.setup(self) - self.reset_nursery() - self.old_objects_pointing_to_young = NULL - # ^^^ the head of a linked list inside the old objects space; it + self.old_objects_pointing_to_young = self.AddressStack() + # ^^^ a list of addresses inside the old objects space; it # may contain static prebuilt objects as well. More precisely, # it lists exactly the old and static objects whose - # GCFLAG_NO_YOUNG_PTRS bit is not set. The 'forw' header field - # of such objects is abused for this linked list; it needs to be - # reset to its correct value when GCFLAG_NO_YOUNG_PTRS is set - # again at the start of a collection. + # GCFLAG_NO_YOUNG_PTRS bit is not set. self.young_objects_with_weakrefs = self.AddressStack() + self.reset_nursery() + + def setup(self): + SemiSpaceGC.setup(self) self.set_nursery_size(self.initial_nursery_size) # the GC is fully setup now. The rest can make use of it. if self.auto_nursery_size: @@ -226,14 +223,11 @@ # the next usage. def reset_young_gcflags(self): - obj = self.old_objects_pointing_to_young - while obj: + oldlist = self.old_objects_pointing_to_young + while oldlist.non_empty(): + obj = oldlist.pop() hdr = self.header(obj) hdr.tid |= GCFLAG_NO_YOUNG_PTRS - nextobj = hdr.forw - self.init_forwarding(obj) - obj = nextobj - self.old_objects_pointing_to_young = NULL def weakrefs_grow_older(self): while self.young_objects_with_weakrefs.non_empty(): @@ -278,19 +272,15 @@ def collect_oldrefs_to_nursery(self): # Follow the old_objects_pointing_to_young list and move the - # young objects they point to out of the nursery. The 'forw' - # fields are reset to their correct value along the way. + # young objects they point to out of the nursery. count = 0 - obj = self.old_objects_pointing_to_young - while obj: + oldlist = self.old_objects_pointing_to_young + while oldlist.non_empty(): count += 1 - nextobj = self.header(obj).forw - self.init_forwarding(obj) + obj = oldlist.pop() self.trace_and_drag_out_of_nursery(obj) - obj = nextobj if DEBUG_PRINT: llop.debug_print(lltype.Void, "collect_oldrefs_to_nursery", count) - self.old_objects_pointing_to_young = NULL def collect_roots_in_nursery(self): # we don't need to trace prebuilt GcStructs during a minor collect: @@ -362,8 +352,7 @@ "nursery object with GCFLAG_NO_YOUNG_PTRS") oldhdr = self.header(addr_struct) if self.is_in_nursery(addr): - oldhdr.forw = self.old_objects_pointing_to_young - self.old_objects_pointing_to_young = addr_struct + self.old_objects_pointing_to_young.append(addr_struct) oldhdr.tid &= ~GCFLAG_NO_YOUNG_PTRS if oldhdr.tid & GCFLAG_NO_HEAP_PTRS: self.move_to_static_roots(addr_struct) Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gc/semispace.py Fri Feb 22 16:19:03 2008 @@ -15,8 +15,9 @@ TYPEID_MASK = 0xffff first_gcflag = 1 << 16 -GCFLAG_IMMORTAL = first_gcflag -GCFLAG_FINALIZATION_ORDERING = first_gcflag << 1 +GCFLAG_FORWARDED = first_gcflag +GCFLAG_IMMORTAL = first_gcflag << 1 +GCFLAG_FINALIZATION_ORDERING = first_gcflag << 2 DEBUG_PRINT = False memoryError = MemoryError() @@ -26,12 +27,14 @@ inline_simple_malloc = True inline_simple_malloc_varsize = True needs_zero_gc_pointers = False - first_unused_gcflag = first_gcflag << 2 + first_unused_gcflag = first_gcflag << 3 total_collection_time = 0.0 total_collection_count = 0 - HDR = lltype.Struct('header', ('forw', llmemory.Address), - ('tid', lltype.Signed)) + HDR = lltype.Struct('header', ('tid', lltype.Signed)) + FORWARDSTUB = lltype.GcStruct('forwarding_stub', + ('forw', llmemory.Address)) + FORWARDSTUBPTR = lltype.Ptr(FORWARDSTUB) def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096, max_space_size=sys.maxint//2+1): @@ -41,6 +44,8 @@ self.gcheaderbuilder = GCHeaderBuilder(self.HDR) self.AddressStack = get_address_stack(chunk_size) self.AddressDeque = get_address_deque(chunk_size) + self.finalizer_lock_count = 0 + self.red_zone = 0 def setup(self): if DEBUG_PRINT: @@ -55,8 +60,6 @@ self.objects_with_finalizers = self.AddressDeque() self.run_finalizers = self.AddressDeque() self.objects_with_weakrefs = self.AddressStack() - self.finalizer_lock_count = 0 - self.red_zone = 0 def disable_finalizers(self): self.finalizer_lock_count += 1 @@ -289,15 +292,13 @@ root.address[0] = self.copy(root.address[0]) def copy(self, obj): - # Objects not living the GC heap have all been initialized by - # setting their 'forw' address so that it points to themselves. - # The logic below will thus simply return 'obj' if 'obj' is prebuilt. if self.is_forwarded(obj): #llop.debug_print(lltype.Void, obj, "already copied to", self.get_forwarding_address(obj)) return self.get_forwarding_address(obj) else: newaddr = self.free - totalsize = self.size_gc_header() + self.get_size(obj) + objsize = self.get_size(obj) + totalsize = self.size_gc_header() + objsize llarena.arena_reserve(newaddr, totalsize) raw_memcopy(obj - self.size_gc_header(), newaddr, totalsize) self.free += totalsize @@ -305,7 +306,7 @@ #llop.debug_print(lltype.Void, obj, "copied to", newobj, # "tid", self.header(obj).tid, # "size", totalsize) - self.set_forwarding_address(obj, newobj) + self.set_forwarding_address(obj, newobj, objsize) return newobj def trace_and_copy(self, obj): @@ -316,14 +317,35 @@ pointer.address[0] = self.copy(pointer.address[0]) def is_forwarded(self, obj): - return self.header(obj).forw != NULL + return self.header(obj).tid & GCFLAG_FORWARDED != 0 + # note: all prebuilt objects also have this flag set def get_forwarding_address(self, obj): - return self.header(obj).forw + tid = self.header(obj).tid + if tid & GCFLAG_IMMORTAL: + return obj # prebuilt objects are "forwarded" to themselves + else: + stub = llmemory.cast_adr_to_ptr(obj, self.FORWARDSTUBPTR) + return stub.forw - def set_forwarding_address(self, obj, newobj): - gc_info = self.header(obj) - gc_info.forw = newobj + def set_forwarding_address(self, obj, newobj, objsize): + # To mark an object as forwarded, we set the GCFLAG_FORWARDED and + # overwrite the object with a FORWARDSTUB. Doing so is a bit + # long-winded on llarena, but it all melts down to two memory + # writes after translation to C. + size_gc_header = self.size_gc_header() + stubsize = llmemory.sizeof(self.FORWARDSTUB) + tid = self.header(obj).tid + ll_assert(tid & GCFLAG_IMMORTAL == 0, "unexpected GCFLAG_IMMORTAL") + ll_assert(tid & GCFLAG_FORWARDED == 0, "unexpected GCFLAG_FORWARDED") + # replace the object at 'obj' with a FORWARDSTUB. + hdraddr = obj - size_gc_header + llarena.arena_reset(hdraddr, size_gc_header + objsize, False) + llarena.arena_reserve(hdraddr, size_gc_header + stubsize) + hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(self.HDR)) + hdr.tid = tid | GCFLAG_FORWARDED + stub = llmemory.cast_adr_to_ptr(obj, self.FORWARDSTUBPTR) + stub.forw = newobj def get_size(self, obj): typeid = self.get_type_id(obj) @@ -340,26 +362,24 @@ return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) def get_type_id(self, addr): - return self.header(addr).tid & TYPEID_MASK + tid = self.header(addr).tid + ll_assert(tid & (GCFLAG_FORWARDED|GCFLAG_IMMORTAL) != GCFLAG_FORWARDED, + "get_type_id on forwarded obj") + # Non-prebuilt forwarded objects are overwritten with a FORWARDSTUB. + # Although calling get_type_id() on a forwarded object works by itself, + # we catch it as an error because it's likely that what is then + # done with the typeid is bogus. + return tid & TYPEID_MASK def init_gc_object(self, addr, typeid, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - #hdr.forw = NULL -- unneeded, the space is initially filled with zero hdr.tid = typeid | flags def init_gc_object_immortal(self, addr, typeid, flags=0): - # immortal objects always have forward to themselves hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.tid = typeid | flags | GCFLAG_IMMORTAL - self.init_forwarding(addr + self.gcheaderbuilder.size_gc_header) - - def init_forwarding(self, obj): - hdr = self.header(obj) - if hdr.tid & GCFLAG_IMMORTAL: - hdr.forw = obj # prebuilt objects point to themselves, - # so that a collection does not move them - else: - hdr.forw = NULL + hdr.tid = typeid | flags | GCFLAG_IMMORTAL | GCFLAG_FORWARDED + # immortal objects always have GCFLAG_FORWARDED set; + # see get_forwarding_address(). def deal_with_objects_with_finalizers(self, scan): # walk over list of objects with finalizers @@ -371,6 +391,7 @@ new_with_finalizer = self.AddressDeque() marked = self.AddressDeque() pending = self.AddressStack() + self.tmpstack = self.AddressStack() while self.objects_with_finalizers.non_empty(): x = self.objects_with_finalizers.popleft() ll_assert(self._finalization_state(x) != 1, @@ -385,11 +406,9 @@ state = self._finalization_state(y) if state == 0: self._bump_finalization_state_from_0_to_1(y) + self.trace(y, self._append_if_nonnull, pending) elif state == 2: - self._bump_finalization_state_from_2_to_3(y) - else: - continue # don't need to recurse inside y - self.trace(y, self._append_if_nonnull, pending) + self._recursively_bump_finalization_state_from_2_to_3(y) scan = self._recursively_bump_finalization_state_from_1_to_2( x, scan) @@ -403,16 +422,11 @@ # we must also fix the state from 2 to 3 here, otherwise # we leave the GCFLAG_FINALIZATION_ORDERING bit behind # which will confuse the next collection - pending.append(x) - while pending.non_empty(): - y = pending.pop() - state = self._finalization_state(y) - if state == 2: - self._bump_finalization_state_from_2_to_3(y) - self.trace(y, self._append_if_nonnull, pending) + self._recursively_bump_finalization_state_from_2_to_3(x) else: new_with_finalizer.append(newx) + self.tmpstack.delete() pending.delete() marked.delete() self.objects_with_finalizers.delete() @@ -445,12 +459,19 @@ hdr = self.header(obj) hdr.tid |= GCFLAG_FINALIZATION_ORDERING - def _bump_finalization_state_from_2_to_3(self, obj): + def _recursively_bump_finalization_state_from_2_to_3(self, obj): ll_assert(self._finalization_state(obj) == 2, "unexpected finalization state != 2") newobj = self.get_forwarding_address(obj) - hdr = self.header(newobj) - hdr.tid &= ~GCFLAG_FINALIZATION_ORDERING + pending = self.tmpstack + ll_assert(not pending.non_empty(), "tmpstack not empty") + pending.append(newobj) + while pending.non_empty(): + y = pending.pop() + hdr = self.header(y) + if hdr.tid & GCFLAG_FINALIZATION_ORDERING: # state 2 ? + hdr.tid &= ~GCFLAG_FINALIZATION_ORDERING # change to state 3 + self.trace(y, self._append_if_nonnull, pending) def _recursively_bump_finalization_state_from_1_to_2(self, obj, scan): # recursively convert objects from state 1 to state 2. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gcheader.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gcheader.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gcheader.py Fri Feb 22 16:19:03 2008 @@ -21,17 +21,17 @@ def header_of_object(self, gcptr): # XXX hackhackhack - gcptr = gcptr._as_obj() + gcptr = gcptr._as_obj(check=False) if isinstance(gcptr, llmemory._gctransformed_wref): - return self.obj2header[gcptr._ptr._as_obj()] + return self.obj2header[gcptr._ptr._as_obj(check=False)] return self.obj2header[gcptr] def object_from_header(headerptr): - return header2obj[headerptr._as_obj()] + return header2obj[headerptr._as_obj(check=False)] object_from_header = staticmethod(object_from_header) def get_header(self, gcptr): - return self.obj2header.get(gcptr._as_obj(), None) + return self.obj2header.get(gcptr._as_obj(check=False), None) def attach_header(self, gcptr, headerptr): gcobj = gcptr._as_obj() Modified: pypy/branch/unified-rtti/pypy/rpython/memory/support.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/support.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/support.py Fri Feb 22 16:19:03 2008 @@ -27,7 +27,10 @@ def get(self): if not self.free_list: - return lltype.malloc(CHUNK, flavor="raw") + # we zero-initialize the chunks to make the translation + # backends happy, but we don't need to do it at run-time. + zero = not we_are_translated() + return lltype.malloc(CHUNK, flavor="raw", zero=zero) result = self.free_list self.free_list = result.next From pedronis at codespeak.net Fri Feb 22 16:31:02 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 22 Feb 2008 16:31:02 +0100 (CET) Subject: [pypy-svn] r51800 - pypy/dist/pypy/doc/discussion Message-ID: <20080222153102.568BD1684F6@codespeak.net> Author: pedronis Date: Fri Feb 22 16:31:00 2008 New Revision: 51800 Modified: pypy/dist/pypy/doc/discussion/ctypes_todo.txt Log: updates Modified: pypy/dist/pypy/doc/discussion/ctypes_todo.txt ============================================================================== --- pypy/dist/pypy/doc/discussion/ctypes_todo.txt (original) +++ pypy/dist/pypy/doc/discussion/ctypes_todo.txt Fri Feb 22 16:31:00 2008 @@ -15,8 +15,15 @@ - there are features, which we don't support like buffer() and array() protocols. - - raw structure/arrays as function arguments/result types are not supported - (even on _rawffi level) + - decide who should create own and free the needed ffitypes for structures by value args, + right now we simple leak them. Structure types etc seem the right place for this. + + - no tests for passing a union by value + + - returning structures by value is still not implemented + + - for some ABIs we will need completely filled ffitypes to do the right thing for passing + structures by value - bitfields are not implemented From tverwaes at codespeak.net Fri Feb 22 16:57:12 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Fri, 22 Feb 2008 16:57:12 +0100 (CET) Subject: [pypy-svn] r51803 - in pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk: . test Message-ID: <20080222155712.C4CEE168469@codespeak.net> Author: tverwaes Date: Fri Feb 22 16:57:10 2008 New Revision: 51803 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py Log: adding tests for shadow changes + basic nameprints for object of which we have a shadow Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py Fri Feb 22 16:57:10 2008 @@ -125,9 +125,8 @@ return "<%s %s>" % (self.__class__.__name__, self) def __str__(self): - from pypy.lang.smalltalk.shadow import ClassShadow - if isinstance(self, W_PointersObject) and isinstance(self._shadow,ClassShadow): - return "%s class" % (self.as_class_get_shadow().name or '?',) + if isinstance(self, W_PointersObject) and self._shadow is not None: + return self._shadow.getname() else: return "a %s" % (self.shadow_of_my_class().name or '?',) @@ -202,9 +201,6 @@ return shadow def as_link_get_shadow(self): - from pypy.lang.smalltalk import classtable - if self.getclass() == classtable.w_Process: - return self.as_process_get_shadow() from pypy.lang.smalltalk.shadow import LinkShadow return self.as_special_get_shadow(LinkShadow) @@ -213,9 +209,6 @@ return self.as_special_get_shadow(SemaphoreShadow) def as_linkedlist_get_shadow(self): - from pypy.lang.smalltalk import classtable - if self.getclass() == classtable.w_Semaphore: - return self.as_semaphore_get_shadow() from pypy.lang.smalltalk.shadow import LinkedListShadow return self.as_special_get_shadow(LinkedListShadow) @@ -240,14 +233,8 @@ return self.as_special_get_shadow(MethodContextShadow) def as_context_get_shadow(self): - from pypy.lang.smalltalk import classtable - if self.getclass() == classtable.w_MethodContext: - return self.as_methodcontext_get_shadow() - elif self.getclass() == classtable.w_BlockContext: - return self.as_blockcontext_get_shadow() - else: - # Should not happen... - raise Exception() + from pypy.lang.smalltalk.shadow import ContextPartShadow + return self.as_special_get_shadow(ContextPartShadow) def equals(self, other): if not isinstance(other, W_PointersObject): Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py Fri Feb 22 16:57:10 2008 @@ -10,6 +10,9 @@ self._w_self = w_self self.invalidate() + def getname(self): + return repr(self) + def invalidate(self): """XXX This should get called whenever the base Smalltalk object changes.""" @@ -56,6 +59,9 @@ self.s_superclass = None # the ClassShadow of the super class self.name = None + def getname(self): + return "%s class" % (self.name or '?',) + def update_shadow(self): from pypy.lang.smalltalk import objtable @@ -409,6 +415,10 @@ def update_shadow(self): pass + # XXX XXX Remove function when fixing superclass to AbstractShadow + def getname(self): + return repr(self) + def w_receiver(self): " Return self of the method, or the method that contains the block " return self.s_home().w_receiver() Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py Fri Feb 22 16:57:10 2008 @@ -205,3 +205,22 @@ s_object.add_last_link(w_last) assert s_object.w_firstlink() == w_first assert s_object.w_lastlink() == w_last + +def test_shadowchanges(): + w_object = model.W_PointersObject(None, 2) + w_o1 = link('a') + w_o2 = link('b') + w_object.store(0, w_o1) + w_object.store(1, w_o2) + s_object = w_object.as_linkedlist_get_shadow() + assert s_object.w_firstlink() == w_o1 + assert s_object.w_lastlink() == w_o2 + assert w_object._shadow == s_object + s_object2 = w_object.as_association_get_shadow() + assert s_object2.key() == w_o1 + assert s_object2.value() == w_o2 + assert w_object._shadow == s_object2 + s_object.check_for_updates() + assert s_object.w_firstlink() == w_o1 + assert s_object.w_lastlink() == w_o2 + assert w_object._shadow == s_object From tverwaes at codespeak.net Fri Feb 22 17:48:48 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Fri, 22 Feb 2008 17:48:48 +0100 (CET) Subject: [pypy-svn] r51805 - pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test Message-ID: <20080222164848.CD1EA168469@codespeak.net> Author: tverwaes Date: Fri Feb 22 17:48:47 2008 New Revision: 51805 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py Log: adding more shadow tests Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py Fri Feb 22 17:48:47 2008 @@ -83,6 +83,7 @@ w_m.bytes = bytes w_m.tempsize = tempsize w_m.argsize = argsize + w_m.literalsize = 2 return w_m def methodcontext(w_sender=objtable.w_nil, pc=1, stackpointer=0, stacksize=5, @@ -108,7 +109,7 @@ w_object.store(constants.MTHDCTX_TEMP_FRAME_START, 'el') return w_object -def test_methodcontext(): +def test_context(): w_m = method() w_object = methodcontext(stackpointer=2, method=w_m) w_object2 = methodcontext(w_sender=w_object) @@ -122,8 +123,8 @@ assert s_object.w_receiver() == 'receiver' s_object2.settemp(0, 'a') s_object2.settemp(1, 'b') - assert s_object2.gettemp(0) == 'a' assert s_object2.gettemp(1) == 'b' + assert s_object2.gettemp(0) == 'a' assert s_object.w_method() == w_m idx = s_object.stackstart() w_object.store(idx + 1, 'f') @@ -139,6 +140,18 @@ assert s_object.pop() == 'f' assert s_object.stackpointer() == s_object.stackstart() +def test_methodcontext(): + w_m = method() + # Point over 2 literals of size 4 + w_object = methodcontext(pc=9,method=w_m) + s_object = w_object.as_methodcontext_get_shadow() + assert s_object.getbytecode() == 97 + assert s_object.getbytecode() == 98 + assert s_object.getbytecode() == 99 + assert s_object.getbytecode() == 100 + assert s_object.getbytecode() == 101 + assert s_object.s_home() == s_object + def process(w_context=methodcontext(), priority=utility.wrap_int(3)): w_object = model.W_PointersObject(None, 4) w_object.store(constants.NEXT_LINK_INDEX, 'foo') From tverwaes at codespeak.net Fri Feb 22 19:24:50 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Fri, 22 Feb 2008 19:24:50 +0100 (CET) Subject: [pypy-svn] r51806 - in pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk: . test Message-ID: <20080222182450.AEC1016851A@codespeak.net> Author: tverwaes Date: Fri Feb 22 19:24:49 2008 New Revision: 51806 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_model.py Log: fixing literalat0 literalatput0, at0 and atput0 for compiledmethods. Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py Fri Feb 22 19:24:49 2008 @@ -454,24 +454,26 @@ def at0(self, index0): from pypy.lang.smalltalk import utility - # XXX Not tested - index0 -= self.headersize() - if index0 < self.getliteralsize(): - self.literalat0(index0) + if index0 <= self.getliteralsize(): + return self.literalat0(index0/constants.BYTES_PER_WORD) else: - index0 = index0 - self.getliteralsize() + # From blue book: + # The literal count indicates the size of the + # CompiledMethod's literal frame. + # This, in turn, indicates where the + # CompiledMethod's bytecodes start. + index0 = index0 - self.getliteralsize() - self.headersize() assert index0 < len(self.bytes) return utility.wrap_int(ord(self.bytes[index0])) def atput0(self, index0, w_value): from pypy.lang.smalltalk import utility - # XXX Not tested - index0 -= self.headersize() - if index0 < self.getliteralsize(): - self.literalatput0(index0, w_value) + if index0 <= self.getliteralsize(): + self.literalatput0(index0/constants.BYTES_PER_WORD, w_value) else: # XXX use to-be-written unwrap_char - index0 = index0 - self.getliteralsize() + index0 = index0 - self.getliteralsize() - self.headersize() + assert index0 < len(self.bytes) self.setchar(index0, chr(utility.unwrap_int(w_value))) def setchar(self, index0, character): Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py Fri Feb 22 19:24:49 2008 @@ -351,7 +351,9 @@ self.w_self().store(constants.PROCESS_MY_LIST_INDEX, w_object) def s_suspended_context(self): - return self.w_self().fetch(constants.PROCESS_SUSPENDED_CONTEXT_INDEX).as_context_get_shadow() + # XXX Can currently only restart context if it is a method context... + # XXX Depends on typechecking ... + return self.w_self().fetch(constants.PROCESS_SUSPENDED_CONTEXT_INDEX).as_methodcontext_get_shadow() def store_w_suspended_context(self, w_object): self.w_self().store(constants.PROCESS_SUSPENDED_CONTEXT_INDEX, w_object) Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py Fri Feb 22 19:24:49 2008 @@ -91,6 +91,7 @@ def test_str_w_object(): w_float_class = get_float_class() + w_float_class.as_class_get_shadow() assert str(w_float_class) == "Float class" assert str(w_float_class.getclass()) == "a Metaclass" #yes, with article assert str(w_float_class.getclass().getclass()) == "Metaclass class" @@ -227,7 +228,7 @@ assert w_false is objtable.w_false def test_runimage(): - #py.test.skip("This method actually runs an image. Fails since no graphical primitives yet") + py.test.skip("This method actually runs an image. Fails since no graphical primitives yet") from pypy.lang.smalltalk.shadow import SemaphoreShadow s_semaphore = SemaphoreShadow(None) s_scheduler = s_semaphore.s_scheduler() @@ -239,7 +240,7 @@ interp.interpret() def test_compile_method(): - py.test.skip("Not working yet.") + #py.test.skip("Not working yet.") sourcecode = """fib ^self < 2 ifTrue: [ 1 ] Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_model.py Fri Feb 22 19:24:49 2008 @@ -1,7 +1,7 @@ import py from pypy.lang.smalltalk import model, shadow, objtable from pypy.lang.smalltalk.shadow import MethodNotFound -from pypy.lang.smalltalk import classtable +from pypy.lang.smalltalk import classtable, utility mockclass = classtable.bootstrap_class @@ -85,3 +85,24 @@ h2 = w_inst.gethash() assert h1 == h2 assert h1 == w_inst.hash + +def test_compiledmethod_fetchbyte(): + w_method = model.W_CompiledMethod() + w_method.bytes = "abc" + w_method.literalsize = 2 + w_method.fetchbyte(9) == ord('a') + w_method.fetchbyte(10) == ord('b') + w_method.fetchbyte(11) == ord('c') + +def test_compiledmethod_at0(): + w_method = model.W_CompiledMethod() + w_method.bytes = "abc" + w_method.header = 100 + w_method.literals = [ 'lit1', 'lit2' ] + w_method.literalsize = 2 + assert utility.unwrap_int(w_method.at0(0)) == 100 + assert w_method.at0(4) == 'lit1' + assert w_method.at0(8) == 'lit2' + assert utility.unwrap_int(w_method.at0(12)) == ord('a') + assert utility.unwrap_int(w_method.at0(13)) == ord('b') + assert utility.unwrap_int(w_method.at0(14)) == ord('c') From tverwaes at codespeak.net Fri Feb 22 19:41:38 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Fri, 22 Feb 2008 19:41:38 +0100 (CET) Subject: [pypy-svn] r51807 - in pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk: . test Message-ID: <20080222184138.56A7A168574@codespeak.net> Author: tverwaes Date: Fri Feb 22 19:41:37 2008 New Revision: 51807 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_model.py Log: adding test for at0 atput0 for compiled methods Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py Fri Feb 22 19:41:37 2008 @@ -468,6 +468,8 @@ def atput0(self, index0, w_value): from pypy.lang.smalltalk import utility + print index0 + print self.getliteralsize() if index0 <= self.getliteralsize(): self.literalatput0(index0/constants.BYTES_PER_WORD, w_value) else: Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_model.py Fri Feb 22 19:41:37 2008 @@ -3,6 +3,13 @@ from pypy.lang.smalltalk.shadow import MethodNotFound from pypy.lang.smalltalk import classtable, utility +def joinbits(values, lengths): + result = 0 + for each, length in reversed(zip(values, lengths)): + result = result << length + result += each + return result + mockclass = classtable.bootstrap_class def test_new(): @@ -106,3 +113,21 @@ assert utility.unwrap_int(w_method.at0(12)) == ord('a') assert utility.unwrap_int(w_method.at0(13)) == ord('b') assert utility.unwrap_int(w_method.at0(14)) == ord('c') + +def test_compiledmethod_atput0(): + w_method = model.W_CompiledMethod(3) + newheader = joinbits([0,2,0,0,0,0],[9,8,1,6,4,1]) + assert w_method.getliteralsize() == 0 + w_method.atput0(0, utility.wrap_int(newheader)) + assert w_method.getliteralsize() == 8 # 2 from new header * BYTES_PER_WORD (= 4) + w_method.atput0(4, 'lit1') + w_method.atput0(8, 'lit2') + w_method.atput0(12, utility.wrap_int(ord('a'))) + w_method.atput0(13, utility.wrap_int(ord('b'))) + w_method.atput0(14, utility.wrap_int(ord('c'))) + assert utility.unwrap_int(w_method.at0(0)) == newheader + assert w_method.at0(4) == 'lit1' + assert w_method.at0(8) == 'lit2' + assert utility.unwrap_int(w_method.at0(12)) == ord('a') + assert utility.unwrap_int(w_method.at0(13)) == ord('b') + assert utility.unwrap_int(w_method.at0(14)) == ord('c') From arigo at codespeak.net Fri Feb 22 22:09:32 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 22 Feb 2008 22:09:32 +0100 (CET) Subject: [pypy-svn] r51812 - in pypy/branch/unified-rtti/pypy/rpython/memory: . gc gctransform test Message-ID: <20080222210932.146C0168441@codespeak.net> Author: arigo Date: Fri Feb 22 22:09:31 2008 New Revision: 51812 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/base.py pypy/branch/unified-rtti/pypy/rpython/memory/gc/generation.py pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py pypy/branch/unified-rtti/pypy/rpython/memory/gc/semispace.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py pypy/branch/unified-rtti/pypy/rpython/memory/gcwrapper.py pypy/branch/unified-rtti/pypy/rpython/memory/lltypelayout.py pypy/branch/unified-rtti/pypy/rpython/memory/test/test_gc.py Log: Progress in updating the Semispace and Generation GCs. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gc/base.py Fri Feb 22 22:09:31 2008 @@ -1,5 +1,4 @@ from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rpython.memory import gcheader, gctypelayout from pypy.rlib.debug import ll_assert class GCBase(object): @@ -9,10 +8,6 @@ needs_zero_gc_pointers = True prebuilt_gc_objects_are_static_roots = True - def __init__(self): - TYPE_INFO = gctypelayout.GCData.TYPE_INFO - self.gcheaderbuilder = gcheader.GCHeaderBuilder(self.HDR, TYPE_INFO) - def set_query_functions(self, is_varsize, has_gcptr_in_varsize, is_gcarrayofgcptr, getfinalizer, Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gc/generation.py Fri Feb 22 22:09:31 2008 @@ -31,13 +31,15 @@ prebuilt_gc_objects_are_static_roots = False first_unused_gcflag = SemiSpaceGC.first_unused_gcflag << 2 - def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, + def __init__(self, gcheaderbuilder, + chunk_size=DEFAULT_CHUNK_SIZE, nursery_size=128, min_nursery_size=128, auto_nursery_size=False, space_size=4096, max_space_size=sys.maxint//2+1): - SemiSpaceGC.__init__(self, chunk_size = chunk_size, + SemiSpaceGC.__init__(self, gcheaderbuilder, + chunk_size = chunk_size, space_size = space_size, max_space_size = max_space_size) assert min_nursery_size <= nursery_size <= space_size // 2 @@ -196,9 +198,9 @@ def init_gc_object(self, addr, typeid, flags=GCFLAG_NO_YOUNG_PTRS): SemiSpaceGC.init_gc_object(self, addr, typeid, flags) - def init_gc_object_immortal(self, addr, typeid, + def init_gc_object_immortal(self, hdr, typeid, flags=GCFLAG_NO_YOUNG_PTRS|GCFLAG_NO_HEAP_PTRS): - SemiSpaceGC.init_gc_object_immortal(self, addr, typeid, flags) + SemiSpaceGC.init_gc_object_immortal(self, hdr, typeid, flags) def semispace_collect(self, size_changing=False): self.reset_young_gcflags() # we are doing a full collection anyway @@ -214,7 +216,7 @@ def trace_and_copy(self, obj): # during a full collect, all objects copied might come from the nursery and # so must have this flag set: - self.header(obj).tid |= GCFLAG_NO_YOUNG_PTRS + self.header(obj).flags |= GCFLAG_NO_YOUNG_PTRS SemiSpaceGC.trace_and_copy(self, obj) # history: this was missing and caused an object to become old but without the # flag set. Such an object is bogus in the sense that the write_barrier doesn't @@ -227,7 +229,7 @@ while oldlist.non_empty(): obj = oldlist.pop() hdr = self.header(obj) - hdr.tid |= GCFLAG_NO_YOUNG_PTRS + hdr.flags |= GCFLAG_NO_YOUNG_PTRS def weakrefs_grow_older(self): while self.young_objects_with_weakrefs.non_empty(): @@ -308,7 +310,7 @@ """obj must not be in the nursery. This copies all the young objects it references out of the nursery. """ - self.header(obj).tid |= GCFLAG_NO_YOUNG_PTRS + self.header(obj).flags |= GCFLAG_NO_YOUNG_PTRS self.trace(obj, self._trace_drag_out, None) def _trace_drag_out(self, pointer, ignored): @@ -336,7 +338,7 @@ self.objects_with_weakrefs.append(obj) def write_barrier(self, oldvalue, newvalue, addr_struct): - if self.header(addr_struct).tid & GCFLAG_NO_YOUNG_PTRS: + if self.header(addr_struct).flags & GCFLAG_NO_YOUNG_PTRS: self.remember_young_pointer(addr_struct, newvalue) def append_to_static_roots(self, pointer, arg): @@ -344,7 +346,7 @@ def move_to_static_roots(self, addr_struct): objhdr = self.header(addr_struct) - objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS + objhdr.flags &= ~GCFLAG_NO_HEAP_PTRS self.trace(addr_struct, self.append_to_static_roots, None) def remember_young_pointer(self, addr_struct, addr): @@ -353,8 +355,8 @@ oldhdr = self.header(addr_struct) if self.is_in_nursery(addr): self.old_objects_pointing_to_young.append(addr_struct) - oldhdr.tid &= ~GCFLAG_NO_YOUNG_PTRS - if oldhdr.tid & GCFLAG_NO_HEAP_PTRS: + oldhdr.flags &= ~GCFLAG_NO_YOUNG_PTRS + if oldhdr.flags & GCFLAG_NO_HEAP_PTRS: self.move_to_static_roots(addr_struct) remember_young_pointer._dont_inline_ = True Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py Fri Feb 22 22:09:31 2008 @@ -74,8 +74,9 @@ POOLNODE.become(lltype.Struct('gc_pool_node', ('linkedlisthdr', HDR), ('nextnode', POOLNODEPTR))) - def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, start_heap_size=4096): - GCBase.__init__(self) + def __init__(self, gcheaderbuilder, + chunk_size=DEFAULT_CHUNK_SIZE, start_heap_size=4096): + self.gcheaderbuilder = gcheaderbuilder self.heap_usage = 0 # at the end of the latest collection self.bytes_malloced = 0 # since the latest collection self.bytes_malloced_threshold = start_heap_size Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gc/semispace.py Fri Feb 22 22:09:31 2008 @@ -1,9 +1,8 @@ from pypy.rpython.lltypesystem.llmemory import raw_malloc, raw_free from pypy.rpython.lltypesystem.llmemory import raw_memcopy, raw_memclear from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage -from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE +from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE, RTTIPTR from pypy.rpython.memory.support import get_address_stack, get_address_deque -from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rlib.objectmodel import free_non_gc_object from pypy.rlib.debug import ll_assert @@ -14,7 +13,7 @@ import sys, os TYPEID_MASK = 0xffff -first_gcflag = 1 << 16 +first_gcflag = 1 GCFLAG_FORWARDED = first_gcflag GCFLAG_IMMORTAL = first_gcflag << 1 GCFLAG_FINALIZATION_ORDERING = first_gcflag << 2 @@ -31,17 +30,21 @@ total_collection_time = 0.0 total_collection_count = 0 - HDR = lltype.Struct('header', ('tid', lltype.Signed)) - FORWARDSTUB = lltype.GcStruct('forwarding_stub', - ('forw', llmemory.Address)) - FORWARDSTUBPTR = lltype.Ptr(FORWARDSTUB) + # header structure: + # + # * typeptr: normally a RTTIPTR, but if the object is forwarded, + # this is used to hold the forwarding address + # * flags: for GCFLAG_XXX + # + HDR = lltype.Struct('header', ('typeptr', llmemory.Address), + ('flags', lltype.Signed)) - def __init__(self, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096, - max_space_size=sys.maxint//2+1): + def __init__(self, gcheaderbuilder, chunk_size=DEFAULT_CHUNK_SIZE, + space_size=4096, max_space_size=sys.maxint//2+1): MovingGCBase.__init__(self) + self.gcheaderbuilder = gcheaderbuilder self.space_size = space_size self.max_space_size = max_space_size - self.gcheaderbuilder = GCHeaderBuilder(self.HDR) self.AddressStack = get_address_stack(chunk_size) self.AddressDeque = get_address_deque(chunk_size) self.finalizer_lock_count = 0 @@ -293,9 +296,10 @@ def copy(self, obj): if self.is_forwarded(obj): - #llop.debug_print(lltype.Void, obj, "already copied to", self.get_forwarding_address(obj)) + # 'obj' already copied to 'self.get_forwarding_address(obj)' return self.get_forwarding_address(obj) else: + # copying 'obj' to 'newobj' newaddr = self.free objsize = self.get_size(obj) totalsize = self.size_gc_header() + objsize @@ -303,9 +307,6 @@ raw_memcopy(obj - self.size_gc_header(), newaddr, totalsize) self.free += totalsize newobj = newaddr + self.size_gc_header() - #llop.debug_print(lltype.Void, obj, "copied to", newobj, - # "tid", self.header(obj).tid, - # "size", totalsize) self.set_forwarding_address(obj, newobj, objsize) return newobj @@ -317,35 +318,26 @@ pointer.address[0] = self.copy(pointer.address[0]) def is_forwarded(self, obj): - return self.header(obj).tid & GCFLAG_FORWARDED != 0 + return self.header(obj).flags & GCFLAG_FORWARDED != 0 # note: all prebuilt objects also have this flag set def get_forwarding_address(self, obj): - tid = self.header(obj).tid - if tid & GCFLAG_IMMORTAL: + hdr = self.header(obj) + flags = hdr.flags + ll_assert(flags & GCFLAG_FORWARDED != 0, "object not forwarded!") + if flags & GCFLAG_IMMORTAL: return obj # prebuilt objects are "forwarded" to themselves else: - stub = llmemory.cast_adr_to_ptr(obj, self.FORWARDSTUBPTR) - return stub.forw + return hdr.typeptr # holds the forwarding address def set_forwarding_address(self, obj, newobj, objsize): - # To mark an object as forwarded, we set the GCFLAG_FORWARDED and - # overwrite the object with a FORWARDSTUB. Doing so is a bit - # long-winded on llarena, but it all melts down to two memory - # writes after translation to C. - size_gc_header = self.size_gc_header() - stubsize = llmemory.sizeof(self.FORWARDSTUB) - tid = self.header(obj).tid - ll_assert(tid & GCFLAG_IMMORTAL == 0, "unexpected GCFLAG_IMMORTAL") - ll_assert(tid & GCFLAG_FORWARDED == 0, "unexpected GCFLAG_FORWARDED") - # replace the object at 'obj' with a FORWARDSTUB. - hdraddr = obj - size_gc_header - llarena.arena_reset(hdraddr, size_gc_header + objsize, False) - llarena.arena_reserve(hdraddr, size_gc_header + stubsize) - hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(self.HDR)) - hdr.tid = tid | GCFLAG_FORWARDED - stub = llmemory.cast_adr_to_ptr(obj, self.FORWARDSTUBPTR) - stub.forw = newobj + # To mark an object as forwarded, we set the GCFLAG_FORWARDED bit. + hdr = self.header(obj) + flags = hdr.flags + ll_assert(flags & GCFLAG_IMMORTAL == 0, "unexpected GCFLAG_IMMORTAL") + ll_assert(flags & GCFLAG_FORWARDED == 0, "unexpected GCFLAG_FORWARDED") + hdr.flags = flags | GCFLAG_FORWARDED + hdr.typeptr = newobj def get_size(self, obj): typeid = self.get_type_id(obj) @@ -362,22 +354,22 @@ return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) def get_type_id(self, addr): - tid = self.header(addr).tid - ll_assert(tid & (GCFLAG_FORWARDED|GCFLAG_IMMORTAL) != GCFLAG_FORWARDED, - "get_type_id on forwarded obj") - # Non-prebuilt forwarded objects are overwritten with a FORWARDSTUB. - # Although calling get_type_id() on a forwarded object works by itself, - # we catch it as an error because it's likely that what is then - # done with the typeid is bogus. - return tid & TYPEID_MASK + hdr = self.header(addr) + # In non-prebuilt forwarded objects, the typeptr is used to hold + # the forwaring address. So far, we don't need to do get_type_id() + # on forwarded objects, so we just assert that nobody tries to. + ll_assert(hdr.flags & (GCFLAG_FORWARDED|GCFLAG_IMMORTAL) + != GCFLAG_FORWARDED, "get_type_id on forwarded obj") + return llmemory.cast_adr_to_ptr(hdr.typeptr, RTTIPTR) def init_gc_object(self, addr, typeid, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.tid = typeid | flags + hdr.typeptr = llmemory.cast_ptr_to_adr(typeid) + hdr.flags = flags - def init_gc_object_immortal(self, addr, typeid, flags=0): - hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.tid = typeid | flags | GCFLAG_IMMORTAL | GCFLAG_FORWARDED + def init_gc_object_immortal(self, hdr, typeid, flags=0): + hdr.typeptr = llmemory.cast_ptr_to_adr(typeid) + hdr.flags = flags | GCFLAG_IMMORTAL | GCFLAG_FORWARDED # immortal objects always have GCFLAG_FORWARDED set; # see get_forwarding_address(). @@ -442,13 +434,13 @@ if self.is_forwarded(obj): newobj = self.get_forwarding_address(obj) hdr = self.header(newobj) - if hdr.tid & GCFLAG_FINALIZATION_ORDERING: + if hdr.flags & GCFLAG_FINALIZATION_ORDERING: return 2 else: return 3 else: hdr = self.header(obj) - if hdr.tid & GCFLAG_FINALIZATION_ORDERING: + if hdr.flags & GCFLAG_FINALIZATION_ORDERING: return 1 else: return 0 @@ -457,7 +449,7 @@ ll_assert(self._finalization_state(obj) == 0, "unexpected finalization state != 0") hdr = self.header(obj) - hdr.tid |= GCFLAG_FINALIZATION_ORDERING + hdr.flags |= GCFLAG_FINALIZATION_ORDERING def _recursively_bump_finalization_state_from_2_to_3(self, obj): ll_assert(self._finalization_state(obj) == 2, @@ -469,8 +461,8 @@ while pending.non_empty(): y = pending.pop() hdr = self.header(y) - if hdr.tid & GCFLAG_FINALIZATION_ORDERING: # state 2 ? - hdr.tid &= ~GCFLAG_FINALIZATION_ORDERING # change to state 3 + if hdr.flags & GCFLAG_FINALIZATION_ORDERING: # state 2 ? + hdr.flags &= ~GCFLAG_FINALIZATION_ORDERING # change to state 3 self.trace(y, self._append_if_nonnull, pending) def _recursively_bump_finalization_state_from_1_to_2(self, obj, scan): Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py Fri Feb 22 22:09:31 2008 @@ -16,7 +16,6 @@ def __init__(self, translator, inline=False): super(BoehmGCTransformer, self).__init__(translator, inline=inline) - self.newgcheaderbuilder(self.HDR, self.TYPEINFO) self.finalizer_funcptrs = {} atomic_mh = mallocHelpers() Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py Fri Feb 22 22:09:31 2008 @@ -95,9 +95,10 @@ use_stackless = False root_stack_depth = 163840 + TYPEINFO = gctypelayout.GCData.TYPE_INFO + def __init__(self, translator): from pypy.rpython.memory.gc.base import choose_gc_from_config - super(FrameworkGCTransformer, self).__init__(translator, inline=True) if hasattr(self, 'GC_PARAMS'): # for tests: the GC choice can be specified as class attributes from pypy.rpython.memory.gc.marksweep import MarkSweepGC @@ -107,13 +108,14 @@ # for regular translation: pick the GC from the config GCClass, GC_PARAMS = choose_gc_from_config(translator.config) - gc = GCClass(**GC_PARAMS) - self.setgcheaderbuilder(gc.gcheaderbuilder) + self.HDR = GCClass.HDR + super(FrameworkGCTransformer, self).__init__(translator, inline=True) + self.layoutbuilder = TransformerLayoutBuilder(self) self.get_type_id = self.layoutbuilder.get_type_id gcdata = gctypelayout.GCData(self.gcheaderbuilder) - gcdata.gc = gc + gcdata.gc = GCClass(self.gcheaderbuilder, **GC_PARAMS) # initialize the following two fields with a random non-NULL address, # to make the annotator happy. The fields are patched in finish() Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/refcounting.py Fri Feb 22 22:09:31 2008 @@ -48,7 +48,6 @@ # create incref, etc graph - self.newgcheaderbuilder(self.HDR, self.TYPEINFO) gchelpers = self.gchelpers gc_header_offset = gchelpers.gc_header_offset HDRPTR = lltype.Ptr(self.HDR) Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py Fri Feb 22 22:09:31 2008 @@ -13,6 +13,7 @@ from pypy.annotation import model as annmodel from pypy.rpython import rmodel, annlowlevel from pypy.rpython.memory import gc +from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rpython.memory.gctransform.support import var_ispyobj from pypy.rpython.annlowlevel import MixLevelHelperAnnotator from pypy.rpython.rtyper import LowLevelOpList @@ -526,14 +527,10 @@ self.stack_malloc_fixedsize_ptr = self.inittime_helper( ll_stack_malloc_fixedsize, [lltype.Signed], llmemory.Address) - def newgcheaderbuilder(self, HDR, TYPEINFO): - from pypy.rpython.memory.gcheader import GCHeaderBuilder - self.setgcheaderbuilder(GCHeaderBuilder(HDR, TYPEINFO)) - - def setgcheaderbuilder(self, gcheaderbuilder): - # at the moment, all GC transformers are based on a GCHeaderBuilder. - self.gcheaderbuilder = gcheaderbuilder - self.gchelpers = GCHelpers(gcheaderbuilder) + # at the moment, all GC transformers are based on a GCHeaderBuilder + # built from self.HDR and self.TYPEINFO. + self.gcheaderbuilder = GCHeaderBuilder(self.HDR, self.TYPEINFO) + self.gchelpers = GCHelpers(self.gcheaderbuilder) if self.translator: self.gc_runtime_type_info_ptr = self.inittime_helper( self.gchelpers.gc_runtime_type_info, [llmemory.Address], Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gcwrapper.py Fri Feb 22 22:09:31 2008 @@ -2,20 +2,23 @@ from pypy.rpython import llinterp from pypy.rpython.annlowlevel import llhelper from pypy.rpython.memory import gctypelayout +from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.objspace.flow.model import Constant class GCManagedHeap(object): def __init__(self, llinterp, flowgraphs, gc_class, GC_PARAMS={}): - self.gc = gc_class(chunk_size = 10, **GC_PARAMS) + TYPEINFO = gctypelayout.GCData.TYPE_INFO + self.gcheaderbuilder = GCHeaderBuilder(gc_class.HDR, TYPEINFO) + self.gc = gc_class(self.gcheaderbuilder, chunk_size = 10, **GC_PARAMS) self.gc.set_root_walker(LLInterpRootWalker(self)) self.llinterp = llinterp self.prepare_graphs(flowgraphs) self.gc.setup() def prepare_graphs(self, flowgraphs): - layoutbuilder = DirectRunLayoutBuilder(self.gc.gcheaderbuilder, + layoutbuilder = DirectRunLayoutBuilder(self.gcheaderbuilder, self.llinterp) self.get_type_id = layoutbuilder.get_type_id layoutbuilder.initialize_gc_query_function(self.gc) Modified: pypy/branch/unified-rtti/pypy/rpython/memory/lltypelayout.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/lltypelayout.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/lltypelayout.py Fri Feb 22 22:09:31 2008 @@ -30,7 +30,7 @@ for name in TYPE._names: layout[name] = curr curr += get_fixed_size(TYPE._flds[name]) - layout["_size"] = curr + layout["_size"] = curr or 1 # sizeof(empty struct) == 1, as in C return layout elif isinstance(TYPE, lltype.Array): return (get_fixed_size(lltype.Signed), get_fixed_size(TYPE.OF)) Modified: pypy/branch/unified-rtti/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/test/test_gc.py Fri Feb 22 22:09:31 2008 @@ -43,6 +43,22 @@ res = ''.join(res.chars) return res + def test_empty_class(self): + class A(object): + pass + def func(): + a1 = A() + a2 = A() + a3 = A() + assert a1 is not a2 + assert a1 is not a3 + assert a2 is not a3 + llop.gc__collect(lltype.Void) + assert a1 is not a2 + assert a1 is not a3 + assert a2 is not a3 + self.interpret(func, []) + def test_llinterp_lists(self): #curr = simulator.current_size def malloc_a_lot(): From niko at codespeak.net Fri Feb 22 23:21:00 2008 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 22 Feb 2008 23:21:00 +0100 (CET) Subject: [pypy-svn] r51813 - in pypy/dist/pypy/translator: cli/test jvm jvm/src/pypy jvm/test oosupport oosupport/test_template Message-ID: <20080222222100.DB9AB16844A@codespeak.net> Author: niko Date: Fri Feb 22 23:20:59 2008 New Revision: 51813 Added: pypy/dist/pypy/translator/jvm/test/test_extreme.py pypy/dist/pypy/translator/oosupport/test_template/extreme.py Modified: pypy/dist/pypy/translator/cli/test/test_snippet.py pypy/dist/pypy/translator/jvm/database.py pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/metavm.py pypy/dist/pypy/translator/jvm/node.py pypy/dist/pypy/translator/jvm/opcodes.py pypy/dist/pypy/translator/jvm/prebuiltnodes.py pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java pypy/dist/pypy/translator/jvm/test/test_snippet.py pypy/dist/pypy/translator/jvm/typesystem.py pypy/dist/pypy/translator/oosupport/function.py pypy/dist/pypy/translator/oosupport/test_template/snippets.py Log: fix two bugs in JVM backend: * llshl takes a long, not an int, for # of bits to shift * rejigger exc. handling so we translate from CLI exceptions in the catch block rather than at the site they are thrown. This allows us to handle StackOverflow and (in theory) OutOfMemory exceptions. Modified: pypy/dist/pypy/translator/cli/test/test_snippet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_snippet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_snippet.py Fri Feb 22 23:20:59 2008 @@ -1,7 +1,11 @@ +import py from pypy.translator.cli.test.runtest import CliTest from pypy.translator.oosupport.test_template.snippets import BaseTestSnippets class TestSnippets(BaseTestSnippets, CliTest): + def test_llshl(self): + py.test.skip('llshl currently broken on CLI') + def test_link_SSA(self): def fn(): lst = [42, 43, 44] Modified: pypy/dist/pypy/translator/jvm/database.py ============================================================================== --- pypy/dist/pypy/translator/jvm/database.py (original) +++ pypy/dist/pypy/translator/jvm/database.py Fri Feb 22 23:20:59 2008 @@ -53,15 +53,23 @@ # Create information about the Main class we will build: # - # note that it will have a static field called 'pypy' that - # points to a PyPy instance. This PyPy instance has been - # paired with the appropriate Interlink implementation - # which allows it to create generated structures. + # It will have two static fields, 'ilink' and 'pypy'. The + # first points to an instance of the interface pypy.Interlink + # which we will be generated. The second points to an instance + # of pypy.PyPy which was created with this Interlink instance. + # + # The Interlink class provides the bridge between static helper + # code and dynamically generated classes. Since there is one + # Main per set of translated code, this also allows multiple + # PyPy interpreters to overlap with one another. # # These are public attributes that are referenced from - # elsewhere in the code. + # elsewhere in the code using + # jvmgen.Generator.push_interlink() and .push_pypy(). self.jPyPyMain = JvmClassType(self._pkg('Main')) self.pypy_field = jvmgen.Field.s(self.jPyPyMain, 'pypy', jPyPy) + self.interlink_field = jvmgen.Field.s(self.jPyPyMain, 'ilink', + jvmtype.jPyPyInterlink) # _________________________________________________________________ # Java String vs Byte Array @@ -509,7 +517,8 @@ # Handle built-in types: if OOT in self.ootype_to_scalar: return self.ootype_to_scalar[OOT] - if isinstance(OOT, lltype.Ptr) and isinstance(t.TO, lltype.OpaqueType): + if (isinstance(OOT, lltype.Ptr) and + isinstance(OOT.TO, lltype.OpaqueType)): return jObject if OOT in self.ootype_to_builtin: return JvmBuiltInType(self, self.ootype_to_builtin[OOT], OOT) Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Fri Feb 22 23:20:59 2008 @@ -733,15 +733,11 @@ if jvartype is jVoid: return opc = LOAD.for_type(jvartype) - self.add_comment(" load_jvm_jar: jvartype=%s varidx=%s" % ( - repr(jvartype), repr(varidx))) self._instr(opc, varidx) def store_jvm_var(self, vartype, varidx): """ Loads from jvm slot #varidx, which is expected to hold a value of type vartype """ - self.add_comment(" store_jvm_jar: vartype=%s varidx=%s" % ( - repr(vartype), repr(varidx))) self._instr(STORE.for_type(vartype), varidx) def load_from_array(self, elemtype): @@ -831,37 +827,30 @@ # __________________________________________________________________ # Exception Handling + # + # You can demarcate regions of code as "try/catch" regions using + # the various functions included here. Either invoke + # try_catch_region(), in which case you must supply all the + # relevant labels, or use the begin_try()/end_try()/begin_catch() + # methods. In the latter case, you define the 3 needed labels as + # you go. Both begin_try() and end_try() must have been invoked + # before begin_catch() is invoked (i.e., the try region must + # appear before the corresponding catch regions). Note that + # end_try() can be called again to reset the end of the try + # region. def begin_try(self): - """ - Begins a try/catch region. Must be followed by a call to end_try() - after the code w/in the try region is complete. - """ self.begintrylbl = self.unique_label("begin_try", mark=True) def end_try(self): - """ - Ends a try/catch region. Must be followed immediately - by a call to begin_catch(). - """ self.endtrylbl = self.unique_label("end_try", mark=True) def begin_catch(self, jexcclsty): - """ - Begins a catch region corresponding to the last try; there can - be more than one call to begin_catch, in which case the last - try region is reused. - 'jexcclsty' --- a JvmType for the class of exception to be caught - """ catchlbl = self.unique_label("catch", mark=True) self.try_catch_region( jexcclsty, self.begintrylbl, self.endtrylbl, catchlbl) - + def end_catch(self): - """ - Ends a catch region. - (Included for CLI compatibility) - """ return def try_catch_region(self, jexcclsty, trystartlbl, tryendlbl, catchlbl): @@ -985,6 +974,11 @@ onto the stack """ self.db.pypy_field.load(self) + def push_interlink(self): + """ Pushes the Interlink object which contains the methods + from prebuildnodes.py onto the stack """ + self.db.interlink_field.load(self) + def get_field(self, CONCRETETYPE, fieldname): clsobj = self.db.pending_class(CONCRETETYPE) fieldobj = clsobj.lookup_field(fieldname) Modified: pypy/dist/pypy/translator/jvm/metavm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/metavm.py (original) +++ pypy/dist/pypy/translator/jvm/metavm.py Fri Feb 22 23:20:59 2008 @@ -62,47 +62,6 @@ jmethod.return_type, op.result) JvmCallMethod = _JvmCallMethod() -class TranslateException(MicroInstruction): - """ Translates an exception into a call of a method on the PyPy object """ - def __init__(self, jexc, pexcmthd, inst): - """ - jexc: the JvmType of the exception - pexcmthd: the name of the method on the PyPy object to call. - The PyPy method must take no arguments, return void, and must - always throw an exception in practice. It would be better to - just find the class to throw normally, but I don't know how. - """ - self.java_exc = jexc - self.pypy_method = jvmgen.Method.v( - jvmtype.jPyPyInterlink, pexcmthd, [], jvmtype.jVoid) - self.instruction = inst - - def render(self, gen, op): - trylbl = gen.unique_label('translate_exc_begin') - catchlbl = gen.unique_label('translate_exc_catch') - donelbl = gen.unique_label('translate_exc_done') - - # try { - gen.mark(trylbl) - self.instruction.render(gen, op) - gen.goto(donelbl) - # } catch (JavaExceptionType) { - gen.mark(catchlbl) - gen.emit(jvmgen.POP) # throw away the exception object - gen.push_pypy() # load the PyPy object - gen.emit(jvmgen.PYPYINTERLINK) # load the interlink field from it - gen.emit(self.pypy_method) # invoke the method - # Note: these instructions will never execute, as we expect - # the pypy_method to throw an exception and not to return. We - # need them here to satisfy the Java verifier, however, as it - # does not know that the pypy_method will never return. - gen.emit(jvmgen.ACONST_NULL) - gen.emit(jvmgen.ATHROW) - # } - gen.mark(donelbl) - - gen.try_catch_region(self.java_exc, trylbl, catchlbl, catchlbl) - class _NewCustomDict(MicroInstruction): def _load_func(self, gen, fn, obj, method_name): db = gen.db @@ -129,23 +88,6 @@ generator.emit(jvmgen.CUSTOMDICTMAKE) NewCustomDict = _NewCustomDict() -#XXX These classes have been adapted to the new -#XXX WeakRef methods, but don't appear to be needed. -#class _CastPtrToWeakAddress(MicroInstruction): -# def render(self, generator, op): -# arg = op.args[0] -# generator.load(arg) -# generator.create_weakref(arg.concretetype) -# generator.store(op.result) -#CastPtrToWeakAddress = _CastPtrToWeakAddress() - -#class _CastWeakAddressToPtr(MicroInstruction): -# def render(self, generator, op): -# RESULTTYPE = op.result.concretetype -# generator.deref_weakref(RESULTTYPE) -#CastWeakAddressToPtr = _CastWeakAddressToPtr() - - CASTS = { # FROM TO (ootype.Signed, ootype.UnsignedLongLong): jvmgen.I2L, Modified: pypy/dist/pypy/translator/jvm/node.py ============================================================================== --- pypy/dist/pypy/translator/jvm/node.py (original) +++ pypy/dist/pypy/translator/jvm/node.py Fri Feb 22 23:20:59 2008 @@ -35,6 +35,7 @@ push_constant import pypy.translator.jvm.generator as jvmgen +import pypy.translator.jvm.typesystem as jvmtype from pypy.translator.jvm.log import log class Node(object): @@ -93,11 +94,12 @@ def render(self, gen): gen.begin_class(gen.db.jPyPyMain, jObject) gen.add_field(gen.db.pypy_field) + gen.add_field(gen.db.interlink_field) # Initialization: # - # 1. Create a PyPy helper class, passing in an appropriate - # interlink instance. + # 1. Create an Interlink instance and a PyPy helper class, and + # store them in the appropriate static fields. # # 2. Run the initialization method for the constant class. # @@ -106,6 +108,8 @@ gen.emit(jvmgen.NEW, jPyPy) gen.emit(jvmgen.DUP) gen.new_with_jtype(gen.db.jInterlinkImplementation) + gen.emit(jvmgen.DUP) + gen.db.interlink_field.store(gen) gen.emit(jvmgen.Method.c(jPyPy, [jPyPyInterlink])) gen.db.pypy_field.store(gen) gen.db.constant_generator.runtime_init(gen) @@ -314,12 +318,54 @@ if cond: self.ilasm.end_try() + def introduce_exception_conversions(self, llexitcases): + + # These are exceptions thrown internally by the JVM. + # If the user is catching an RPython exception that corresponds + # to one of these cases, we introduce a translation block which + # catches the corresponding JVM exception and re-throwns the + # RPython one. A better solution would be to find a way to + # make the RPython class the same as the JVM class, but that is + # currently hindered by the presence of a few fields (meta) on + # the Object class. + translation_table = [ + (ZeroDivisionError, jvmtype.jArithmeticException), + (RuntimeError, jvmtype.jStackOverflowError), + (MemoryError, jvmtype.jOutOfMemoryError), + ] + + for pyexccls, jexcty in translation_table: + for llexitcase in llexitcases: + assert issubclass(llexitcase, BaseException) + if issubclass(llexitcase, pyexccls): + # Generate some converter code like: + # try { ... } + # catch (OutOfStackError e) { + # jPyPyMain.ilink.throwRuntimeError(); + # throw null; + # } + # The "throw null" will never execute, it's just + # there to make the verifier happy, since it doesn't + # realize that Interlink's throwXyzError() methods + # never return. At the end we invoke end_try() again + # so as to extend the encompassing try/catch region + # to include this code, thus allowing the RPython + # exception to be caught by the normal handlers. + self.ilasm.begin_catch(jexcty) + self.ilasm.push_interlink() + interlink_method = jvmgen.Method.v( + jPyPyInterlink, "throw"+pyexccls.__name__, [], jVoid) + self.ilasm.emit(interlink_method) + self.ilasm.emit(jvmgen.ACONST_NULL) + self.ilasm.emit(jvmgen.ATHROW) + self.ilasm.end_try() + def begin_catch(self, llexitcase): ll_meta_exc = llexitcase ll_exc = ll_meta_exc._inst.class_._INSTANCE jtype = self.cts.lltype_to_cts(ll_exc) assert jtype.throwable # SHOULD only try to catch subtypes of Exception - self.ilasm.begin_catch(jtype) + self.ilasm.begin_catch(jtype) def end_catch(self, exit_lbl): self.ilasm.goto(exit_lbl) Modified: pypy/dist/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/jvm/opcodes.py (original) +++ pypy/dist/pypy/translator/jvm/opcodes.py Fri Feb 22 23:20:59 2008 @@ -10,7 +10,7 @@ SetField, GetField, DownCast, RuntimeNew, OOString, OOUnicode, \ CastTo, PushPrimitive from pypy.translator.jvm.metavm import \ - IndirectCall, JvmCallMethod, TranslateException, NewCustomDict, \ + IndirectCall, JvmCallMethod, NewCustomDict, \ CastPrimitive, PushPyPy from pypy.rpython.ootypesystem import ootype @@ -47,10 +47,11 @@ return res def _check_zer(op): - return [TranslateException( - jvmtype.jArithmeticException, - 'throwZeroDivisionError', - _proc(op))] + # Note: we convert from Java's ArithmeticException to RPython's + # ZeroDivisionError in the *catch* code, not here where the + # exception is generated. See introduce_exception_conversions() + # in node.py for details. + return op def _check_ovf(op): return op @@ -213,7 +214,7 @@ 'llong_ge': 'long_greater_equals', 'llong_and': jvmgen.LAND, 'llong_or': jvmgen.LOR, - 'llong_lshift': jvmgen.LSHL, + 'llong_lshift': [PushAllArgs, jvmgen.L2I, jvmgen.LSHL, StoreResult], # XXX - do we care about shifts of >(1<<32) bits?? 'llong_rshift': [PushAllArgs, jvmgen.L2I, jvmgen.LSHR, StoreResult], 'llong_xor': jvmgen.LXOR, 'llong_floordiv_ovf': jvmgen.LDIV, # these can't overflow! Modified: pypy/dist/pypy/translator/jvm/prebuiltnodes.py ============================================================================== --- pypy/dist/pypy/translator/jvm/prebuiltnodes.py (original) +++ pypy/dist/pypy/translator/jvm/prebuiltnodes.py Fri Feb 22 23:20:59 2008 @@ -22,6 +22,10 @@ raise OverflowError @with_types([]) +def throwRuntimeError(): + raise RuntimeError + + at with_types([]) def throwValueError(): raise ValueError Modified: pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java Fri Feb 22 23:20:59 2008 @@ -13,6 +13,7 @@ public void throwZeroDivisionError(); public void throwIndexError(); public void throwOverflowError(); + public void throwRuntimeError(); public void throwValueError(); public void throwUnicodeDecodeError(); public void throwOSError(int errCode); Added: pypy/dist/pypy/translator/jvm/test/test_extreme.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/test/test_extreme.py Fri Feb 22 23:20:59 2008 @@ -0,0 +1,5 @@ +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.translator.oosupport.test_template.extreme import BaseTestExtreme + +class TestExtreme(BaseTestExtreme, JvmTest): + pass Modified: pypy/dist/pypy/translator/jvm/test/test_snippet.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_snippet.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_snippet.py Fri Feb 22 23:20:59 2008 @@ -1,3 +1,4 @@ +import sys from pypy.translator.jvm.test.runtest import JvmTest from pypy.translator.oosupport.test_template.snippets import BaseTestSnippets Modified: pypy/dist/pypy/translator/jvm/typesystem.py ============================================================================== --- pypy/dist/pypy/translator/jvm/typesystem.py (original) +++ pypy/dist/pypy/translator/jvm/typesystem.py Fri Feb 22 23:20:59 2008 @@ -201,6 +201,8 @@ jPyPyRecordFloatFloat = JvmClassType('pypy.RecordFloatFloat') jPyPyAbstractMethodException = JvmClassType('pypy.AbstractMethodException') +jStackOverflowError = JvmClassType('java.lang.StackOverflowError', throwable=True) +jOutOfMemoryError = JvmClassType('java.lang.OutOfMemoryError', throwable=True) jArithmeticException = JvmClassType('java.lang.ArithmeticException', throwable=True) class JvmScalarType(JvmType): Modified: pypy/dist/pypy/translator/oosupport/function.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/function.py (original) +++ pypy/dist/pypy/translator/oosupport/function.py Fri Feb 22 23:20:59 2008 @@ -176,6 +176,13 @@ else: assert False, "No non-exceptional case from exc_handling block" + # give the backend a chance to see all the exceptions that might + # be caught here. For ex., JVM uses this to convert between + # built-in JVM exceptions to their RPython equivalents + if anyHandler: + self.introduce_exception_conversions( + [link.exitcase for link in block.exits if link.exitcase]) + # catch the exception and dispatch to the appropriate block for link in block.exits: if link.exitcase is None: @@ -192,6 +199,11 @@ self.after_except_block() + def introduce_exception_conversions(self, llexitcases): + """ Called before any catch blocks are emitted with the full set of + exceptions that might be caught """ + return + def after_except_block(self): pass Added: pypy/dist/pypy/translator/oosupport/test_template/extreme.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/oosupport/test_template/extreme.py Fri Feb 22 23:20:59 2008 @@ -0,0 +1,24 @@ +import sys, py + +class BaseTestExtreme: + + def test_memoryerror_due_to_oom(self): + py.test.skip("can't get MemoryError except block to show up") + def fn(): + try: + lst = [] + for i in range(sys.maxint): lst.append(i) + except MemoryError: + return "OK" + assert self.interpret(fn, []) == "OK" + + def test_runtimeerror_due_to_stack_overflow(self): + def loop(): + loop() + def fn(): + try: + loop() + except RuntimeError, e: + return "OK" + assert self.interpret(fn, []) == "OK" + Modified: pypy/dist/pypy/translator/oosupport/test_template/snippets.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/test_template/snippets.py (original) +++ pypy/dist/pypy/translator/oosupport/test_template/snippets.py Fri Feb 22 23:20:59 2008 @@ -1,4 +1,5 @@ from pypy.translator.test import snippet as s +from pypy.rlib.rarithmetic import r_longlong # ----------------------------------------------------------------- @@ -42,6 +43,12 @@ return x+y assert self.interpret(fn, [4,7]) == 11 + def test_llshl(self): + def fn(a, b): + return a << b + assert self.interpret(fn, [r_longlong(1), 52]) == (1<<52) + assert self.interpret(fn, [r_longlong(1), r_longlong(52)]) == (1<<52) + def test_manipulate(self): def fn(x,y): obj = SimplestObject() From arigo at codespeak.net Sat Feb 23 10:53:41 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 23 Feb 2008 10:53:41 +0100 (CET) Subject: [pypy-svn] r51815 - in pypy/branch/unified-rtti/pypy: rpython/memory/gc rpython/memory/gctransform translator/c Message-ID: <20080223095341.DA8A9168433@codespeak.net> Author: arigo Date: Sat Feb 23 10:53:39 2008 New Revision: 51815 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/base.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py pypy/branch/unified-rtti/pypy/translator/c/gc.py pypy/branch/unified-rtti/pypy/translator/c/node.py Log: * Type fixes. * Database ordering issues, as usual. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gc/base.py Sat Feb 23 10:53:39 2008 @@ -224,8 +224,9 @@ if not target: freeentry = i else: - ll_assert(self.get_type_id(llmemory.cast_ptr_to_adr(target)) - > 0, "bogus weakref in compute_id()") + targetadr = llmemory.cast_ptr_to_adr(target) + ll_assert(bool(self.get_type_id(targetadr)), + "bogus weakref in compute_id()") # record this entry in the dict adr = llmemory.cast_ptr_to_adr(target) self.object_id_dict[adr] = i Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py Sat Feb 23 10:53:39 2008 @@ -233,7 +233,7 @@ s_True = annmodel.SomeBool(); s_True .const = True self.malloc_fast_ptr = getfn( malloc_fast, - [s_gc, annmodel.SomeInteger(nonneg=True), + [s_gc, annmodel.SomePtr(RTTIPTR), annmodel.SomeInteger(nonneg=True), s_True, s_False, s_False], s_gcref, @@ -284,13 +284,13 @@ self.coalloc_clear_ptr = getfn( GCClass.coalloc_fixedsize_clear.im_func, [s_gc, annmodel.SomeAddress(), - annmodel.SomeInteger(nonneg=True), + annmodel.SomePtr(RTTIPTR), annmodel.SomeInteger(nonneg=True)], s_gcref, inline=True) self.coalloc_varsize_clear_ptr = getfn( GCClass.coalloc_varsize_clear.im_func, - [s_gc, annmodel.SomeAddress()] + - [annmodel.SomeInteger(nonneg=True) for i in range(5)], + [s_gc, annmodel.SomeAddress(), annmodel.SomePtr(RTTIPTR)] + + [annmodel.SomeInteger(nonneg=True) for i in range(4)], s_gcref, inline=True) else: self.coalloc_clear_ptr = self.coalloc_varsize_clear_ptr = None @@ -329,9 +329,23 @@ def consider_constant(self, TYPE, value): self.layoutbuilder.consider_constant(TYPE, value, self.gcdata.gc) - def initialize_typeinfo(self, typeinfo, rtti, TYPE): - raise Exception("for now, the layoutbuilder should have found " - "all possible GC types - got %r" % (TYPE,)) + def convert_rtti(self, rtti): + # xxx a bit indirect + rtti = rtti._as_ptr() + try: + return self.gcheaderbuilder.typeinfo_from_rtti(rtti) + except KeyError: + try: + TYPE = lltype.getGcTypeForRtti(rtti) + except ValueError: + # ignore rtti's not attached anywhere, e.g. in the + # vtable of raw-flavored RPython classes + typeinfo = self.gcheaderbuilder.new_typeinfo(rtti) + else: + rtti1 = self.layoutbuilder.get_type_id(TYPE) + assert rtti1 == rtti + typeinfo = self.gcheaderbuilder.typeinfo_from_rtti(rtti) + return typeinfo #def get_type_id(self, TYPE): # this method is attached to the instance and redirects to Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/transform.py Sat Feb 23 10:53:39 2008 @@ -481,6 +481,8 @@ # A bit of manual inlining... hdraddr = objaddr - gc_header_offset rtti = llmemory.cast_adr_to_ptr(hdraddr, HDRPTR).typeptr + if lltype.typeOf(rtti) is llmemory.Address: + rtti = llmemory.cast_adr_to_ptr(rtti, RTTIPTR) ll_assert(bool(rtti), "NULL rtti") return rtti gh.gc_runtime_type_info = gc_runtime_type_info @@ -491,6 +493,8 @@ # A bit of manual inlining... hdraddr = objaddr - gc_header_offset rtti = llmemory.cast_adr_to_ptr(hdraddr, HDRPTR).typeptr + if lltype.typeOf(rtti) is llmemory.Address: + rtti = llmemory.cast_adr_to_ptr(rtti, RTTIPTR) ll_assert(bool(rtti), "NULL rtti") return gcheaderbuilder.cast_rtti_to_typeinfo(rtti) gh.typeof = typeof @@ -679,7 +683,10 @@ p = value._as_ptr() if not self.gcheaderbuilder.get_header(p): hdr = self.gcheaderbuilder.new_header(p) - hdr.typeptr = self.gcheaderbuilder.getRtti(TYPE) + rtti = self.gcheaderbuilder.getRtti(TYPE) + if lltype.typeOf(hdr).TO.typeptr is llmemory.Address: + rtti = llmemory.cast_ptr_to_adr(rtti) + hdr.typeptr = rtti self.initialize_constant_header(hdr, TYPE, value) def initialize_constant_header(self, hdr, TYPE, value): Modified: pypy/branch/unified-rtti/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/gc.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/gc.py Sat Feb 23 10:53:39 2008 @@ -42,6 +42,23 @@ def array_gcheader_initdata(self, defnode): return self.common_gcheader_initdata(defnode) + def enum_gcheader_dependencies(self, TYPE): + if TYPE._gckind != 'gc': + return [] + # make sure that the rtti object of the TYPE is seen by the + # database early, i.e. before finish_helpers() on the + # gctransformer. In particular, this should follow the + # ll_finalizer helper function stored in the typeinfo. + gct = self.db.gctransformer + rtti = gct.gcheaderbuilder.getRtti(TYPE) + result = [rtti] + # The ll_finalizer helpers are delayed func pointers computed + # only in finish_helpers(). But we need to follow the regular + # destructor before finish_helpers(), in case it uses new types. + if rtti is not None and rtti.destructor_funcptr is not None: + result.append(rtti.destructor_funcptr) + return result + def struct_after_definition(self, defnode): return [] Modified: pypy/branch/unified-rtti/pypy/translator/c/node.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/node.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/node.py Sat Feb 23 10:53:39 2008 @@ -68,6 +68,8 @@ db = self.db STRUCT = self.STRUCT varlength = self.varlength + for ptr in db.gcpolicy.enum_gcheader_dependencies(STRUCT): + db.get(ptr) if needs_gcheader(self.STRUCT): for fname, T in db.gcpolicy.struct_gcheader_definition(self): self.fields.append((fname, db.gettype(T, who_asks=self))) @@ -196,6 +198,8 @@ return # setup() was already called, likely by __init__ db = self.db ARRAY = self.ARRAY + for ptr in db.gcpolicy.enum_gcheader_dependencies(ARRAY): + db.get(ptr) if needs_gcheader(ARRAY): for fname, T in db.gcpolicy.array_gcheader_definition(self): self.gcfields.append((fname, db.gettype(T, who_asks=self))) From arigo at codespeak.net Sat Feb 23 10:59:08 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 23 Feb 2008 10:59:08 +0100 (CET) Subject: [pypy-svn] r51816 - pypy/branch/unified-rtti/pypy/rpython/memory/test Message-ID: <20080223095908.4763216843E@codespeak.net> Author: arigo Date: Sat Feb 23 10:59:07 2008 New Revision: 51816 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/test/test_gctypelayout.py Log: Fix test. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/test/test_gctypelayout.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/test/test_gctypelayout.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/test/test_gctypelayout.py Sat Feb 23 10:59:07 2008 @@ -1,5 +1,6 @@ from pypy.rpython.memory.gctypelayout import TypeLayoutBuilder, GCData from pypy.rpython.memory.gctypelayout import offsets_to_gc_pointers +from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rpython.lltypesystem import lltype def getname(T): @@ -32,11 +33,14 @@ def test_layout_builder(): # XXX a very minimal test - layoutbuilder = TypeLayoutBuilder() + HDR = lltype.Struct("HDR") + TYPEINFO = GCData.TYPE_INFO + gcheaderbuilder = GCHeaderBuilder(HDR, TYPEINFO) + layoutbuilder = TypeLayoutBuilder(gcheaderbuilder) for T1, T2 in [(GC_A, GC_S), (GC_A2, GC_S2), (GC_S3, GC_S2)]: tid1 = layoutbuilder.get_type_id(T1) tid2 = layoutbuilder.get_type_id(T2) - gcdata = GCData(layoutbuilder.type_info_list) + gcdata = GCData(gcheaderbuilder) lst1 = gcdata.q_varsize_offsets_to_gcpointers_in_var_part(tid1) lst2 = gcdata.q_offsets_to_gc_pointers(tid2) assert len(lst1) == len(lst2) From arigo at codespeak.net Sat Feb 23 11:05:54 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 23 Feb 2008 11:05:54 +0100 (CET) Subject: [pypy-svn] r51817 - pypy/branch/unified-rtti/pypy/rpython/memory/gc Message-ID: <20080223100554.D8A54168440@codespeak.net> Author: arigo Date: Sat Feb 23 11:05:53 2008 New Revision: 51817 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py Log: Typo. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gc/marksweep.py Sat Feb 23 11:05:53 2008 @@ -623,6 +623,7 @@ hdr.setmark2() # mark all objects from malloced_list oldobjects.append(llmemory.cast_ptr_to_adr(hdr)) hdr = next + del hdr # a stack of addresses of places that still points to old objects # and that must possibly be fixed to point to a new copy @@ -640,7 +641,7 @@ continue # ignore objects that were not in the malloced_list newhdr = oldhdr.getnext() # abused to point to the copy if not newhdr: - typeid = hdr.typeptr + typeid = oldhdr.typeptr size = self.fixed_size(typeid) # XXX! collect() at the beginning if the free heap is low if self.is_varsize(typeid): From arigo at codespeak.net Sat Feb 23 11:28:01 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 23 Feb 2008 11:28:01 +0100 (CET) Subject: [pypy-svn] r51818 - in pypy/branch/unified-rtti/pypy: rpython/memory/gctransform translator/c translator/c/test Message-ID: <20080223102801.CA77216842B@codespeak.net> Author: arigo Date: Sat Feb 23 11:28:01 2008 New Revision: 51818 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py pypy/branch/unified-rtti/pypy/translator/c/database.py pypy/branch/unified-rtti/pypy/translator/c/gc.py pypy/branch/unified-rtti/pypy/translator/c/node.py pypy/branch/unified-rtti/pypy/translator/c/test/test_newgc.py Log: Yay! Kill an obscure hack that was failing anyway. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py Sat Feb 23 11:28:01 2008 @@ -182,3 +182,7 @@ plink = lltype.malloc(WEAKLINK, immortal=True) plink[0] = llmemory.cast_ptr_to_adr(targetptr) return plink + +BoehmGCTransformer.WEAKREFTYPE = WEAKLINK +BoehmGCTransformer.convert_weakref_to = staticmethod(convert_weakref_to) + Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/framework.py Sat Feb 23 11:28:01 2008 @@ -13,7 +13,7 @@ from pypy.rpython import annlowlevel from pypy.rpython.rbuiltin import gen_cast from pypy.rpython.memory.gctypelayout import ll_weakref_deref, WEAKREF -from pypy.rpython.memory.gctypelayout import convert_weakref_to, WEAKREFPTR +from pypy.rpython.memory.gctypelayout import WEAKREFPTR from pypy.rpython.memory.gctransform.log import log from pypy.tool.sourcetools import func_with_new_name from pypy.rpython.lltypesystem.lloperation import llop @@ -97,6 +97,9 @@ TYPEINFO = gctypelayout.GCData.TYPE_INFO + WEAKREFTYPE = WEAKREF + convert_weakref_to = staticmethod(gctypelayout.convert_weakref_to) + def __init__(self, translator): from pypy.rpython.memory.gc.base import choose_gc_from_config if hasattr(self, 'GC_PARAMS'): Modified: pypy/branch/unified-rtti/pypy/translator/c/database.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/database.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/database.py Sat Feb 23 11:28:01 2008 @@ -1,9 +1,9 @@ from pypy.rpython.lltypesystem.lltype import \ - Primitive, Ptr, typeOf, RuntimeTypeInfo, RuntimeTypeInfoType, \ + Primitive, Ptr, typeOf, RuntimeTypeInfoType, \ Struct, Array, FuncType, PyObject, Void, \ ContainerType, OpaqueType, FixedSizeArray, _uninitialized from pypy.rpython.lltypesystem import lltype -from pypy.rpython.lltypesystem.llmemory import Address, WeakRef, _WeakRefType +from pypy.rpython.lltypesystem.llmemory import Address, _WeakRefType from pypy.rpython.lltypesystem.rffi import CConstant from pypy.tool.sourcetools import valid_identifier from pypy.translator.c.primitive import PrimitiveName, PrimitiveType @@ -85,14 +85,12 @@ node = ArrayDefNode(self, T, varlength) elif isinstance(T, OpaqueType) and T.hints.get("render_structure", False): node = ExtTypeOpaqueDefNode(self, T) - elif T == WeakRef: - REALT = self.gcpolicy.get_real_weakref_type() - node = self.gettypedefnode(REALT) - elif T == RuntimeTypeInfo: - REALT = self.gcpolicy.get_real_rtti_type() - node = self.gettypedefnode(REALT) else: - raise NoCorrespondingNode("don't know about %r" % (T,)) + REALT = self.gcpolicy.convert_type(T) + if REALT != T: + node = self.gettypedefnode(REALT) + else: + raise NoCorrespondingNode("don't know about %r" % (T,)) self.structdefnodes[key] = node self.pendingsetupnodes.append(node) return node @@ -131,9 +129,7 @@ argtypes = ', '.join(argtypes) or 'void' return resulttype.replace('@', '(@)(%s)' % argtypes) elif isinstance(T, OpaqueType): - if T == RuntimeTypeInfo: - return self.gcpolicy.rtti_type() - elif T.hints.get("render_structure", False): + if T.hints.get("render_structure", False): node = self.gettypedefnode(T, varlength=varlength) if who_asks is not None: who_asks.dependencies[node] = True @@ -147,10 +143,12 @@ else: raise Exception("don't know about type %r" % (T,)) - def getcontainernode(self, container, _dont_write_c_code=True, **buildkwds): + def getcontainernode(self, container, **buildkwds): try: node = self.containernodes[container] except KeyError: + originalcontainer = container + container = self.gcpolicy.convert_prebuilt_object(container) T = typeOf(container) if isinstance(T, (lltype.Array, lltype.Struct)): if self.gctransformer is not None: @@ -158,10 +156,7 @@ nodefactory = ContainerNodeFactory[T.__class__] node = nodefactory(self, T, container, **buildkwds) self.containernodes[container] = node - # _dont_write_c_code should only be False for a hack in - # weakrefnode_factory() - if not _dont_write_c_code: - return node + self.containernodes[originalcontainer] = node kind = getattr(node, 'nodekind', '?') self.containerstats[kind] = self.containerstats.get(kind, 0) + 1 self.containerlist.append(node) Modified: pypy/branch/unified-rtti/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/gc.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/gc.py Sat Feb 23 11:28:01 2008 @@ -74,12 +74,24 @@ def gc_startup_code(self): return [] - # for rtti node - def get_real_rtti_type(self): - return self.db.gctransformer.gcheaderbuilder.TYPEINFO + # support for mapping weakref and rtti objects and types - def convert_rtti(self, obj): - return self.db.gctransformer.convert_rtti(obj) + def convert_type(self, TYPE): + if TYPE == lltype.RuntimeTypeInfo: + return self.db.gctransformer.gcheaderbuilder.TYPEINFO + elif TYPE == llmemory.WeakRef: + return self.db.gctransformer.WEAKREFTYPE + else: + return TYPE + + def convert_prebuilt_object(self, obj): + if isinstance(obj, lltype._rtti): + return self.db.gctransformer.convert_rtti(obj)._obj + elif isinstance(obj, llmemory._wref): + ptarget = obj._dereference() + return self.db.gctransformer.convert_weakref_to(ptarget)._obj + else: + return obj def OP_GC_PUSH_ALIVE_PYOBJ(self, funcgen, op): expr = funcgen.expr(op.args[0]) @@ -130,12 +142,6 @@ yield 'GC_all_interior_pointers = 0;' yield 'boehm_gc_startup_code();' - def get_real_weakref_type(self): - return boehm.WEAKLINK - - def convert_weakref_to(self, ptarget): - return boehm.convert_weakref_to(ptarget) - def OP_GC__COLLECT(self, funcgen, op): return 'GC_gcollect();' @@ -165,12 +171,6 @@ fnptr = self.db.gctransformer.frameworkgc_setup_ptr.value yield '%s();' % (self.db.get(fnptr),) - def get_real_weakref_type(self): - return framework.WEAKREF - - def convert_weakref_to(self, ptarget): - return framework.convert_weakref_to(ptarget) - def OP_GC_RELOAD_POSSIBLY_MOVED(self, funcgen, op): args = [funcgen.expr(v) for v in op.args] return '%s = %s; /* for moving GCs */' % (args[1], args[0]) Modified: pypy/branch/unified-rtti/pypy/translator/c/node.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/node.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/node.py Sat Feb 23 11:28:01 2008 @@ -875,34 +875,6 @@ def implementation(self): return [] -def weakrefnode_factory(db, T, obj): - assert isinstance(obj, llmemory._wref) - ptarget = obj._dereference() - wrapper = db.gcpolicy.convert_weakref_to(ptarget) - container = wrapper._obj - obj._converted_weakref = container # hack for genllvm :-/ - return db.getcontainernode(container, _dont_write_c_code=False) - -class RttiNode(ContainerNode): - nodekind = 'rtti' - - def __init__(self, db, T, obj): - assert isinstance(obj, lltype._rtti) - wrapper = db.gcpolicy.convert_rtti(obj) - self.realobj = wrapper._obj - self.realnode = db.getcontainernode(self.realobj, - _dont_write_c_code=False) - ContainerNode.__init__(self, db, T, obj) - - def basename(self): - return self.realnode.basename() - - def enum_dependencies(self): - return self.realnode.enum_dependencies() - - def initializationexpr(self, decoration=''): - return self.realnode.initializationexpr(decoration) - ContainerNodeFactory = { Struct: StructNode, @@ -913,6 +885,4 @@ FuncType: FuncNode, OpaqueType: opaquenode_factory, PyObjectType: PyObjectNode, - llmemory._WeakRefType: weakrefnode_factory, - lltype.RuntimeTypeInfoType: RttiNode, } Modified: pypy/branch/unified-rtti/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/test/test_newgc.py Sat Feb 23 11:28:01 2008 @@ -147,9 +147,8 @@ def test_del_basic(): for gcpolicy in ["ref"]: #, "framework"]: - S = lltype.GcStruct('S', ('x', lltype.Signed)) - TRASH = lltype.GcStruct('TRASH', ('x', lltype.Signed)) - lltype.attachRuntimeTypeInfo(S) + rtti = lltype.malloc(lltype.RuntimeTypeInfo, immortal=True) + S = lltype.GcStruct('S', ('x', lltype.Signed), runtime_type_info=rtti) GLOBAL = lltype.Struct('GLOBAL', ('x', lltype.Signed)) glob = lltype.malloc(GLOBAL, immortal=True) def destructor(s): @@ -170,7 +169,7 @@ t.buildannotator().build_types(entrypoint, [int]) rtyper = t.buildrtyper() destrptr = rtyper.annotate_helper_fn(destructor, [lltype.Ptr(S)]) - rtyper.attachRuntimeTypeInfoFunc(S, type_info_S, destrptr=destrptr) + rtti.destructor_funcptr = destrptr rtyper.specialize() fn = compile_func(entrypoint, None, t, gcpolicy=gcpolicy) From arigo at codespeak.net Sat Feb 23 11:46:47 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 23 Feb 2008 11:46:47 +0100 (CET) Subject: [pypy-svn] r51819 - pypy/branch/unified-rtti/pypy/translator/c Message-ID: <20080223104647.11B5C168443@codespeak.net> Author: arigo Date: Sat Feb 23 11:46:46 2008 New Revision: 51819 Modified: pypy/branch/unified-rtti/pypy/translator/c/database.py pypy/branch/unified-rtti/pypy/translator/c/node.py Log: Fix -- all vtables were generated as just their type_info header, without the rest part. Modified: pypy/branch/unified-rtti/pypy/translator/c/database.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/database.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/database.py Sat Feb 23 11:46:46 2008 @@ -154,7 +154,8 @@ if self.gctransformer is not None: self.gctransformer.consider_constant(T, container) nodefactory = ContainerNodeFactory[T.__class__] - node = nodefactory(self, T, container, **buildkwds) + parentlink = lltype.parentlink(originalcontainer) # typeinfo/rtti + node = nodefactory(self, T, container, parentlink, **buildkwds) self.containernodes[container] = node self.containernodes[originalcontainer] = node kind = getattr(node, 'nodekind', '?') Modified: pypy/branch/unified-rtti/pypy/translator/c/node.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/node.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/node.py Sat Feb 23 11:46:46 2008 @@ -2,7 +2,7 @@ from pypy.rpython.lltypesystem.lltype import \ Struct, Array, FixedSizeArray, FuncType, PyObjectType, typeOf, \ GcStruct, GcArray, ContainerType, \ - parentlink, Ptr, PyObject, Void, OpaqueType, Float, \ + Ptr, PyObject, Void, OpaqueType, Float, \ RuntimeTypeInfo, getRuntimeTypeInfo, Char, _subarray from pypy.rpython.lltypesystem import lltype, llmemory from pypy.translator.c.funcgen import FunctionCodeGenerator @@ -426,14 +426,14 @@ name ptrname globalcontainer""".split() - def __init__(self, db, T, obj): + def __init__(self, db, T, obj, parentlink): self.db = db self.T = T self.obj = obj #self.dependencies = {} self.typename = db.gettype(T) #, who_asks=self) self.implementationtypename = db.gettype(T, varlength=self.getlength()) - parent, parentindex = parentlink(obj) + parent, parentindex = parentlink if parent is None: self.name = db.namespace.uniquename('g_' + self.basename()) self.globalcontainer = True @@ -535,8 +535,8 @@ if USESLOTS: __slots__ = () - def __init__(self, db, T, obj): - ContainerNode.__init__(self, db, T, obj) + def __init__(self, db, T, obj, parentlink): + ContainerNode.__init__(self, db, T, obj, parentlink) if barebonearray(T): self.ptrname = self.name @@ -598,8 +598,8 @@ if USESLOTS: __slots__ = () - def __init__(self, db, T, obj): - ContainerNode.__init__(self, db, T, obj) + def __init__(self, db, T, obj, parentlink): + ContainerNode.__init__(self, db, T, obj, parentlink) if not isinstance(obj, _subarray): # XXX hackish self.ptrname = self.name @@ -661,7 +661,7 @@ # there not so many node of this kind, slots should not # be necessary - def __init__(self, db, T, obj, forcename=None): + def __init__(self, db, T, obj, parentlink, forcename=None): self.globalcontainer = True self.db = db self.T = T @@ -839,7 +839,7 @@ typename = 'PyObject @' implementationtypename = 'PyObject *@' - def __init__(self, db, T, obj): + def __init__(self, db, T, obj, parentlink): # obj is a _pyobject here; obj.value is the underlying CPython object self.db = db self.T = T From arigo at codespeak.net Sat Feb 23 12:12:14 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 23 Feb 2008 12:12:14 +0100 (CET) Subject: [pypy-svn] r51820 - pypy/branch/unified-rtti/pypy/translator/c Message-ID: <20080223111214.1507C16842F@codespeak.net> Author: arigo Date: Sat Feb 23 12:12:13 2008 New Revision: 51820 Modified: pypy/branch/unified-rtti/pypy/translator/c/gc.py pypy/branch/unified-rtti/pypy/translator/c/node.py Log: Database ordering issue, take #8317. Modified: pypy/branch/unified-rtti/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/gc.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/gc.py Sat Feb 23 12:12:13 2008 @@ -42,22 +42,14 @@ def array_gcheader_initdata(self, defnode): return self.common_gcheader_initdata(defnode) - def enum_gcheader_dependencies(self, TYPE): - if TYPE._gckind != 'gc': - return [] - # make sure that the rtti object of the TYPE is seen by the - # database early, i.e. before finish_helpers() on the - # gctransformer. In particular, this should follow the - # ll_finalizer helper function stored in the typeinfo. - gct = self.db.gctransformer - rtti = gct.gcheaderbuilder.getRtti(TYPE) - result = [rtti] - # The ll_finalizer helpers are delayed func pointers computed - # only in finish_helpers(). But we need to follow the regular - # destructor before finish_helpers(), in case it uses new types. - if rtti is not None and rtti.destructor_funcptr is not None: - result.append(rtti.destructor_funcptr) - return result + def follow_rtti_dependencies(self, rtti): + # Issue: the typeinfo corresponding to the rtti contains, as + # destructor, a pointer to some ll_finalizer helper. However it + # is a delayed func pointer computed only in finish_helpers(). + # But we need to follow the regular destructor before + # finish_helpers(), in case it uses new types. + if rtti.destructor_funcptr is not None: + self.db.get(rtti.destructor_funcptr) def struct_after_definition(self, defnode): return [] @@ -86,6 +78,7 @@ def convert_prebuilt_object(self, obj): if isinstance(obj, lltype._rtti): + self.follow_rtti_dependencies(obj) return self.db.gctransformer.convert_rtti(obj)._obj elif isinstance(obj, llmemory._wref): ptarget = obj._dereference() Modified: pypy/branch/unified-rtti/pypy/translator/c/node.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/node.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/node.py Sat Feb 23 12:12:13 2008 @@ -68,8 +68,6 @@ db = self.db STRUCT = self.STRUCT varlength = self.varlength - for ptr in db.gcpolicy.enum_gcheader_dependencies(STRUCT): - db.get(ptr) if needs_gcheader(self.STRUCT): for fname, T in db.gcpolicy.struct_gcheader_definition(self): self.fields.append((fname, db.gettype(T, who_asks=self))) @@ -198,8 +196,6 @@ return # setup() was already called, likely by __init__ db = self.db ARRAY = self.ARRAY - for ptr in db.gcpolicy.enum_gcheader_dependencies(ARRAY): - db.get(ptr) if needs_gcheader(ARRAY): for fname, T in db.gcpolicy.array_gcheader_definition(self): self.gcfields.append((fname, db.gettype(T, who_asks=self))) From arigo at codespeak.net Sat Feb 23 13:48:12 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 23 Feb 2008 13:48:12 +0100 (CET) Subject: [pypy-svn] r51821 - in pypy/branch/unified-rtti/pypy: rpython/lltypesystem/test rpython/test translator/c/test Message-ID: <20080223124812.F212F16847A@codespeak.net> Author: arigo Date: Sat Feb 23 13:48:10 2008 New Revision: 51821 Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_llarena.py pypy/branch/unified-rtti/pypy/rpython/test/test_rptr.py pypy/branch/unified-rtti/pypy/translator/c/test/test_genc.py Log: Fix more tests. Modified: pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/lltypesystem/test/test_llarena.py Sat Feb 23 13:48:10 2008 @@ -213,9 +213,10 @@ def test_replace_object_with_stub(): from pypy.rpython.memory.gcheader import GCHeaderBuilder HDR = lltype.Struct('HDR', ('x', lltype.Signed)) + TYPEINFO = lltype.Struct('TYPEINFO') S = lltype.GcStruct('S', ('y', lltype.Signed), ('z', lltype.Signed)) STUB = lltype.GcStruct('STUB', ('t', lltype.Char)) - gcheaderbuilder = GCHeaderBuilder(HDR) + gcheaderbuilder = GCHeaderBuilder(HDR, TYPEINFO) size_gc_header = gcheaderbuilder.size_gc_header a = arena_malloc(50, True) Modified: pypy/branch/unified-rtti/pypy/rpython/test/test_rptr.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/test/test_rptr.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/test/test_rptr.py Sat Feb 23 13:48:10 2008 @@ -36,8 +36,8 @@ assert s.ll_ptrtype == PS2 def test_runtime_type_info(): - S = GcStruct('s', ('x', Signed)) - attachRuntimeTypeInfo(S) + rtti = malloc(RuntimeTypeInfo, immortal=True) + S = GcStruct('s', ('x', Signed), runtime_type_info=rtti) def ll_example(p): return (runtime_type_info(p), runtime_type_info(p) == getRuntimeTypeInfo(S)) Modified: pypy/branch/unified-rtti/pypy/translator/c/test/test_genc.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/test/test_genc.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/test/test_genc.py Sat Feb 23 13:48:10 2008 @@ -138,41 +138,18 @@ def test_runtime_type_info(): - S = GcStruct('s', ('is_actually_s1', Bool)) - S1 = GcStruct('s1', ('sub', S)) - attachRuntimeTypeInfo(S) - attachRuntimeTypeInfo(S1) - def rtti_S(p): - if p.is_actually_s1: - return getRuntimeTypeInfo(S1) - else: - return getRuntimeTypeInfo(S) - def rtti_S1(p): - return getRuntimeTypeInfo(S1) + rtti = malloc(RuntimeTypeInfo, immortal=True) + rtti1 = malloc(RuntimeTypeInfo, immortal=True) + S = GcStruct('s', runtime_type_info=rtti) + S1 = GcStruct('s1', ('sub', S), runtime_type_info=rtti1) def does_stuff(): p = malloc(S) - p.is_actually_s1 = False p1 = malloc(S1) - p1.sub.is_actually_s1 = True # and no crash when p and p1 are decref'ed return None - t = TranslationContext() - t.buildannotator().build_types(does_stuff, []) - rtyper = t.buildrtyper() - rtyper.attachRuntimeTypeInfoFunc(S, rtti_S) - rtyper.attachRuntimeTypeInfoFunc(S1, rtti_S1) - rtyper.specialize() - #t.view() - from pypy.translator.c import genc - t.config.translation.countmallocs = True - builder = genc.CExtModuleBuilder(t, does_stuff, config=t.config) - builder.generate_source() - builder.compile() - f1 = builder.get_entry_point() + f1 = compile(does_stuff, []) f1() - mallocs, frees = builder.get_malloc_counters()() - assert mallocs == frees def test_str(): From arigo at codespeak.net Sat Feb 23 13:50:54 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 23 Feb 2008 13:50:54 +0100 (CET) Subject: [pypy-svn] r51822 - pypy/branch/unified-rtti/pypy/translator/c Message-ID: <20080223125054.CB83E168473@codespeak.net> Author: arigo Date: Sat Feb 23 13:50:52 2008 New Revision: 51822 Modified: pypy/branch/unified-rtti/pypy/translator/c/genc.py Log: Don't depend on trackgcroot.py to be an executable file. (e.g. it's not the cause with py.test --dist) Modified: pypy/branch/unified-rtti/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/genc.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/genc.py Sat Feb 23 13:50:52 2008 @@ -835,7 +835,7 @@ \t$(CC) $(CFLAGS) -o $@ -S $< $(INCLUDEDIRS) gcmaptable.s: $(ASMFILES) -\t$(PYPYDIR)/translator/c/gcc/trackgcroot.py $(ASMFILES) > $@ || (rm -f $@ && exit 1) +\tpython $(PYPYDIR)/translator/c/gcc/trackgcroot.py $(ASMFILES) > $@ || (rm -f $@ && exit 1) clean: \trm -f $(OBJECTS) $(TARGET) From arigo at codespeak.net Sat Feb 23 14:14:14 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 23 Feb 2008 14:14:14 +0100 (CET) Subject: [pypy-svn] r51823 - pypy/branch/unified-rtti/pypy/translator/c/test Message-ID: <20080223131414.4A933168473@codespeak.net> Author: arigo Date: Sat Feb 23 14:14:13 2008 New Revision: 51823 Modified: pypy/branch/unified-rtti/pypy/translator/c/test/test_boehm.py Log: A baseline test already fails with Boehm. Seems we don't have many such tests for Boehm (they are all run with refcounting...) Modified: pypy/branch/unified-rtti/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/branch/unified-rtti/pypy/translator/c/test/test_boehm.py (original) +++ pypy/branch/unified-rtti/pypy/translator/c/test/test_boehm.py Sat Feb 23 14:14:13 2008 @@ -63,6 +63,19 @@ fn = self.getcompiled(malloc_a_lot) fn() + def test_classattribute(self): + from pypy.translator.test import snippet + def classattr(): + ok1 = snippet.classattribute(1) == 123 + ok2 = snippet.classattribute(2) == 456 + ok3 = snippet.classattribute(3) == 789 + ok4 = snippet.classattribute(4) == 789 + ok5 = snippet.classattribute(5) == 101112 + return (ok1 + ok2*10 + ok3*100 + ok4*1000 + ok5*10000) + fn = self.getcompiled(classattr) + res = fn() + assert res == 11111 + def test__del__(self): from pypy.rpython.lltypesystem.lloperation import llop class State: From niko at codespeak.net Sat Feb 23 14:30:45 2008 From: niko at codespeak.net (niko at codespeak.net) Date: Sat, 23 Feb 2008 14:30:45 +0100 (CET) Subject: [pypy-svn] r51824 - in pypy/dist/pypy/translator: jvm jvm/src/pypy oosupport/test_template Message-ID: <20080223133045.CA93416844A@codespeak.net> Author: niko Date: Sat Feb 23 14:30:45 2008 New Revision: 51824 Modified: pypy/dist/pypy/translator/jvm/prebuiltnodes.py pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java pypy/dist/pypy/translator/oosupport/test_template/extreme.py Log: fix out of memory error and incldue a include a test Modified: pypy/dist/pypy/translator/jvm/prebuiltnodes.py ============================================================================== --- pypy/dist/pypy/translator/jvm/prebuiltnodes.py (original) +++ pypy/dist/pypy/translator/jvm/prebuiltnodes.py Sat Feb 23 14:30:45 2008 @@ -26,6 +26,10 @@ raise RuntimeError @with_types([]) +def throwMemoryError(): + raise MemoryError + + at with_types([]) def throwValueError(): raise ValueError Modified: pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java ============================================================================== --- pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java (original) +++ pypy/dist/pypy/translator/jvm/src/pypy/Interlink.java Sat Feb 23 14:30:45 2008 @@ -14,6 +14,7 @@ public void throwIndexError(); public void throwOverflowError(); public void throwRuntimeError(); + public void throwMemoryError(); public void throwValueError(); public void throwUnicodeDecodeError(); public void throwOSError(int errCode); Modified: pypy/dist/pypy/translator/oosupport/test_template/extreme.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/test_template/extreme.py (original) +++ pypy/dist/pypy/translator/oosupport/test_template/extreme.py Sat Feb 23 14:30:45 2008 @@ -3,13 +3,16 @@ class BaseTestExtreme: def test_memoryerror_due_to_oom(self): - py.test.skip("can't get MemoryError except block to show up") + def relentless_memory_consumption_machine(): + lst = [] + while True: lst.append([]) + def fn(): try: - lst = [] - for i in range(sys.maxint): lst.append(i) + relentless_memory_consumption_machine() except MemoryError: return "OK" + return "How much memory do you HAVE??" assert self.interpret(fn, []) == "OK" def test_runtimeerror_due_to_stack_overflow(self): From cfbolz at codespeak.net Sun Feb 24 11:57:18 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 24 Feb 2008 11:57:18 +0100 (CET) Subject: [pypy-svn] r51831 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter Message-ID: <20080224105718.1879F168414@codespeak.net> Author: cfbolz Date: Sun Feb 24 11:57:15 2008 New Revision: 51831 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Log: start supporting promotion in the rainbow interpreter Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Sun Feb 24 11:57:15 2008 @@ -26,6 +26,7 @@ self.raise_analyzer = hannotator.exceptiontransformer.raise_analyzer self.all_graphs = {} # mapping graph to bytecode self.unfinished_graphs = [] + self.num_global_mergepoints = 0 def can_raise(self, op): return self.raise_analyzer.analyze(op) @@ -45,8 +46,9 @@ self.arrayfielddescs = [] self.interiordescs = [] self.oopspecdescs = [] + self.promotiondescs = [] self.called_bytecodes = [] - self.num_mergepoints = 0 + self.num_local_mergepoints = 0 self.graph_color = self.graph_calling_color(graph) self.nonrainbow_functions = [] self.is_portal = is_portal @@ -74,13 +76,17 @@ self.interiordesc_positions = {} # mapping (fnobj, can_raise) to index self.oopspecdesc_positions = {} + # mapping (fnobj, can_raise) to index + self.promotiondesc_positions = {} # mapping graphs to index self.graph_positions = {} # mapping fnobjs to index self.nonrainbow_positions = {} self.graph = graph - self.entrymap = flowmodel.mkentrymap(graph) + self.mergepoint_set = {} + self.compute_merge_points() + self.make_bytecode_block(graph.startblock) assert self.current_block is None bytecode = self.all_graphs[graph] @@ -95,13 +101,16 @@ self.arrayfielddescs, self.interiordescs, self.oopspecdescs, + self.promotiondescs, self.called_bytecodes, - self.num_mergepoints, + self.num_local_mergepoints, self.graph_color, self.nonrainbow_functions, self.is_portal) if is_portal: self.finish_all_graphs() + self.interpreter.set_num_global_mergepoints( + self.num_global_mergepoints) return bytecode def finish_all_graphs(self): @@ -109,6 +118,43 @@ graph = self.unfinished_graphs.pop() self.make_bytecode(graph, is_portal=False) + def compute_merge_points(self): + entrymap = flowmodel.mkentrymap(self.graph) + startblock = self.graph.startblock + global_merge_blocks = {} + for block in self.graph.iterblocks(): + if not block.operations: + continue + op = block.operations[0] + hashint = False + cand = 0 + if (op.opname == 'hint' and + op.args[1].value == {'global_merge_point': True}): + hashint = True + if block is startblock or len(entrymap[block]) > 1: + global_merge_blocks[block] = True + cand += 1 + else: + prevblock = entrymap[block][0].prevblock + if len(entrymap[prevblock]) > 1: + global_merge_blocks[prevblock] = True + cand += 1 + assert not hashint or cand==1, ( + "ambigous global merge point hint: %r" % block) + for op in block.operations[1:]: + assert not (op.opname == 'hint' and + op.args[1].value == {'global_merge_point': True}), ( + "stranded global merge point hint: %r" % block) + + for block, links in entrymap.items(): + if len(links) > 1 and block is not self.graph.returnblock: + if block in global_merge_blocks: + self.mergepoint_set[block] = 'global' + else: + self.mergepoint_set[block] = 'local' + if startblock in global_merge_blocks: + self.mergepoint_set[startblock] = 'global' + def make_bytecode_block(self, block, insert_goto=False): if block in self.seen_blocks: if insert_goto: @@ -185,10 +231,8 @@ def insert_merges(self, block): if block is self.graph.returnblock: return - if len(self.entrymap[block]) <= 1: + if block not in self.mergepoint_set: return - num = self.num_mergepoints - self.num_mergepoints += 1 # make keydesc key = () for arg in self.sort_by_color(block.inputargs)[1]: @@ -202,7 +246,16 @@ self.keydescs.append(KeyDesc(self.RGenOp, *key)) else: keyindex = self.keydesc_positions[key] - self.emit("merge") + + kind = self.mergepoint_set[block] + if kind == "global": + self.emit("guard_global_merge") + num = self.num_global_mergepoints + self.num_global_mergepoints += 1 + else: + num = self.num_local_mergepoints + self.num_local_mergepoints += 1 + self.emit("%s_merge" % kind) self.emit(num) self.emit(keyindex) @@ -367,6 +420,16 @@ self.oopspecdesc_positions[key] = result return result + def promotiondesc_position(self, TYPE): + ERASED = self.RGenOp.erasedType(TYPE) + if ERASED in self.promotiondesc_positions: + return self.promotiondesc_positions[ERASED] + promotiondesc = rtimeshift.PromotionDesc(ERASED, self.interpreter) + result = len(self.promotiondescs) + self.promotiondescs.append(promotiondesc) + self.promotiondesc_positions[ERASED] = result + return result + def graph_position(self, graph): if graph in self.graph_positions: return self.graph_positions[graph] @@ -491,6 +554,18 @@ else: self.register_greenvar(result, self.green_position(arg)) return + if "promote" in hints: + if self.varcolor(arg) == "green": + self.register_greenvar(result, self.green_position(arg)) + return + self.emit("promote") + self.emit(self.serialize_oparg("red", arg)) + self.emit(self.promotiondesc_position(arg.concretetype)) + self.register_greenvar(result) + return + if "global_merge_point" in hints: + return # the compute_merge_points function already cared + XXX def args_of_call(self, args, colored_as): result = [] Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Sun Feb 24 11:57:15 2008 @@ -1,7 +1,7 @@ from pypy.rlib.rarithmetic import intmask from pypy.rlib.unroll import unrolling_iterable from pypy.jit.timeshifter import rtimeshift, rcontainer -from pypy.jit.timeshifter.greenkey import empty_key, GreenKey +from pypy.jit.timeshifter.greenkey import empty_key, GreenKey, newgreendict from pypy.rpython.lltypesystem import lltype class JitCode(object): @@ -19,7 +19,8 @@ def __init__(self, name, code, constants, typekinds, redboxclasses, keydescs, structtypedescs, fielddescs, arrayfielddescs, - interiordescs, oopspecdescs, called_bytecodes, num_mergepoints, + interiordescs, oopspecdescs, promotiondescs, + called_bytecodes, num_mergepoints, graph_color, nonrainbow_functions, is_portal): self.name = name self.code = code @@ -32,6 +33,7 @@ self.arrayfielddescs = arrayfielddescs self.interiordescs = interiordescs self.oopspecdescs = oopspecdescs + self.promotiondescs = promotiondescs self.called_bytecodes = called_bytecodes self.num_mergepoints = num_mergepoints self.graph_color = graph_color @@ -47,6 +49,29 @@ pass STOP = STOP() + +class RainbowResumer(rtimeshift.Resumer): + def __init__(self, interpreter, frame): + self.interpreter = interpreter + self.bytecode = frame.bytecode + self.pc = frame.pc + + def resume(self, jitstate, resuming): + interpreter = self.interpreter + dispatchqueue = rtimeshift.DispatchQueue(self.bytecode.num_mergepoints) + dispatchqueue.resuming = resuming + jitstate.frame.dispatchqueue = dispatchqueue + interpreter.newjitstate(jitstate) + interpreter.frame.pc = self.pc + interpreter.frame.bytecode = self.bytecode + interpreter.frame.local_green = jitstate.greens + jitstate.frame.dispatchqueue = dispatchqueue + interpreter.bytecode_loop() + finaljitstate = interpreter.jitstate + if finaljitstate is not None: + interpreter.finish_jitstate(interpreter.portalstate.sigtoken) + + class JitInterpreter(object): def __init__(self, exceptiondesc, RGenOp): self.exceptiondesc = exceptiondesc @@ -56,8 +81,21 @@ self.jitstate = None self.queue = None self.rgenop = RGenOp() + self.portalstate = None + self.num_global_mergepoints = -1 + self.global_state_dicts = None self._add_implemented_opcodes() + def set_portalstate(self, portalstate): + assert self.portalstate is None + self.portalstate = portalstate + + def set_num_global_mergepoints(self, num_global_mergepoints): + assert self.num_global_mergepoints == -1 + self.num_global_mergepoints = num_global_mergepoints + dicts = [newgreendict() for i in range(self.num_global_mergepoints)] + self.global_state_dicts = dicts + def run(self, jitstate, bytecode, greenargs, redargs, start_bytecode_loop=True): self.jitstate = jitstate @@ -72,6 +110,14 @@ self.bytecode_loop() return self.jitstate + def resume(self, jitstate, greenargs, redargs): + self.newjitstate(jitstate) + self.frame.local_boxes = redargs + self.frame.local_green = greenargs + self.frame.gc = rtimeshift.getresumepoint(jitstate) + self.bytecode_loop() + return self.jitstate + def fresh_jitstate(self, builder): return rtimeshift.JITState(builder, None, self.exceptiondesc.null_exc_type_box, @@ -180,6 +226,15 @@ def get_redarg(self): return self.frame.local_boxes[self.load_2byte()] + + def get_greenkey(self): + keydescnum = self.load_2byte() + if keydescnum == -1: + return empty_key + else: + keydesc = self.frame.bytecode.keydescs[keydescnum] + return GreenKey(self.frame.local_green[:keydesc.nb_vals], keydesc) + def red_result(self, box): self.frame.local_boxes.append(box) @@ -287,20 +342,40 @@ newgreens.append(self.get_greenarg()) self.frame.local_green = newgreens - def opimpl_merge(self): + def opimpl_local_merge(self): mergepointnum = self.load_2byte() - keydescnum = self.load_2byte() - if keydescnum == -1: - key = empty_key - else: - keydesc = self.frame.bytecode.keydescs[keydescnum] - key = GreenKey(self.frame.local_green[:keydesc.nb_vals], keydesc) + key = self.get_greenkey() states_dic = self.queue.local_caches[mergepointnum] done = rtimeshift.retrieve_jitstate_for_merge(states_dic, self.jitstate, key, None) if done: return self.dispatch() + def opimpl_global_merge(self): + mergepointnum = self.load_2byte() + key = self.get_greenkey() + states_dic = self.global_state_dicts[mergepointnum] + global_resumer = RainbowResumer(self, self.frame) + done = rtimeshift.retrieve_jitstate_for_merge(states_dic, self.jitstate, + key, global_resumer) + if done: + return self.dispatch() + + def opimpl_guard_global_merge(self): + rtimeshift.guard_global_merge(self.jitstate, self.frame.pc) + return self.dispatch() + + def opimpl_promote(self): + promotebox = self.get_redarg() + promotiondescnum = self.load_2byte() + promotiondesc = self.frame.bytecode.promotiondescs[promotiondescnum] + done = rtimeshift.promote(self.jitstate, promotebox, promotiondesc) + if done: + return self.dispatch() + gv_switchvar = promotebox.getgenvar(self.jitstate) + assert gv_switchvar.is_const + self.green_result(gv_switchvar) + def opimpl_red_direct_call(self): greenargs = self.get_green_varargs() redargs = self.get_red_varargs() Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py Sun Feb 24 11:57:15 2008 @@ -54,11 +54,13 @@ self.sigtoken = self.RGenOp.sigToken(self.RESIDUAL_FUNCTYPE) def make_state_instance(self): - state = self.PortalState(self.interpreter) + portalbytecode = self.codewriter.all_graphs[self.portalgraph] + state = self.PortalState(self.interpreter, portalbytecode) def portal_entry(*args): return state.portal_entry(*args) self.state = state self.portal_entry = portal_entry + self.interpreter.set_portalstate(state) def mutate_origportalgraph(self): # XXX @@ -90,7 +92,7 @@ if lowleveltype.TO._hints.get('virtualizable', False): redportargdesccls = RedVirtualizableStructPortalArgDesc else: - redportargdesccls = RedStructPortalArgDesc + redportargdesccls = RedPortalArgDesc return redportargdesccls(lowleveltype, self.RGenOp) @@ -98,10 +100,12 @@ portal_jitcode, rtyper): args_specification = unrolling_iterable(args_specification) class PortalState(object): - def __init__(self, interpreter): + def __init__(self, interpreter, portalbytecode): self.cache = {} self.graph_compilation_queue = [] self.interpreter = interpreter + self.portalbytecode = portalbytecode + self.sigtoken = sigtoken def compile_more_functions(self): while self.graph_compilation_queue: @@ -185,6 +189,17 @@ self.compile_more_functions() return gv_generated + + def make_dummy_args(self): + redargs = () + greenargs = () + for color, _, make_arg_redbox in args_specification: + if color == "green": + greenargs += (None, ) + else: + redargs += (None, ) + return list(greenargs), list(redargs) + # debug helpers def readportal(self, *args): i = 0 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Sun Feb 24 11:57:15 2008 @@ -1,11 +1,12 @@ import py from pypy.translator.translator import TranslationContext, graphof +from pypy.jit.codegen.llgraph.rgenop import RGenOp as LLRGenOp from pypy.jit.hintannotator.annotator import HintAnnotator from pypy.jit.hintannotator.policy import StopAtXPolicy, HintAnnotatorPolicy from pypy.jit.hintannotator.model import SomeLLAbstractConstant, OriginFlags from pypy.jit.hintannotator.model import originalconcretetype from pypy.jit.rainbow.codewriter import BytecodeWriter, label, tlabel, assemble -from pypy.jit.codegen.llgraph.rgenop import RGenOp as LLRGenOp +from pypy.jit.rainbow.portal import PortalRewriter from pypy.jit.rainbow.test.test_serializegraph import AbstractSerializationTest from pypy.jit.timeshifter import rtimeshift, rvalue from pypy.rpython.lltypesystem import lltype, rstr @@ -98,6 +99,7 @@ t = hannotator.translator graph2 = graphof(t, portal) self.graph = graph2 + self.maingraph = graphof(rtyper.annotator.translator, func) writer = BytecodeWriter(t, hannotator, self.RGenOp) jitcode = writer.make_bytecode(graph2) # the bytecode writer can ask for llhelpers about lists and dicts @@ -117,6 +119,19 @@ for i, ll_val in enumerate(values): color = writer.varcolor(graph2.startblock.inputargs[i]) argcolors.append(color) + + + # rewire the original portal + + rewriter = PortalRewriter(self.hintannotator, self.rtyper, self.RGenOp, + writer) + self.rewriter = rewriter + origportalgraph = graphof(self.rtyper.annotator.translator, portal) + portalgraph = graphof(t, portal) + rewriter.rewrite(origportalgraph=origportalgraph, + portalgraph=portalgraph, + view = conftest.option.view and self.small) + self.writer = writer self.jitcode = jitcode self.argcolors = argcolors Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Sun Feb 24 11:57:15 2008 @@ -40,22 +40,8 @@ policy=policy, inline=inline, backendoptimize=backendoptimize) - t = self.rtyper.annotator.translator - self.maingraph = graphof(t, main) - - # rewire the original portal - - rewriter = PortalRewriter(self.hintannotator, self.rtyper, self.RGenOp, - self.writer) - self.rewriter = rewriter - origportalgraph = graphof(t, portal) - portalgraph = graphof(self.hintannotator.translator, portal) - rewriter.rewrite(origportalgraph=origportalgraph, - portalgraph=portalgraph, - view = conftest.option.view and self.small) - if conftest.option.view and self.small: - t.view() + self.rtyper.annotator.translator.view() # Populate the cache if len(self._cache_order) >= 3: Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Sun Feb 24 11:57:15 2008 @@ -160,7 +160,7 @@ "make_new_redvars", 1, 2, "make_new_greenvars", 0, label("after"), - "merge", 0, -1, + "local_merge", 0, -1, "make_redbox", -1, 0, "red_int_add", 1, 0, "make_new_redvars", 1, 2, @@ -193,7 +193,7 @@ "make_new_redvars", 2, 0, 1, "make_new_greenvars", 0, label("while"), - "merge", 0, -1, + "local_merge", 0, -1, "red_int_is_true", 0, "red_goto_iftrue", 2, tlabel("body"), "make_new_redvars", 1, 0, Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Sun Feb 24 11:57:15 2008 @@ -4,10 +4,11 @@ from pypy.jit.hintannotator.model import originalconcretetype from pypy.jit.timeshifter import rvalue, rcontainer, rvirtualizable from pypy.jit.timeshifter.greenkey import newgreendict, empty_key +from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable from pypy.rpython.annlowlevel import cachedtype, base_ptr_lltype from pypy.rpython.annlowlevel import cast_instance_to_base_ptr -from pypy.rpython.annlowlevel import cast_base_ptr_to_instance +from pypy.rpython.annlowlevel import cast_base_ptr_to_instance, llhelper FOLDABLE_GREEN_OPS = dict.fromkeys(lloperation.enum_foldable_ops()) FOLDABLE_GREEN_OPS['getfield'] = None @@ -263,8 +264,11 @@ incoming[i].genvar = linkargs[i] return newblock -def return_marker(jitstate, resuming): - raise AssertionError("shouldn't get here") +class Resumer(object): + def resume(self, jitstate, resumepoint): + raise NotImplementedError("abstract base class") + +return_marker = Resumer() def start_new_block(states_dic, jitstate, key, global_resumer, index=-1): memo = rvalue.freeze_memo() @@ -293,10 +297,10 @@ dispatchqueue.clearlocalcaches() jitstate.promotion_path = PromotionPathMergesToSee(node, 0) #debug_print(lltype.Void, "PROMOTION ROOT") -start_new_block._annspecialcase_ = "specialize:arglltype(2)" def retrieve_jitstate_for_merge(states_dic, jitstate, key, global_resumer, force_merge=False): + # global_resumer is always None, just not for global merge points if jitstate.virtualizables: jitstate.enter_block_sweep_virtualizables() if key not in states_dic: @@ -663,6 +667,10 @@ self.incoming_gv = incoming_gv self.promotion_path = promotion_path + # hack for testing: make the llinterpreter believe this is a Ptr to base + # instance + _TYPE = base_ptr_lltype() + class AbstractPromotionPath(object): cut_limit = False @@ -692,7 +700,7 @@ jitstate.greens = self.greens_gv assert jitstate.frame.backframe is None resuminginfo.merges_to_see() - self.global_resumer(jitstate, resuminginfo) + self.global_resumer.resume(jitstate, resuminginfo) builder.show_incremental_progress() class PromotionPathNode(AbstractPromotionPath): @@ -742,37 +750,47 @@ MC_IGNORE_UNTIL_RETURN = -1 MC_CALL_NOT_TAKEN = -2 +# for testing purposes +def _cast_base_ptr_to_promotion_point(ptr): + if we_are_translated(): + return cast_base_ptr_to_instance(PromotionPoint, ptr) + else: + return ptr + +def _cast_promotion_point_to_base_ptr(instance): + assert isinstance(instance, PromotionPoint) + if we_are_translated(): + return cast_instance_to_base_ptr(instance) + else: + return instance + class PromotionDesc: __metaclass__ = cachedtype - def __init__(self, ERASED, hrtyper): - state = hrtyper.portalstate + def __init__(self, ERASED, interpreter): + self.exceptiondesc = interpreter.exceptiondesc def ll_continue_compilation(promotion_point_ptr, value): try: - promotion_point = cast_base_ptr_to_instance( - PromotionPoint, promotion_point_ptr) + promotion_point = _cast_base_ptr_to_promotion_point( + promotion_point_ptr) path = [None] root = promotion_point.promotion_path.follow_path(path) gv_value = root.rgenop.genconst(value) resuminginfo = ResumingInfo(promotion_point, gv_value, path) root.continue_compilation(resuminginfo) - state.compile_more_functions() + interpreter.portalstate.compile_more_functions() except Exception, e: + if not we_are_translated(): + import pdb; pdb.set_trace() lloperation.llop.debug_fatalerror( lltype.Void, "compilation-time error %s" % e) + self.ll_continue_compilation = ll_continue_compilation - fnptr = hrtyper.annhelper.delayedfunction( - ll_continue_compilation, - [annmodel.SomePtr(base_ptr_lltype()), - annmodel.lltype_to_annotation(ERASED)], - annmodel.s_None, needtype=True) - RGenOp = hrtyper.RGenOp - self.exceptiondesc = hrtyper.exceptiondesc - self.gv_continue_compilation = RGenOp.constPrebuiltGlobal(fnptr) - self.sigtoken = RGenOp.sigToken(lltype.typeOf(fnptr).TO) -## self.PROMOTION_POINT = r_PromotionPoint.lowleveltype + FUNCTYPE = lltype.FuncType([base_ptr_lltype(), ERASED], lltype.Void) + self.FUNCPTRTYPE = lltype.Ptr(FUNCTYPE) + self.sigtoken = interpreter.rgenop.sigToken(FUNCTYPE) def _freeze_(self): return True @@ -796,7 +814,7 @@ pm = PromotionPoint(flexswitch, incoming_gv, jitstate.promotion_path) #debug_print(lltype.Void, "PROMOTE") - ll_pm = cast_instance_to_base_ptr(pm) + ll_pm = _cast_promotion_point_to_base_ptr(pm) gv_pm = default_builder.rgenop.genconst(ll_pm) gv_switchvar = promotebox.genvar exceptiondesc = promotiondesc.exceptiondesc @@ -806,9 +824,12 @@ exceptiondesc.gv_null_exc_type ) exceptiondesc.genop_set_exc_value(default_builder, exceptiondesc.gv_null_exc_value) + fnptr = llhelper(promotiondesc.FUNCPTRTYPE, + promotiondesc.ll_continue_compilation) + gv_continue_compilation = default_builder.rgenop.genconst(fnptr) default_builder.genop_call(promotiondesc.sigtoken, - promotiondesc.gv_continue_compilation, - [gv_pm, gv_switchvar]) + gv_continue_compilation, + [gv_pm, gv_switchvar]) exceptiondesc.genop_set_exc_type (default_builder, gv_exc_type ) exceptiondesc.genop_set_exc_value(default_builder, gv_exc_value) linkargs = [] From cfbolz at codespeak.net Sun Feb 24 12:52:52 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 24 Feb 2008 12:52:52 +0100 (CET) Subject: [pypy-svn] r51832 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080224115252.79C01168419@codespeak.net> Author: cfbolz Date: Sun Feb 24 12:52:48 2008 New Revision: 51832 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Log: implement hint(reverse_split_queue), some passing tests Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Sun Feb 24 12:52:48 2008 @@ -565,6 +565,9 @@ return if "global_merge_point" in hints: return # the compute_merge_points function already cared + if "reverse_split_queue" in hints: + self.emit("reverse_split_queue") + return XXX def args_of_call(self, args, colored_as): Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Sun Feb 24 12:52:48 2008 @@ -376,6 +376,9 @@ assert gv_switchvar.is_const self.green_result(gv_switchvar) + def opimpl_reverse_split_queue(self): + rtimeshift.reverse_split_queue(self.frame.dispatchqueue) + def opimpl_red_direct_call(self): greenargs = self.get_green_varargs() redargs = self.get_red_varargs() Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py Sun Feb 24 12:52:48 2008 @@ -190,16 +190,6 @@ return gv_generated - def make_dummy_args(self): - redargs = () - greenargs = () - for color, _, make_arg_redbox in args_specification: - if color == "green": - greenargs += (None, ) - else: - redargs += (None, ) - return list(greenargs), list(redargs) - # debug helpers def readportal(self, *args): i = 0 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Sun Feb 24 12:52:48 2008 @@ -119,7 +119,6 @@ self.check_insns({}) def test_multiple_portal_calls(self): - py.test.skip("promote not implemented") def ll_function(n): hint(None, global_merge_point=True) k = n From tverwaes at codespeak.net Sun Feb 24 13:31:37 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Sun, 24 Feb 2008 13:31:37 +0100 (CET) Subject: [pypy-svn] r51833 - in pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk: . test Message-ID: <20080224123137.0E0FE168404@codespeak.net> Author: tverwaes Date: Sun Feb 24 13:31:37 2008 New Revision: 51833 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_model.py Log: fixing equal according to blue book + adding test Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py Sun Feb 24 13:31:37 2008 @@ -42,11 +42,11 @@ def shadow_of_my_class(self): return self.getclass().as_class_get_shadow() - def shallow_equals(self,other): + def pointer_equals(self,other): return self == other def equals(self, other): - return self.shallow_equals(other) + return self.pointer_equals(other) class W_SmallInteger(W_Object): __slots__ = ('value',) # the only allowed slot here @@ -67,7 +67,7 @@ def __repr__(self): return "W_SmallInteger(%d)" % self.value - def shallow_equals(self, other): + def equals(self, other): if not isinstance(other, W_SmallInteger): return False return self.value == other.value @@ -89,7 +89,7 @@ def __repr__(self): return "W_Float(%f)" % self.value - def shallow_equals(self, other): + def equals(self, other): if not isinstance(other, W_Float): return False return self.value == other.value @@ -236,18 +236,6 @@ from pypy.lang.smalltalk.shadow import ContextPartShadow return self.as_special_get_shadow(ContextPartShadow) - def equals(self, other): - if not isinstance(other, W_PointersObject): - return False - if not other.getclass() == self.getclass(): - return False - if not other.size() == self.size(): - return False - for i in range(self.size()): - if not other.fetch(i).shallow_equals(self.fetch(i)): - return False - return True - class W_BytesObject(W_AbstractObjectWithClassReference): def __init__(self, w_class, size): W_AbstractObjectWithClassReference.__init__(self, w_class) @@ -288,11 +276,6 @@ return False return True - def shallow_equals(self, other): - if not isinstance(other,W_BytesObject): - return False - return self.bytes == other.bytes - class W_WordsObject(W_AbstractObjectWithClassReference): def __init__(self, w_class, size): W_AbstractObjectWithClassReference.__init__(self, w_class) @@ -319,11 +302,6 @@ return (W_AbstractObjectWithClassReference.invariant(self) and isinstance(self.words, list)) - def shallow_equals(self, other): - if not isinstance(other,W_WordsObject): - return False - return self.words == other.words - class W_CompiledMethod(W_AbstractObjectWithIdentityHash): """My instances are methods suitable for interpretation by the virtual machine. This is the only class in the system whose instances intermix both indexable pointer fields and indexable integer fields. Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_model.py Sun Feb 24 13:31:37 2008 @@ -131,3 +131,30 @@ assert utility.unwrap_int(w_method.at0(12)) == ord('a') assert utility.unwrap_int(w_method.at0(13)) == ord('b') assert utility.unwrap_int(w_method.at0(14)) == ord('c') + +def test_equals(w_o1=model.W_PointersObject(None,0), w_o2=None): + if w_o2 is None: + w_o2 = w_o1 + assert w_o1.equals(w_o2) + assert w_o2.equals(w_o1) + +def test_not_equals(w_o1=model.W_PointersObject(None,0),w_o2=model.W_PointersObject(None,0)): + assert not w_o1.equals(w_o2) + assert not w_o2.equals(w_o1) + w_o2 = model.W_SmallInteger(2) + assert not w_o1.equals(w_o2) + assert not w_o2.equals(w_o1) + w_o2 = model.W_Float(5.5) + assert not w_o1.equals(w_o2) + assert not w_o2.equals(w_o1) + +def test_intfloat_equals(): + test_equals(model.W_SmallInteger(1), model.W_SmallInteger(1)) + test_equals(model.W_SmallInteger(100), model.W_SmallInteger(100)) + test_equals(model.W_Float(1.100), model.W_Float(1.100)) + +def test_intfloat_notequals(): + test_not_equals(model.W_SmallInteger(1), model.W_Float(1)) + test_not_equals(model.W_Float(100), model.W_SmallInteger(100)) + test_not_equals(model.W_Float(1.100), model.W_Float(1.200)) + test_not_equals(model.W_SmallInteger(101), model.W_SmallInteger(100)) From tverwaes at codespeak.net Sun Feb 24 14:54:44 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Sun, 24 Feb 2008 14:54:44 +0100 (CET) Subject: [pypy-svn] r51834 - in pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk: . test Message-ID: <20080224135444.350C7168431@codespeak.net> Author: tverwaes Date: Sun Feb 24 14:54:42 2008 New Revision: 51834 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_model.py Log: fixing equivalent. for PointersObjects compares pointers unless chars. In that case they are unwrapped and their value is compared. Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py Sun Feb 24 14:54:42 2008 @@ -134,6 +134,18 @@ return (W_AbstractObjectWithIdentityHash.invariant(self) and isinstance(self.w_class, W_PointersObject)) + def equals(self, w_other): + # Special case: + # Chars are not compared by pointer but by char-value. + from pypy.lang.smalltalk.classtable import w_Character + from pypy.lang.smalltalk import utility + if self.getclass() == w_Character: + if w_other.getclass() != w_Character: + return False + else: + return utility.unwrap_char(self) == utility.unwrap_char(w_other) + else: + return self.pointer_equals(w_other) class W_PointersObject(W_AbstractObjectWithClassReference): """ The normal object """ @@ -276,6 +288,11 @@ return False return True + def equals(self, other): + if not isinstance(other, W_BytesObject): + return False + return self.bytes == other.bytes + class W_WordsObject(W_AbstractObjectWithClassReference): def __init__(self, w_class, size): W_AbstractObjectWithClassReference.__init__(self, w_class) Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_model.py Sun Feb 24 14:54:42 2008 @@ -158,3 +158,12 @@ test_not_equals(model.W_Float(100), model.W_SmallInteger(100)) test_not_equals(model.W_Float(1.100), model.W_Float(1.200)) test_not_equals(model.W_SmallInteger(101), model.W_SmallInteger(100)) + +def test_charequals(): + test_equals(utility.wrap_char('a'), utility.wrap_char('a')) + test_equals(utility.wrap_char('d'), utility.wrap_char('d')) + +def test_not_charequals(): + test_not_equals(utility.wrap_char('a'), utility.wrap_char('d')) + test_not_equals(utility.wrap_char('d'), utility.wrap_int(3)) + test_not_equals(utility.wrap_char('d'), utility.wrap_float(3.0)) From tverwaes at codespeak.net Sun Feb 24 18:19:34 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Sun, 24 Feb 2008 18:19:34 +0100 (CET) Subject: [pypy-svn] r51836 - in pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk: . test Message-ID: <20080224171934.0501E1683DA@codespeak.net> Author: tverwaes Date: Sun Feb 24 18:19:33 2008 New Revision: 51836 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/objtable.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/utility.py Log: adding untested become primitive implementation. All objects saved globally; should become weakref array Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py Sun Feb 24 18:19:33 2008 @@ -145,7 +145,7 @@ self.push(interp.TWO) def pushActiveContextBytecode(self, interp): - self.push(self) + self.push(self.w_self()) def duplicateTopBytecode(self, interp): self.push(self.top()) Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py Sun Feb 24 18:19:33 2008 @@ -48,6 +48,9 @@ def equals(self, other): return self.pointer_equals(other) + def become(self, w_old, w_new): + pass + class W_SmallInteger(W_Object): __slots__ = ('value',) # the only allowed slot here @@ -122,7 +125,7 @@ return self.w_class def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self) + return "<%s %s at %r>" % (self.__class__.__name__, self, self.gethash()) def __str__(self): if isinstance(self, W_PointersObject) and self._shadow is not None: @@ -137,6 +140,7 @@ def equals(self, w_other): # Special case: # Chars are not compared by pointer but by char-value. + # XXX Check comment at utility.unwrap_char ... is related. from pypy.lang.smalltalk.classtable import w_Character from pypy.lang.smalltalk import utility if self.getclass() == w_Character: @@ -147,6 +151,10 @@ else: return self.pointer_equals(w_other) + def become(self, w_old, w_new): + if self.w_class == w_old: + self.w_class = w_new + class W_PointersObject(W_AbstractObjectWithClassReference): """ The normal object """ @@ -248,6 +256,12 @@ from pypy.lang.smalltalk.shadow import ContextPartShadow return self.as_special_get_shadow(ContextPartShadow) + def become(self, w_old, w_new): + W_AbstractObjectWithClassReference.become(self, w_old, w_new) + for i in range(len(self._vars)): + if self.fetch(i) == w_old: + self.store(i, w_new) + class W_BytesObject(W_AbstractObjectWithClassReference): def __init__(self, w_class, size): W_AbstractObjectWithClassReference.__init__(self, w_class) @@ -319,6 +333,8 @@ return (W_AbstractObjectWithClassReference.invariant(self) and isinstance(self.words, list)) +# XXX Shouldn't compiledmethod have class reference for subclassed compiled +# methods? class W_CompiledMethod(W_AbstractObjectWithIdentityHash): """My instances are methods suitable for interpretation by the virtual machine. This is the only class in the system whose instances intermix both indexable pointer fields and indexable integer fields. @@ -370,8 +386,11 @@ return w_literal.as_string() # XXX performance issue here def create_frame(self, receiver, arguments, sender = None): + from pypy.lang.smalltalk import objtable assert len(arguments) == self.argsize - return W_MethodContext(self, receiver, arguments, sender) + w_new = W_MethodContext(self, receiver, arguments, sender) + objtable.objects += [w_new] + return w_new def __str__(self): from pypy.lang.smalltalk.interpreter import BYTECODE_TABLE @@ -463,8 +482,6 @@ def atput0(self, index0, w_value): from pypy.lang.smalltalk import utility - print index0 - print self.getliteralsize() if index0 <= self.getliteralsize(): self.literalatput0(index0/constants.BYTES_PER_WORD, w_value) else: @@ -558,6 +575,15 @@ # Invalid! raise IndexError + def become(self, w_old, w_new): + if self._s_sender is not None and self._s_sender.w_self() == w_old: + self._s_sender = w_new.as_context_get_shadow() + for i in range(len(self._stack)): + if self._stack[i] == w_old: + self._stack[i] = w_new + if self._s_home is not None and self._s_home.w_self() == w_old: + self._s_home = w_new.as_context_get_shadow() + def stackstart(self): return constants.MTHDCTX_TEMP_FRAME_START @@ -623,7 +649,7 @@ res = self.stack()[start:] del self.stack()[start:] return res - + class W_BlockContext(W_ContextPart): def __init__(self, s_home, s_sender, argcnt, initialip): @@ -652,7 +678,7 @@ elif index == constants.BLKCTX_INITIAL_IP_INDEX: return utility.wrap_int(self.initialip) elif index == constants.BLKCTX_HOME_INDEX: - return self.s_home() + return self.s_home().w_self() elif index >= constants.BLKCTX_TEMP_FRAME_START: stack_index = len(self.stack()) - index - 1 return self.stack()[stack_index] @@ -700,6 +726,26 @@ def store_w_receiver(self, w_receiver): self._w_receiver = w_receiver + def become(self, w_old, w_new): + W_ContextPart.become(self, w_old, w_new) + for i in range(len(self.temps)): + if self.temps[i] == w_old: + self.temps[i] = w_new + if w_old == self._w_receiver: + self._w_receiver = w_new + if w_old == self._w_method: + self._w_method = w_new + + def at0(self, index0): + # XXX to test + # Fetches in temppart (variable size) + return self.fetch(index0+constants.MTHDCTX_TEMP_FRAME_START) + + def atput0(self, index0, w_v): + # XXX to test + # Stores in temppart (variable size) + return self.store(index0+constants.MTHDCTX_TEMP_FRAME_START, w_v) + def fetch(self, index): if index == constants.MTHDCTX_METHOD: return self.w_method() @@ -749,6 +795,8 @@ def w_method(self): return self._w_method + def size(self): + return len(self.temps) + len(self.stack()) # Use black magic to create w_nil without running the constructor, # thus allowing it to be used even in the constructor of its own Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/objtable.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/objtable.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/objtable.py Sun Feb 24 18:19:33 2008 @@ -8,15 +8,15 @@ def wrap_char_table(): global CharacterTable def bld_char(i): - w_cinst = classtable.w_Character.as_class_get_shadow().new() + w_cinst = classtable.w_Character.as_class_get_shadow().new(store=False) w_cinst.store(constants.CHARACTER_VALUE_INDEX, model.W_SmallInteger(i)) return w_cinst CharacterTable = [bld_char(i) for i in range(256)] wrap_char_table() -w_true = classtable.classtable['w_True'].as_class_get_shadow().new() -w_false = classtable.classtable['w_False'].as_class_get_shadow().new() +w_true = classtable.classtable['w_True'].as_class_get_shadow().new(store=False) +w_false = classtable.classtable['w_False'].as_class_get_shadow().new(store=False) w_minus_one = model.W_SmallInteger(-1) w_zero = model.W_SmallInteger(0) w_one = model.W_SmallInteger(1) @@ -29,6 +29,8 @@ w_nil.w_class = classtable.classtable['w_UndefinedObject'] objtable = {} +# XXX Should be table of weakref, contains all objects in the system +objects = [] for name in constants.objects_in_special_object_table: try: Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py Sun Feb 24 18:19:33 2008 @@ -490,9 +490,16 @@ # ___________________________________________________________________________ # Squeak Miscellaneous Primitives (128-149) +BECOME = 128 FULL_GC = 130 INC_GC = 131 + at expose_primitive(BECOME, unwrap_spec=[object, object]) +def func(interp, w_rcvr, w_new): + for w_object in objtable.objects: + w_object.become(w_rcvr, w_new) + return w_rcvr + def fake_bytes_left(): return utility.wrap_int(2**20) # XXX we don't know how to do this :-( Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py Sun Feb 24 18:19:33 2008 @@ -144,12 +144,15 @@ else: self.s_superclass = w_superclass.as_class_get_shadow() - def new(self, extrasize=0): + # XXX check better way to store objects + # XXX storing is necessary for "become" which loops over all pointers + # XXX and replaces old pointers with new pointers + def new(self, extrasize=0, store=True): from pypy.lang.smalltalk import classtable w_cls = self.w_self() if w_cls == classtable.w_BlockContext: - return model.W_BlockContext(None, None, 0, 0) + w_new = model.W_BlockContext(None, None, 0, 0) elif w_cls == classtable.w_MethodContext: # From slang: Contexts must only be created with newForMethod: # raise error.PrimitiveFailedError @@ -160,18 +163,22 @@ # The mini.image -however- doesn't overwrite this method, and as # far as I was able to trace it back, it -does- call this method. from pypy.rlib.objectmodel import instantiate - return instantiate(model.W_MethodContext) + w_new = instantiate(model.W_MethodContext) if self.instance_kind == POINTERS: - return model.W_PointersObject(w_cls, self.instance_size+extrasize) + w_new = model.W_PointersObject(w_cls, self.instance_size+extrasize) elif self.instance_kind == WORDS: - return model.W_WordsObject(w_cls, extrasize) + w_new = model.W_WordsObject(w_cls, extrasize) elif self.instance_kind == BYTES: - return model.W_BytesObject(w_cls, extrasize) + w_new = model.W_BytesObject(w_cls, extrasize) elif self.instance_kind == COMPILED_METHOD: - return model.W_CompiledMethod(extrasize) + w_new = model.W_CompiledMethod(extrasize) else: raise NotImplementedError(self.instance_kind) + if store: + from pypy.lang.smalltalk import objtable + objtable.objects += [w_new] + return w_new # _______________________________________________________________ # Methods for querying the format word, taken from the blue book: Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py Sun Feb 24 18:19:33 2008 @@ -202,8 +202,8 @@ reader.chunks[reader.specialobjectspointer] .g_object.pointers] - self.objects = [chunk.g_object.w_object for chunk in reader.chunklist] - from pypy.lang.smalltalk import constants, objtable + from pypy.lang.smalltalk import objtable + objtable.objects = [chunk.g_object.w_object for chunk in reader.chunklist] for name, idx in constants.objects_in_special_object_table.items(): objtable.objtable["w_" + name] = self.special_objects[idx] Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py Sun Feb 24 18:19:33 2008 @@ -57,7 +57,7 @@ def test_number_of_objects(): image = get_image() - objects = image.objects + objects = objtable.objects assert len(objects) > 0 assert 15000 < len(objects) < 16000 @@ -76,7 +76,7 @@ def test_invariant(): image = get_image() - for each in image.objects: + for each in objtable.objects: each.invariant() def test_float_class_size(): Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/utility.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/utility.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/utility.py Sun Feb 24 18:19:33 2008 @@ -10,6 +10,8 @@ return w_value.value raise UnwrappingError("expected a W_SmallInteger, got %s" % (w_value,)) +# XXX maybe chars are stored in char-table for uniqueness? This would solve +# the equal problem... def unwrap_char(w_char): from pypy.lang.smalltalk import classtable, objtable, constants w_class = w_char.getclass() From tverwaes at codespeak.net Sun Feb 24 22:43:04 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Sun, 24 Feb 2008 22:43:04 +0100 (CET) Subject: [pypy-svn] r51837 - in pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk: . test Message-ID: <20080224214304.1CDEF168400@codespeak.net> Author: tverwaes Date: Sun Feb 24 22:43:02 2008 New Revision: 51837 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py Log: got the compiling of fib + call to work!!! Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py Sun Feb 24 22:43:02 2008 @@ -165,10 +165,12 @@ self._vars = [w_nil] * size def at0(self, index0): - return self.fetch(index0) + # To test, at0 = in varsize part + return self.fetch(index0+self.instsize()) def atput0(self, index0, w_value): - self.store(index0, w_value) + # To test, at0 = in varsize part + self.store(index0+self.instsize(), w_value) def fetch(self, n0): return self._vars[n0] Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py Sun Feb 24 22:43:02 2008 @@ -246,7 +246,7 @@ ifTrue: [ 1 ] ifFalse: [ (self - 1) fib + (self - 2) fib ]""" perform(w(10).getclass(), "compile:classified:notifying:", w(sourcecode), w('pypy'), w(None)) - assert perform(w(10), "fib") == w(89) + assert perform(w(10), "fib").equals(w(89)) def w(any): if any is None: @@ -278,5 +278,5 @@ interp.step() #print interp.s_active_context.stack except interpreter.ReturnFromTopLevel, e: - return e.object.value + return e.object From tverwaes at codespeak.net Sun Feb 24 23:10:03 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Sun, 24 Feb 2008 23:10:03 +0100 (CET) Subject: [pypy-svn] r51838 - in pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk: . test Message-ID: <20080224221003.C54CA168433@codespeak.net> Author: tverwaes Date: Sun Feb 24 23:10:02 2008 New Revision: 51838 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/constants.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/utility.py Log: using chartable from special objects array when available Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/constants.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/constants.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/constants.py Sun Feb 24 23:10:02 2008 @@ -149,6 +149,7 @@ "nil": SO_NIL, "true": SO_TRUE, "false": SO_FALSE, + "charactertable": SO_CHARACTER_TABLE_ARRAY, "schedulerassociationpointer" : SO_SCHEDULERASSOCIATIONPOINTER, } Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py Sun Feb 24 23:10:02 2008 @@ -138,18 +138,7 @@ isinstance(self.w_class, W_PointersObject)) def equals(self, w_other): - # Special case: - # Chars are not compared by pointer but by char-value. - # XXX Check comment at utility.unwrap_char ... is related. - from pypy.lang.smalltalk.classtable import w_Character - from pypy.lang.smalltalk import utility - if self.getclass() == w_Character: - if w_other.getclass() != w_Character: - return False - else: - return utility.unwrap_char(self) == utility.unwrap_char(w_other) - else: - return self.pointer_equals(w_other) + return self.pointer_equals(w_other) def become(self, w_old, w_new): if self.w_class == w_old: Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/squeakimage.py Sun Feb 24 23:10:02 2008 @@ -207,6 +207,8 @@ for name, idx in constants.objects_in_special_object_table.items(): objtable.objtable["w_" + name] = self.special_objects[idx] + objtable.CharacterTable = objtable.objtable["w_charactertable"] + def special(self, index): return self.special_objects[index] Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py Sun Feb 24 23:10:02 2008 @@ -240,7 +240,6 @@ interp.interpret() def test_compile_method(): - #py.test.skip("Not working yet.") sourcecode = """fib ^self < 2 ifTrue: [ 1 ] Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/utility.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/utility.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/utility.py Sun Feb 24 23:10:02 2008 @@ -52,7 +52,11 @@ def wrap_char(c): from pypy.lang.smalltalk.objtable import CharacterTable - return CharacterTable[ord(c)] + try: + return CharacterTable.fetch(ord(c)) + except Exception: + # XXX For now fall back to old version of charactertable + return CharacterTable[ord(c)] def wrap_bool(bool): from pypy.lang.smalltalk import objtable From tverwaes at codespeak.net Sun Feb 24 23:13:51 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Sun, 24 Feb 2008 23:13:51 +0100 (CET) Subject: [pypy-svn] r51839 - pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk Message-ID: <20080224221351.E3A0E168449@codespeak.net> Author: tverwaes Date: Sun Feb 24 23:13:51 2008 New Revision: 51839 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/objtable.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/utility.py Log: always use pointersobject for chartable (even when homebrewed) Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/objtable.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/objtable.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/objtable.py Sun Feb 24 23:13:51 2008 @@ -12,7 +12,10 @@ w_cinst.store(constants.CHARACTER_VALUE_INDEX, model.W_SmallInteger(i)) return w_cinst - CharacterTable = [bld_char(i) for i in range(256)] + CharacterTable = model.W_PointersObject(classtable.classtable['w_Array'], 256) + for i in range(256): + CharacterTable.atput0(i, bld_char(i)) + wrap_char_table() w_true = classtable.classtable['w_True'].as_class_get_shadow().new(store=False) Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/utility.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/utility.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/utility.py Sun Feb 24 23:13:51 2008 @@ -52,11 +52,7 @@ def wrap_char(c): from pypy.lang.smalltalk.objtable import CharacterTable - try: - return CharacterTable.fetch(ord(c)) - except Exception: - # XXX For now fall back to old version of charactertable - return CharacterTable[ord(c)] + return CharacterTable.fetch(ord(c)) def wrap_bool(bool): from pypy.lang.smalltalk import objtable From cfbolz at codespeak.net Mon Feb 25 00:09:19 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 25 Feb 2008 00:09:19 +0100 (CET) Subject: [pypy-svn] r51841 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080224230919.657731683FB@codespeak.net> Author: cfbolz Date: Mon Feb 25 00:09:17 2008 New Revision: 51841 Added: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_promotion.py (contents, props changed) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: oops, forgot to svn add the promotion tests. A lot of them pass. Some other small changes. Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 25 00:09:17 2008 @@ -638,6 +638,8 @@ self.emit(pos) self.register_greenvar(op.result) return + elif kind == "residual": + XXX targets = dict(self.graphs_from(op)) assert len(targets) == 1 targetgraph, = targets.values() Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Mon Feb 25 00:09:17 2008 @@ -64,7 +64,7 @@ interpreter.newjitstate(jitstate) interpreter.frame.pc = self.pc interpreter.frame.bytecode = self.bytecode - interpreter.frame.local_green = jitstate.greens + interpreter.frame.local_green = [] jitstate.frame.dispatchqueue = dispatchqueue interpreter.bytecode_loop() finaljitstate = interpreter.jitstate @@ -226,7 +226,6 @@ def get_redarg(self): return self.frame.local_boxes[self.load_2byte()] - def get_greenkey(self): keydescnum = self.load_2byte() if keydescnum == -1: Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 25 00:09:17 2008 @@ -13,7 +13,7 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.rpython.module.support import LLSupport from pypy.annotation import model as annmodel -from pypy.objspace.flow.model import summary +from pypy.objspace.flow.model import summary, Variable from pypy.rlib.jit import hint from pypy.rlib.objectmodel import keepalive_until_here from pypy import conftest @@ -212,6 +212,28 @@ for opname, count in counts.items(): assert self.insns.get(opname, 0) == count + def check_oops(self, expected=None, **counts): + if not self.on_llgraph: + return + oops = {} + for block in self.residual_graph.iterblocks(): + for op in block.operations: + if op.opname == 'direct_call': + f = getattr(op.args[0].value._obj, "_callable", None) + if hasattr(f, 'oopspec'): + name, _ = f.oopspec.split('(', 1) + oops[name] = oops.get(name, 0) + 1 + if expected is not None: + assert oops == expected + for name, count in counts.items(): + assert oops.get(name, 0) == count + def check_flexswitches(self, expected_count): + count = 0 + for block in self.residual_graph.iterblocks(): + if (isinstance(block.exitswitch, Variable) and + block.exitswitch.concretetype is lltype.Signed): + count += 1 + assert count == expected_count class SimpleTests(InterpretationTest): def test_simple_fixed(self): Added: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_promotion.py ============================================================================== --- (empty file) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_promotion.py Mon Feb 25 00:09:17 2008 @@ -0,0 +1,414 @@ +import py +from pypy.rpython.lltypesystem import lltype +from pypy.jit.rainbow.test.test_interpreter import InterpretationTest +from pypy.jit.rainbow.test.test_interpreter import StopAtXPolicy +from pypy.jit.rainbow.test.test_interpreter import P_NOVIRTUAL +from pypy.jit.rainbow.test.test_vlist import P_OOPSPEC +from pypy.rlib.jit import hint +from pypy.rpython.module.support import LLSupport + +class TestPromotion(InterpretationTest): + type_system = "lltype" + small = True + + def test_simple_promotion(self): + def ll_two(k): + return (k+1)*2 + def ll_function(n): + hint(None, global_merge_point=True) + k = hint(n, promote=True) + k = ll_two(k) + return hint(k, variable=True) + + # easy case: no promotion needed + res = self.interpret(ll_function, [20], [0]) + assert res == 42 + self.check_insns({}) + + # the real test: with promotion + res = self.interpret(ll_function, [20], []) + assert res == 42 + self.check_insns(int_add=0, int_mul=0) + + def test_many_promotions(self): + def ll_two(k): + return k*k + def ll_function(n, total): + while n > 0: + hint(None, global_merge_point=True) + k = hint(n, promote=True) + k = ll_two(k) + total += hint(k, variable=True) + n -= 1 + return total + + res = self.interpret(ll_function, [10, 0], [], policy=P_NOVIRTUAL) + assert res == ll_function(10, 0) + self.check_insns(int_add=10, int_mul=0) + + def test_promote_after_call(self): + S = lltype.GcStruct('S', ('x', lltype.Signed)) + def ll_two(k, s): + if k > 5: + s.x = 20 + else: + s.x = 10 + def ll_function(n): + hint(None, global_merge_point=True) + s = lltype.malloc(S) + ll_two(n, s) + k = hint(n, promote=True) + k *= 17 + return hint(k, variable=True) + s.x + + res = self.interpret(ll_function, [4], [], policy=P_NOVIRTUAL) + assert res == 4*17 + 10 + self.check_insns(int_mul=0, int_add=1) + + def test_promote_after_yellow_call(self): + S = lltype.GcStruct('S', ('x', lltype.Signed)) + def ll_two(k, s): + if k > 5: + s.x = 20*k + return 7 + else: + s.x = 10*k + return 9 + + def ll_function(n): + hint(None, global_merge_point=True) + s = lltype.malloc(S) + c = ll_two(n, s) + k = hint(s.x, promote=True) + k += c + return hint(k, variable=True) + + res = self.interpret(ll_function, [4], [], policy=P_NOVIRTUAL) + assert res == 49 + self.check_insns(int_add=0) + + def test_promote_inside_call(self): + def ll_two(n): + k = hint(n, promote=True) + k *= 17 + return hint(k, variable=True) + def ll_function(n): + hint(None, global_merge_point=True) + return ll_two(n + 1) - 1 + + res = self.interpret(ll_function, [10], [], policy=P_NOVIRTUAL) + assert res == 186 + self.check_insns(int_add=1, int_mul=0, int_sub=0) + + def test_two_promotions(self): + def ll_function(n, m): + hint(None, global_merge_point=True) + n1 = hint(n, promote=True) + m1 = hint(m, promote=True) + s1 = n1 + m1 + return hint(s1, variable=True) + + res = self.interpret(ll_function, [40, 2], [], policy=P_NOVIRTUAL) + assert res == 42 + self.check_insns(int_add=0) + + def test_merge_then_promote(self): + S = lltype.GcStruct('S', ('x', lltype.Signed)) + def ll_two(n): + s = lltype.malloc(S) + if n < 0: + s.x = 10 + else: + s.x = 20 + k = hint(s.x, promote=True) + k *= 17 + return hint(k, variable=True) + def ll_function(n): + hint(None, global_merge_point=True) + return ll_two(n) + + res = self.interpret(ll_function, [3], [], policy=P_NOVIRTUAL) + assert res == 340 + self.check_insns(int_lt=1, int_mul=0) + + def test_vstruct_unfreeze(self): + S = lltype.GcStruct('S', ('x', lltype.Signed)) + def ll_two(k): + return (k+1)*2 + def ll_function(n): + hint(None, global_merge_point=True) + s = lltype.malloc(S) + s.x = n + k = hint(n, promote=True) + k = ll_two(k) + return hint(k, variable=True) + s.x + + # easy case: no promotion needed + res = self.interpret(ll_function, [20], [0], policy=P_NOVIRTUAL) + assert res == 62 + self.check_insns({}) + + # the real test: with promotion + res = self.interpret(ll_function, [20], [], policy=P_NOVIRTUAL) + assert res == 62 + self.check_insns(int_add=0, int_mul=0) + + def test_more_promotes(self): + py.test.skip("not working yet") + S = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed)) + def ll_two(s, i, m): + if i > 4: + s.x += i + return 10 + else: + s.y = i + return s.x + m + def ll_three(s, k): + k = hint(k, promote=True) + if s.x > 6: + k *= hint(s.y, promote=True) + return k + else: + return hint(1, concrete=True) + def ll_function(n, m): + s = lltype.malloc(S) + s.x = 0 + s.y = 0 + i = 0 + while i < n: + hint(None, global_merge_point=True) + k = ll_two(s, i, m) + if m & 1: + k *= 3 + else: + s.y += 1 + j = ll_three(s, k) + j = hint(j, variable=True) + i += j + return s.x + s.y * 17 + + res = self.interpret(ll_function, [100, 2], [], policy=P_NOVIRTUAL) + assert res == ll_function(100, 2) + + def test_mixed_merges(self): + def ll_function(x, y, z, k): + if x: + while x > 0: + hint(None, global_merge_point=True) + if y < 0: + y = -y + hint(None, reverse_split_queue=True) + return y + else: + n = 10 + while n: + n -= 1 + y = hint(y, promote=True) + y *= 2 + y = hint(y, variable=True) + x -= 1 + else: + if z < 0: + z = -z + else: + k = 3 + y = y + z*k + return y + + res = self.interpret(ll_function, [6, 3, 2, 2], [3], policy=P_NOVIRTUAL) + + assert res == ll_function(6, 3, 2, 2) + + def test_green_across_global_mp(self): + py.test.skip("void vars not handled correctly") + def ll_function(n1, n2, n3, n4, total): + while n2: + hint(None, global_merge_point=True) + total += n3 + hint(n4, concrete=True) + hint(n3, concrete=True) + hint(n2, concrete=True) + hint(n1, concrete=True) + n2 -= 1 + return total + void = lambda s: None + ll_function.convert_arguments = [void, int, int, void, int] + + res = self.interpret(ll_function, [None, 4, 3, None, 100], [0], + policy=P_NOVIRTUAL) + assert res == ll_function(None, 4, 3, None, 100) + + def test_remembers_across_mp(self): + def ll_function(x, flag): + hint(None, global_merge_point=True) + hint(x.field, promote=True) + m = x.field + if flag: + m += 1 * flag + else: + m += 2 + flag + hint(x.field, promote=True) + return m + x.field + + S = lltype.GcStruct('S', ('field', lltype.Signed), + hints={'immutable': True}) + + def struct_S(string): + s = lltype.malloc(S) + s.field = int(string) + return s + ll_function.convert_arguments = [struct_S, int] + + res = self.interpret(ll_function, ["20", 0], [], policy=P_NOVIRTUAL) + assert res == 42 + self.check_flexswitches(1) + + def test_virtual_list_copy(self): + def ll_function(x, y): + hint(None, global_merge_point=True) + l = [y] * x + size = len(l) + size = hint(size, promote=True) + vl = [0] * size + i = 0 + while i < size: + hint(i, concrete=True) + vl[i] = l[i] + i = i + 1 + return len(vl) + res = self.interpret(ll_function, [6, 5], [], policy=P_OOPSPEC) + assert res == 6 + self.check_oops(**{'newlist': 1, 'list.len': 1}) + + def test_promote_bug_1(self): + def ll_function(x, y, z): + a = 17 + while True: + hint(None, global_merge_point=True) + y += 1 + + if a != 17: + z = -z + + if z > 0: + b = 1 - z + else: + b = 2 + y = -y + if b == 2: + hint(z, promote=True) + return y + z + a + a += z + + assert ll_function(1, 5, 8) == 22 + res = self.interpret(ll_function, [1, 5, 8], [], + policy=P_NOVIRTUAL) + assert res == 22 + + def test_raise_result_mixup(self): + py.test.skip("residual calls not supported") + def w(x): + pass + class E(Exception): + def __init__(self, x): + self.x = x + def o(x): + if x < 0: + e = E(x) + w(e) + raise e + return x + def ll_function(c, x): + i = 0 + while True: + hint(None, global_merge_point=True) + op = c[i] + hint(op, concrete=True) + if op == 'e': + break + elif op == 'o': + x = o(x) + x = hint(x, promote=True) + i = x + r = hint(i, variable=True) + return r + ll_function.convert_arguments = [LLSupport.to_rstr, int] + + assert ll_function("oe", 1) == 1 + + res = self.interpret(ll_function, ["oe", 1], [], + policy=StopAtXPolicy(w)) + res == 1 + + def test_raise_result_mixup_some_more(self): + py.test.skip("residual calls not supported") + def w(x): + if x > 1000: + return None + else: + return E(x) + class E(Exception): + def __init__(self, x): + self.x = x + def o(x): + if x < 0: + e = w(x) + raise e + return x + def ll_function(c, x): + i = 0 + while True: + hint(None, global_merge_point=True) + op = c[i] + hint(op, concrete=True) + if op == 'e': + break + elif op == 'o': + x = o(x) + x = hint(x, promote=True) + i = x + r = hint(i, variable=True) + return r + ll_function.convert_arguments = [LLSupport.to_rstr, int] + + assert ll_function("oe", 1) == 1 + + res = self.interpret(ll_function, ["oe", 1], [], + policy=StopAtXPolicy(w)) + res == 1 + + def test_promote_in_yellow_call(self): + def ll_two(n): + n = hint(n, promote=True) + return n + 2 + + def ll_function(n): + hint(None, global_merge_point=True) + c = ll_two(n) + return hint(c, variable=True) + + res = self.interpret(ll_function, [4], [], policy=P_NOVIRTUAL) + assert res == 6 + self.check_insns(int_add=0) + + def test_two_promotions_in_call(self): + def ll_two(n, m): + if n < 1: + return m + else: + return n + + def ll_one(n, m): + n = ll_two(n, m) + n = hint(n, promote=True) + m = hint(m, promote=True) + return hint(n + m, variable=True) + + def ll_function(n, m): + hint(None, global_merge_point=True) + c = ll_one(n, m) + return c + + res = self.interpret(ll_function, [4, 7], [], policy=P_NOVIRTUAL) + assert res == 11 + self.check_insns(int_add=0) From jared.grubb at codespeak.net Mon Feb 25 06:34:17 2008 From: jared.grubb at codespeak.net (jared.grubb at codespeak.net) Date: Mon, 25 Feb 2008 06:34:17 +0100 (CET) Subject: [pypy-svn] r51843 - in pypy/dist/pypy/rlib/parsing: . test Message-ID: <20080225053417.B9115168408@codespeak.net> Author: jared.grubb Date: Mon Feb 25 06:34:15 2008 New Revision: 51843 Modified: pypy/dist/pypy/rlib/parsing/deterministic.py pypy/dist/pypy/rlib/parsing/regexparse.py pypy/dist/pypy/rlib/parsing/test/test_deterministic.py Log: parsing/deterministic.py: add comments, make some code simpler Modified: pypy/dist/pypy/rlib/parsing/deterministic.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/deterministic.py (original) +++ pypy/dist/pypy/rlib/parsing/deterministic.py Mon Feb 25 06:34:15 2008 @@ -6,22 +6,49 @@ from sets import Set as set, ImmutableSet as frozenset def compress_char_set(chars): + """Take the character list and compress runs of adjacent + characters; the result is a list of the first character in + a run and the number of chars following, sorted with longer + runs first. + + Example: 'abc' => [('a', 3)] + Example: 'abcmxyz' => [('a',3),('x',3),('m',1)]""" + # Find the runs. Creates a list like [['a',3],['m',1],['x',3]] chars = list(chars) chars.sort() - result = [chars[0], 1] + result = [[chars[0], 1]] for a, b in zip(chars[:-1], chars[1:]): if ord(a) == ord(b) - 1: - result.append(result.pop() + 1) + # Found adjacent characters, increment counter + result[-1][1] += 1 else: - result.append(b) - result.append(1) - real_result = [] - for i in range(len(result) // 2): - real_result.append((result[i * 2 + 1], result[i * 2])) - real_result.sort() - real_result = zip(*zip(*real_result)[::-1]) + # Found a 'hole', so create a new entry + result += [[b, 1]] + + # Change the above list into a list of sorted tuples + real_result = [(c,l) for [c,l] in result] + real_result.sort(key=lambda (l,c): (-c,l)) return real_result +def make_nice_charset_repr(chars): + # Compress the letters & digits + letters = set(chars) & set("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + therest = set(chars) - letters - set('-') + charranges = compress_char_set(letters) + result = [] + for a, num in charranges: + if num == 1: + result.append(a) + elif num==2: # 'ab' better than 'a-b' + result.append(a) + result.append(chr(ord(a)+1)) + else: + result.append("%s-%s" % (repr(a)[1:-1], repr(chr(ord(a) + num - 1))[1:-1])) + result += [repr(c)[1:-1] for c in therest] + if '-' in chars: + result += ['\\-'] + return "".join(result) + class LexerError(Exception): def __init__(self, input, state, source_pos): self.input = input @@ -36,18 +63,6 @@ result.append("LexerError") return "\n".join(result) -def make_nice_charset_repr(chars): - charranges = compress_char_set(chars) - result = [] - for a, num in charranges: - if num == 1: - result.append(a) - if a == "-": - result.append("\\-") - else: - result.append("%s-%s" % (repr(a)[1:-1], repr(chr(ord(a) + num - 1))[1:-1])) - return "".join(result) - class DFA(object): def __init__(self, num_states=0, transitions=None, final_states=None, unmergeable_states=None, names=None): @@ -81,8 +96,9 @@ if name is None: name = str(state) self.names.append(name) - return self.num_states - 1 + return state + # DFA returns transitions like a dict() def __setitem__(self, (state, input), next_state): self.transitions[state, input] = next_state @@ -94,7 +110,7 @@ def get_all_chars(self): all_chars = set() - for state, input in self.transitions: + for (state, input) in self.transitions: all_chars.add(input) return all_chars @@ -448,25 +464,27 @@ return result def epsilon_closure(self, states): - result = set(states) + """Return the epsilon-closure of 'states'.""" + closure = set(states) # states are in closure, by definition stack = list(states) while stack: state = stack.pop() + # Get all next_state s.t. state->next_state is marked epsilon (None): for next_state in self.transitions.get(state, {}).get(None, set()): - if next_state not in result: - result.add(next_state) - stack.append(next_state) - return result + if next_state not in closure: + closure.add(next_state) + stack.append(next_state) # Need to find eps-cl of next_state + return closure def make_deterministic(self, name_precedence=None): fda = DFA() set_to_state = {} stack = [] - def get_state(states): + def get_dfa_state(states): states = self.epsilon_closure(states) frozenstates = frozenset(states) if frozenstates in set_to_state: - return set_to_state[frozenstates] + return set_to_state[frozenstates] # already created this state if states == self.start_states: assert not set_to_state final = bool( @@ -495,7 +513,7 @@ name, final, unmergeable) stack.append((result, states)) return result - startstate = get_state(self.start_states) + startstate = get_dfa_state(self.start_states) while stack: fdastate, ndastates = stack.pop() chars_to_states = {} @@ -506,7 +524,7 @@ for char, states in chars_to_states.iteritems(): if char is None: continue - fda[fdastate, char] = get_state(states) + fda[fdastate, char] = get_dfa_state(states) return fda def update(self, other): Modified: pypy/dist/pypy/rlib/parsing/regexparse.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/regexparse.py (original) +++ pypy/dist/pypy/rlib/parsing/regexparse.py Mon Feb 25 06:34:15 2008 @@ -161,8 +161,7 @@ return r def make_runner(regex, view=False): - p = RegexParser(regex) - r = p.parse() + r = parse_regex(regex) nfa = r.make_automaton() dfa = nfa.make_deterministic() if view: Modified: pypy/dist/pypy/rlib/parsing/test/test_deterministic.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/test/test_deterministic.py (original) +++ pypy/dist/pypy/rlib/parsing/test/test_deterministic.py Mon Feb 25 06:34:15 2008 @@ -155,6 +155,19 @@ fda = a.make_deterministic() def test_compress_char_set(): + import string assert compress_char_set("ace") == [('a', 1), ('c', 1), ('e', 1)] assert compress_char_set("abcdefg") == [('a', 7)] assert compress_char_set("ABCabc") == [('A', 3), ('a', 3)] + assert compress_char_set("zycba") == [('a',3), ('y',2)] + assert compress_char_set(string.ascii_letters) == [('A', 26), ('a', 26)] + assert compress_char_set(string.printable) == [(' ', 95), ('\t', 5)] + +def test_make_nice_charset_repr(): + import string + assert make_nice_charset_repr("ace") == 'ace' + assert make_nice_charset_repr("abcdefg") == 'a-g' + assert make_nice_charset_repr("ABCabc") == 'A-Ca-c' + assert make_nice_charset_repr("zycba") == 'a-cyz' + assert make_nice_charset_repr(string.ascii_letters) == 'A-Za-z' + assert make_nice_charset_repr(string.printable) == 'A-Za-z0-9\\t\\x0b\\n\\r\\x0c! #"%$\'&)(+*,/.;:=@[]\\\\_^`{}|~\\-' From cfbolz at codespeak.net Mon Feb 25 10:15:33 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 25 Feb 2008 10:15:33 +0100 (CET) Subject: [pypy-svn] r51844 - pypy/branch/jit-refactoring/pypy/jit/rainbow Message-ID: <20080225091533.449161683F3@codespeak.net> Author: cfbolz Date: Mon Feb 25 10:15:32 2008 New Revision: 51844 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Log: use a decorator to load all the arguments for the bytecode implementations Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 25 10:15:32 2008 @@ -459,8 +459,8 @@ k = 0 for ARG in argiter: if ARG == lltype.Void: - # XXX terrible hack arg = voidargs[k] + # XXX terrible hack if not we_are_translated(): arg._TYPE = lltype.Void args += (arg, ) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Mon Feb 25 10:15:32 2008 @@ -72,6 +72,80 @@ interpreter.finish_jitstate(interpreter.portalstate.sigtoken) +def arguments(*argtypes, **kwargs): + result = kwargs.pop("returns", None) + assert not kwargs + argtypes = unrolling_iterable(argtypes) + def decorator(func): + def wrapped(self): + args = (self, ) + for argspec in argtypes: + if argspec == "red": + args += (self.get_redarg(), ) + elif argspec == "green": + args += (self.get_greenarg(), ) + elif argspec == "kind": + args += (self.frame.bytecode.typekinds[self.load_2byte()], ) + elif argspec == "jumptarget": + args += (self.load_4byte(), ) + elif argspec == "bool": + args += (self.load_bool(), ) + elif argspec == "redboxcls": + args += (self.frame.bytecode.redboxclasses[self.load_2byte()], ) + elif argspec == "2byte": + args += (self.load_2byte(), ) + elif argspec == "greenkey": + args += (self.get_greenkey(), ) + elif argspec == "promotiondesc": + promotiondescnum = self.load_2byte() + promotiondesc = self.frame.bytecode.promotiondescs[promotiondescnum] + args += (promotiondesc, ) + elif argspec == "green_varargs": + args += (self.get_green_varargs(), ) + elif argspec == "red_varargs": + args += (self.get_red_varargs(), ) + elif argspec == "bytecode": + bytecodenum = self.load_2byte() + args += (self.frame.bytecode.called_bytecodes[bytecodenum], ) + elif argspec == "nonrainbow_function": + index = self.load_2byte() + function = self.frame.bytecode.nonrainbow_functions[index] + args += (function, ) + elif argspec == "oopspec": + oopspecindex = self.load_2byte() + oopspec = self.frame.bytecode.oopspecdescs[oopspecindex] + args += (oopspec, ) + elif argspec == "structtypedesc": + td = self.frame.bytecode.structtypedescs[self.load_2byte()] + args += (td, ) + elif argspec == "arraydesc": + td = self.frame.bytecode.arrayfielddescs[self.load_2byte()] + args += (td, ) + elif argspec == "fielddesc": + d = self.frame.bytecode.fielddescs[self.load_2byte()] + args += (d, ) + elif argspec == "interiordesc": + d = self.frame.bytecode.interiordescs[self.load_2byte()] + args += (d, ) + else: + assert 0, "unknown argtype declaration" + val = func(*args) + if result is not None: + if result == "red": + self.red_result(val) + elif result == "green": + self.green_result(val) + elif result == "green_from_red": + self.green_result_from_red(val) + else: + assert 0, "unknown result declaration" + return + return val + wrapped.func_name = "wrap_" + func.func_name + return wrapped + return decorator + + class JitInterpreter(object): def __init__(self, exceptiondesc, RGenOp): self.exceptiondesc = exceptiondesc @@ -257,60 +331,54 @@ self.frame = None # operation implementations - def opimpl_make_redbox(self): - genconst = self.get_greenarg() - typeindex = self.load_2byte() - kind = self.frame.bytecode.typekinds[typeindex] - redboxcls = self.frame.bytecode.redboxclasses[typeindex] - self.red_result(redboxcls(kind, genconst)) + @arguments("green", "2byte", returns="red") + def opimpl_make_redbox(self, genconst, typeid): + redboxcls = self.frame.bytecode.redboxclasses[typeid] + kind = self.frame.bytecode.typekinds[typeid] + return redboxcls(kind, genconst) - def opimpl_goto(self): - target = self.load_4byte() + @arguments("jumptarget") + def opimpl_goto(self, target): self.frame.pc = target - def opimpl_green_goto_iftrue(self): - genconst = self.get_greenarg() - target = self.load_4byte() + @arguments("green", "jumptarget") + def opimpl_green_goto_iftrue(self, genconst, target): arg = genconst.revealconst(lltype.Bool) if arg: self.frame.pc = target - def opimpl_red_goto_iftrue(self): - switchbox = self.get_redarg() - target = self.load_4byte() + @arguments("red", "jumptarget") + def opimpl_red_goto_iftrue(self, switchbox, target): # XXX not sure about passing no green vars descision = rtimeshift.split(self.jitstate, switchbox, self.frame.pc) if descision: self.frame.pc = target - def opimpl_red_goto_ifptrnonzero(self): - reverse = self.load_bool() - ptrbox = self.get_redarg() - switchbox = self.get_redarg() - target = self.load_4byte() + @arguments("bool", "red", "red", "jumptarget") + def opimpl_red_goto_ifptrnonzero(self, reverse, ptrbox, switchbox, target): # XXX not sure about passing no green vars descision = rtimeshift.split_ptr_nonzero(self.jitstate, switchbox, self.frame.pc, ptrbox, reverse) if descision: self.frame.pc = target - def opimpl_red_ptr_nonzero(self, reverse=False): - ptrbox = self.get_redarg() - resultbox = rtimeshift.genptrnonzero(self.jitstate, ptrbox, reverse) - self.red_result(resultbox) - - def opimpl_red_ptr_iszero(self): - self.opimpl_red_ptr_nonzero(reverse=True) - - def opimpl_red_ptr_eq(self, reverse=False): - ptrbox1 = self.get_redarg() - ptrbox2 = self.get_redarg() - resultbox = rtimeshift.genptreq(self.jitstate, ptrbox1, - ptrbox2, reverse) - self.red_result(resultbox) - - def opimpl_red_ptr_ne(self): - self.opimpl_red_ptr_eq(reverse=True) + @arguments("red", returns="red") + def opimpl_red_ptr_nonzero(self, ptrbox): + return rtimeshift.genptrnonzero(self.jitstate, ptrbox, False) + + @arguments("red", returns="red") + def opimpl_red_ptr_iszero(self, ptrbox): + return rtimeshift.genptrnonzero(self.jitstate, ptrbox, True) + + @arguments("red", "red", returns="red") + def opimpl_red_ptr_eq(self, ptrbox1, ptrbox2): + return rtimeshift.genptreq(self.jitstate, ptrbox1, + ptrbox2, False) + + @arguments("red", "red", returns="red") + def opimpl_red_ptr_ne(self, ptrbox1, ptrbox2): + return rtimeshift.genptreq(self.jitstate, ptrbox1, + ptrbox2, True) def opimpl_red_return(self): rtimeshift.save_return(self.jitstate) @@ -341,18 +409,16 @@ newgreens.append(self.get_greenarg()) self.frame.local_green = newgreens - def opimpl_local_merge(self): - mergepointnum = self.load_2byte() - key = self.get_greenkey() + @arguments("2byte", "greenkey") + def opimpl_local_merge(self, mergepointnum, key): states_dic = self.queue.local_caches[mergepointnum] done = rtimeshift.retrieve_jitstate_for_merge(states_dic, self.jitstate, key, None) if done: return self.dispatch() - def opimpl_global_merge(self): - mergepointnum = self.load_2byte() - key = self.get_greenkey() + @arguments("2byte", "greenkey") + def opimpl_global_merge(self, mergepointnum, key): states_dic = self.global_state_dicts[mergepointnum] global_resumer = RainbowResumer(self, self.frame) done = rtimeshift.retrieve_jitstate_for_merge(states_dic, self.jitstate, @@ -364,10 +430,8 @@ rtimeshift.guard_global_merge(self.jitstate, self.frame.pc) return self.dispatch() - def opimpl_promote(self): - promotebox = self.get_redarg() - promotiondescnum = self.load_2byte() - promotiondesc = self.frame.bytecode.promotiondescs[promotiondescnum] + @arguments("red", "promotiondesc") + def opimpl_promote(self, promotebox, promotiondesc): done = rtimeshift.promote(self.jitstate, promotebox, promotiondesc) if done: return self.dispatch() @@ -378,11 +442,8 @@ def opimpl_reverse_split_queue(self): rtimeshift.reverse_split_queue(self.frame.dispatchqueue) - def opimpl_red_direct_call(self): - greenargs = self.get_green_varargs() - redargs = self.get_red_varargs() - bytecodenum = self.load_2byte() - targetbytecode = self.frame.bytecode.called_bytecodes[bytecodenum] + @arguments("green_varargs", "red_varargs", "bytecode") + def opimpl_red_direct_call(self, greenargs, redargs, targetbytecode): self.run(self.jitstate, targetbytecode, greenargs, redargs, start_bytecode_loop=False) # this frame will be resumed later in the next bytecode, which is @@ -394,17 +455,12 @@ self.frame.local_green) assert newjitstate is self.jitstate - def opimpl_green_direct_call(self): - greenargs = self.get_green_varargs() - index = self.load_2byte() - function = self.frame.bytecode.nonrainbow_functions[index] + @arguments("green_varargs", "nonrainbow_function") + def opimpl_green_direct_call(self, greenargs, function): function(self, greenargs) - def opimpl_yellow_direct_call(self): - greenargs = self.get_green_varargs() - redargs = self.get_red_varargs() - bytecodenum = self.load_2byte() - targetbytecode = self.frame.bytecode.called_bytecodes[bytecodenum] + @arguments("green_varargs", "red_varargs", "bytecode") + def opimpl_yellow_direct_call(self, greenargs, redargs, targetbytecode): self.run(self.jitstate, targetbytecode, greenargs, redargs, start_bytecode_loop=False) # this frame will be resumed later in the next bytecode, which is @@ -416,164 +472,121 @@ self.frame.local_green) assert newjitstate is self.jitstate + @arguments(returns="green") def opimpl_yellow_retrieve_result(self): # XXX all this jitstate.greens business is a bit messy - self.green_result(self.jitstate.greens[0]) + return self.jitstate.greens[0] - def opimpl_red_oopspec_call_0(self): - oopspecindex = self.load_2byte() - deepfrozen = self.load_bool() - oopspec = self.frame.bytecode.oopspecdescs[oopspecindex] - result = oopspec.ll_handler(self.jitstate, oopspec, deepfrozen) - self.red_result(result) - - def opimpl_red_oopspec_call_1(self): - oopspecindex = self.load_2byte() - deepfrozen = self.load_bool() - arg1 = self.get_redarg() - oopspec = self.frame.bytecode.oopspecdescs[oopspecindex] - result = oopspec.ll_handler(self.jitstate, oopspec, deepfrozen, arg1) - self.red_result(result) - - def opimpl_red_oopspec_call_2(self): - oopspecindex = self.load_2byte() - deepfrozen = self.load_bool() - arg1 = self.get_redarg() - arg2 = self.get_redarg() - oopspec = self.frame.bytecode.oopspecdescs[oopspecindex] - result = oopspec.ll_handler(self.jitstate, oopspec, deepfrozen, arg1, arg2) - self.red_result(result) - - def opimpl_red_oopspec_call_3(self): - oopspecindex = self.load_2byte() - deepfrozen = self.load_bool() - arg1 = self.get_redarg() - arg2 = self.get_redarg() - arg3 = self.get_redarg() - oopspec = self.frame.bytecode.oopspecdescs[oopspecindex] - result = oopspec.ll_handler(self.jitstate, oopspec, deepfrozen, arg1, arg2, arg3) - self.red_result(result) + @arguments("oopspec", "bool", returns="red") + def opimpl_red_oopspec_call_0(self, oopspec, deepfrozen): + return oopspec.ll_handler(self.jitstate, oopspec, deepfrozen) + + @arguments("oopspec", "bool", "red", returns="red") + def opimpl_red_oopspec_call_1(self, oopspec, deepfrozen, arg1): + return oopspec.ll_handler(self.jitstate, oopspec, deepfrozen, arg1) + + @arguments("oopspec", "bool", "red", "red", returns="red") + def opimpl_red_oopspec_call_2(self, oopspec, deepfrozen, arg1, arg2): + return oopspec.ll_handler(self.jitstate, oopspec, deepfrozen, arg1, arg2) + + @arguments("oopspec", "bool", "red", "red", "red", returns="red") + def opimpl_red_oopspec_call_3(self, oopspec, deepfrozen, arg1, arg2, arg3): + return oopspec.ll_handler(self.jitstate, oopspec, deepfrozen, arg1, arg2, arg3) # exceptions + @arguments(returns="red") def opimpl_read_exctype(self): - box = rtimeshift.getexctypebox(self.jitstate) - self.red_result(box) + return rtimeshift.getexctypebox(self.jitstate) + @arguments(returns="red") def opimpl_read_excvalue(self): - box = rtimeshift.getexcvaluebox(self.jitstate) + return rtimeshift.getexcvaluebox(self.jitstate) self.red_result(box) - def opimpl_write_exctype(self): - typebox = self.get_redarg() + @arguments("red") + def opimpl_write_exctype(self, typebox): rtimeshift.setexctypebox(self.jitstate, typebox) - def opimpl_write_excvalue(self): - valuebox = self.get_redarg() + @arguments("red") + def opimpl_write_excvalue(self, valuebox): rtimeshift.setexcvaluebox(self.jitstate, valuebox) # structs and arrays - def opimpl_red_malloc(self): - structtypedesc = self.frame.bytecode.structtypedescs[self.load_2byte()] + @arguments("structtypedesc", returns="red") + def opimpl_red_malloc(self, structtypedesc): redbox = rcontainer.create(self.jitstate, structtypedesc) - self.red_result(redbox) + return redbox - def opimpl_red_malloc_varsize_struct(self): - structtypedesc = self.frame.bytecode.structtypedescs[self.load_2byte()] - sizebox = self.get_redarg() + @arguments("structtypedesc", "red", returns="red") + def opimpl_red_malloc_varsize_struct(self, structtypedesc, sizebox): redbox = rcontainer.create_varsize(self.jitstate, structtypedesc, sizebox) - self.red_result(redbox) + return redbox + + @arguments("arraydesc", "red", returns="red") + def opimpl_red_malloc_varsize_array(self, arraytypedesc, sizebox): + return rtimeshift.genmalloc_varsize(self.jitstate, arraytypedesc, + sizebox) + + @arguments("red", "fielddesc", "bool", returns="red") + def opimpl_red_getfield(self, structbox, fielddesc, deepfrozen): + return rtimeshift.gengetfield(self.jitstate, deepfrozen, fielddesc, + structbox) + + @arguments("red", "fielddesc", "red") + def opimpl_red_setfield(self, destbox, fielddesc, valuebox): + rtimeshift.gensetfield(self.jitstate, fielddesc, destbox, + valuebox) + + @arguments("red", "arraydesc", "red", "bool", returns="red") + def opimpl_red_getarrayitem(self, arraybox, fielddesc, indexbox, deepfrozen): + return rtimeshift.gengetarrayitem(self.jitstate, deepfrozen, fielddesc, + arraybox, indexbox) + + @arguments("red", "arraydesc", "red", "red") + def opimpl_red_setarrayitem(self, destbox, fielddesc, indexbox, valuebox): + rtimeshift.gensetarrayitem(self.jitstate, fielddesc, destbox, + indexbox, valuebox) + + @arguments("red", "arraydesc", returns="red") + def opimpl_red_getarraysize(self, arraybox, fielddesc): + return rtimeshift.gengetarraysize(self.jitstate, fielddesc, arraybox) + + @arguments("red", "interiordesc", "bool", "red_varargs", returns="red") + def opimpl_red_getinteriorfield(self, structbox, interiordesc, deepfrozen, + indexboxes): + return interiordesc.gengetinteriorfield(self.jitstate, deepfrozen, + structbox, indexboxes) + + @arguments("red", "interiordesc", "bool", "red_varargs", + returns="green_from_red") + def opimpl_green_getinteriorfield(self, structbox, interiordesc, deepfrozen, + indexboxes): + # XXX make a green version that does not use the constant folding of + # the red one + return interiordesc.gengetinteriorfield(self.jitstate, deepfrozen, + structbox, indexboxes) - def opimpl_red_malloc_varsize_array(self): - arraytypedesc = self.frame.bytecode.arrayfielddescs[self.load_2byte()] - sizebox = self.get_redarg() - redbox = rtimeshift.genmalloc_varsize(self.jitstate, arraytypedesc, - sizebox) - self.red_result(redbox) - - def opimpl_red_getfield(self): - structbox = self.get_redarg() - fielddesc = self.frame.bytecode.fielddescs[self.load_2byte()] - deepfrozen = self.load_bool() - resbox = rtimeshift.gengetfield(self.jitstate, deepfrozen, fielddesc, - structbox) - self.red_result(resbox) - - def opimpl_red_setfield(self): - destbox = self.get_redarg() - fielddesc = self.frame.bytecode.fielddescs[self.load_2byte()] - valuebox = self.get_redarg() - resbox = rtimeshift.gensetfield(self.jitstate, fielddesc, destbox, - valuebox) - - def opimpl_red_getarrayitem(self): - arraybox = self.get_redarg() - fielddesc = self.frame.bytecode.arrayfielddescs[self.load_2byte()] - indexbox = self.get_redarg() - deepfrozen = self.load_bool() - resbox = rtimeshift.gengetarrayitem(self.jitstate, deepfrozen, fielddesc, - arraybox, indexbox) - self.red_result(resbox) - - def opimpl_red_setarrayitem(self): - destbox = self.get_redarg() - fielddesc = self.frame.bytecode.arrayfielddescs[self.load_2byte()] - indexbox = self.get_redarg() - valuebox = self.get_redarg() - resbox = rtimeshift.gensetarrayitem(self.jitstate, fielddesc, destbox, - indexbox, valuebox) - - def opimpl_red_getarraysize(self): - arraybox = self.get_redarg() - fielddesc = self.frame.bytecode.arrayfielddescs[self.load_2byte()] - resbox = rtimeshift.gengetarraysize(self.jitstate, fielddesc, arraybox) - self.red_result(resbox) - - def opimpl_red_getinteriorfield(self): - structbox = self.get_redarg() - interiordesc = self.frame.bytecode.interiordescs[self.load_2byte()] - deepfrozen = self.load_bool() - indexboxes = self.get_red_varargs() - resultbox = interiordesc.gengetinteriorfield(self.jitstate, deepfrozen, - structbox, indexboxes) - self.red_result(resultbox) - - def opimpl_green_getinteriorfield(self): - structbox = self.get_redarg() - interiordesc = self.frame.bytecode.interiordescs[self.load_2byte()] - deepfrozen = self.load_bool() - indexboxes = self.get_red_varargs() - resultbox = interiordesc.gengetinteriorfield(self.jitstate, deepfrozen, - structbox, indexboxes) - self.green_result_from_red(resultbox) - - def opimpl_red_setinteriorfield(self): - destbox = self.get_redarg() - interiordesc = self.frame.bytecode.interiordescs[self.load_2byte()] - indexboxes = self.get_red_varargs() - valuebox = self.get_redarg() + @arguments("red", "interiordesc", "red_varargs", "red") + def opimpl_red_setinteriorfield(self, destbox, interiordesc, indexboxes, + valuebox): interiordesc.gensetinteriorfield(self.jitstate, destbox, valuebox, indexboxes) - def opimpl_red_getinteriorarraysize(self): - arraybox = self.get_redarg() - interiordesc = self.frame.bytecode.interiordescs[self.load_2byte()] - indexboxes = self.get_red_varargs() - resultbox = interiordesc.gengetinteriorarraysize( + @arguments("red", "interiordesc", "red_varargs", returns="red") + def opimpl_red_getinteriorarraysize(self, arraybox, interiordesc, indexboxes): + return interiordesc.gengetinteriorarraysize( self.jitstate, arraybox, indexboxes) - self.red_result(resultbox) - def opimpl_green_getinteriorarraysize(self): + @arguments("red", "interiordesc", "red_varargs", returns="green_from_red") + def opimpl_green_getinteriorarraysize(self, arraybox, interiordesc, + indexboxes): # XXX make a green version that does not use the constant folding of # the red one - arraybox = self.get_redarg() - interiordesc = self.frame.bytecode.interiordescs[self.load_2byte()] - indexboxes = self.get_red_varargs() - resultbox = interiordesc.gengetinteriorarraysize( + return interiordesc.gengetinteriorarraysize( self.jitstate, arraybox, indexboxes) - self.green_result_from_red(resultbox) # ____________________________________________________________ # construction-time interface From cfbolz at codespeak.net Mon Feb 25 13:36:08 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 25 Feb 2008 13:36:08 +0100 (CET) Subject: [pypy-svn] r51846 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter Message-ID: <20080225123608.8AF6D168449@codespeak.net> Author: cfbolz Date: Mon Feb 25 13:36:07 2008 New Revision: 51846 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Log: implement residual red calls Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 25 13:36:07 2008 @@ -1,6 +1,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.objectmodel import we_are_translated from pypy.objspace.flow import model as flowmodel +from pypy.rpython.annlowlevel import cachedtype from pypy.rpython.lltypesystem import lltype from pypy.jit.hintannotator.model import originalconcretetype from pypy.jit.hintannotator import model as hintmodel @@ -11,6 +12,49 @@ from pypy.translator.backendopt.removenoops import remove_same_as +class CallDesc: + __metaclass__ = cachedtype + + def __init__(self, RGenOp, FUNCTYPE, voidargs=()): + self.sigtoken = RGenOp.sigToken(FUNCTYPE.TO) + self.result_kind = RGenOp.kindToken(FUNCTYPE.TO.RESULT) + # xxx what if the result is virtualizable? + self.redboxbuilder = rvalue.ll_redboxbuilder(FUNCTYPE.TO.RESULT) + whatever_return_value = FUNCTYPE.TO.RESULT._defl() + numargs = len(FUNCTYPE.TO.ARGS) + argiter = unrolling_iterable(FUNCTYPE.TO.ARGS) + def green_call(interpreter, fnptr_gv, greenargs): + fnptr = fnptr_gv.revealconst(FUNCTYPE) + assert len(greenargs) + len(voidargs) == numargs + args = () + j = 0 + k = 0 + for ARG in argiter: + if ARG == lltype.Void: + arg = voidargs[k] + # XXX terrible hack + if not we_are_translated(): + arg._TYPE = lltype.Void + args += (arg, ) + k += 1 + else: + genconst = greenargs[j] + arg = genconst.revealconst(ARG) + args += (arg, ) + j += 1 + rgenop = interpreter.jitstate.curbuilder.rgenop + try: + result = rgenop.genconst(fnptr(*args)) + except Exception, e: + XXX # set exception + return rgenop.genconst(whatever_return_value) + interpreter.green_result(result) + self.green_call = green_call + + def _freeze_(self): + return True + + class BytecodeWriter(object): def __init__(self, t, hannotator, RGenOp): self.translator = t @@ -50,7 +94,7 @@ self.called_bytecodes = [] self.num_local_mergepoints = 0 self.graph_color = self.graph_calling_color(graph) - self.nonrainbow_functions = [] + self.calldescs = [] self.is_portal = is_portal # mapping constant -> index in constants self.const_positions = {} @@ -81,7 +125,7 @@ # mapping graphs to index self.graph_positions = {} # mapping fnobjs to index - self.nonrainbow_positions = {} + self.calldesc_positions = {} self.graph = graph self.mergepoint_set = {} @@ -105,7 +149,7 @@ self.called_bytecodes, self.num_local_mergepoints, self.graph_color, - self.nonrainbow_functions, + self.calldescs, self.is_portal) if is_portal: self.finish_all_graphs() @@ -347,8 +391,8 @@ def redvar_position(self, arg): return self.redvar_positions[arg] - def register_greenvar(self, arg, where=None): - assert isinstance(arg, flowmodel.Variable) + def register_greenvar(self, arg, where=None, check=True): + assert isinstance(arg, flowmodel.Variable) or not check if where is None: where = self.free_green[self.current_block] self.free_green[self.current_block] += 1 @@ -444,41 +488,14 @@ self.graph_positions[graph] = index return index - def nonrainbow_position(self, fnptr, *voidargs): - fn = fnptr._obj - key = fn, voidargs - if key in self.nonrainbow_positions: - return self.nonrainbow_positions[key] - FUNCTYPE = lltype.typeOf(fn) - argiter = unrolling_iterable(FUNCTYPE.ARGS) - numargs = len(FUNCTYPE.ARGS) - def call_normal_function(interpreter, greenargs): - assert len(greenargs) + len(voidargs) == numargs - args = () - j = 0 - k = 0 - for ARG in argiter: - if ARG == lltype.Void: - arg = voidargs[k] - # XXX terrible hack - if not we_are_translated(): - arg._TYPE = lltype.Void - args += (arg, ) - k += 1 - else: - genconst = greenargs[j] - arg = genconst.revealconst(ARG) - args += (arg, ) - j += 1 - rgenop = interpreter.jitstate.curbuilder.rgenop - try: - result = rgenop.genconst(fnptr(*args)) - except Exception, e: - XXX # need to create a default result and set exception - interpreter.green_result(result) - result = len(self.nonrainbow_functions) - self.nonrainbow_functions.append(call_normal_function) - self.nonrainbow_positions[key] = result + def calldesc_position(self, FUNCTYPE, *voidargs): + key = FUNCTYPE, voidargs + if key in self.calldesc_positions: + return self.calldesc_positions[key] + result = len(self.calldescs) + self.calldescs.append( + CallDesc(self.RGenOp, FUNCTYPE, voidargs)) + self.calldesc_positions[key] = result return result def interiordesc(self, op, PTRTYPE, nb_offsets): @@ -584,12 +601,12 @@ pass def serialize_op_direct_call(self, op): - kind, exc = self.guess_call_kind(op) - print op, kind, exc + kind, withexc = self.guess_call_kind(op) + print op, kind, withexc if kind == "oopspec": from pypy.jit.timeshifter.oop import Index fnobj = op.args[0].value._obj - oopspecdescindex = self.oopspecdesc_position(fnobj, exc) + oopspecdescindex = self.oopspecdesc_position(fnobj, withexc) oopspecdesc = self.oopspecdescs[oopspecdescindex] opargs = op.args[1:] args_v = [] @@ -627,19 +644,42 @@ elif kind == "green": voidargs = [const.value for const in op.args[1:] if const.concretetype == lltype.Void] - pos = self.nonrainbow_position(op.args[0].value, *voidargs) + fnptr = op.args[0] + pos = self.calldesc_position(lltype.typeOf(fnptr.value), *voidargs) + func = self.serialize_oparg("green", fnptr) emitted_args = [] for v in op.args[1:]: if v.concretetype != lltype.Void: emitted_args.append(self.serialize_oparg("green", v)) self.emit("green_direct_call") + self.emit(func, pos) self.emit(len(emitted_args)) self.emit(*emitted_args) - self.emit(pos) self.register_greenvar(op.result) return elif kind == "residual": - XXX + fnptr = op.args[0] + pos = self.calldesc_position(lltype.typeOf(fnptr.value)) + func = self.serialize_oparg("red", fnptr) + emitted_args = [] + for v in op.args[1:]: + emitted_args.append(self.serialize_oparg("red", v)) + self.emit("red_residual_direct_call") + self.emit(func, pos, withexc, len(emitted_args), *emitted_args) + self.register_redvar(op.result) + pos = self.register_redvar(("residual_flags_red", op.args[0])) + self.emit("promote") + self.emit(pos) + self.emit(self.promotiondesc_position(lltype.Signed)) + self.register_greenvar(("residual_flags_green", op.args[0]), check=False) + self.emit("residual_fetch", True, pos) + return + elif kind == "rpyexc_raise": + emitted_args = [] + for v in op.args[1:]: + emitted_args.append(self.serialize_oparg("red", v)) + self.emit("setexception", *emitted_args) + return targets = dict(self.graphs_from(op)) assert len(targets) == 1 targetgraph, = targets.values() @@ -703,7 +743,7 @@ pass def serialize_op_getfield(self, op): - assert self.opcolor(op) == "red" + color = self.opcolor(op) args = op.args if args[0] == self.exceptiondesc.cexcdata: # reading one of the exception boxes (exc_type or exc_value) @@ -731,9 +771,11 @@ fielddescindex = self.fielddesc_position(PTRTYPE.TO, fieldname) if fielddescindex == -1: # Void field return - self.emit("red_getfield", index, fielddescindex, deepfrozen) - self.register_redvar(op.result) - + self.emit("%s_getfield" % (color, ), index, fielddescindex, deepfrozen) + if color == "red": + self.register_redvar(op.result) + else: + self.register_greenvar(op.result) def serialize_op_setfield(self, op): args = op.args Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Mon Feb 25 13:36:07 2008 @@ -21,7 +21,7 @@ keydescs, structtypedescs, fielddescs, arrayfielddescs, interiordescs, oopspecdescs, promotiondescs, called_bytecodes, num_mergepoints, - graph_color, nonrainbow_functions, is_portal): + graph_color, calldescs, is_portal): self.name = name self.code = code self.constants = constants @@ -37,7 +37,7 @@ self.called_bytecodes = called_bytecodes self.num_mergepoints = num_mergepoints self.graph_color = graph_color - self.nonrainbow_functions = nonrainbow_functions + self.calldescs = calldescs self.is_portal = is_portal def _freeze_(self): @@ -71,7 +71,6 @@ if finaljitstate is not None: interpreter.finish_jitstate(interpreter.portalstate.sigtoken) - def arguments(*argtypes, **kwargs): result = kwargs.pop("returns", None) assert not kwargs @@ -107,9 +106,9 @@ elif argspec == "bytecode": bytecodenum = self.load_2byte() args += (self.frame.bytecode.called_bytecodes[bytecodenum], ) - elif argspec == "nonrainbow_function": + elif argspec == "calldesc": index = self.load_2byte() - function = self.frame.bytecode.nonrainbow_functions[index] + function = self.frame.bytecode.calldescs[index] args += (function, ) elif argspec == "oopspec": oopspecindex = self.load_2byte() @@ -455,9 +454,9 @@ self.frame.local_green) assert newjitstate is self.jitstate - @arguments("green_varargs", "nonrainbow_function") - def opimpl_green_direct_call(self, greenargs, function): - function(self, greenargs) + @arguments("green", "calldesc", "green_varargs") + def opimpl_green_direct_call(self, fnptr_gv, calldesc, greenargs): + calldesc.green_call(self, fnptr_gv, greenargs) @arguments("green_varargs", "red_varargs", "bytecode") def opimpl_yellow_direct_call(self, greenargs, redargs, targetbytecode): @@ -493,6 +492,24 @@ def opimpl_red_oopspec_call_3(self, oopspec, deepfrozen, arg1, arg2, arg3): return oopspec.ll_handler(self.jitstate, oopspec, deepfrozen, arg1, arg2, arg3) + @arguments("red", "calldesc", "bool", "red_varargs") + def opimpl_red_residual_direct_call(self, funcbox, calldesc, withexc, redargs): + result = rtimeshift.gen_residual_call(self.jitstate, calldesc, + funcbox, redargs) + self.red_result(result) + if withexc: + exceptiondesc = self.exceptiondesc + else: + exceptiondesc = None + flagbox = rtimeshift.after_residual_call(self.jitstate, + exceptiondesc, True) + self.red_result(flagbox) + + @arguments("bool", "red") + def opimpl_residual_fetch(self, check_forced, flagbox): + rtimeshift.residual_fetch(self.jitstate, self.exceptiondesc, + check_forced, flagbox) + # exceptions @@ -513,6 +530,10 @@ def opimpl_write_excvalue(self, valuebox): rtimeshift.setexcvaluebox(self.jitstate, valuebox) + @arguments("red", "red") + def opimpl_setexception(self, typebox, valuebox): + rtimeshift.setexception(self.jitstate, typebox, valuebox) + # structs and arrays @arguments("structtypedesc", returns="red") @@ -536,6 +557,11 @@ return rtimeshift.gengetfield(self.jitstate, deepfrozen, fielddesc, structbox) + @arguments("red", "fielddesc", "bool", returns="green_from_red") + def opimpl_green_getfield(self, structbox, fielddesc, deepfrozen): + return rtimeshift.gengetfield(self.jitstate, deepfrozen, fielddesc, + structbox) + @arguments("red", "fielddesc", "red") def opimpl_red_setfield(self, destbox, fielddesc, valuebox): rtimeshift.gensetfield(self.jitstate, fielddesc, destbox, Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 25 13:36:07 2008 @@ -198,7 +198,8 @@ self.residual_graph = graph if conftest.option.view: graph.show() - llinterp = LLInterpreter(self.rtyper) + llinterp = LLInterpreter( + self.rtyper, exc_data_ptr=writer.exceptiondesc.exc_data_ptr) res = llinterp.eval_graph(graph, residualargs) return res @@ -236,6 +237,8 @@ assert count == expected_count class SimpleTests(InterpretationTest): + small = True + def test_simple_fixed(self): py.test.skip("green return") def ll_function(x, y): @@ -991,7 +994,6 @@ self.check_insns({}) def test_residual_red_call(self): - py.test.skip("needs promote") def g(x): return x+1 @@ -1003,7 +1005,6 @@ self.check_insns(int_add=0) def test_residual_red_call_with_exc(self): - py.test.skip("needs promote") def h(x): if x > 0: return x+1 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Mon Feb 25 13:36:07 2008 @@ -246,14 +246,15 @@ writer, jitcode = self.serialize(ll_function, [int]) assert jitcode.code == assemble(writer.interpreter, - "green_direct_call", 1, 0, 0, + "green_direct_call", -1, 0, 1, 0, "make_redbox", 1, 0, "make_new_redvars", 1, 0, "make_new_greenvars", 0, "red_return") assert jitcode.is_portal assert len(jitcode.called_bytecodes) == 0 - assert len(jitcode.nonrainbow_functions) == 1 + assert len(jitcode.calldescs) == 1 + assert len(jitcode.constants) == 1 def test_yellow_call(self): def ll_two(x): Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/greenkey.py Mon Feb 25 13:36:07 2008 @@ -19,15 +19,17 @@ self.hash = lambda self: 0 self.compare = lambda self, other: True - TARGETTYPES = [] - for TYPE in TYPES: + index_TYPE = [] + for i, TYPE in enumerate(TYPES): # XXX more cases? TARGET = lltype.Signed - if TYPE == lltype.Float: + if TYPE == lltype.Void: + continue + elif TYPE == lltype.Float: TARGET = TYPE - TARGETTYPES.append(TARGET) + index_TYPE.append((i, TARGET)) - iterator = unrolling_iterable(enumerate(TARGETTYPES)) + iterator = unrolling_iterable(index_TYPE) length = len(TYPES) def greenhash(self): retval = 0x345678 Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Mon Feb 25 13:36:07 2008 @@ -566,37 +566,16 @@ ##def ll_gvar_from_constant(jitstate, ll_value): ## return jitstate.curbuilder.rgenop.genconst(ll_value) -class CallDesc: - __metaclass__ = cachedtype - - def __init__(self, RGenOp, FUNCTYPE): - self.sigtoken = RGenOp.sigToken(FUNCTYPE) - self.result_kind = RGenOp.kindToken(FUNCTYPE.RESULT) - # xxx what if the result is virtualizable? - self.redboxbuilder = rvalue.ll_redboxbuilder(FUNCTYPE.RESULT) - whatever_return_value = FUNCTYPE.RESULT._defl() - def green_call(jitstate, fnptr, *args): - try: - result = fnptr(*args) - except Exception, e: - jitstate.residual_exception(e) - result = whatever_return_value - return result - self.green_call = green_call - - def _freeze_(self): - return True -def gen_residual_call(jitstate, calldesc, funcbox): +def gen_residual_call(jitstate, calldesc, funcbox, argboxes): builder = jitstate.curbuilder gv_funcbox = funcbox.getgenvar(jitstate) - argboxes = jitstate.frame.local_boxes args_gv = [argbox.getgenvar(jitstate) for argbox in argboxes] jitstate.prepare_for_residual_call() gv_result = builder.genop_call(calldesc.sigtoken, gv_funcbox, args_gv) return calldesc.redboxbuilder(calldesc.result_kind, gv_result) -def ll_after_residual_call(jitstate, exceptiondesc, check_forced): +def after_residual_call(jitstate, exceptiondesc, check_forced): builder = jitstate.curbuilder if check_forced: gv_flags = jitstate.check_forced_after_residual_call() From cfbolz at codespeak.net Mon Feb 25 14:44:00 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 25 Feb 2008 14:44:00 +0100 (CET) Subject: [pypy-svn] r51851 - pypy/branch/jit-refactoring/pypy/jit/rainbow Message-ID: <20080225134400.1EB7D168442@codespeak.net> Author: cfbolz Date: Mon Feb 25 14:43:58 2008 New Revision: 51851 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Log: small cleanup Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 25 14:43:58 2008 @@ -552,40 +552,44 @@ hints = op.args[1].value arg = op.args[0] result = op.result - if "concrete" in hints: - assert self.hannotator.binding(arg).is_green() - assert self.hannotator.binding(result).is_green() + assert len(hints) == 1 + hint = hints.keys()[0] + handler = getattr(self, "handle_%s_hint" % (hint, )) + return handler(op, arg, result) + + def handle_concrete_hint(self, op, arg, result): + assert self.hannotator.binding(arg).is_green() + assert self.hannotator.binding(result).is_green() + self.register_greenvar(result, self.green_position(arg)) + + def handle_variable_hint(self, op, arg, result): + assert not self.hannotator.binding(result).is_green() + if self.hannotator.binding(arg).is_green(): + resultindex = self.convert_to_red(arg) + self.register_redvar(result, resultindex) + else: + self.register_redvar(result, self.redvar_position(arg)) + + def handle_deepfreeze_hint(self, op, arg, result): + if self.varcolor(result) == "red": + self.register_redvar(result, self.redvar_position(arg)) + else: + self.register_greenvar(result, self.green_position(arg)) + + def handle_promote_hint(self, op, arg, result): + if self.varcolor(arg) == "green": self.register_greenvar(result, self.green_position(arg)) return - if "variable" in hints: - assert not self.hannotator.binding(result).is_green() - if self.hannotator.binding(arg).is_green(): - resultindex = self.convert_to_red(arg) - self.register_redvar(result, resultindex) - else: - self.register_redvar(result, self.redvar_position(arg)) - return - if "deepfreeze" in hints: - if self.varcolor(result) == "red": - self.register_redvar(result, self.redvar_position(arg)) - else: - self.register_greenvar(result, self.green_position(arg)) - return - if "promote" in hints: - if self.varcolor(arg) == "green": - self.register_greenvar(result, self.green_position(arg)) - return - self.emit("promote") - self.emit(self.serialize_oparg("red", arg)) - self.emit(self.promotiondesc_position(arg.concretetype)) - self.register_greenvar(result) - return - if "global_merge_point" in hints: - return # the compute_merge_points function already cared - if "reverse_split_queue" in hints: - self.emit("reverse_split_queue") - return - XXX + self.emit("promote") + self.emit(self.serialize_oparg("red", arg)) + self.emit(self.promotiondesc_position(arg.concretetype)) + self.register_greenvar(result) + + def handle_global_merge_point_hint(self, op, arg, result): + return # the compute_merge_points function already cared + + def handle_reverse_split_queue_hint(self, op, arg, result): + self.emit("reverse_split_queue") def args_of_call(self, args, colored_as): result = [] From cfbolz at codespeak.net Mon Feb 25 14:58:01 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 25 Feb 2008 14:58:01 +0100 (CET) Subject: [pypy-svn] r51852 - pypy/branch/jit-refactoring/pypy/jit/rainbow Message-ID: <20080225135801.6A8F7168441@codespeak.net> Author: cfbolz Date: Mon Feb 25 14:57:59 2008 New Revision: 51852 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Log: more cleanups Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 25 14:57:59 2008 @@ -606,109 +606,117 @@ def serialize_op_direct_call(self, op): kind, withexc = self.guess_call_kind(op) + handler = getattr(self, "handle_%s_call" % (kind, )) print op, kind, withexc - if kind == "oopspec": - from pypy.jit.timeshifter.oop import Index - fnobj = op.args[0].value._obj - oopspecdescindex = self.oopspecdesc_position(fnobj, withexc) - oopspecdesc = self.oopspecdescs[oopspecdescindex] - opargs = op.args[1:] - args_v = [] - args = [] - for obj in oopspecdesc.argtuple: - if isinstance(obj, Index): - v = opargs[obj.n] - else: - v = flowmodel.Constant(obj, lltype.typeOf(obj)) - args_v.append(v) - args.append(self.serialize_oparg("red", v)) - - ll_handler = oopspecdesc.ll_handler - couldfold = oopspecdesc.couldfold - missing_args = ((ll_handler.func_code.co_argcount - 3) - - len(oopspecdesc.argtuple)) - assert missing_args >= 0 - if missing_args > 0: - assert (ll_handler.func_defaults[-missing_args:] == - (None,) * missing_args) - - if oopspecdesc.is_method: - hs_self = self.hannotator.binding( - opargs[oopspecdesc.argtuple[0].n]) - deepfrozen = hs_self.deepfrozen + return handler(op, withexc) + + def handle_oopspec_call(self, op, withexc): + from pypy.jit.timeshifter.oop import Index + fnobj = op.args[0].value._obj + oopspecdescindex = self.oopspecdesc_position(fnobj, withexc) + oopspecdesc = self.oopspecdescs[oopspecdescindex] + opargs = op.args[1:] + args_v = [] + args = [] + for obj in oopspecdesc.argtuple: + if isinstance(obj, Index): + v = opargs[obj.n] else: - deepfrozen = False + v = flowmodel.Constant(obj, lltype.typeOf(obj)) + args_v.append(v) + args.append(self.serialize_oparg("red", v)) + + ll_handler = oopspecdesc.ll_handler + couldfold = oopspecdesc.couldfold + missing_args = ((ll_handler.func_code.co_argcount - 3) - + len(oopspecdesc.argtuple)) + assert missing_args >= 0 + if missing_args > 0: + assert (ll_handler.func_defaults[-missing_args:] == + (None,) * missing_args) + + if oopspecdesc.is_method: + hs_self = self.hannotator.binding( + opargs[oopspecdesc.argtuple[0].n]) + deepfrozen = hs_self.deepfrozen + else: + deepfrozen = False + + self.emit("red_oopspec_call_%s" % (len(args), )) + self.emit(oopspecdescindex) + self.emit(deepfrozen) + self.emit(*args) + self.register_redvar(op.result) - self.emit("red_oopspec_call_%s" % (len(args), )) - self.emit(oopspecdescindex) - self.emit(deepfrozen) - self.emit(*args) - self.register_redvar(op.result) - return - elif kind == "green": - voidargs = [const.value for const in op.args[1:] - if const.concretetype == lltype.Void] - fnptr = op.args[0] - pos = self.calldesc_position(lltype.typeOf(fnptr.value), *voidargs) - func = self.serialize_oparg("green", fnptr) - emitted_args = [] - for v in op.args[1:]: - if v.concretetype != lltype.Void: - emitted_args.append(self.serialize_oparg("green", v)) - self.emit("green_direct_call") - self.emit(func, pos) - self.emit(len(emitted_args)) - self.emit(*emitted_args) - self.register_greenvar(op.result) - return - elif kind == "residual": - fnptr = op.args[0] - pos = self.calldesc_position(lltype.typeOf(fnptr.value)) - func = self.serialize_oparg("red", fnptr) - emitted_args = [] - for v in op.args[1:]: - emitted_args.append(self.serialize_oparg("red", v)) - self.emit("red_residual_direct_call") - self.emit(func, pos, withexc, len(emitted_args), *emitted_args) + def handle_green_call(self, op, withexc): + voidargs = [const.value for const in op.args[1:] + if const.concretetype == lltype.Void] + fnptr = op.args[0] + pos = self.calldesc_position(lltype.typeOf(fnptr.value), *voidargs) + func = self.serialize_oparg("green", fnptr) + emitted_args = [] + for v in op.args[1:]: + if v.concretetype != lltype.Void: + emitted_args.append(self.serialize_oparg("green", v)) + self.emit("green_direct_call") + self.emit(func, pos) + self.emit(len(emitted_args)) + self.emit(*emitted_args) + self.register_greenvar(op.result) + + def handle_residual_call(self, op, withexc): + fnptr = op.args[0] + pos = self.calldesc_position(lltype.typeOf(fnptr.value)) + func = self.serialize_oparg("red", fnptr) + emitted_args = [] + for v in op.args[1:]: + emitted_args.append(self.serialize_oparg("red", v)) + self.emit("red_residual_direct_call") + self.emit(func, pos, withexc, len(emitted_args), *emitted_args) + self.register_redvar(op.result) + pos = self.register_redvar(("residual_flags_red", op.args[0])) + self.emit("promote") + self.emit(pos) + self.emit(self.promotiondesc_position(lltype.Signed)) + self.register_greenvar(("residual_flags_green", op.args[0]), check=False) + self.emit("residual_fetch", True, pos) + + def handle_rpyexc_raise_call(self, op, withexc): + emitted_args = [] + for v in op.args[1:]: + emitted_args.append(self.serialize_oparg("red", v)) + self.emit("setexception", *emitted_args) + + def handle_red_call(self, op, withexc, kind="red"): + targets = dict(self.graphs_from(op)) + assert len(targets) == 1 + targetgraph, = targets.values() + graphindex = self.graph_position(targetgraph) + args = targetgraph.getargs() + emitted_args = self.args_of_call(op.args[1:], args) + self.emit("red_direct_call") + self.emit(*emitted_args) + self.emit(graphindex) + if kind == "red": self.register_redvar(op.result) - pos = self.register_redvar(("residual_flags_red", op.args[0])) - self.emit("promote") - self.emit(pos) - self.emit(self.promotiondesc_position(lltype.Signed)) - self.register_greenvar(("residual_flags_green", op.args[0]), check=False) - self.emit("residual_fetch", True, pos) - return - elif kind == "rpyexc_raise": - emitted_args = [] - for v in op.args[1:]: - emitted_args.append(self.serialize_oparg("red", v)) - self.emit("setexception", *emitted_args) - return + self.emit("red_after_direct_call") + + def handle_gray_call(self, op, withexc): + return self.handle_red_call(op, withexc, "gray") + + def handle_yellow_call(self, op, withexc): targets = dict(self.graphs_from(op)) assert len(targets) == 1 targetgraph, = targets.values() - if kind == "red" or kind == "gray": - graphindex = self.graph_position(targetgraph) - args = targetgraph.getargs() - emitted_args = self.args_of_call(op.args[1:], args) - self.emit("red_direct_call") - self.emit(*emitted_args) - self.emit(graphindex) - if kind == "red": - self.register_redvar(op.result) - self.emit("red_after_direct_call") - elif kind == "yellow": - graphindex = self.graph_position(targetgraph) - args = targetgraph.getargs() - emitted_args = self.args_of_call(op.args[1:], args) - self.emit("yellow_direct_call") - self.emit(*emitted_args) - self.emit(graphindex) - self.emit("yellow_after_direct_call") - self.emit("yellow_retrieve_result") - self.register_greenvar(op.result) - else: - XXX + graphindex = self.graph_position(targetgraph) + args = targetgraph.getargs() + emitted_args = self.args_of_call(op.args[1:], args) + self.emit("yellow_direct_call") + self.emit(*emitted_args) + self.emit(graphindex) + self.emit("yellow_after_direct_call") + self.emit("yellow_retrieve_result") + self.register_greenvar(op.result) def serialize_op_indirect_call(self, op): XXX From arigo at codespeak.net Mon Feb 25 15:01:31 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 25 Feb 2008 15:01:31 +0100 (CET) Subject: [pypy-svn] r51853 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080225140131.D3172168449@codespeak.net> Author: arigo Date: Mon Feb 25 15:01:29 2008 New Revision: 51853 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Log: Unused import. Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Mon Feb 25 15:01:29 2008 @@ -3,7 +3,6 @@ from pypy import conftest from pypy.translator.translator import graphof from pypy.jit.timeshifter.test.test_timeshift import TestLLType as TSTestLLType, getargtypes -from pypy.jit.rainbow.portal import PortalRewriter from pypy.jit.rainbow.test.test_interpreter import P_NOVIRTUAL, StopAtXPolicy from pypy.jit.rainbow.test.test_interpreter import hannotate, InterpretationTest from pypy.jit.rainbow.test.test_vlist import P_OOPSPEC From arigo at codespeak.net Mon Feb 25 17:43:37 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 25 Feb 2008 17:43:37 +0100 (CET) Subject: [pypy-svn] r51854 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080225164337.3CE6C168459@codespeak.net> Author: arigo Date: Mon Feb 25 17:43:35 2008 New Revision: 51854 Added: pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py (contents, props changed) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Log: A (hackish) pretty-printer for JitCode objects created by codewriter.py. This is not really a disassembler, as it is based on the source list. Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 25 17:43:35 2008 @@ -134,8 +134,10 @@ self.make_bytecode_block(graph.startblock) assert self.current_block is None bytecode = self.all_graphs[graph] + labelpos = {} + code = assemble_labelpos(labelpos, self.interpreter, *self.assembler) bytecode.__init__(graph.name, - assemble(self.interpreter, *self.assembler), + code, self.constants, self.typekinds, self.redboxclasses, @@ -151,6 +153,9 @@ self.graph_color, self.calldescs, self.is_portal) + bytecode._source = self.assembler + bytecode._interpreter = self.interpreter + bytecode._labelpos = labelpos if is_portal: self.finish_all_graphs() self.interpreter.set_num_global_mergepoints( @@ -1033,9 +1038,8 @@ def __repr__(self): return "tlabel(%r)" % (self.name, ) -def assemble(interpreter, *args): +def assemble_labelpos(labelpos, interpreter, *args): result = [] - labelpos = {} def emit_2byte(index): result.append(chr((index >> 8) & 0xff)) result.append(chr(index & 0xff)) @@ -1065,3 +1069,6 @@ result[i + 2] = chr((index >> 8) & 0xff) result[i + 3] = chr(index & 0xff) return "".join(result) + +def assemble(interpreter, *args): + return assemble_labelpos({}, interpreter, *args) Added: pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py ============================================================================== --- (empty file) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py Mon Feb 25 17:43:35 2008 @@ -0,0 +1,166 @@ +from pypy.jit.rainbow import codewriter + + +class SourceIterator: + + def __init__(self, jitcode, source, interpreter, labelpos): + self.jitcode = jitcode + self.source = source + self.interpreter = interpreter + self.labelpos = labelpos + self.index = 0 + self.pc = 0 + + def finished(self): + return self.index == len(self.source) + + def peek(self): + return self.source[self.index] + + def get(self, expected_type, bytes_count): + arg = self.source[self.index] + assert isinstance(arg, expected_type) + self.index += 1 + self.pc += bytes_count + return arg + + def get_opname(self): + return self.get(str, 2) + + def load_2byte(self): + return self.get(int, 2) + + def load_bool(self): + return self.get(bool, 1) + + def get_greenarg(self): + i = self.load_2byte() + if i < 0: + return self.jitcode.constants[~i] + return CustomRepr('g%d' % i) + + def get_green_varargs(self): + greenargs = [] + num = self.load_2byte() + for i in range(num): + greenargs.append(self.get_greenarg()) + return greenargs + + def get_red_varargs(self): + redargs = [] + num = self.load_2byte() + for i in range(num): + redargs.append(self.get_redarg()) + return redargs + + def get_redarg(self): + return CustomRepr('r%d' % self.get(int, 2)) + + def get_greenkey(self): + keydescnum = self.load_2byte() + if keydescnum == -1: + return None + else: + keydesc = self.jitcode.keydescs[keydescnum] + return keydesc + + def load_jumptarget(self): + tlbl = self.get(codewriter.tlabel, 4) + return self.labelpos[tlbl.name] + + +class CustomRepr: + def __init__(self, s): + self.s = s + def __repr__(self): + return self.s + + +def dump_bytecode(jitcode, file=None): + # XXX this is not really a disassembler, but just a pretty-printer + # for the '_source' attribute that codewriter.py attaches + source = jitcode._source + interpreter = jitcode._interpreter + labelpos = jitcode._labelpos + print >> file, 'JITCODE %r' % (jitcode.name,) + + src = SourceIterator(jitcode, source, interpreter, labelpos) + noblankline = {0: True} + while not src.finished(): + arg = src.peek() + if isinstance(arg, str): + startpc = src.pc + opname = src.get_opname() + opcode = interpreter.find_opcode(opname) + opimpl = interpreter.opcode_implementations[opcode] + argtypes = opimpl.argspec + resulttype = opimpl.resultspec + args = [] + + for argspec in argtypes: + if argspec == "red": + args.append(src.get_redarg()) + elif argspec == "green": + args.append(src.get_greenarg()) + elif argspec == "kind": + args.append(jitcode.typekinds[src.load_2byte()]) + elif argspec == "jumptarget": + args.append(src.load_jumptarget()) + elif argspec == "bool": + args.append(src.load_bool()) + elif argspec == "redboxcls": + args.append(jitcode.redboxclasses[src.load_2byte()]) + elif argspec == "2byte": + args.append(src.load_2byte()) + elif argspec == "greenkey": + args.append(src.get_greenkey()) + elif argspec == "promotiondesc": + promotiondescnum = src.load_2byte() + promotiondesc = jitcode.promotiondescs[promotiondescnum] + args.append(promotiondesc) + elif argspec == "green_varargs": + args.append(src.get_green_varargs()) + elif argspec == "red_varargs": + args.append(src.get_red_varargs()) + elif argspec == "bytecode": + bytecodenum = src.load_2byte() + called_bytecode = jitcode.called_bytecodes[bytecodenum] + args.append(called_bytecode.name) + elif argspec == "calldesc": + index = src.load_2byte() + function = jitcode.calldescs[index] + args.append(function) + elif argspec == "oopspec": + oopspecindex = src.load_2byte() + oopspec = jitcode.oopspecdescs[oopspecindex] + args.append(oopspec) + elif argspec == "structtypedesc": + td = jitcode.structtypedescs[src.load_2byte()] + args.append(td) + elif argspec == "arraydesc": + td = jitcode.arrayfielddescs[src.load_2byte()] + args.append(td) + elif argspec == "fielddesc": + d = jitcode.fielddescs[src.load_2byte()] + args.append(d) + elif argspec == "interiordesc": + d = jitcode.interiordescs[src.load_2byte()] + args.append(d) + else: + assert 0, "unknown argtype declaration" + + args = map(str, args) + # XXX we should print the result from resultspec too, + # but it's not obvious how to do that + line = '%5d | %-20s %s' % (startpc, opname, ', '.join(args)) + print >> file, line.rstrip() + elif isinstance(arg, codewriter.label): + if src.pc not in noblankline: # no duplicate blank lines + print >> file, '%5s |' % '' + noblankline[src.pc] = True + src.index += 1 + else: + assert 0, "unexpected object: %r" % (arg,) + + if src.pc != len(jitcode.code): + print >> file, 'WARNING: the pc column is bogus! fix dump.py!' Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Mon Feb 25 17:43:35 2008 @@ -43,6 +43,10 @@ def _freeze_(self): return True + def dump(self, file=None): + from pypy.jit.rainbow import dump + dump.dump_bytecode(self, file=file) + SIGN_EXTEND2 = 1 << 15 class STOP(object): @@ -141,6 +145,8 @@ return return val wrapped.func_name = "wrap_" + func.func_name + wrapped.argspec = argtypes + wrapped.resultspec = result return wrapped return decorator @@ -379,22 +385,26 @@ return rtimeshift.genptreq(self.jitstate, ptrbox1, ptrbox2, True) + @arguments() def opimpl_red_return(self): rtimeshift.save_return(self.jitstate) return self.dispatch() + @arguments() def opimpl_gray_return(self): rtimeshift.save_return(self.jitstate) return self.dispatch() + @arguments() def opimpl_yellow_return(self): # save the greens to make the return value findable by collect_split rtimeshift.save_greens(self.jitstate, self.frame.local_green) rtimeshift.save_return(self.jitstate) return self.dispatch() - def opimpl_make_new_redvars(self): - self.frame.local_boxes = self.get_red_varargs() + @arguments("red_varargs") + def opimpl_make_new_redvars(self, local_boxes): + self.frame.local_boxes = local_boxes def opimpl_make_new_greenvars(self): # an opcode with a variable number of args @@ -407,6 +417,8 @@ for i in range(num): newgreens.append(self.get_greenarg()) self.frame.local_green = newgreens + opimpl_make_new_greenvars.argspec = ("green_varargs",) # for dump.py + opimpl_make_new_greenvars.resultspec = None @arguments("2byte", "greenkey") def opimpl_local_merge(self, mergepointnum, key): @@ -425,6 +437,7 @@ if done: return self.dispatch() + @arguments() def opimpl_guard_global_merge(self): rtimeshift.guard_global_merge(self.jitstate, self.frame.pc) return self.dispatch() @@ -438,6 +451,7 @@ assert gv_switchvar.is_const self.green_result(gv_switchvar) + @arguments() def opimpl_reverse_split_queue(self): rtimeshift.reverse_split_queue(self.frame.dispatchqueue) @@ -448,6 +462,7 @@ # this frame will be resumed later in the next bytecode, which is # red_after_direct_call + @arguments() def opimpl_red_after_direct_call(self): newjitstate = rtimeshift.collect_split( self.jitstate, self.frame.pc, @@ -465,6 +480,7 @@ # this frame will be resumed later in the next bytecode, which is # yellow_after_direct_call + @arguments() def opimpl_yellow_after_direct_call(self): newjitstate = rtimeshift.collect_split( self.jitstate, self.frame.pc, @@ -640,6 +656,8 @@ args += (arg, ) result = self.rgenop.genconst(opdesc.llop(*args)) self.green_result(result) + implementation.argspec = ("green",) * len(list(numargs)) + implementation.resultspec = "green" elif color == "red": if opdesc.nb_args == 1: impl = rtimeshift.ll_gen1 @@ -653,6 +671,8 @@ args += (self.get_redarg(), ) result = impl(*args) self.red_result(result) + implementation.argspec = ("red",) * len(list(numargs)) + implementation.resultspec = "red" else: assert 0, "unknown color" implementation.func_name = "opimpl_%s_%s" % (color, opdesc.opname) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Mon Feb 25 17:43:35 2008 @@ -1,3 +1,4 @@ +import py from pypy.translator.translator import TranslationContext, graphof from pypy.jit.hintannotator.annotator import HintAnnotator from pypy.jit.hintannotator.policy import StopAtXPolicy, HintAnnotatorPolicy @@ -210,6 +211,48 @@ assert jitcode.is_portal assert len(jitcode.called_bytecodes) == 0 + def test_dump_loop(self): + def f(x): + r = 0 + while x: + r += x + x -= 1 + return r + writer, jitcode = self.serialize(f, [int]) + import StringIO + output = StringIO.StringIO() + jitcode.dump(file=output) + result = output.getvalue().rstrip() + print '-' * 40 + print result + print '-' * 40 + # xxx slightly fragile test, it will break whenever we tweak dump.py + expected = """\ +JITCODE 'f' + 0 | make_redbox (0), 0 + 6 | make_new_redvars [r0, r1] + 14 | make_new_greenvars [] + | + 18 | local_merge 0, None + 24 | red_int_is_true r0 + 28 | red_goto_iftrue r2, 48 + 36 | make_new_redvars [r1] + 42 | make_new_greenvars [] + | + 46 | red_return + | + 48 | make_new_redvars [r0, r1] + 56 | make_new_greenvars [] + | + 60 | red_int_add r1, r0 + 66 | make_redbox (1), 0 + 72 | red_int_sub r0, r3 + 78 | make_new_redvars [r4, r2] + 86 | make_new_greenvars [] + 90 | goto 18 + """.rstrip() + assert result == expected + def test_call(self): def g(x): return x + 1 From tverwaes at codespeak.net Mon Feb 25 17:58:34 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Mon, 25 Feb 2008 17:58:34 +0100 (CET) Subject: [pypy-svn] r51855 - in pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk: . test Message-ID: <20080225165834.C1D6716845B@codespeak.net> Author: tverwaes Date: Mon Feb 25 17:58:32 2008 New Revision: 51855 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_interpreter.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_primitives.py Log: first wave of code cleanup Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py Mon Feb 25 17:58:32 2008 @@ -26,12 +26,15 @@ ONE = objtable.w_one TWO = objtable.w_two - _s_last_active_context = None + _w_last_active_context = None def __init__(self): - self.s_active_context = None + self.w_active_context = None self.cnt = 0 + def s_active_context(self): + return self.w_active_context.as_context_get_shadow() + def interpret(self): try: while True: @@ -43,30 +46,29 @@ return (not objectmodel.we_are_translated()) and option.bc_trace def step(self): - next = self.s_active_context.getNextBytecode() + next = self.s_active_context().getNextBytecode() # we_are_translated returns false on top of CPython and true when # translating the interpreter if not objectmodel.we_are_translated(): bytecodeimpl = BYTECODE_TABLE[next] - if self._s_last_active_context != self.s_active_context: + if self._w_last_active_context != self.w_active_context: cnt = 0 - p = self.s_active_context + p = self.w_active_context # AK make method - while p is not None: + while p is not objtable.w_nil: cnt += 1 - p = p.s_sender() + p = p.as_context_get_shadow().w_sender() self._last_indent = " " * cnt - self._s_last_active_context = self.s_active_context + self._w_last_active_context = self.w_active_context if self.should_trace(): - print "%sStack=%s" % ( self._last_indent, - repr(self.s_active_context.stack()),) + repr(self.s_active_context().stack()),) print "%sBytecode at %d (%d:%s):" % ( self._last_indent, - self.s_active_context.pc(), + self.s_active_context().pc(), next, bytecodeimpl.__name__,) - bytecodeimpl(self.s_active_context, self) + bytecodeimpl(self.w_active_context, self) else: # this is a performance optimization: when translating the # interpreter, the bytecode dispatching is not implemented as a @@ -74,7 +76,7 @@ # below produces the switch (by being unrolled). for code, bytecodeimpl in unrolling_bytecode_table: if code == next: - bytecodeimpl(self.s_active_context, self) + bytecodeimpl(self.w_active_context, self) break @@ -201,33 +203,33 @@ print "PRIMITIVE FAILED: %d %s" % (method.primitive, selector,) pass # ignore this error and fall back to the Smalltalk version arguments = self.pop_and_return_n(argcount) - interp.s_active_context = method.create_frame(receiver, arguments, self) + interp.w_active_context = method.create_frame(receiver, arguments, self) self.pop() - def _return(self, object, interp, s_return_to): + def _return(self, object, interp, w_return_to): # for tests, when returning from the top-level context - if s_return_to is None: + if w_return_to is objtable.w_nil: raise ReturnFromTopLevel(object) - s_return_to.push(object) - interp.s_active_context = s_return_to + w_return_to.as_context_get_shadow().push(object) + interp.w_active_context = w_return_to def returnReceiver(self, interp): - self._return(self.w_receiver(), interp, self.s_home().s_sender()) + self._return(self.w_receiver(), interp, self.s_home().w_sender()) def returnTrue(self, interp): - self._return(interp.TRUE, interp, self.s_home().s_sender()) + self._return(interp.TRUE, interp, self.s_home().w_sender()) def returnFalse(self, interp): - self._return(interp.FALSE, interp, self.s_home().s_sender()) + self._return(interp.FALSE, interp, self.s_home().w_sender()) def returnNil(self, interp): - self._return(interp.NIL, interp, self.s_home().s_sender()) + self._return(interp.NIL, interp, self.s_home().w_sender()) def returnTopFromMethod(self, interp): - self._return(self.top(), interp, self.s_home().s_sender()) + self._return(self.top(), interp, self.s_home().w_sender()) def returnTopFromBlock(self, interp): - self._return(self.top(), interp, self.s_sender()) + self._return(self.top(), interp, self.w_sender()) def unknownBytecode(self, interp): raise MissingBytecode("unknownBytecode") Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py Mon Feb 25 17:58:32 2008 @@ -490,13 +490,13 @@ __metaclass__ = extendabletype - def __init__(self, s_home, s_sender): + def __init__(self, w_home, w_sender): self._stack = [] self._pc = 0 #assert isinstance(s_home, W_MethodContext) - self._s_home = s_home + self._w_home = w_home #assert w_sender is None or isinstance(w_sender, W_ContextPart) - self._s_sender = s_sender + self._w_sender = w_sender def as_context_get_shadow(self): # Backward compatibility @@ -519,15 +519,24 @@ " Return self of the method, or the method that contains the block " return self.s_home().w_receiver() + def w_home(self): + return self._w_home + def s_home(self): - return self._s_home + return self.w_home().as_methodcontext_get_shadow() + + def store_w_home(self, w_home): + self._w_home = w_home - def s_sender(self): - if self._s_sender: - return self._s_sender + def w_sender(self): + if self._w_sender is not None: + return self._w_sender + else: + from pypy.lang.smalltalk import objtable + return objtable.w_nil - def store_s_sender(self, s_sender): - self._s_sender = s_sender + def store_w_sender(self, w_sender): + self._w_sender = w_sender def stackpointer(self): return len(self.stack()) + self.stackstart() - 1 @@ -537,11 +546,7 @@ def fetch(self, index): from pypy.lang.smalltalk import utility, objtable if index == constants.CTXPART_SENDER_INDEX: - sender = self.s_sender() - if sender is None: - return objtable.w_nil - else: - return sender.w_self() + return self.w_sender() elif index == constants.CTXPART_PC_INDEX: return utility.wrap_int(self.pc()) elif index == constants.CTXPART_STACKP_INDEX: @@ -555,7 +560,7 @@ from pypy.lang.smalltalk import utility, objtable if index == constants.CTXPART_SENDER_INDEX: if w_value != objtable.w_nil: - self._s_sender = w_value.as_context_get_shadow() + self.store_w_sender(w_value) elif index == constants.CTXPART_PC_INDEX: self._pc = utility.unwrap_int(w_value) elif index == constants.CTXPART_STACKP_INDEX: @@ -567,13 +572,13 @@ raise IndexError def become(self, w_old, w_new): - if self._s_sender is not None and self._s_sender.w_self() == w_old: - self._s_sender = w_new.as_context_get_shadow() + if self.w_sender() == w_old: + self.store_w_sender(w_new) for i in range(len(self._stack)): if self._stack[i] == w_old: self._stack[i] = w_new - if self._s_home is not None and self._s_home.w_self() == w_old: - self._s_home = w_new.as_context_get_shadow() + if self.w_home() == w_old: + self.store_w_home(w_new) def stackstart(self): return constants.MTHDCTX_TEMP_FRAME_START @@ -643,8 +648,8 @@ class W_BlockContext(W_ContextPart): - def __init__(self, s_home, s_sender, argcnt, initialip): - W_ContextPart.__init__(self, s_home, s_sender) + def __init__(self, w_home, w_sender, argcnt, initialip): + W_ContextPart.__init__(self, w_home, w_sender) self.argcnt = argcnt self._initialip = initialip @@ -669,7 +674,7 @@ elif index == constants.BLKCTX_INITIAL_IP_INDEX: return utility.wrap_int(self.initialip) elif index == constants.BLKCTX_HOME_INDEX: - return self.s_home().w_self() + return self.w_home() elif index >= constants.BLKCTX_TEMP_FRAME_START: stack_index = len(self.stack()) - index - 1 return self.stack()[stack_index] @@ -685,7 +690,7 @@ elif index == constants.BLKCTX_INITIAL_IP_INDEX: self.pc = utility.unwrap_int(value) elif index == constants.BLKCTX_HOME_INDEX: - self._s_home = value.as_methodcontext_get_shadow() + self.store_w_home(value) elif index >= constants.BLKCTX_TEMP_FRAME_START: stack_index = len(self.stack()) - index - 1 self.stack()[stack_index] = value @@ -697,8 +702,8 @@ class W_MethodContext(W_ContextPart): def __init__(self, w_method, w_receiver, - arguments, s_sender=None): - W_ContextPart.__init__(self, self, s_sender) + arguments, w_sender=None): + W_ContextPart.__init__(self, self, w_sender) self._w_method = w_method self._w_receiver = w_receiver self.temps = arguments + [w_nil] * w_method.tempsize @@ -743,7 +748,7 @@ elif index == constants.MTHDCTX_RECEIVER_MAP: # what is this thing? return w_nil elif index == constants.MTHDCTX_RECEIVER: - return self._w_receiver + return self.w_receiver() elif index >= constants.MTHDCTX_TEMP_FRAME_START: # First set of indices are temporary variables: offset = index - constants.MTHDCTX_TEMP_FRAME_START @@ -763,7 +768,7 @@ elif index == constants.MTHDCTX_RECEIVER_MAP: # what is this thing? pass elif index == constants.MTHDCTX_RECEIVER: - self._w_receiver = w_object + self.store_w_receiver(w_object) elif index >= constants.MTHDCTX_TEMP_FRAME_START: # First set of indices are temporary variables: offset = index - constants.MTHDCTX_TEMP_FRAME_START Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py Mon Feb 25 17:58:32 2008 @@ -68,7 +68,7 @@ w_result = func(interp, argument_count_m1) if not no_result: assert w_result is not None - interp.s_active_context.push(w_result) + interp.w_active_context.as_context_get_shadow().push(w_result) return w_result else: len_unwrap_spec = len(unwrap_spec) @@ -77,14 +77,15 @@ unrolling_unwrap_spec = unrolling_iterable(enumerate(unwrap_spec)) def wrapped(interp, argument_count_m1): argument_count = argument_count_m1 + 1 # to account for the rcvr - frame = interp.s_active_context + frame = interp.w_active_context + s_frame = frame.as_context_get_shadow() assert argument_count == len_unwrap_spec - if frame.stackpointer() - frame.stackstart() + 1 < len_unwrap_spec: + if s_frame.stackpointer() - s_frame.stackstart() + 1 < len_unwrap_spec: raise PrimitiveFailedError() args = () for i, spec in unrolling_unwrap_spec: index = len_unwrap_spec - 1 - i - w_arg = frame.peek(index) + w_arg = s_frame.peek(index) if spec is int: args += (utility.unwrap_int(w_arg), ) elif spec is index1_0: @@ -101,10 +102,13 @@ raise NotImplementedError( "unknown unwrap_spec %s" % (spec, )) w_result = func(interp, *args) - frame.pop_n(len_unwrap_spec) # only if no exception occurs! + # After calling primitive, reload context-shadow in case it + # needs to be updated + new_s_frame = interp.w_active_context.as_context_get_shadow() + frame.as_context_get_shadow().pop_n(len_unwrap_spec) # only if no exception occurs! if not no_result: assert w_result is not None - interp.s_active_context.push(w_result) + new_s_frame.push(w_result) wrapped.func_name = "wrap_prim_" + name prim_table[code] = wrapped prim_table_implemented_only.append((code, wrapped)) @@ -618,7 +622,7 @@ @expose_primitive(PRIMITIVE_BLOCK_COPY, unwrap_spec=[object, int]) def func(interp, w_context, argcnt): - frame = interp.s_active_context + frame = interp.w_active_context.as_context_get_shadow() # From B.B.: If receiver is a MethodContext, then it becomes # the new BlockContext's home context. Otherwise, the home @@ -626,20 +630,20 @@ # Note that in our impl, MethodContext.w_home == self if not isinstance(w_context, model.W_ContextPart): raise PrimitiveFailedError() - s_method_context = w_context.as_context_get_shadow().s_home() + w_method_context = w_context.as_context_get_shadow().w_home() # The block bytecodes are stored inline: so we skip past the # byteodes to invoke this primitive to find them (hence +2) initialip = frame.pc() + 2 - s_new_context = model.W_BlockContext( - s_method_context, None, argcnt, initialip) - return s_new_context.w_self() + w_new_context = model.W_BlockContext( + w_method_context, None, argcnt, initialip) + return w_new_context def finalize_block_ctx(interp, s_block_ctx, frame): # Set some fields s_block_ctx.store_pc(s_block_ctx.initialip()) - s_block_ctx.store_s_sender(frame) - interp.s_active_context = s_block_ctx + s_block_ctx.store_w_sender(frame) + interp.w_active_context = s_block_ctx.w_self() @expose_primitive(PRIMITIVE_VALUE, no_result=True) def func(interp, argument_count): @@ -649,7 +653,7 @@ # Rcvr | Arg 0 | Arg1 | Arg 2 # - frame = interp.s_active_context + frame = interp.w_active_context.as_context_get_shadow() # Validate that we have a block on the stack and that it received # the proper number of arguments: @@ -667,7 +671,7 @@ s_block_ctx.push_all(block_args) frame.pop() - finalize_block_ctx(interp, s_block_ctx, frame) + finalize_block_ctx(interp, s_block_ctx, frame.w_self()) @expose_primitive(PRIMITIVE_VALUE_WITH_ARGS, unwrap_spec=[object, object], no_result=True) @@ -687,8 +691,7 @@ # XXX Check original logic. Image does not test this anyway # because falls back to value + internal implementation - - finalize_block_ctx(interp, s_block_ctx, interp.s_active_context) + finalize_block_ctx(interp, s_block_ctx, interp.w_active_context) @expose_primitive(PRIMITIVE_PERFORM) def func(interp, argcount): @@ -701,11 +704,11 @@ w_method = w_rcvr.shadow_of_my_class().lookup(sel) assert w_method - s_frame = w_method.create_frame(w_rcvr, + w_frame = w_method.create_frame(w_rcvr, [w_args.fetch(i) for i in range(w_args.size())]) - s_frame.store_s_sender(interp.s_active_context) - interp.s_active_context = s_frame + w_frame.as_context_get_shadow().store_w_sender(interp.w_active_context) + interp.w_active_context = w_frame @expose_primitive(PRIMITIVE_SIGNAL, unwrap_spec=[object]) Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py Mon Feb 25 17:58:32 2008 @@ -432,17 +432,19 @@ " Return self of the method, or the method that contains the block " return self.s_home().w_receiver() + def w_sender(self): + return self.w_self().fetch(constants.CTXPART_SENDER_INDEX) + def s_sender(self): - # XXX XXX from pypy.lang.smalltalk import objtable - w_sender = self.w_self().fetch(constants.CTXPART_SENDER_INDEX) + w_sender = self.w_sender() if w_sender == objtable.w_nil: return None else: return w_sender.as_context_get_shadow() - def store_s_sender(self, s_sender): - self.w_self().store(constants.CTXPART_SENDER_INDEX, s_sender.w_self()) + def store_w_sender(self, s_sender): + self.w_self().store(constants.CTXPART_SENDER_INDEX, w_sender) def pc(self): return utility.unwrap_int(self.w_self().fetch(constants.CTXPART_PC_INDEX)) @@ -536,9 +538,12 @@ def initialip(self): return utility.unwrap_int(self.w_self().fetch(constants.BLKCTX_INITIAL_IP_INDEX)) - def s_home(self): - return self.w_self().fetch(constants.BLKCTX_HOME_INDEX).as_methodcontext_get_shadow() + def w_home(self): + return self.w_self().fetch(constants.BLKCTX_HOME_INDEX) + def s_home(self): + return self.w_home().as_methodcontext_get_shadow() + def stackstart(self): return constants.BLKCTX_TEMP_FRAME_START @@ -561,6 +566,9 @@ def settemp(self, index, w_value): self.w_self().store(constants.MTHDCTX_TEMP_FRAME_START + index, w_value) + def w_home(self): + return self.w_self() + def s_home(self): return self Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_interpreter.py Mon Feb 25 17:58:32 2008 @@ -74,7 +74,7 @@ w_method.tempsize=1 w_frame = w_method.create_frame(receiver, ["foo", "bar"]) interp = interpreter.Interpreter() - interp.s_active_context = w_frame.as_methodcontext_get_shadow() + interp.w_active_context = w_frame return interp def test_create_frame(): @@ -95,7 +95,7 @@ def test_push_pop(): interp = new_interpreter("") - frame = interp.s_active_context + frame = interp.s_active_context() frame.push(12) frame.push(34) frame.push(56) @@ -116,7 +116,7 @@ def test_pushReceiverBytecode(): interp = new_interpreter(pushReceiverBytecode) interp.step() - assert interp.s_active_context.top() == interp.s_active_context.w_receiver() + assert interp.s_active_context().top() == interp.w_active_context.as_methodcontext_get_shadow().w_receiver() def test_pushReceiverVariableBytecode(bytecode = (pushReceiverVariableBytecode(0) + pushReceiverVariableBytecode(1) + @@ -129,27 +129,27 @@ interp.step() interp.step() interp.step() - assert interp.s_active_context.stack() == ["egg", "bar", "baz"] + assert interp.s_active_context().stack() == ["egg", "bar", "baz"] def test_pushTemporaryVariableBytecode(bytecode=(pushTemporaryVariableBytecode(0) + pushTemporaryVariableBytecode(1) + pushTemporaryVariableBytecode(2))): interp = new_interpreter(bytecode) - interp.s_active_context.settemp(2, "temp") + interp.w_active_context.as_methodcontext_get_shadow().settemp(2, "temp") interp.step() interp.step() interp.step() - assert interp.s_active_context.stack() == ["foo", "bar", "temp"] + assert interp.s_active_context().stack() == ["foo", "bar", "temp"] def test_pushLiteralConstantBytecode(bytecode=pushLiteralConstantBytecode(0) + pushLiteralConstantBytecode(1) + pushLiteralConstantBytecode(2)): interp = new_interpreter(bytecode) - interp.s_active_context.w_method().literals = fakeliterals("a", "b", "c") + interp.w_active_context.as_methodcontext_get_shadow().w_method().literals = fakeliterals("a", "b", "c") interp.step() interp.step() interp.step() - assert interp.s_active_context.stack() == [fakesymbol("a"), + assert interp.s_active_context().stack() == [fakesymbol("a"), fakesymbol("b"), fakesymbol("c")] @@ -158,9 +158,9 @@ w_association.store(0, "mykey") w_association.store(1, "myvalue") interp = new_interpreter(bytecode) - interp.s_active_context.w_method().literals = fakeliterals(w_association) + interp.w_active_context.as_methodcontext_get_shadow().w_method().literals = fakeliterals(w_association) interp.step() - assert interp.s_active_context.stack() == ["myvalue"] + assert interp.s_active_context().stack() == ["myvalue"] def test_storeAndPopReceiverVariableBytecode(bytecode=storeAndPopReceiverVariableBytecode, popped=True): @@ -168,13 +168,13 @@ for index in range(8): w_object = shadow.new() interp = new_interpreter(pushConstantTrueBytecode + bytecode(index)) - interp.s_active_context.store_w_receiver(w_object) + interp.w_active_context.as_methodcontext_get_shadow().store_w_receiver(w_object) interp.step() interp.step() if popped: - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().stack() == [] else: - assert interp.s_active_context.stack() == [interp.TRUE] + assert interp.s_active_context().stack() == [interp.TRUE] for test_index in range(8): if test_index == index: @@ -185,167 +185,167 @@ def test_storeAndPopTemporaryVariableBytecode(bytecode=storeAndPopTemporaryVariableBytecode): for index in range(8): interp = new_interpreter(pushConstantTrueBytecode + bytecode(index)) - interp.s_active_context.temps = [None] * 8 + interp.w_active_context.as_methodcontext_get_shadow().temps = [None] * 8 interp.step() interp.step() - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().stack() == [] for test_index in range(8): if test_index == index: - assert interp.s_active_context.temps[test_index] == interp.TRUE + assert interp.w_active_context.as_methodcontext_get_shadow().temps[test_index] == interp.TRUE else: - assert interp.s_active_context.temps[test_index] == None + assert interp.w_active_context.as_methodcontext_get_shadow().temps[test_index] == None def test_pushConstantTrueBytecode(): interp = new_interpreter(pushConstantTrueBytecode) interp.step() - assert interp.s_active_context.pop() == interp.TRUE - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().pop() == interp.TRUE + assert interp.s_active_context().stack() == [] def test_pushConstantFalseBytecode(): interp = new_interpreter(pushConstantFalseBytecode) interp.step() - assert interp.s_active_context.pop() == interp.FALSE - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().pop() == interp.FALSE + assert interp.s_active_context().stack() == [] def test_pushConstantNilBytecode(): interp = new_interpreter(pushConstantNilBytecode) interp.step() - assert interp.s_active_context.pop() == interp.NIL - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().pop() == interp.NIL + assert interp.s_active_context().stack() == [] def test_pushConstantMinusOneBytecode(): interp = new_interpreter(pushConstantMinusOneBytecode) interp.step() - assert interp.s_active_context.pop() == interp.MINUS_ONE - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().pop() == interp.MINUS_ONE + assert interp.s_active_context().stack() == [] def test_pushConstantZeroBytecode(): interp = new_interpreter(pushConstantZeroBytecode) interp.step() - assert interp.s_active_context.pop() == interp.ZERO - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().pop() == interp.ZERO + assert interp.s_active_context().stack() == [] def test_pushConstantOneBytecode(): interp = new_interpreter(pushConstantOneBytecode) interp.step() - assert interp.s_active_context.pop() == interp.ONE - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().pop() == interp.ONE + assert interp.s_active_context().stack() == [] def test_pushConstantTwoBytecode(): interp = new_interpreter(pushConstantTwoBytecode) interp.step() - assert interp.s_active_context.pop() == interp.TWO - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().pop() == interp.TWO + assert interp.s_active_context().stack() == [] def test_pushActiveContextBytecode(): interp = new_interpreter(pushActiveContextBytecode) interp.step() - assert interp.s_active_context.pop() == interp.s_active_context.w_self() - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().pop() == interp.w_active_context + assert interp.s_active_context().stack() == [] def test_duplicateTopBytecode(): interp = new_interpreter(pushConstantZeroBytecode + duplicateTopBytecode) interp.step() interp.step() - assert interp.s_active_context.stack() == [interp.ZERO, interp.ZERO] + assert interp.s_active_context().stack() == [interp.ZERO, interp.ZERO] def test_bytecodePrimBitAnd(): interp = new_interpreter(pushConstantOneBytecode + pushConstantTwoBytecode + bytecodePrimBitAnd) interp.step() interp.step() interp.step() - assert interp.s_active_context.pop().value == 0 - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().pop().value == 0 + assert interp.s_active_context().stack() == [] def test_bytecodePrimBitOr(): interp = new_interpreter(pushConstantOneBytecode + pushConstantTwoBytecode + bytecodePrimBitOr) interp.step() interp.step() interp.step() - assert interp.s_active_context.pop().value == 3 - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().pop().value == 3 + assert interp.s_active_context().stack() == [] def test_bytecodePrimBitShift(): interp = new_interpreter(pushConstantOneBytecode + pushConstantTwoBytecode + bytecodePrimBitShift) interp.step() interp.step() interp.step() - assert interp.s_active_context.pop().value == 4 - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().pop().value == 4 + assert interp.s_active_context().stack() == [] def test_bytecodePrimClass(): interp = new_interpreter(pushConstantOneBytecode + bytecodePrimClass) interp.step() interp.step() - assert interp.s_active_context.pop() == classtable.w_SmallInteger - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().pop() == classtable.w_SmallInteger + assert interp.s_active_context().stack() == [] def test_bytecodePrimSubtract(): interp = new_interpreter(pushConstantOneBytecode + pushConstantTwoBytecode + bytecodePrimSubtract) interp.step() interp.step() interp.step() - assert interp.s_active_context.pop().value == -1 - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().pop().value == -1 + assert interp.s_active_context().stack() == [] def test_bytecodePrimMultiply(): interp = new_interpreter(pushConstantMinusOneBytecode + pushConstantTwoBytecode + bytecodePrimMultiply) interp.step() interp.step() interp.step() - assert interp.s_active_context.pop().value == -2 - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().pop().value == -2 + assert interp.s_active_context().stack() == [] def test_bytecodePrimDivide(): interp = new_interpreter(pushConstantTwoBytecode + pushConstantMinusOneBytecode + bytecodePrimDivide) interp.step() interp.step() interp.step() - assert interp.s_active_context.pop().value == -2 - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().pop().value == -2 + assert interp.s_active_context().stack() == [] def test_bytecodePrimDiv(): interp = new_interpreter(pushConstantTwoBytecode + pushConstantMinusOneBytecode + bytecodePrimDiv) interp.step() interp.step() interp.step() - assert interp.s_active_context.pop().value == -2 - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().pop().value == -2 + assert interp.s_active_context().stack() == [] def test_bytecodePrimMod(): interp = new_interpreter(pushConstantTwoBytecode + pushConstantMinusOneBytecode + bytecodePrimMod) interp.step() interp.step() interp.step() - assert interp.s_active_context.pop().value == 0 - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().pop().value == 0 + assert interp.s_active_context().stack() == [] def test_bytecodePrimEquivalent(): interp = new_interpreter(pushConstantTwoBytecode + pushConstantMinusOneBytecode + bytecodePrimEquivalent) interp.step() interp.step() interp.step() - assert interp.s_active_context.pop() == interpreter.Interpreter.FALSE - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().pop() == interpreter.Interpreter.FALSE + assert interp.s_active_context().stack() == [] interp = new_interpreter(pushConstantOneBytecode + pushConstantOneBytecode + bytecodePrimEquivalent) interp.step() interp.step() interp.step() - assert interp.s_active_context.pop() == interpreter.Interpreter.TRUE - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().pop() == interpreter.Interpreter.TRUE + assert interp.s_active_context().stack() == [] def test_bytecodePrimNew(): w_fakeclassclass = mockclass(10, name='fakeclassclass') w_fakeclass = mockclass(1, name='fakeclass', varsized=False, w_metaclass=w_fakeclassclass) interp = new_interpreter(bytecodePrimNew) - interp.s_active_context.push(w_fakeclass) + interp.s_active_context().push(w_fakeclass) run_with_faked_methods( [[w_fakeclassclass, primitives.NEW, 0, "new"]], interp.step) - w_fakeinst = interp.s_active_context.pop() - assert interp.s_active_context.stack() == [] + w_fakeinst = interp.s_active_context().pop() + assert interp.s_active_context().stack() == [] assert w_fakeinst.getclass() == w_fakeclass assert w_fakeinst.size() == 1 @@ -354,13 +354,13 @@ w_fakeclass = mockclass(1, name='fakeclass', varsized=True, w_metaclass=w_fakeclassclass) interp = new_interpreter(bytecodePrimNewWithArg) - interp.s_active_context.push(w_fakeclass) - interp.s_active_context.push(interpreter.Interpreter.TWO) + interp.s_active_context().push(w_fakeclass) + interp.s_active_context().push(interpreter.Interpreter.TWO) run_with_faked_methods( [[w_fakeclassclass, primitives.NEW_WITH_ARG, 1, "new:"]], interp.step) - w_fakeinst = interp.s_active_context.pop() - assert interp.s_active_context.stack() == [] + w_fakeinst = interp.s_active_context().pop() + assert interp.s_active_context().stack() == [] assert w_fakeinst.getclass() == w_fakeclass assert w_fakeinst.size() == 3 @@ -368,12 +368,12 @@ w_fakeclass = mockclass(2, name='fakeclass', varsized=True) w_fakeinst = w_fakeclass.as_class_get_shadow().new(5) interp = new_interpreter(bytecodePrimSize) - interp.s_active_context.push(w_fakeinst) + interp.s_active_context().push(w_fakeinst) run_with_faked_methods( [[w_fakeclass, primitives.SIZE, 0, "size"]], interp.step) - assert interp.s_active_context.pop().value == 5 - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().pop().value == 5 + assert interp.s_active_context().stack() == [] # w_class - the class from which the method is going to be called # (and on which it is going to be installed) @@ -390,19 +390,19 @@ w_method.bytes = pushConstantOneBytecode + bytecode shadow.installmethod("foo", w_method) interp = new_interpreter(bytecodes) - interp.s_active_context.w_method().literals = fakeliterals("foo") - interp.s_active_context.push(w_object) - callerContext = interp.s_active_context - interp.step() - assert interp.s_active_context.s_sender() == callerContext - assert interp.s_active_context.stack() == [] - assert interp.s_active_context.w_receiver() == w_object - assert interp.s_active_context.w_method() == shadow.methoddict["foo"] + interp.w_active_context.as_methodcontext_get_shadow().w_method().literals = fakeliterals("foo") + interp.s_active_context().push(w_object) + callerContext = interp.w_active_context + interp.step() + assert interp.s_active_context().w_sender() == callerContext + assert interp.s_active_context().stack() == [] + assert interp.w_active_context.as_methodcontext_get_shadow().w_receiver() == w_object + assert interp.w_active_context.as_methodcontext_get_shadow().w_method() == shadow.methoddict["foo"] assert callerContext.stack() == [] interp.step() interp.step() - assert interp.s_active_context == callerContext - assert interp.s_active_context.stack() == [result] + assert interp.w_active_context == callerContext + assert interp.s_active_context().stack() == [result] def test_sendLiteralSelectorBytecode(): w_class = mockclass(0) @@ -420,9 +420,9 @@ shadow.installmethod("fib:", method) w_object = shadow.new() interp = new_interpreter(sendLiteralSelectorBytecode(16) + returnTopFromMethod) - interp.s_active_context.w_method().literals = fakeliterals("fib:") - interp.s_active_context.push(w_object) - interp.s_active_context.push(wrap_int(8)) + interp.w_active_context.as_methodcontext_get_shadow().w_method().literals = fakeliterals("fib:") + interp.s_active_context().push(w_object) + interp.s_active_context().push(wrap_int(8)) result = interp.interpret() assert unwrap_int(result) == 34 @@ -430,14 +430,14 @@ def test(): interp = new_interpreter(sendLiteralSelectorBytecode(1 + 16)) - interp.s_active_context.w_method().literals = fakeliterals("foo", "sub") - interp.s_active_context.push(wrap_int(50)) - interp.s_active_context.push(wrap_int(8)) - callerContext = interp.s_active_context - interp.step() - assert interp.s_active_context is callerContext - assert len(interp.s_active_context.stack()) == 1 - w_result = interp.s_active_context.pop() + interp.w_active_context.as_methodcontext_get_shadow().w_method().literals = fakeliterals("foo", "sub") + interp.s_active_context().push(wrap_int(50)) + interp.s_active_context().push(wrap_int(8)) + callerContext = interp.w_active_context + interp.step() + assert interp.w_active_context is callerContext + assert len(interp.s_active_context().stack()) == 1 + w_result = interp.s_active_context().pop() assert unwrap_int(w_result) == 42 run_with_faked_methods( @@ -447,58 +447,58 @@ def test_longJumpIfTrue(): interp = new_interpreter(longJumpIfTrue(0) + chr(15) + longJumpIfTrue(0) + chr(15)) - interp.s_active_context.push(interp.FALSE) - pc = interp.s_active_context.pc() + 2 + interp.s_active_context().push(interp.FALSE) + pc = interp.w_active_context.pc() + 2 interp.step() - assert interp.s_active_context.pc() == pc - interp.s_active_context.push(interp.TRUE) - pc = interp.s_active_context.pc() + 2 + assert interp.s_active_context().pc() == pc + interp.s_active_context().push(interp.TRUE) + pc = interp.s_active_context().pc() + 2 interp.step() - assert interp.s_active_context.pc() == pc + 15 + assert interp.s_active_context().pc() == pc + 15 def test_longJumpIfFalse(): interp = new_interpreter(pushConstantTrueBytecode + longJumpIfFalse(0) + chr(15) + pushConstantFalseBytecode + longJumpIfFalse(0) + chr(15)) interp.step() - pc = interp.s_active_context.pc() + 2 + pc = interp.s_active_context().pc() + 2 interp.step() - assert interp.s_active_context.pc() == pc + assert interp.s_active_context().pc() == pc interp.step() - pc = interp.s_active_context.pc() + 2 + pc = interp.s_active_context().pc() + 2 interp.step() - assert interp.s_active_context.pc() == pc + 15 + assert interp.s_active_context().pc() == pc + 15 def test_longUnconditionalJump(): interp = new_interpreter(longUnconditionalJump(4) + chr(15)) - pc = interp.s_active_context.pc() + 2 + pc = interp.s_active_context().pc() + 2 interp.step() - assert interp.s_active_context.pc() == pc + 15 + assert interp.s_active_context().pc() == pc + 15 def test_shortUnconditionalJump(): interp = new_interpreter(chr(145)) - pc = interp.s_active_context.pc() + 1 + pc = interp.s_active_context().pc() + 1 interp.step() - assert interp.s_active_context.pc() == pc + 2 + assert interp.s_active_context().pc() == pc + 2 def test_shortConditionalJump(): interp = new_interpreter(pushConstantTrueBytecode + shortConditionalJump(3) + pushConstantFalseBytecode + shortConditionalJump(3)) interp.step() - pc = interp.s_active_context.pc() + 1 + pc = interp.s_active_context().pc() + 1 interp.step() - assert interp.s_active_context.pc() == pc + assert interp.s_active_context().pc() == pc interp.step() - pc = interp.s_active_context.pc() + 1 + pc = interp.s_active_context().pc() + 1 interp.step() - assert interp.s_active_context.pc() == pc + 4 + assert interp.s_active_context().pc() == pc + 4 def test_popStackBytecode(): interp = new_interpreter(pushConstantTrueBytecode + popStackBytecode) interp.step() - assert interp.s_active_context.stack() == [interp.TRUE] + assert interp.s_active_context().stack() == [interp.TRUE] interp.step() - assert interp.s_active_context.stack() == [] + assert interp.s_active_context().stack() == [] def test_extendedPushBytecode(): test_pushReceiverVariableBytecode(extendedPushBytecode + chr((0<<6) + 0) + @@ -520,7 +520,7 @@ w_association.store(0, "mykey") w_association.store(1, "myvalue") interp = new_interpreter(pushConstantOneBytecode + bytecode) - interp.s_active_context.w_method().literals = fakeliterals(w_association) + interp.w_active_context.as_methodcontext_get_shadow().w_method().literals = fakeliterals(w_association) interp.step() interp.step() assert w_association.fetch(1) == interp.ONE @@ -545,13 +545,13 @@ shadow.installmethod("+", w_method) w_object = shadow.new() - interp.s_active_context.push(w_object) - interp.s_active_context.push(interp.ONE) + interp.s_active_context().push(w_object) + interp.s_active_context().push(interp.ONE) interp.step() - assert interp.s_active_context.w_method() == shadow.methoddict["+"] - assert interp.s_active_context.w_receiver() is w_object - assert interp.s_active_context.gettemp(0) == interp.ONE - assert interp.s_active_context.stack() == [] + assert interp.w_active_context.as_methodcontext_get_shadow().w_method() == shadow.methoddict["+"] + assert interp.s_active_context().w_receiver() is w_object + assert interp.w_active_context.as_methodcontext_get_shadow().gettemp(0) == interp.ONE + assert interp.s_active_context().stack() == [] def test_bytecodePrimBool(): interp = new_interpreter(bytecodePrimLessThan + @@ -561,10 +561,10 @@ bytecodePrimEqual + bytecodePrimNotEqual) for i in range(6): - interp.s_active_context.push(interp.ONE) - interp.s_active_context.push(interp.TWO) + interp.s_active_context().push(interp.ONE) + interp.s_active_context().push(interp.TWO) interp.step() - assert interp.s_active_context.stack() == [interp.TRUE, interp.FALSE, + assert interp.s_active_context().stack() == [interp.TRUE, interp.FALSE, interp.TRUE, interp.FALSE, interp.FALSE, interp.TRUE] @@ -593,18 +593,18 @@ meth1.literals = fakeliterals("foo") meth2.literals = fakeliterals("foo") interp = new_interpreter(bytecodes) - interp.s_active_context.w_method().literals = fakeliterals("foo") - interp.s_active_context.push(w_object) + interp.w_active_context.as_methodcontext_get_shadow().w_method().literals = fakeliterals("foo") + interp.s_active_context().push(w_object) interp.step() for w_specificclass in [w_super, w_supersuper]: - callerContext = interp.s_active_context + callerContext = interp.w_active_context interp.step() interp.step() - assert interp.s_active_context.s_sender() == callerContext - assert interp.s_active_context.stack() == [] - assert interp.s_active_context.w_receiver() == w_object + assert interp.s_active_context().w_sender() == callerContext + assert interp.s_active_context().stack() == [] + assert interp.w_active_context.as_methodcontext_get_shadow().w_receiver() == w_object meth = w_specificclass.as_class_get_shadow().methoddict["foo"] - assert interp.s_active_context.w_method() == meth + assert interp.w_active_context.as_methodcontext_get_shadow().w_method() == meth assert callerContext.stack() == [] def test_secondExtendedSendBytecode(): @@ -639,7 +639,7 @@ def interpret_bc(bcodes, literals, receiver=objtable.w_nil): bcode = "".join([chr(x) for x in bcodes]) interp = new_interpreter(bcode, receiver=receiver) - interp.s_active_context.w_method().literals = literals + interp.w_active_context.as_methodcontext_get_shadow().w_method().literals = literals return interp.interpret() # tests: bytecodePrimValue & bytecodePrimValueWithArg Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py Mon Feb 25 17:58:32 2008 @@ -201,15 +201,15 @@ w_method = s_class.lookup("abs") assert w_method - s_frame = w_method.create_frame(w_object, []) - interp.s_active_context = s_frame + w_frame = w_method.create_frame(w_object, []) + interp.w_active_context = w_frame print w_method while True: try: interp.step() - print interp.s_active_context.stack + print interp.s_active_context().stack() except interpreter.ReturnFromTopLevel, e: assert e.object.value == abs(int) return @@ -236,7 +236,7 @@ s_ctx = s_ap.s_suspended_context() s_ap.store_w_suspended_context(objtable.w_nil) interp = interpreter.Interpreter() - interp.s_active_context = s_ctx + interp.w_active_context = s_ctx.w_self() interp.interpret() def test_compile_method(): @@ -270,8 +270,8 @@ s_class = w_receiver.shadow_of_my_class() w_method = s_class.lookup(selector) assert w_method - s_frame = w_method.create_frame(w_receiver, list(arguments_w)) - interp.s_active_context = s_frame + w_frame = w_method.create_frame(w_receiver, list(arguments_w)) + interp.w_active_context = w_frame while True: try: interp.step() Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_primitives.py Mon Feb 25 17:58:32 2008 @@ -25,24 +25,24 @@ mapped_stack = [wrap(x) for x in stack] frame = MockFrame(mapped_stack) interp = interpreter.Interpreter() - interp.s_active_context = frame + interp.w_active_context = frame return (interp, len(stack)) def prim(code, stack): interp, argument_count = mock(stack) prim_table[code](interp, argument_count-1) - res = interp.s_active_context.pop() - assert not len(interp.s_active_context.stack()) # check args are consumed + res = interp.s_active_context().pop() + assert not len(interp.s_active_context().stack()) # check args are consumed return res def prim_fails(code, stack): interp, argument_count = mock(stack) - orig_stack = list(interp.s_active_context.stack()) + orig_stack = list(interp.s_active_context().stack()) try: prim_table[code](interp, argument_count-1) py.test.fail("Expected PrimitiveFailedError") except PrimitiveFailedError: - assert interp.s_active_context.stack() == orig_stack + assert interp.s_active_context().stack() == orig_stack # smallinteger tests def test_small_int_add(): From tverwaes at codespeak.net Mon Feb 25 18:12:31 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Mon, 25 Feb 2008 18:12:31 +0100 (CET) Subject: [pypy-svn] r51856 - pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test Message-ID: <20080225171231.E0E2216847A@codespeak.net> Author: tverwaes Date: Mon Feb 25 18:12:31 2008 New Revision: 51856 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_interpreter.py Log: fixing previously skipped tests Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_interpreter.py Mon Feb 25 18:12:31 2008 @@ -742,7 +742,6 @@ def test_bc_primBytecodeAt_with_instvars(): # ^ self at: 1 - py.test.skip("Broken, fix me") w_fakeclass = mockclass(1, name='fakeclass', varsized=True) w_fakeinst = w_fakeclass.as_class_get_shadow().new(1) w_fakeinst.store(0, wrap_char("a")) # static slot 0: instance variable @@ -758,7 +757,6 @@ def test_bc_primBytecodeAtPut_with_instvars(): # ^ self at: 1 put: #b - py.test.skip("Broken, fix me") w_fakeclass = mockclass(1, name='fakeclass', varsized=True) w_fakeinst = w_fakeclass.as_class_get_shadow().new(1) w_fakeinst.store(0, wrap_char("a")) # static slot 0: instance variable @@ -779,15 +777,13 @@ # ^ self objectAt: 2. yields the first literal (22) # ^ self objectAt: 2 put: 3. changes the first literal to 3 # ^ self objectAt: 2. yields the new first literal (3) - py.test.skip("Broken, fix me") - prim_meth = model.W_CompiledMethod(0) + prim_meth = model.W_CompiledMethod(header=1024) prim_meth.literals = fakeliterals(22) - mhs = fakesymbol("methodheader") oal = fakeliterals("objectAt:") oalp = fakeliterals("objectAt:put:", 3) def test(): assert interpret_bc( - [112, 118, 224, 124], oal, receiver=prim_meth) == mhs + [112, 118, 224, 124], oal, receiver=prim_meth).value == 1024 assert interpret_bc( [112, 119, 224, 124], oal, receiver=prim_meth).value == 22 assert interpret_bc( From arigo at codespeak.net Mon Feb 25 20:00:53 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 25 Feb 2008 20:00:53 +0100 (CET) Subject: [pypy-svn] r51857 - pypy/branch/jit-refactoring/pypy/jit/rainbow Message-ID: <20080225190053.1F6551683E5@codespeak.net> Author: arigo Date: Mon Feb 25 20:00:48 2008 New Revision: 51857 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Log: Fix comment. Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Mon Feb 25 20:00:48 2008 @@ -407,8 +407,8 @@ self.frame.local_boxes = local_boxes def opimpl_make_new_greenvars(self): - # an opcode with a variable number of args - # num_args arg_old_1 arg_new_1 ... + # this uses a "green_varargs" argument, but we do the decoding + # manually for the fast case num = self.load_2byte() if num == 0 and len(self.frame.local_green) == 0: # fast (very common) case From arigo at codespeak.net Mon Feb 25 20:29:05 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 25 Feb 2008 20:29:05 +0100 (CET) Subject: [pypy-svn] r51858 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080225192905.18EF81684C6@codespeak.net> Author: arigo Date: Mon Feb 25 20:29:04 2008 New Revision: 51858 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Log: In dump(), show where the operations put their result. This is done with the help of comments inserted in the "assembler" source by codewriter.py. Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 25 20:29:04 2008 @@ -220,9 +220,9 @@ self.emit(label(block)) reds, greens = self.sort_by_color(block.inputargs) for arg in reds: - self.register_redvar(arg) + self.register_redvar(arg, verbose=False) for arg in greens: - self.register_greenvar(arg) + self.register_greenvar(arg, verbose=False) self.insert_merges(block) for op in block.operations: self.serialize_op(op) @@ -364,10 +364,10 @@ # already converted return self.redvar_positions[arg, block] self.emit("make_redbox") - resultindex = self.register_redvar((arg, block)) argindex = self.green_position(arg) self.emit(argindex) self.emit(self.type_position(arg.concretetype)) + resultindex = self.register_redvar((arg, block)) return resultindex def opcolor(self, op): @@ -385,22 +385,26 @@ color = "red" return color - def register_redvar(self, arg, where=-1): + def register_redvar(self, arg, where=-1, verbose=True): assert arg not in self.redvar_positions if where == -1: where = self.free_red[self.current_block] self.free_red[self.current_block] += 1 + if verbose: + self.emit('# => r%d' % (where,)) self.redvar_positions[arg] = where return where def redvar_position(self, arg): return self.redvar_positions[arg] - def register_greenvar(self, arg, where=None, check=True): + def register_greenvar(self, arg, where=None, check=True, verbose=True): assert isinstance(arg, flowmodel.Variable) or not check if where is None: where = self.free_green[self.current_block] self.free_green[self.current_block] += 1 + if verbose: + self.emit('# => g%d' % (where,)) self.greenvar_positions[arg] = where return where @@ -1045,6 +1049,8 @@ result.append(chr(index & 0xff)) for arg in args: if isinstance(arg, str): + if arg.startswith('#'): # skip comments + continue opcode = interpreter.find_opcode(arg) assert opcode >= 0, "unknown opcode %s" % (arg, ) emit_2byte(opcode) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py Mon Feb 25 20:29:04 2008 @@ -66,7 +66,7 @@ def load_jumptarget(self): tlbl = self.get(codewriter.tlabel, 4) - return self.labelpos[tlbl.name] + return 'pc: %d' % self.labelpos[tlbl.name] class CustomRepr: @@ -94,7 +94,6 @@ opcode = interpreter.find_opcode(opname) opimpl = interpreter.opcode_implementations[opcode] argtypes = opimpl.argspec - resulttype = opimpl.resultspec args = [] for argspec in argtypes: @@ -150,9 +149,19 @@ assert 0, "unknown argtype declaration" args = map(str, args) - # XXX we should print the result from resultspec too, - # but it's not obvious how to do that - line = '%5d | %-20s %s' % (startpc, opname, ', '.join(args)) + + comments = [] + while (not src.finished() and isinstance(src.peek(), str) + and src.peek().startswith('#')): + # comment, used to tell where the result of the previous + # operation goes + comments.append(src.get(str, 0)[1:].strip()) + + if startpc == 0: + startpc = 'pc: 0' + line = '%5s | %-20s %-16s %s' % (startpc, opname, + ', '.join(args), + ', '.join(comments)) print >> file, line.rstrip() elif isinstance(arg, codewriter.label): if src.pc not in noblankline: # no duplicate blank lines Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Mon Feb 25 20:29:04 2008 @@ -145,7 +145,7 @@ return return val wrapped.func_name = "wrap_" + func.func_name - wrapped.argspec = argtypes + wrapped.argspec = tuple(argtypes) wrapped.resultspec = result return wrapped return decorator Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_serializegraph.py Mon Feb 25 20:29:04 2008 @@ -229,13 +229,13 @@ # xxx slightly fragile test, it will break whenever we tweak dump.py expected = """\ JITCODE 'f' - 0 | make_redbox (0), 0 +pc: 0 | make_redbox (0), 0 => r1 6 | make_new_redvars [r0, r1] 14 | make_new_greenvars [] | 18 | local_merge 0, None - 24 | red_int_is_true r0 - 28 | red_goto_iftrue r2, 48 + 24 | red_int_is_true r0 => r2 + 28 | red_goto_iftrue r2, pc: 48 36 | make_new_redvars [r1] 42 | make_new_greenvars [] | @@ -244,12 +244,12 @@ 48 | make_new_redvars [r0, r1] 56 | make_new_greenvars [] | - 60 | red_int_add r1, r0 - 66 | make_redbox (1), 0 - 72 | red_int_sub r0, r3 + 60 | red_int_add r1, r0 => r2 + 66 | make_redbox (1), 0 => r3 + 72 | red_int_sub r0, r3 => r4 78 | make_new_redvars [r4, r2] 86 | make_new_greenvars [] - 90 | goto 18 + 90 | goto pc: 18 """.rstrip() assert result == expected From arigo at codespeak.net Mon Feb 25 20:36:34 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 25 Feb 2008 20:36:34 +0100 (CET) Subject: [pypy-svn] r51859 - pypy/branch/jit-refactoring/pypy/jit/rainbow Message-ID: <20080225193634.D01C51684CD@codespeak.net> Author: arigo Date: Mon Feb 25 20:36:29 2008 New Revision: 51859 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py Log: * Don't crash if the called_bytecode is uninitialized yet. * Show where to put a call to dump() for nice results. Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 25 20:36:29 2008 @@ -156,6 +156,7 @@ bytecode._source = self.assembler bytecode._interpreter = self.interpreter bytecode._labelpos = labelpos + #bytecode.dump() if is_portal: self.finish_all_graphs() self.interpreter.set_num_global_mergepoints( Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py Mon Feb 25 20:36:29 2008 @@ -124,7 +124,7 @@ elif argspec == "bytecode": bytecodenum = src.load_2byte() called_bytecode = jitcode.called_bytecodes[bytecodenum] - args.append(called_bytecode.name) + args.append(getattr(called_bytecode, 'name', '?')) elif argspec == "calldesc": index = src.load_2byte() function = jitcode.calldescs[index] From arigo at codespeak.net Mon Feb 25 21:04:05 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 25 Feb 2008 21:04:05 +0100 (CET) Subject: [pypy-svn] r51860 - pypy/dist/pypy/doc/discussion Message-ID: <20080225200405.817A6168477@codespeak.net> Author: arigo Date: Mon Feb 25 21:04:03 2008 New Revision: 51860 Modified: pypy/dist/pypy/doc/discussion/jit-refactoring-plan.txt Log: Rename the hints and clarify some points. Modified: pypy/dist/pypy/doc/discussion/jit-refactoring-plan.txt ============================================================================== --- pypy/dist/pypy/doc/discussion/jit-refactoring-plan.txt (original) +++ pypy/dist/pypy/doc/discussion/jit-refactoring-plan.txt Mon Feb 25 21:04:03 2008 @@ -50,14 +50,15 @@ ++++++++++++++++ We'd replace portals and global merge points with the following variant: -two hints, "jit_can_enter" and "jit_can_leave", which are where the -execution can go from interpreter to JITted and back. The idea is that -"jit_can_leave" is present at the beginning of the main interpreter loop --- i.e. it is a global merge point. +two hints, "can_enter_jit" and "global_merge_point", which are where the +execution can go from interpreter to JITted and back. As before, +"global_merge_point" is present at the beginning of the main interpreter +loop; in this model it has the additional meaning of being where the +JIT can be *left* in order to go back to regular interpretation. -The other hint, "jit_can_enter", is the place where some lightweight +The other hint, "can_enter_jit", is the place where some lightweight profiling occurs in order to know if we should enter the JIT. It's -important to not have one "jit_can_enter" for each opcode -- that's a +important to not have one "can_enter_jit" for each opcode -- that's a too heavy slow-down for regularly interpreted code (but it would be correct too). A probably reasonable idea is to put it in the opcodes that close loops (JUMP_ABSOLUTE, CONTINUE). This would make the regular @@ -65,30 +66,30 @@ often executed. (In time, the JIT should follow calls too, so that means that the functions called by loops also get JITted.) -If the profiling in "jit_can_enter" finds out we should start JITting, +If the profiling in "can_enter_jit" finds out we should start JITting, it calls the JIT, which compiles and executes some machine code, which makes the current function frame progress, maybe to its end or not, but -at least to an opcode boundary; so when the call done by "jit_can_enter" +at least to an opcode boundary; so when the call done by "can_enter_jit" returns the regular interpreter can simply continue from the new next -opcode. For this reason it's necessary to put "jit_can_enter" and -"jit_can_leave" next to each other, control-flow-wise -- -i.e. "jit_can_enter" should be at the end of JUMP_ABSOLUTE and CONTINUE, -so that they are immediately followed by the global "jit_can_leave". +opcode. For this reason it's necessary to put "can_enter_jit" and +"global_merge_point" next to each other, control-flow-wise -- +i.e. "can_enter_jit" should be at the end of JUMP_ABSOLUTE and CONTINUE, +so that they are immediately followed by the "global_merge_point". -Note that "jit_can_enter", in the regular interpreter, has another goal +Note that "can_enter_jit", in the regular interpreter, has another goal too: it should quickly check if machine code was already emitted for the next opcode, and if so, jump to it -- i.e. do a call to it. As above the call to the machine code will make the current function execution -progress and when it returns we can go on interpreter it. +progress and when it returns we can go on interpreting it. PyPy contains some custom logic to virtualize the frame and the value stack; in this new model it should go somewhere related to -"jit_can_enter". +"can_enter_jit". -The "jit_can_enter" hint becomes nothing in the rainbow interpreter's -bytecode. Conversely, the "jit_can_leave" hint becomes nothing in +The "can_enter_jit" hint becomes nothing in the rainbow interpreter's +bytecode. Conversely, the "global_merge_point" hint becomes nothing in the regular interpreter, but an important bytecode in the rainbow -bytecode -- a global merge point. +bytecode. Very lazy code generation ++++++++++++++++++++++++++++ @@ -109,7 +110,7 @@ then compile a bit more, and wait; and progress like this. In this model we get the nice effect that in a Python-level loop, we would end up compiling only the loop instead of the whole function that contains -it: indeed, the "jit_can_enter" profiling only triggers on the start of +it: indeed, the "can_enter_jit" profiling only triggers on the start of the loop, and the stop-early logic means that the path that exits the loop is cold and will not be compiled. @@ -131,9 +132,9 @@ The "something" in question, the fall-back rainbow interpreter, is quite slow, but only runs until the end of the current opcode and can directly perform all nested calls instead of interpreting them. When it reaches -the "jit_can_leave", it then returns; as described in the "hints" +the "global_merge_point", it then returns; as described in the "hints" section this should be a return from the initial call to the JIT or the -machine code -- a call which was in "jit_can_enter" in the regular +machine code -- a call which was in "can_enter_jit" in the regular interpreter. So the control flow is now in the regular interpreter, which can go on interpreting at its normal speed from there. @@ -174,7 +175,9 @@ JITState in this simplified model, so no need for the careful red/yellow call logic (at least for now). Residual calls, like now, would be followed by the equivalent of a promotion, checking if the residual call -caused an exception or forced devirtualization. +caused an exception or forced devirtualization (though we could +immediately compile the expected common case, which is no exception and +no forcing). About local merge points: in this model of a single JITState, I vaguely suspect that it gives better results to have *less* local merge points, @@ -185,7 +188,7 @@ Random improvement ideas ++++++++++++++++++++++++++++++++ -- in the global merge point "jit_can_leave", so far we'd +- in the "global_merge_point", so far we'd record one state snapshot for each opcode; instead, we can use the idea implemented in the flow object space of only recording the state at the beginning of an opcode that actually @@ -204,11 +207,12 @@ the compact Paths as step two. - compiling of more code: we could tweak the flexswitch - interface. For example, instead of "please add a new path", + interface of the JIT backends. + For example, instead of "please add a new path", it would make sense to have an API "please override the switch completely so that it has this new set of paths". -- we also need a "jit_can_enter" at the end of the stack +- we also need a "can_enter_jit" at the end of the stack unroller corresponding to CONTINUE, for the case where the "continue" statement was in a try:finally:. This is not necessarily a problem, just a note that we have to allow From cfbolz at codespeak.net Mon Feb 25 21:24:46 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 25 Feb 2008 21:24:46 +0100 (CET) Subject: [pypy-svn] r51861 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080225202446.5146A16845F@codespeak.net> Author: cfbolz Date: Mon Feb 25 21:24:44 2008 New Revision: 51861 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: first indirect_call test passes Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 25 21:24:44 2008 @@ -2,7 +2,7 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.objspace.flow import model as flowmodel from pypy.rpython.annlowlevel import cachedtype -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, llmemory from pypy.jit.hintannotator.model import originalconcretetype from pypy.jit.hintannotator import model as hintmodel from pypy.jit.timeshifter import rtimeshift, rvalue, rcontainer, exception @@ -54,6 +54,30 @@ def _freeze_(self): return True +class IndirectCallsetDesc(object): + __metaclass__ = cachedtype + + def __init__(self, graph2tsgraph, codewriter): + + keys = [] + values = [] + common_args_r = None + for graph, tsgraph in graph2tsgraph: + fnptr = codewriter.rtyper.getcallable(graph) + keys.append(llmemory.cast_ptr_to_adr(fnptr)) + values.append(codewriter.get_jitcode(tsgraph)) + + def bytecode_for_address(fnaddress): + # XXX optimize + for i in range(len(keys)): + if keys[i] == fnaddress: + return values[i] + + self.bytecode_for_address = bytecode_for_address + + self.graphs = [graph for (graph, tsgraph) in graph2tsgraph] + self.jitcodes = values + class BytecodeWriter(object): def __init__(self, t, hannotator, RGenOp): @@ -71,6 +95,7 @@ self.all_graphs = {} # mapping graph to bytecode self.unfinished_graphs = [] self.num_global_mergepoints = 0 + self.ptr_to_jitcode = {} def can_raise(self, op): return self.raise_analyzer.analyze(op) @@ -95,6 +120,7 @@ self.num_local_mergepoints = 0 self.graph_color = self.graph_calling_color(graph) self.calldescs = [] + self.indirectcalldescs = [] self.is_portal = is_portal # mapping constant -> index in constants self.const_positions = {} @@ -126,6 +152,8 @@ self.graph_positions = {} # mapping fnobjs to index self.calldesc_positions = {} + # mapping fnobjs to index + self.indirectcalldesc_positions = {} self.graph = graph self.mergepoint_set = {} @@ -152,6 +180,7 @@ self.num_local_mergepoints, self.graph_color, self.calldescs, + self.indirectcalldescs, self.is_portal) bytecode._source = self.assembler bytecode._interpreter = self.interpreter @@ -163,6 +192,14 @@ self.num_global_mergepoints) return bytecode + def get_jitcode(self, graph): + if graph in self.all_graphs: + return self.all_graphs[graph] + bytecode = JitCode.__new__(JitCode) + self.all_graphs[graph] = bytecode + self.unfinished_graphs.append(graph) + return bytecode + def finish_all_graphs(self): while self.unfinished_graphs: graph = self.unfinished_graphs.pop() @@ -487,12 +524,7 @@ def graph_position(self, graph): if graph in self.graph_positions: return self.graph_positions[graph] - if graph in self.all_graphs: - bytecode = self.all_graphs[graph] - else: - bytecode = JitCode.__new__(JitCode) - self.all_graphs[graph] = bytecode - self.unfinished_graphs.append(graph) + bytecode = self.get_jitcode(graph) index = len(self.called_bytecodes) self.called_bytecodes.append(bytecode) self.graph_positions[graph] = index @@ -508,6 +540,27 @@ self.calldesc_positions[key] = result return result + def indirectcalldesc_position(self, graph2code): + key = graph2code.items() + key.sort() + key = tuple(key) + if key in self.indirectcalldesc_positions: + return self.indirectcalldesc_positions[key] + callset = IndirectCallsetDesc(key, self) + for i in range(len(key) + 1, 0, -1): + subkey = key[:i] + if subkey in self.indirectcalldesc_positions: + result = self.indirectcalldesc_positions[subkey] + self.indirectcalldescs[result] = callset + break + else: + result = len(self.indirectcalldescs) + self.indirectcalldescs.append(callset) + for i in range(len(key) + 1, 0, -1): + subkey = key[:i] + self.indirectcalldesc_positions[subkey] = result + return result + def interiordesc(self, op, PTRTYPE, nb_offsets): path = [] CONTAINER = PTRTYPE.TO @@ -620,6 +673,41 @@ print op, kind, withexc return handler(op, withexc) + def serialize_op_indirect_call(self, op): + kind, withexc = self.guess_call_kind(op) + if kind == "red": + XXX + if kind == "yellow": + targets = dict(self.graphs_from(op)) + fnptrindex = self.serialize_oparg("red", op.args[0]) + self.emit("goto_if_constant", fnptrindex, tlabel(("direct call", op))) + emitted_args = [] + for v in op.args[1:-1]: + emitted_args.append(self.serialize_oparg("red", v)) + self.emit("red_residual_call") + calldescindex = self.calldesc_position(op.args[0].concretetype) + self.emit(fnptrindex, calldescindex, withexc) + self.emit(len(emitted_args), *emitted_args) + self.emit(self.promotiondesc_position(lltype.Signed)) + self.emit("goto", tlabel(("after indirect call", op))) + + self.emit(label(("direct call", op))) + args = targets.values()[0].getargs() + emitted_args = self.args_of_call(op.args[1:-1], args) + self.emit("indirect_call_const") + self.emit(*emitted_args) + setdescindex = self.indirectcalldesc_position(targets) + self.emit(fnptrindex, setdescindex) + self.emit("yellow_after_direct_call") + self.emit("yellow_retrieve_result_as_red") + self.emit(self.type_position(op.result.concretetype)) + + + self.emit(label(("after indirect call", op))) + self.register_redvar(op.result) + return + XXX + def handle_oopspec_call(self, op, withexc): from pypy.jit.timeshifter.oop import Index fnobj = op.args[0].value._obj @@ -683,13 +771,8 @@ emitted_args.append(self.serialize_oparg("red", v)) self.emit("red_residual_direct_call") self.emit(func, pos, withexc, len(emitted_args), *emitted_args) - self.register_redvar(op.result) - pos = self.register_redvar(("residual_flags_red", op.args[0])) - self.emit("promote") - self.emit(pos) self.emit(self.promotiondesc_position(lltype.Signed)) - self.register_greenvar(("residual_flags_green", op.args[0]), check=False) - self.emit("residual_fetch", True, pos) + self.register_redvar(op.result) def handle_rpyexc_raise_call(self, op, withexc): emitted_args = [] @@ -728,8 +811,6 @@ self.emit("yellow_retrieve_result") self.register_greenvar(op.result) - def serialize_op_indirect_call(self, op): - XXX def serialize_op_malloc(self, op): index = self.structtypedesc_position(op.args[0].value) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/dump.py Mon Feb 25 21:24:44 2008 @@ -129,6 +129,10 @@ index = src.load_2byte() function = jitcode.calldescs[index] args.append(function) + elif argspec == "indirectcalldesc": + index = src.load_2byte() + function = jitcode.indirectcalldescs[index] + args.append(function) elif argspec == "oopspec": oopspecindex = src.load_2byte() oopspec = jitcode.oopspecdescs[oopspecindex] Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Mon Feb 25 21:24:44 2008 @@ -2,7 +2,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.jit.timeshifter import rtimeshift, rcontainer from pypy.jit.timeshifter.greenkey import empty_key, GreenKey, newgreendict -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, llmemory class JitCode(object): """ @@ -21,7 +21,7 @@ keydescs, structtypedescs, fielddescs, arrayfielddescs, interiordescs, oopspecdescs, promotiondescs, called_bytecodes, num_mergepoints, - graph_color, calldescs, is_portal): + graph_color, calldescs, indirectcalldescs, is_portal): self.name = name self.code = code self.constants = constants @@ -38,6 +38,7 @@ self.num_mergepoints = num_mergepoints self.graph_color = graph_color self.calldescs = calldescs + self.indirectcalldescs = indirectcalldescs self.is_portal = is_portal def _freeze_(self): @@ -114,6 +115,10 @@ index = self.load_2byte() function = self.frame.bytecode.calldescs[index] args += (function, ) + elif argspec == "indirectcalldesc": + index = self.load_2byte() + function = self.frame.bytecode.indirectcalldescs[index] + args += (function, ) elif argspec == "oopspec": oopspecindex = self.load_2byte() oopspec = self.frame.bytecode.oopspecdescs[oopspecindex] @@ -367,6 +372,11 @@ if descision: self.frame.pc = target + @arguments("red", "jumptarget") + def opimpl_goto_if_constant(self, valuebox, target): + if valuebox.is_constant(): + self.frame.pc = target + @arguments("red", returns="red") def opimpl_red_ptr_nonzero(self, ptrbox): return rtimeshift.genptrnonzero(self.jitstate, ptrbox, False) @@ -480,6 +490,15 @@ # this frame will be resumed later in the next bytecode, which is # yellow_after_direct_call + @arguments("green_varargs", "red_varargs", "red", "indirectcalldesc") + def opimpl_indirect_call_const(self, greenargs, redargs, + funcptrbox, callset): + gv = funcptrbox.getgenvar(self.jitstate) + addr = gv.revealconst(llmemory.Address) + bytecode = callset.bytecode_for_address(addr) + self.run(self.jitstate, bytecode, greenargs, redargs, + start_bytecode_loop=False) + @arguments() def opimpl_yellow_after_direct_call(self): newjitstate = rtimeshift.collect_split( @@ -492,6 +511,13 @@ # XXX all this jitstate.greens business is a bit messy return self.jitstate.greens[0] + @arguments("2byte", returns="red") + def opimpl_yellow_retrieve_result_as_red(self, typeid): + # XXX all this jitstate.greens business is a bit messy + redboxcls = self.frame.bytecode.redboxclasses[typeid] + kind = self.frame.bytecode.typekinds[typeid] + return redboxcls(kind, self.jitstate.greens[0]) + @arguments("oopspec", "bool", returns="red") def opimpl_red_oopspec_call_0(self, oopspec, deepfrozen): return oopspec.ll_handler(self.jitstate, oopspec, deepfrozen) @@ -508,8 +534,9 @@ def opimpl_red_oopspec_call_3(self, oopspec, deepfrozen, arg1, arg2, arg3): return oopspec.ll_handler(self.jitstate, oopspec, deepfrozen, arg1, arg2, arg3) - @arguments("red", "calldesc", "bool", "red_varargs") - def opimpl_red_residual_direct_call(self, funcbox, calldesc, withexc, redargs): + @arguments("red", "calldesc", "bool", "red_varargs", "promotiondesc") + def opimpl_red_residual_call(self, funcbox, calldesc, withexc, + redargs, promotiondesc): result = rtimeshift.gen_residual_call(self.jitstate, calldesc, funcbox, redargs) self.red_result(result) @@ -519,13 +546,13 @@ exceptiondesc = None flagbox = rtimeshift.after_residual_call(self.jitstate, exceptiondesc, True) - self.red_result(flagbox) - - @arguments("bool", "red") - def opimpl_residual_fetch(self, check_forced, flagbox): + done = rtimeshift.promote(self.jitstate, flagbox, promotiondesc) + if done: + return self.dispatch() + gv_flag = flagbox.getgenvar(self.jitstate) + assert gv_flag.is_const rtimeshift.residual_fetch(self.jitstate, self.exceptiondesc, - check_forced, flagbox) - + True, flagbox) # exceptions Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 25 21:24:44 2008 @@ -1040,7 +1040,6 @@ assert res == 212 def test_simple_meth(self): - py.test.skip("needs promote") class Base(object): def m(self): raise NotImplementedError From cfbolz at codespeak.net Mon Feb 25 21:28:43 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 25 Feb 2008 21:28:43 +0100 (CET) Subject: [pypy-svn] r51862 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080225202843.B17F81683DF@codespeak.net> Author: cfbolz Date: Mon Feb 25 21:28:43 2008 New Revision: 51862 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: more indirect_call tests passing Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 25 21:28:43 2008 @@ -675,38 +675,38 @@ def serialize_op_indirect_call(self, op): kind, withexc = self.guess_call_kind(op) + targets = dict(self.graphs_from(op)) + fnptrindex = self.serialize_oparg("red", op.args[0]) + self.emit("goto_if_constant", fnptrindex, tlabel(("direct call", op))) + emitted_args = [] + for v in op.args[1:-1]: + emitted_args.append(self.serialize_oparg("red", v)) + self.emit("red_residual_call") + calldescindex = self.calldesc_position(op.args[0].concretetype) + self.emit(fnptrindex, calldescindex, withexc) + self.emit(len(emitted_args), *emitted_args) + self.emit(self.promotiondesc_position(lltype.Signed)) + self.emit("goto", tlabel(("after indirect call", op))) + + self.emit(label(("direct call", op))) + args = targets.values()[0].getargs() + emitted_args = self.args_of_call(op.args[1:-1], args) + self.emit("indirect_call_const") + self.emit(*emitted_args) + setdescindex = self.indirectcalldesc_position(targets) + self.emit(fnptrindex, setdescindex) + if kind == "red": - XXX - if kind == "yellow": - targets = dict(self.graphs_from(op)) - fnptrindex = self.serialize_oparg("red", op.args[0]) - self.emit("goto_if_constant", fnptrindex, tlabel(("direct call", op))) - emitted_args = [] - for v in op.args[1:-1]: - emitted_args.append(self.serialize_oparg("red", v)) - self.emit("red_residual_call") - calldescindex = self.calldesc_position(op.args[0].concretetype) - self.emit(fnptrindex, calldescindex, withexc) - self.emit(len(emitted_args), *emitted_args) - self.emit(self.promotiondesc_position(lltype.Signed)) - self.emit("goto", tlabel(("after indirect call", op))) - - self.emit(label(("direct call", op))) - args = targets.values()[0].getargs() - emitted_args = self.args_of_call(op.args[1:-1], args) - self.emit("indirect_call_const") - self.emit(*emitted_args) - setdescindex = self.indirectcalldesc_position(targets) - self.emit(fnptrindex, setdescindex) + self.emit("red_after_direct_call") + elif kind == "yellow": self.emit("yellow_after_direct_call") self.emit("yellow_retrieve_result_as_red") self.emit(self.type_position(op.result.concretetype)) - + else: + XXX - self.emit(label(("after indirect call", op))) - self.register_redvar(op.result) - return - XXX + self.emit(label(("after indirect call", op))) + self.register_redvar(op.result) def handle_oopspec_call(self, op, withexc): from pypy.jit.timeshifter.oop import Index Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Mon Feb 25 21:28:43 2008 @@ -1066,7 +1066,6 @@ self.check_insns(indirect_call=0) def test_simple_red_meth(self): - py.test.skip("needs promote") class Base(object): def m(self, n): raise NotImplementedError @@ -1089,7 +1088,6 @@ self.check_insns({'int_mul': 1}) def test_simple_red_meth_vars_around(self): - py.test.skip("needs promote") class Base(object): def m(self, n): raise NotImplementedError From cfbolz at codespeak.net Mon Feb 25 21:34:37 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 25 Feb 2008 21:34:37 +0100 (CET) Subject: [pypy-svn] r51863 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter/test Message-ID: <20080225203437.586841684C9@codespeak.net> Author: cfbolz Date: Mon Feb 25 21:34:36 2008 New Revision: 51863 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py Log: one more test that just passes. typo Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 25 21:34:36 2008 @@ -769,7 +769,7 @@ emitted_args = [] for v in op.args[1:]: emitted_args.append(self.serialize_oparg("red", v)) - self.emit("red_residual_direct_call") + self.emit("red_residual_call") self.emit(func, pos, withexc, len(emitted_args), *emitted_args) self.emit(self.promotiondesc_position(lltype.Signed)) self.register_redvar(op.result) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Mon Feb 25 21:34:36 2008 @@ -137,3 +137,78 @@ assert res == 68 self.check_insns(int_floordiv=1, int_mul=0) + def test_method_call_nonpromote(self): + class Base(object): + pass + class Int(Base): + def __init__(self, n): + self.n = n + def double(self): + return Int(self.n * 2) + def get(self): + return self.n + class Str(Base): + def __init__(self, s): + self.s = s + def double(self): + return Str(self.s + self.s) + def get(self): + return ord(self.s[4]) + + def ll_main(n): + if n > 0: + o = Int(n) + else: + o = Str('123') + return ll_function(o) + + def ll_function(o): + hint(None, global_merge_point=True) + return o.double().get() + + res = self.timeshift_from_portal(ll_main, ll_function, [5], policy=P_NOVIRTUAL) + assert res == 10 + self.check_insns(indirect_call=2) + + res = self.timeshift_from_portal(ll_main, ll_function, [0], policy=P_NOVIRTUAL) + assert res == ord('2') + self.check_insns(indirect_call=2) + + def test_method_call_promote(self): + py.test.skip("not working yet") + class Base(object): + pass + class Int(Base): + def __init__(self, n): + self.n = n + def double(self): + return Int(self.n * 2) + def get(self): + return self.n + class Str(Base): + def __init__(self, s): + self.s = s + def double(self): + return Str(self.s + self.s) + def get(self): + return ord(self.s[4]) + + def ll_main(n): + if n > 0: + o = Int(n) + else: + o = Str('123') + return ll_function(o) + + def ll_function(o): + hint(None, global_merge_point=True) + hint(o.__class__, promote=True) + return o.double().get() + + res = self.timeshift_from_portal(ll_main, ll_function, [5], policy=P_NOVIRTUAL) + assert res == 10 + self.check_insns(indirect_call=0) + + res = self.timeshift_from_portal(ll_main, ll_function, [0], policy=P_NOVIRTUAL) + assert res == ord('2') + self.check_insns(indirect_call=0) Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py Mon Feb 25 21:34:36 2008 @@ -198,80 +198,6 @@ policy=P_OOPSPEC) assert not res - def test_method_call_nonpromote(self): - class Base(object): - pass - class Int(Base): - def __init__(self, n): - self.n = n - def double(self): - return Int(self.n * 2) - def get(self): - return self.n - class Str(Base): - def __init__(self, s): - self.s = s - def double(self): - return Str(self.s + self.s) - def get(self): - return ord(self.s[4]) - - def ll_main(n): - if n > 0: - o = Int(n) - else: - o = Str('123') - return ll_function(o) - - def ll_function(o): - hint(None, global_merge_point=True) - return o.double().get() - - res = self.timeshift_from_portal(ll_main, ll_function, [5], policy=P_NOVIRTUAL) - assert res == 10 - self.check_insns(indirect_call=2) - - res = self.timeshift_from_portal(ll_main, ll_function, [0], policy=P_NOVIRTUAL) - assert res == ord('2') - self.check_insns(indirect_call=2) - - def test_method_call_promote(self): - class Base(object): - pass - class Int(Base): - def __init__(self, n): - self.n = n - def double(self): - return Int(self.n * 2) - def get(self): - return self.n - class Str(Base): - def __init__(self, s): - self.s = s - def double(self): - return Str(self.s + self.s) - def get(self): - return ord(self.s[4]) - - def ll_main(n): - if n > 0: - o = Int(n) - else: - o = Str('123') - return ll_function(o) - - def ll_function(o): - hint(None, global_merge_point=True) - hint(o.__class__, promote=True) - return o.double().get() - - res = self.timeshift_from_portal(ll_main, ll_function, [5], policy=P_NOVIRTUAL) - assert res == 10 - self.check_insns(indirect_call=0) - - res = self.timeshift_from_portal(ll_main, ll_function, [0], policy=P_NOVIRTUAL) - assert res == ord('2') - self.check_insns(indirect_call=0) def test_virt_obj_method_call_promote(self): class Base(object): From arigo at codespeak.net Mon Feb 25 21:53:02 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 25 Feb 2008 21:53:02 +0100 (CET) Subject: [pypy-svn] r51864 - pypy/branch/jit-refactoring/pypy/jit/rainbow Message-ID: <20080225205302.ADD951684CD@codespeak.net> Author: arigo Date: Mon Feb 25 21:52:54 2008 New Revision: 51864 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Log: dump.py is a bit fragile: register_redvar() should be called exactly after all arguments for the current operation have been emitted, and before the next operation or label starts... Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 25 21:52:54 2008 @@ -705,8 +705,8 @@ else: XXX - self.emit(label(("after indirect call", op))) self.register_redvar(op.result) + self.emit(label(("after indirect call", op))) def handle_oopspec_call(self, op, withexc): from pypy.jit.timeshifter.oop import Index From cfbolz at codespeak.net Mon Feb 25 22:00:17 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 25 Feb 2008 22:00:17 +0100 (CET) Subject: [pypy-svn] r51865 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter timeshifter/test Message-ID: <20080225210017.562A41684D0@codespeak.net> Author: cfbolz Date: Mon Feb 25 22:00:13 2008 New Revision: 51865 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py Log: move over some tests and fix problems I found trying to run them Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Mon Feb 25 22:00:13 2008 @@ -936,14 +936,18 @@ indexboxindex, valboxindex) def serialize_op_getarraysize(self, op): + color = self.opcolor(op) arrayvar, = op.args PTRTYPE = arrayvar.concretetype if PTRTYPE.TO.OF is lltype.Void: return fielddescindex = self.arrayfielddesc_position(PTRTYPE.TO) arrayindex = self.serialize_oparg("red", arrayvar) - self.emit("red_getarraysize", arrayindex, fielddescindex) - self.register_redvar(op.result) + self.emit("%s_getarraysize" % (color, ), arrayindex, fielddescindex) + if color == "red": + self.register_redvar(op.result) + else: + self.register_greenvar(op.result) def serialize_op_getinteriorfield(self, op): color = self.opcolor(op) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Mon Feb 25 22:00:13 2008 @@ -624,6 +624,10 @@ def opimpl_red_getarraysize(self, arraybox, fielddesc): return rtimeshift.gengetarraysize(self.jitstate, fielddesc, arraybox) + @arguments("red", "arraydesc", returns="green_from_red") + def opimpl_green_getarraysize(self, arraybox, fielddesc): + return rtimeshift.gengetarraysize(self.jitstate, fielddesc, arraybox) + @arguments("red", "interiordesc", "bool", "red_varargs", returns="red") def opimpl_red_getinteriorfield(self, structbox, interiordesc, deepfrozen, indexboxes): Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py Mon Feb 25 22:00:13 2008 @@ -5,7 +5,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rpython.annlowlevel import llhelper, cachedtype from pypy.rpython.llinterp import LLInterpreter -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, llmemory # graph transformations for transforming the portal graph(s) class PortalRewriter(object): Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Mon Feb 25 22:00:13 2008 @@ -137,6 +137,65 @@ assert res == 68 self.check_insns(int_floordiv=1, int_mul=0) + def test_dfa_compile(self): + py.test.skip("not working yet") + from pypy.lang.automata.dfa import getautomaton, convertdfa, recognizetable + a = getautomaton() + dfatable, final_states = convertdfa(a) + def main(gets): + s = ["aaaaaaaaaab", "aaaa"][gets] + return recognizetable(dfatable, s, final_states) + + # must backendoptimize to remove the mallocs related + # to the interior ptrs + res = self.timeshift_from_portal(main, recognizetable, [0], + policy=P_NOVIRTUAL, + backendoptimize=True) + assert res + + res = self.timeshift_from_portal(main, recognizetable, [1], + policy=P_NOVIRTUAL, + backendoptimize=True) + assert not res + + def test_dfa_compile2(self): + py.test.skip("not working yet") + from pypy.lang.automata.dfa import getautomaton, convertagain, recognizeparts + more = [convertagain(getautomaton()), convertagain(getautomaton())] + def main(gets, gets2): + alltrans, final_states = more[gets2] + s = ["aaaaaaaaaab", "aaaa"][gets] + return recognizeparts(alltrans, final_states, s) + + # must backendoptimize to remove the mallocs related + # to the interior ptrs + res = self.timeshift_from_portal(main, recognizeparts, [0, 0], + policy=P_NOVIRTUAL, + backendoptimize=True) + assert res + + # XXX unfortunately we have to create a new version each time - because of pbc + res = self.timeshift_from_portal(main, recognizeparts, [1, 0], + policy=P_NOVIRTUAL, + backendoptimize=True) + assert not res + + def test_dfa_compile3(self): + py.test.skip("not working yet") + from pypy.lang.automata.dfa import getautomaton, recognize3 + def main(gets): + auto = getautomaton() + s = ["aaaaaaaaaab", "aaaa"][gets] + return recognize3(auto, s) + + res = self.timeshift_from_portal(main, recognize3, [0], + policy=P_OOPSPEC) + assert res + + res = self.timeshift_from_portal(main, recognize3, [1], + policy=P_OOPSPEC) + assert not res + def test_method_call_nonpromote(self): class Base(object): pass Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Mon Feb 25 22:00:13 2008 @@ -203,7 +203,7 @@ resgv = fielddesc.getarraysize_if_non_null( jitstate, argbox.getgenvar(jitstate)) if resgv is not None: - return fielddesc.makebox(jitstate, resgv) + return rvalue.redboxbuilder_int(fielddesc.indexkind, resgv) genvar = jitstate.curbuilder.genop_getarraysize( fielddesc.arraytoken, argbox.getgenvar(jitstate)) Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py Mon Feb 25 22:00:13 2008 @@ -142,61 +142,6 @@ class TestPortal(PortalTest): - def test_dfa_compile(self): - from pypy.lang.automata.dfa import getautomaton, convertdfa, recognizetable - a = getautomaton() - dfatable, final_states = convertdfa(a) - def main(gets): - s = ["aaaaaaaaaab", "aaaa"][gets] - return recognizetable(dfatable, s, final_states) - - # must backendoptimize to remove the mallocs related - # to the interior ptrs - res = self.timeshift_from_portal(main, recognizetable, [0], - policy=P_NOVIRTUAL, - backendoptimize=True) - assert res - - res = self.timeshift_from_portal(main, recognizetable, [1], - policy=P_NOVIRTUAL, - backendoptimize=True) - assert not res - - def test_dfa_compile2(self): - from pypy.lang.automata.dfa import getautomaton, convertagain, recognizeparts - more = [convertagain(getautomaton()), convertagain(getautomaton())] - def main(gets, gets2): - alltrans, final_states = more[gets2] - s = ["aaaaaaaaaab", "aaaa"][gets] - return recognizeparts(alltrans, final_states, s) - - # must backendoptimize to remove the mallocs related - # to the interior ptrs - res = self.timeshift_from_portal(main, recognizeparts, [0, 0], - policy=P_NOVIRTUAL, - backendoptimize=True) - assert res - - # XXX unfortunately we have to create a new version each time - because of pbc - res = self.timeshift_from_portal(main, recognizeparts, [1, 0], - policy=P_NOVIRTUAL, - backendoptimize=True) - assert not res - - def test_dfa_compile3(self): - from pypy.lang.automata.dfa import getautomaton, recognize3 - def main(gets): - auto = getautomaton() - s = ["aaaaaaaaaab", "aaaa"][gets] - return recognize3(auto, s) - - res = self.timeshift_from_portal(main, recognize3, [0], - policy=P_OOPSPEC) - assert res - - res = self.timeshift_from_portal(main, recognize3, [1], - policy=P_OOPSPEC) - assert not res def test_virt_obj_method_call_promote(self): From arigo at codespeak.net Mon Feb 25 22:01:00 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 25 Feb 2008 22:01:00 +0100 (CET) Subject: [pypy-svn] r51866 - pypy/branch/jit-refactoring/pypy/jit/timeshifter Message-ID: <20080225210100.CFA291684D3@codespeak.net> Author: arigo Date: Mon Feb 25 22:00:58 2008 New Revision: 51866 Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Log: A more useful pdb invocation. Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Mon Feb 25 22:00:58 2008 @@ -762,7 +762,9 @@ interpreter.portalstate.compile_more_functions() except Exception, e: if not we_are_translated(): - import pdb; pdb.set_trace() + import sys, pdb + print >> sys.stderr, "\n*** Error in ll_continue_compilation ***" + pdb.post_mortem(sys.exc_info()[2]) lloperation.llop.debug_fatalerror( lltype.Void, "compilation-time error %s" % e) self.ll_continue_compilation = ll_continue_compilation From tverwaes at codespeak.net Mon Feb 25 23:32:11 2008 From: tverwaes at codespeak.net (tverwaes at codespeak.net) Date: Mon, 25 Feb 2008 23:32:11 +0100 (CET) Subject: [pypy-svn] r51867 - in pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk: . test Message-ID: <20080225223211.CB135168477@codespeak.net> Author: tverwaes Date: Mon Feb 25 23:32:10 2008 New Revision: 51867 Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_interpreter.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_primitives.py pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py Log: slowed down the implementaton but made it more flexible by removing W_*Context* and fully going in the direction of shadows. Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/interpreter.py Mon Feb 25 23:32:10 2008 @@ -1,7 +1,6 @@ import py from pypy.lang.smalltalk import model, constants, primitives from pypy.lang.smalltalk import objtable -from pypy.lang.smalltalk.model import W_ContextPart from pypy.lang.smalltalk.shadow import ContextPartShadow from pypy.lang.smalltalk.conftest import option from pypy.rlib import objectmodel, unroll @@ -68,7 +67,7 @@ self._last_indent, self.s_active_context().pc(), next, bytecodeimpl.__name__,) - bytecodeimpl(self.w_active_context, self) + bytecodeimpl(self.s_active_context(), self) else: # this is a performance optimization: when translating the # interpreter, the bytecode dispatching is not implemented as a @@ -76,7 +75,7 @@ # below produces the switch (by being unrolled). for code, bytecodeimpl in unrolling_bytecode_table: if code == next: - bytecodeimpl(self.w_active_context, self) + bytecodeimpl(self.s_active_context(), self) break @@ -87,10 +86,10 @@ # ___________________________________________________________________________ # Bytecode Implementations: # -# "self" is always a W_ContextPart instance. +# "self" is always a ContextPartShadow instance. -# __extend__ adds new methods to the W_ContextPart class -class __extend__(W_ContextPart): +# __extend__ adds new methods to the ContextPartShadow class +class __extend__(ContextPartShadow): # push bytecodes def pushReceiverVariableBytecode(self, interp): index = self.currentBytecode & 15 @@ -203,7 +202,7 @@ print "PRIMITIVE FAILED: %d %s" % (method.primitive, selector,) pass # ignore this error and fall back to the Smalltalk version arguments = self.pop_and_return_n(argcount) - interp.w_active_context = method.create_frame(receiver, arguments, self) + interp.w_active_context = method.create_frame(receiver, arguments, self.w_self()) self.pop() def _return(self, object, interp, w_return_to): @@ -513,77 +512,77 @@ BYTECODE_RANGES = [ - ( 0, 15, W_ContextPart.pushReceiverVariableBytecode), - ( 16, 31, W_ContextPart.pushTemporaryVariableBytecode), - ( 32, 63, W_ContextPart.pushLiteralConstantBytecode), - ( 64, 95, W_ContextPart.pushLiteralVariableBytecode), - ( 96, 103, W_ContextPart.storeAndPopReceiverVariableBytecode), - (104, 111, W_ContextPart.storeAndPopTemporaryVariableBytecode), - (112, W_ContextPart.pushReceiverBytecode), - (113, W_ContextPart.pushConstantTrueBytecode), - (114, W_ContextPart.pushConstantFalseBytecode), - (115, W_ContextPart.pushConstantNilBytecode), - (116, W_ContextPart.pushConstantMinusOneBytecode), - (117, W_ContextPart.pushConstantZeroBytecode), - (118, W_ContextPart.pushConstantOneBytecode), - (119, W_ContextPart.pushConstantTwoBytecode), - (120, W_ContextPart.returnReceiver), - (121, W_ContextPart.returnTrue), - (122, W_ContextPart.returnFalse), - (123, W_ContextPart.returnNil), - (124, W_ContextPart.returnTopFromMethod), - (125, W_ContextPart.returnTopFromBlock), - (126, W_ContextPart.unknownBytecode), - (127, W_ContextPart.unknownBytecode), - (128, W_ContextPart.extendedPushBytecode), - (129, W_ContextPart.extendedStoreBytecode), - (130, W_ContextPart.extendedStoreAndPopBytecode), - (131, W_ContextPart.singleExtendedSendBytecode), - (132, W_ContextPart.doubleExtendedDoAnythingBytecode), - (133, W_ContextPart.singleExtendedSuperBytecode), - (134, W_ContextPart.secondExtendedSendBytecode), - (135, W_ContextPart.popStackBytecode), - (136, W_ContextPart.duplicateTopBytecode), - (137, W_ContextPart.pushActiveContextBytecode), - (138, 143, W_ContextPart.experimentalBytecode), - (144, 151, W_ContextPart.shortUnconditionalJump), - (152, 159, W_ContextPart.shortConditionalJump), - (160, 167, W_ContextPart.longUnconditionalJump), - (168, 171, W_ContextPart.longJumpIfTrue), - (172, 175, W_ContextPart.longJumpIfFalse), - (176, W_ContextPart.bytecodePrimAdd), - (177, W_ContextPart.bytecodePrimSubtract), - (178, W_ContextPart.bytecodePrimLessThan), - (179, W_ContextPart.bytecodePrimGreaterThan), - (180, W_ContextPart.bytecodePrimLessOrEqual), - (181, W_ContextPart.bytecodePrimGreaterOrEqual), - (182, W_ContextPart.bytecodePrimEqual), - (183, W_ContextPart.bytecodePrimNotEqual), - (184, W_ContextPart.bytecodePrimMultiply), - (185, W_ContextPart.bytecodePrimDivide), - (186, W_ContextPart.bytecodePrimMod), - (187, W_ContextPart.bytecodePrimMakePoint), - (188, W_ContextPart.bytecodePrimBitShift), - (189, W_ContextPart.bytecodePrimDiv), - (190, W_ContextPart.bytecodePrimBitAnd), - (191, W_ContextPart.bytecodePrimBitOr), - (192, W_ContextPart.bytecodePrimAt), - (193, W_ContextPart.bytecodePrimAtPut), - (194, W_ContextPart.bytecodePrimSize), - (195, W_ContextPart.bytecodePrimNext), - (196, W_ContextPart.bytecodePrimNextPut), - (197, W_ContextPart.bytecodePrimAtEnd), - (198, W_ContextPart.bytecodePrimEquivalent), - (199, W_ContextPart.bytecodePrimClass), - (200, W_ContextPart.bytecodePrimBlockCopy), - (201, W_ContextPart.bytecodePrimValue), - (202, W_ContextPart.bytecodePrimValueWithArg), - (203, W_ContextPart.bytecodePrimDo), - (204, W_ContextPart.bytecodePrimNew), - (205, W_ContextPart.bytecodePrimNewWithArg), - (206, W_ContextPart.bytecodePrimPointX), - (207, W_ContextPart.bytecodePrimPointY), - (208, 255, W_ContextPart.sendLiteralSelectorBytecode), + ( 0, 15, ContextPartShadow.pushReceiverVariableBytecode), + ( 16, 31, ContextPartShadow.pushTemporaryVariableBytecode), + ( 32, 63, ContextPartShadow.pushLiteralConstantBytecode), + ( 64, 95, ContextPartShadow.pushLiteralVariableBytecode), + ( 96, 103, ContextPartShadow.storeAndPopReceiverVariableBytecode), + (104, 111, ContextPartShadow.storeAndPopTemporaryVariableBytecode), + (112, ContextPartShadow.pushReceiverBytecode), + (113, ContextPartShadow.pushConstantTrueBytecode), + (114, ContextPartShadow.pushConstantFalseBytecode), + (115, ContextPartShadow.pushConstantNilBytecode), + (116, ContextPartShadow.pushConstantMinusOneBytecode), + (117, ContextPartShadow.pushConstantZeroBytecode), + (118, ContextPartShadow.pushConstantOneBytecode), + (119, ContextPartShadow.pushConstantTwoBytecode), + (120, ContextPartShadow.returnReceiver), + (121, ContextPartShadow.returnTrue), + (122, ContextPartShadow.returnFalse), + (123, ContextPartShadow.returnNil), + (124, ContextPartShadow.returnTopFromMethod), + (125, ContextPartShadow.returnTopFromBlock), + (126, ContextPartShadow.unknownBytecode), + (127, ContextPartShadow.unknownBytecode), + (128, ContextPartShadow.extendedPushBytecode), + (129, ContextPartShadow.extendedStoreBytecode), + (130, ContextPartShadow.extendedStoreAndPopBytecode), + (131, ContextPartShadow.singleExtendedSendBytecode), + (132, ContextPartShadow.doubleExtendedDoAnythingBytecode), + (133, ContextPartShadow.singleExtendedSuperBytecode), + (134, ContextPartShadow.secondExtendedSendBytecode), + (135, ContextPartShadow.popStackBytecode), + (136, ContextPartShadow.duplicateTopBytecode), + (137, ContextPartShadow.pushActiveContextBytecode), + (138, 143, ContextPartShadow.experimentalBytecode), + (144, 151, ContextPartShadow.shortUnconditionalJump), + (152, 159, ContextPartShadow.shortConditionalJump), + (160, 167, ContextPartShadow.longUnconditionalJump), + (168, 171, ContextPartShadow.longJumpIfTrue), + (172, 175, ContextPartShadow.longJumpIfFalse), + (176, ContextPartShadow.bytecodePrimAdd), + (177, ContextPartShadow.bytecodePrimSubtract), + (178, ContextPartShadow.bytecodePrimLessThan), + (179, ContextPartShadow.bytecodePrimGreaterThan), + (180, ContextPartShadow.bytecodePrimLessOrEqual), + (181, ContextPartShadow.bytecodePrimGreaterOrEqual), + (182, ContextPartShadow.bytecodePrimEqual), + (183, ContextPartShadow.bytecodePrimNotEqual), + (184, ContextPartShadow.bytecodePrimMultiply), + (185, ContextPartShadow.bytecodePrimDivide), + (186, ContextPartShadow.bytecodePrimMod), + (187, ContextPartShadow.bytecodePrimMakePoint), + (188, ContextPartShadow.bytecodePrimBitShift), + (189, ContextPartShadow.bytecodePrimDiv), + (190, ContextPartShadow.bytecodePrimBitAnd), + (191, ContextPartShadow.bytecodePrimBitOr), + (192, ContextPartShadow.bytecodePrimAt), + (193, ContextPartShadow.bytecodePrimAtPut), + (194, ContextPartShadow.bytecodePrimSize), + (195, ContextPartShadow.bytecodePrimNext), + (196, ContextPartShadow.bytecodePrimNextPut), + (197, ContextPartShadow.bytecodePrimAtEnd), + (198, ContextPartShadow.bytecodePrimEquivalent), + (199, ContextPartShadow.bytecodePrimClass), + (200, ContextPartShadow.bytecodePrimBlockCopy), + (201, ContextPartShadow.bytecodePrimValue), + (202, ContextPartShadow.bytecodePrimValueWithArg), + (203, ContextPartShadow.bytecodePrimDo), + (204, ContextPartShadow.bytecodePrimNew), + (205, ContextPartShadow.bytecodePrimNewWithArg), + (206, ContextPartShadow.bytecodePrimPointX), + (207, ContextPartShadow.bytecodePrimPointY), + (208, 255, ContextPartShadow.sendLiteralSelectorBytecode), ] Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/model.py Mon Feb 25 23:32:10 2008 @@ -125,7 +125,7 @@ return self.w_class def __repr__(self): - return "<%s %s at %r>" % (self.__class__.__name__, self, self.gethash()) + return "<%s %s>" % (self.__class__.__name__, self) def __str__(self): if isinstance(self, W_PointersObject) and self._shadow is not None: @@ -380,7 +380,6 @@ from pypy.lang.smalltalk import objtable assert len(arguments) == self.argsize w_new = W_MethodContext(self, receiver, arguments, sender) - objtable.objects += [w_new] return w_new def __str__(self): @@ -432,6 +431,7 @@ self.tempsize = tempsize self.primitive = primitive self.w_compiledin = None + self.islarge = islarge def literalat0(self, index0): if index0 == 0: @@ -486,313 +486,33 @@ self.bytes = (self.bytes[:index0] + character + self.bytes[index0 + 1:]) -class W_ContextPart(W_AbstractObjectWithIdentityHash): - - __metaclass__ = extendabletype - - def __init__(self, w_home, w_sender): - self._stack = [] - self._pc = 0 - #assert isinstance(s_home, W_MethodContext) - self._w_home = w_home - #assert w_sender is None or isinstance(w_sender, W_ContextPart) - self._w_sender = w_sender - - def as_context_get_shadow(self): - # Backward compatibility - return self - - def w_self(self): - # Backward compatibility - return self - - def pc(self): - return self._pc - - def stack(self): - return self._stack - - def store_pc(self, pc): - self._pc = pc - - def w_receiver(self): - " Return self of the method, or the method that contains the block " - return self.s_home().w_receiver() - - def w_home(self): - return self._w_home - - def s_home(self): - return self.w_home().as_methodcontext_get_shadow() - - def store_w_home(self, w_home): - self._w_home = w_home - - def w_sender(self): - if self._w_sender is not None: - return self._w_sender - else: - from pypy.lang.smalltalk import objtable - return objtable.w_nil - - def store_w_sender(self, w_sender): - self._w_sender = w_sender - - def stackpointer(self): - return len(self.stack()) + self.stackstart() - 1 - # ______________________________________________________________________ - # Imitate the primitive accessors - - def fetch(self, index): - from pypy.lang.smalltalk import utility, objtable - if index == constants.CTXPART_SENDER_INDEX: - return self.w_sender() - elif index == constants.CTXPART_PC_INDEX: - return utility.wrap_int(self.pc()) - elif index == constants.CTXPART_STACKP_INDEX: - return utility.wrap_int(self.stackpointer()) - - # Invalid! - raise IndexError - - def store(self, index, w_value): - # XXX Untested code... - from pypy.lang.smalltalk import utility, objtable - if index == constants.CTXPART_SENDER_INDEX: - if w_value != objtable.w_nil: - self.store_w_sender(w_value) - elif index == constants.CTXPART_PC_INDEX: - self._pc = utility.unwrap_int(w_value) - elif index == constants.CTXPART_STACKP_INDEX: - size = utility.unwrap_int(w_value) - size = 1 + size - self.stackstart() - self._stack = [objtable.w_nil] * size - else: - # Invalid! - raise IndexError - - def become(self, w_old, w_new): - if self.w_sender() == w_old: - self.store_w_sender(w_new) - for i in range(len(self._stack)): - if self._stack[i] == w_old: - self._stack[i] = w_new - if self.w_home() == w_old: - self.store_w_home(w_new) - - def stackstart(self): - return constants.MTHDCTX_TEMP_FRAME_START - - # ______________________________________________________________________ - # Method that contains the bytecode for this method/block context - - def w_method(self): - return self.s_home().w_method() - - def getbytecode(self): - pc = self.pc() - bytecode = self.w_method().bytes[pc] - currentBytecode = ord(bytecode) - self.store_pc(pc + 1) - return currentBytecode - - def getNextBytecode(self): - self.currentBytecode = self.getbytecode() - return self.currentBytecode - - # ______________________________________________________________________ - # Temporary Variables - # - # Are always fetched relative to the home method context. - - def gettemp(self, index): - return self.s_home().gettemp(index) - - def settemp(self, index, w_value): - self.s_home().settemp(index, w_value) - - # ______________________________________________________________________ - # Stack Manipulation - - def pop(self): - return self.stack().pop() - - def push(self, w_v): - assert w_v - self.stack().append(w_v) - - def push_all(self, lst): - " Equivalent to 'for x in lst: self.push(x)' where x is a lst " - assert None not in lst - self._stack += lst - - def top(self): - return self.peek(0) - - def peek(self, idx): - return self.stack()[-(idx+1)] - - def pop_n(self, n): - assert n >= 0 - start = len(self.stack()) - n - assert start >= 0 # XXX what if this fails? - del self.stack()[start:] - - def pop_and_return_n(self, n): - assert n >= 0 - start = len(self.stack()) - n - assert start >= 0 # XXX what if this fails? - res = self.stack()[start:] - del self.stack()[start:] - return res - -class W_BlockContext(W_ContextPart): - - def __init__(self, w_home, w_sender, argcnt, initialip): - W_ContextPart.__init__(self, w_home, w_sender) - self.argcnt = argcnt - self._initialip = initialip - - def initialip(self): - return self._initialip - - def expected_argument_count(self): - return self.argcnt - - def getclass(self): - from pypy.lang.smalltalk.classtable import w_BlockContext - return w_BlockContext - - def as_blockcontext_get_shadow(self): - # Backwards compatibility - return self - - def fetch(self, index): - from pypy.lang.smalltalk import utility - if index == constants.BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX: - return utility.wrap_int(self.argcnt) - elif index == constants.BLKCTX_INITIAL_IP_INDEX: - return utility.wrap_int(self.initialip) - elif index == constants.BLKCTX_HOME_INDEX: - return self.w_home() - elif index >= constants.BLKCTX_TEMP_FRAME_START: - stack_index = len(self.stack()) - index - 1 - return self.stack()[stack_index] - else: - return W_ContextPart.fetch(self, index) - - def store(self, index, value): - # THIS IS ALL UNTESTED CODE and we're a bit unhappy about it - # because it crashd the translation N+4 times :-( - from pypy.lang.smalltalk import utility - if index == constants.BLKCTX_BLOCK_ARGUMENT_COUNT_INDEX: - self.argcnt = utility.unwrap_int(value) - elif index == constants.BLKCTX_INITIAL_IP_INDEX: - self.pc = utility.unwrap_int(value) - elif index == constants.BLKCTX_HOME_INDEX: - self.store_w_home(value) - elif index >= constants.BLKCTX_TEMP_FRAME_START: - stack_index = len(self.stack()) - index - 1 - self.stack()[stack_index] = value - else: - W_ContextPart.store(self, index, value) - - def stackstart(self): - return constants.BLKCTX_TEMP_FRAME_START - -class W_MethodContext(W_ContextPart): - def __init__(self, w_method, w_receiver, - arguments, w_sender=None): - W_ContextPart.__init__(self, self, w_sender) - self._w_method = w_method - self._w_receiver = w_receiver - self.temps = arguments + [w_nil] * w_method.tempsize - - def as_methodcontext_get_shadow(self): - # Backwards compatibility - return self - - def getclass(self): - from pypy.lang.smalltalk.classtable import w_MethodContext - return w_MethodContext - - def w_receiver(self): - return self._w_receiver - - def store_w_receiver(self, w_receiver): - self._w_receiver = w_receiver - - def become(self, w_old, w_new): - W_ContextPart.become(self, w_old, w_new) - for i in range(len(self.temps)): - if self.temps[i] == w_old: - self.temps[i] = w_new - if w_old == self._w_receiver: - self._w_receiver = w_new - if w_old == self._w_method: - self._w_method = w_new - - def at0(self, index0): - # XXX to test - # Fetches in temppart (variable size) - return self.fetch(index0+constants.MTHDCTX_TEMP_FRAME_START) - - def atput0(self, index0, w_v): - # XXX to test - # Stores in temppart (variable size) - return self.store(index0+constants.MTHDCTX_TEMP_FRAME_START, w_v) - - def fetch(self, index): - if index == constants.MTHDCTX_METHOD: - return self.w_method() - elif index == constants.MTHDCTX_RECEIVER_MAP: # what is this thing? - return w_nil - elif index == constants.MTHDCTX_RECEIVER: - return self.w_receiver() - elif index >= constants.MTHDCTX_TEMP_FRAME_START: - # First set of indices are temporary variables: - offset = index - constants.MTHDCTX_TEMP_FRAME_START - if offset < len(self.temps): - return self.temps[offset] - - # After that comes the stack: - offset -= len(self.temps) - stack_index = len(self.stack()) - offset - 1 - return self.stack()[stack_index] - else: - return W_ContextPart.fetch(self, index) - - def store(self, index, w_object): - if index == constants.MTHDCTX_METHOD: - self._w_method = w_object - elif index == constants.MTHDCTX_RECEIVER_MAP: # what is this thing? - pass - elif index == constants.MTHDCTX_RECEIVER: - self.store_w_receiver(w_object) - elif index >= constants.MTHDCTX_TEMP_FRAME_START: - # First set of indices are temporary variables: - offset = index - constants.MTHDCTX_TEMP_FRAME_START - if offset < len(self.temps): - self.temps[offset] = w_object - - # After that comes the stack: - offset -= len(self.temps) - stack_index = len(self.stack()) - offset - 1 - self.stack()[stack_index] = w_object - else: - W_ContextPart.store(self, index, w_object) - - def gettemp(self, idx): - return self.temps[idx] - - def settemp(self, idx, w_value): - self.temps[idx] = w_value - - def w_method(self): - return self._w_method - - def size(self): - return len(self.temps) + len(self.stack()) +def W_BlockContext(w_home, w_sender, argcnt, initialip): + from pypy.lang.smalltalk.classtable import w_BlockContext + w_result = W_PointersObject(w_BlockContext, w_home.size()) + s_result = w_result.as_blockcontext_get_shadow() + s_result.store_expected_argument_count(argcnt) + s_result.store_initialip(initialip) + s_result.store_w_home(w_home) + s_result.store_stackpointer(s_result.stackstart()) + return w_result + +def W_MethodContext(w_method, w_receiver, + arguments, w_sender=None): + from pypy.lang.smalltalk.classtable import w_MethodContext + # From blue book: normal mc have place for 12 temps+maxstack + # mc for methods with islarge flag turned on 32 + size = 12 + w_method.islarge * 20 + w_method.argsize + w_result = w_MethodContext.as_class_get_shadow().new(size) + s_result = w_result.as_methodcontext_get_shadow() + s_result.store_w_method(w_method) + if w_sender: + s_result.store_w_sender(w_sender) + s_result.store_w_receiver(w_receiver) + s_result.store_pc(w_method.getliteralsize()+1) + for i in range(len(arguments)): + s_result.settemp(i, arguments[i]) + s_result.store_stackpointer(s_result.stackstart()) + return w_result # Use black magic to create w_nil without running the constructor, # thus allowing it to be used even in the constructor of its own Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/primitives.py Mon Feb 25 23:32:10 2008 @@ -68,7 +68,7 @@ w_result = func(interp, argument_count_m1) if not no_result: assert w_result is not None - interp.w_active_context.as_context_get_shadow().push(w_result) + interp.s_active_context().push(w_result) return w_result else: len_unwrap_spec = len(unwrap_spec) @@ -104,7 +104,7 @@ w_result = func(interp, *args) # After calling primitive, reload context-shadow in case it # needs to be updated - new_s_frame = interp.w_active_context.as_context_get_shadow() + new_s_frame = interp.s_active_context() frame.as_context_get_shadow().pop_n(len_unwrap_spec) # only if no exception occurs! if not no_result: assert w_result is not None @@ -397,7 +397,6 @@ "Stores a value into a fixed field from the object, and fails otherwise" s_class = w_rcvr.shadow_of_my_class() assert_bounds(n0, 0, s_class.instsize()) - # only pointers have non-0 size # XXX Now MethodContext is still own format, leave #assert isinstance(w_rcvr, model.W_PointersObject) w_rcvr.store(n0, w_value) @@ -622,21 +621,19 @@ @expose_primitive(PRIMITIVE_BLOCK_COPY, unwrap_spec=[object, int]) def func(interp, w_context, argcnt): - frame = interp.w_active_context.as_context_get_shadow() + frame = interp.s_active_context() # From B.B.: If receiver is a MethodContext, then it becomes # the new BlockContext's home context. Otherwise, the home # context of the receiver is used for the new BlockContext. # Note that in our impl, MethodContext.w_home == self - if not isinstance(w_context, model.W_ContextPart): - raise PrimitiveFailedError() w_method_context = w_context.as_context_get_shadow().w_home() # The block bytecodes are stored inline: so we skip past the # byteodes to invoke this primitive to find them (hence +2) initialip = frame.pc() + 2 w_new_context = model.W_BlockContext( - w_method_context, None, argcnt, initialip) + w_method_context, objtable.w_nil, argcnt, initialip) return w_new_context def finalize_block_ctx(interp, s_block_ctx, frame): @@ -653,14 +650,16 @@ # Rcvr | Arg 0 | Arg1 | Arg 2 # - frame = interp.w_active_context.as_context_get_shadow() + frame = interp.s_active_context() # Validate that we have a block on the stack and that it received # the proper number of arguments: w_block_ctx = frame.peek(argument_count) if w_block_ctx.getclass() != classtable.w_BlockContext: raise PrimitiveFailedError() + s_block_ctx = w_block_ctx.as_blockcontext_get_shadow() + exp_arg_cnt = s_block_ctx.expected_argument_count() if argument_count != exp_arg_cnt: # exp_arg_cnt doesn't count self raise PrimitiveFailedError() @@ -668,6 +667,9 @@ # Initialize the block stack with the arguments that were # pushed. Also pop the receiver. block_args = frame.pop_and_return_n(exp_arg_cnt) + + # Reset stack of blockcontext to [] + s_block_ctx.store_stackpointer(s_block_ctx.stackstart()) s_block_ctx.push_all(block_args) frame.pop() @@ -710,7 +712,6 @@ w_frame.as_context_get_shadow().store_w_sender(interp.w_active_context) interp.w_active_context = w_frame - @expose_primitive(PRIMITIVE_SIGNAL, unwrap_spec=[object]) def func(interp, w_rcvr): if w_rcvr.getclass() != classtable.classtable['w_Semaphore']: Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/shadow.py Mon Feb 25 23:32:10 2008 @@ -150,21 +150,6 @@ def new(self, extrasize=0, store=True): from pypy.lang.smalltalk import classtable w_cls = self.w_self() - - if w_cls == classtable.w_BlockContext: - w_new = model.W_BlockContext(None, None, 0, 0) - elif w_cls == classtable.w_MethodContext: - # From slang: Contexts must only be created with newForMethod: - # raise error.PrimitiveFailedError - # XXX XXX XXX XXX - # The above text is bogous. This does not come from slang but a - # method implementation of ContextPart in Squeak3.9 which - # overwrites the default compiledmethod to give this error. - # The mini.image -however- doesn't overwrite this method, and as - # far as I was able to trace it back, it -does- call this method. - from pypy.rlib.objectmodel import instantiate - w_new = instantiate(model.W_MethodContext) - if self.instance_kind == POINTERS: w_new = model.W_PointersObject(w_cls, self.instance_size+extrasize) elif self.instance_kind == WORDS: @@ -391,43 +376,16 @@ def process_lists(self): return self.w_self().fetch(constants.SCHEDULER_PROCESS_LISTS_INDEX) -# XXX This should be changed once everything works using contextshadows -# current hack to make the class extension-system for bytecode lookup happy... -# Should be AbstractShadow -# XXX XXX -class ContextPartShadow(model.W_ContextPart): +class ContextPartShadow(AbstractShadow): - #__metaclass__ = extendabletype + __metaclass__ = extendabletype def __init__(self, w_self): - self._w_self = w_self - self.invalidate() + AbstractShadow.__init__(self, w_self) def s_home(self): raise NotImplementedError() - # XXX XXX Remove function when fixing superclass to AbstractShadow - def w_self(self): - return self._w_self - - # XXX XXX Remove function when fixing superclass to AbstractShadow - def invalidate(self): - self.invalid = True - - # XXX XXX Remove function when fixing superclass to AbstractShadow - def check_for_updates(self): - if self.invalid: - self.w_self().setshadow(self) - self.update_shadow() - - # XXX XXX Remove function when fixing superclass to AbstractShadow - def update_shadow(self): - pass - - # XXX XXX Remove function when fixing superclass to AbstractShadow - def getname(self): - return repr(self) - def w_receiver(self): " Return self of the method, or the method that contains the block " return self.s_home().w_receiver() @@ -443,7 +401,7 @@ else: return w_sender.as_context_get_shadow() - def store_w_sender(self, s_sender): + def store_w_sender(self, w_sender): self.w_self().store(constants.CTXPART_SENDER_INDEX, w_sender) def pc(self): @@ -516,7 +474,8 @@ self.store_stackpointer(self.stackpointer() - n) def stack(self): - return [self.w_self().fetch(i) for i in range(self.stackstart(), self.stackpointer() + 1)] + # fetch = 1-based + return [self.w_self().fetch(i) for i in range(self.stackstart() + 1, self.stackpointer() + 1)] def pop_and_return_n(self, n): self.pop_n(n) @@ -538,6 +497,13 @@ def initialip(self): return utility.unwrap_int(self.w_self().fetch(constants.BLKCTX_INITIAL_IP_INDEX)) + def store_initialip(self, initialip): + self.w_self().store(constants.BLKCTX_INITIAL_IP_INDEX, + utility.wrap_int(initialip)) + + def store_w_home(self, w_home): + self.w_self().store(constants.BLKCTX_HOME_INDEX, w_home) + def w_home(self): return self.w_self().fetch(constants.BLKCTX_HOME_INDEX) @@ -545,7 +511,8 @@ return self.w_home().as_methodcontext_get_shadow() def stackstart(self): - return constants.BLKCTX_TEMP_FRAME_START + return (constants.BLKCTX_TEMP_FRAME_START + + self.expected_argument_count()) class MethodContextShadow(ContextPartShadow): def __init__(self, w_self): @@ -554,6 +521,9 @@ def w_method(self): return self.w_self().fetch(constants.MTHDCTX_METHOD) + def store_w_method(self, w_method): + return self.w_self().store(constants.MTHDCTX_METHOD, w_method) + def w_receiver(self): return self.w_self().fetch(constants.MTHDCTX_RECEIVER) @@ -573,4 +543,6 @@ return self def stackstart(self): - return constants.MTHDCTX_TEMP_FRAME_START + return (constants.MTHDCTX_TEMP_FRAME_START + + self.w_method().argsize + + self.w_method().tempsize) Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_interpreter.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_interpreter.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_interpreter.py Mon Feb 25 23:32:10 2008 @@ -69,9 +69,10 @@ def new_interpreter(bytes, receiver=objtable.w_nil): assert isinstance(bytes, str) w_method = model.W_CompiledMethod(len(bytes)) + w_method.islarge = 1 w_method.bytes = bytes w_method.argsize=2 - w_method.tempsize=1 + w_method.tempsize=8 w_frame = w_method.create_frame(receiver, ["foo", "bar"]) interp = interpreter.Interpreter() interp.w_active_context = w_frame @@ -80,18 +81,20 @@ def test_create_frame(): w_method = model.W_CompiledMethod(len("hello")) w_method.bytes="hello" + w_method.islarge = 1 w_method.argsize=2 - w_method.tempsize=1 + w_method.tempsize=8 w_frame = w_method.create_frame("receiver", ["foo", "bar"]) - assert w_frame.w_receiver() == "receiver" - assert w_frame.gettemp(0) == "foo" - assert w_frame.gettemp(1) == "bar" - assert w_frame.gettemp(2) is objtable.w_nil - w_frame.settemp(2, "spam") - assert w_frame.gettemp(2) == "spam" - assert w_frame.getNextBytecode() == ord("h") - assert w_frame.getNextBytecode() == ord("e") - assert w_frame.getNextBytecode() == ord("l") + s_frame = w_frame.as_context_get_shadow() + assert s_frame.w_receiver() == "receiver" + assert s_frame.gettemp(0) == "foo" + assert s_frame.gettemp(1) == "bar" + assert s_frame.gettemp(2) is objtable.w_nil + s_frame.settemp(2, "spam") + assert s_frame.gettemp(2) == "spam" + assert s_frame.getNextBytecode() == ord("h") + assert s_frame.getNextBytecode() == ord("e") + assert s_frame.getNextBytecode() == ord("l") def test_push_pop(): interp = new_interpreter("") @@ -185,15 +188,17 @@ def test_storeAndPopTemporaryVariableBytecode(bytecode=storeAndPopTemporaryVariableBytecode): for index in range(8): interp = new_interpreter(pushConstantTrueBytecode + bytecode(index)) - interp.w_active_context.as_methodcontext_get_shadow().temps = [None] * 8 + #interp.w_active_context.as_methodcontext_get_shadow().temps = [None] * 8 interp.step() interp.step() assert interp.s_active_context().stack() == [] + interp.w_active_context.as_methodcontext_get_shadow() for test_index in range(8): + print interp.w_active_context._vars if test_index == index: - assert interp.w_active_context.as_methodcontext_get_shadow().temps[test_index] == interp.TRUE + assert interp.s_active_context().gettemp(test_index) == interp.TRUE else: - assert interp.w_active_context.as_methodcontext_get_shadow().temps[test_index] == None + assert interp.s_active_context().gettemp(test_index) != interp.TRUE def test_pushConstantTrueBytecode(): interp = new_interpreter(pushConstantTrueBytecode) @@ -398,7 +403,7 @@ assert interp.s_active_context().stack() == [] assert interp.w_active_context.as_methodcontext_get_shadow().w_receiver() == w_object assert interp.w_active_context.as_methodcontext_get_shadow().w_method() == shadow.methoddict["foo"] - assert callerContext.stack() == [] + assert callerContext.as_context_get_shadow().stack() == [] interp.step() interp.step() assert interp.w_active_context == callerContext @@ -448,7 +453,7 @@ def test_longJumpIfTrue(): interp = new_interpreter(longJumpIfTrue(0) + chr(15) + longJumpIfTrue(0) + chr(15)) interp.s_active_context().push(interp.FALSE) - pc = interp.w_active_context.pc() + 2 + pc = interp.s_active_context().pc() + 2 interp.step() assert interp.s_active_context().pc() == pc interp.s_active_context().push(interp.TRUE) @@ -605,7 +610,7 @@ assert interp.w_active_context.as_methodcontext_get_shadow().w_receiver() == w_object meth = w_specificclass.as_class_get_shadow().methoddict["foo"] assert interp.w_active_context.as_methodcontext_get_shadow().w_method() == meth - assert callerContext.stack() == [] + assert callerContext.as_context_get_shadow().stack() == [] def test_secondExtendedSendBytecode(): w_class = mockclass(0) Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_miniimage.py Mon Feb 25 23:32:10 2008 @@ -204,12 +204,9 @@ w_frame = w_method.create_frame(w_object, []) interp.w_active_context = w_frame - print w_method - while True: try: interp.step() - print interp.s_active_context().stack() except interpreter.ReturnFromTopLevel, e: assert e.object.value == abs(int) return Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_primitives.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_primitives.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_primitives.py Mon Feb 25 23:32:10 2008 @@ -9,9 +9,12 @@ mockclass = classtable.bootstrap_class -class MockFrame(model.W_MethodContext): +class MockFrame(model.W_PointersObject): def __init__(self, stack): - self._stack = stack + self._vars = [None] * 6 + stack + s_self = self.as_blockcontext_get_shadow() + s_self.store_expected_argument_count(0) + s_self.store_stackpointer(s_self.stackstart()+len(stack)-1) def wrap(x): if isinstance(x, int): return utility.wrap_int(x) Modified: pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py ============================================================================== --- pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py (original) +++ pypy/branch/smalltalk-shadow-changes/pypy/lang/smalltalk/test/test_shadow.py Mon Feb 25 23:32:10 2008 @@ -88,7 +88,7 @@ def methodcontext(w_sender=objtable.w_nil, pc=1, stackpointer=0, stacksize=5, method=method()): - stackstart = 7 # (len notation, not idx notation) + stackstart = 6+method.argsize+method.tempsize # (len notation, not idx notation) w_object = model.W_PointersObject(classtable.w_MethodContext, stackstart+stacksize) w_object.store(constants.CTXPART_SENDER_INDEX, w_sender) w_object.store(constants.CTXPART_PC_INDEX, utility.wrap_int(pc)) @@ -98,20 +98,12 @@ w_object.store(constants.MTHDCTX_RECEIVER_MAP, '???') w_object.store(constants.MTHDCTX_RECEIVER, 'receiver') - # XXX Might want to check the realness of the next assumption, - # XXX made by hooking into the suspended thread of the image. - # XXX it seems the only possibility, and using this assumption - # XXX it actually runs... - # Weirdly enough, undependant from the size of the tempsize and - # argsize, the stackpointer can point to anything starting from - # the temp_frame_start. That's why stacks always print all elements - # including possible "temps or args" w_object.store(constants.MTHDCTX_TEMP_FRAME_START, 'el') return w_object def test_context(): w_m = method() - w_object = methodcontext(stackpointer=2, method=w_m) + w_object = methodcontext(stackpointer=3, method=w_m) w_object2 = methodcontext(w_sender=w_object) s_object = w_object.as_methodcontext_get_shadow() s_object2 = w_object2.as_methodcontext_get_shadow() @@ -131,7 +123,7 @@ w_object.store(idx + 2, 'g') w_object.store(idx + 3, 'h') assert s_object.top() == 'h' - assert s_object.stack() == ['el', 'f', 'g', 'h' ] + assert s_object.stack() == ['f', 'g', 'h' ] s_object.push('i') assert s_object.top() == 'i' assert s_object.peek(1) == 'h' From arigo at codespeak.net Tue Feb 26 10:01:10 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 26 Feb 2008 10:01:10 +0100 (CET) Subject: [pypy-svn] r51869 - pypy/branch/unified-rtti/pypy/rpython/memory/gctransform Message-ID: <20080226090110.9C402168052@codespeak.net> Author: arigo Date: Tue Feb 26 10:01:08 2008 New Revision: 51869 Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py Log: Fix for Boehm. Modified: pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/branch/unified-rtti/pypy/rpython/memory/gctransform/boehm.py Tue Feb 26 10:01:08 2008 @@ -20,25 +20,44 @@ atomic_mh = mallocHelpers() atomic_mh.allocate = lambda size: llop.boehm_malloc_atomic(llmemory.Address, size) - ll_malloc_fixedsize_atomic = atomic_mh._ll_malloc_fixedsize - mh = mallocHelpers() mh.allocate = lambda size: llop.boehm_malloc(llmemory.Address, size) - ll_malloc_fixedsize = mh._ll_malloc_fixedsize + HDRPTR = lltype.Ptr(self.HDR) + + def ll_malloc_fixedsize_rtti(size, rtti): + raw = mh._ll_malloc_fixedsize(size) + llmemory.cast_adr_to_ptr(raw, HDRPTR).typeptr = rtti + return raw + + def ll_malloc_fixedsize_atomic_rtti(size, rtti): + raw = atomic_mh._ll_malloc_fixedsize(size) + llmemory.cast_adr_to_ptr(raw, HDRPTR).typeptr = rtti + return raw # XXX, do we need/want an atomic version of this function? - ll_malloc_varsize_no_length = mh.ll_malloc_varsize_no_length - ll_malloc_varsize = mh.ll_malloc_varsize + def ll_malloc_varsize_no_length_rtti(length, size, itemsize, rtti): + raw = mh.ll_malloc_varsize_no_length(length, size, itemsize) + llmemory.cast_adr_to_ptr(raw, HDRPTR).typeptr = rtti + return raw + + def ll_malloc_varsize_rtti(length, size, itemsize, lengthoffset, rtti): + raw = mh.ll_malloc_varsize(length, size, itemsize, lengthoffset) + llmemory.cast_adr_to_ptr(raw, HDRPTR).typeptr = rtti + return raw if self.translator: self.malloc_fixedsize_ptr = self.inittime_helper( - ll_malloc_fixedsize, [lltype.Signed], llmemory.Address) + ll_malloc_fixedsize_rtti, + [lltype.Signed, RTTIPTR], llmemory.Address) self.malloc_fixedsize_atomic_ptr = self.inittime_helper( - ll_malloc_fixedsize_atomic, [lltype.Signed], llmemory.Address) + ll_malloc_fixedsize_atomic_rtti, + [lltype.Signed, RTTIPTR], llmemory.Address) self.malloc_varsize_no_length_ptr = self.inittime_helper( - ll_malloc_varsize_no_length, [lltype.Signed]*3, llmemory.Address, inline=False) + ll_malloc_varsize_no_length_rtti, + [lltype.Signed]*3 + [RTTIPTR], llmemory.Address, inline=False) self.malloc_varsize_ptr = self.inittime_helper( - ll_malloc_varsize, [lltype.Signed]*4, llmemory.Address, inline=False) + ll_malloc_varsize_rtti, + [lltype.Signed]*4 + [RTTIPTR], llmemory.Address, inline=False) self.weakref_create_ptr = self.inittime_helper( ll_weakref_create, [llmemory.Address], llmemory.WeakRefPtr, inline=False) @@ -62,8 +81,9 @@ funcptr = self.malloc_fixedsize_atomic_ptr else: funcptr = self.malloc_fixedsize_ptr + c_rtti = rmodel.inputconst(RTTIPTR, self.gcheaderbuilder.getRtti(TYPE)) v_raw = hop.genop("direct_call", - [funcptr, c_size], + [funcptr, c_size, c_rtti], resulttype=llmemory.Address) finalizer_ptr = self.finalizer_funcptr_for_type(TYPE) if finalizer_ptr: @@ -75,15 +95,17 @@ def gct_fv_gc_malloc_varsize(self, hop, flags, TYPE, v_length, c_const_size, c_item_size, c_offset_to_length): # XXX same behavior for zero=True: in theory that's wrong + c_rtti = rmodel.inputconst(RTTIPTR, self.gcheaderbuilder.getRtti(TYPE)) if c_offset_to_length is None: v_raw = hop.genop("direct_call", [self.malloc_varsize_no_length_ptr, v_length, - c_const_size, c_item_size], + c_const_size, c_item_size, c_rtti], resulttype=llmemory.Address) else: v_raw = hop.genop("direct_call", [self.malloc_varsize_ptr, v_length, - c_const_size, c_item_size, c_offset_to_length], + c_const_size, c_item_size, c_offset_to_length, + c_rtti], resulttype=llmemory.Address) return v_raw From arigo at codespeak.net Tue Feb 26 10:07:05 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 26 Feb 2008 10:07:05 +0100 (CET) Subject: [pypy-svn] r51870 - in pypy/dist/pypy: rpython/memory rpython/memory/gc rpython/memory/gctransform translator/c translator/c/src Message-ID: <20080226090705.2063C168052@codespeak.net> Author: arigo Date: Tue Feb 26 10:07:04 2008 New Revision: 51870 Modified: pypy/dist/pypy/rpython/memory/gc/base.py pypy/dist/pypy/rpython/memory/gc/semispace.py pypy/dist/pypy/rpython/memory/gctransform/boehm.py pypy/dist/pypy/rpython/memory/gctransform/framework.py pypy/dist/pypy/rpython/memory/gctransform/refcounting.py pypy/dist/pypy/rpython/memory/gcwrapper.py pypy/dist/pypy/translator/c/gc.py pypy/dist/pypy/translator/c/src/mem.h Log: Define MALLOC_ZERO_FILLED to a sensible value provided by the gc transformer. This allows the semispace and generation GCs to set it to 1. Modified: pypy/dist/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/base.py (original) +++ pypy/dist/pypy/rpython/memory/gc/base.py Tue Feb 26 10:07:04 2008 @@ -5,7 +5,7 @@ _alloc_flavor_ = "raw" moving_gc = False needs_write_barrier = False - needs_zero_gc_pointers = True + malloc_zero_filled = False prebuilt_gc_objects_are_static_roots = True def set_query_functions(self, is_varsize, has_gcptr_in_varsize, Modified: pypy/dist/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/dist/pypy/rpython/memory/gc/semispace.py Tue Feb 26 10:07:04 2008 @@ -26,7 +26,7 @@ _alloc_flavor_ = "raw" inline_simple_malloc = True inline_simple_malloc_varsize = True - needs_zero_gc_pointers = False + malloc_zero_filled = True first_unused_gcflag = first_gcflag << 3 total_collection_time = 0.0 total_collection_count = 0 Modified: pypy/dist/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/boehm.py Tue Feb 26 10:07:04 2008 @@ -6,6 +6,7 @@ from pypy.rpython.lltypesystem.lloperation import llop class BoehmGCTransformer(GCTransformer): + malloc_zero_filled = True FINALIZER_PTR = lltype.Ptr(lltype.FuncType([llmemory.Address], lltype.Void)) def __init__(self, translator, inline=False): Modified: pypy/dist/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/framework.py Tue Feb 26 10:07:04 2008 @@ -330,7 +330,7 @@ s_gc = self.translator.annotator.bookkeeper.valueoftype(GCClass) r_gc = self.translator.rtyper.getrepr(s_gc) self.c_const_gc = rmodel.inputconst(r_gc, self.gcdata.gc) - self.needs_zero_gc_pointers = GCClass.needs_zero_gc_pointers + self.malloc_zero_filled = GCClass.malloc_zero_filled HDR = self._gc_HDR = self.gcdata.gc.gcheaderbuilder.HDR self._gc_fields = fields = [] @@ -550,7 +550,7 @@ resultvar=op.result) def gct_zero_gc_pointers_inside(self, hop): - if self.needs_zero_gc_pointers: + if not self.malloc_zero_filled: v_ob = hop.spaceop.args[0] TYPE = v_ob.concretetype.TO gen_zero_gc_pointers(TYPE, v_ob, hop.llops) Modified: pypy/dist/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/refcounting.py Tue Feb 26 10:07:04 2008 @@ -32,6 +32,7 @@ ADDRESS_VOID_FUNC = lltype.FuncType([llmemory.Address], lltype.Void) class RefcountingGCTransformer(GCTransformer): + malloc_zero_filled = True HDR = lltype.Struct("header", ("refcount", lltype.Signed)) Modified: pypy/dist/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/dist/pypy/rpython/memory/gcwrapper.py Tue Feb 26 10:07:04 2008 @@ -36,7 +36,7 @@ typeid = self.get_type_id(TYPE) addr = self.gc.malloc(typeid, n, zero=zero) result = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(TYPE)) - if self.gc.needs_zero_gc_pointers: + if not self.gc.malloc_zero_filled: gctypelayout.zero_gc_pointers(result) return result else: @@ -52,7 +52,7 @@ addr = self.gc.malloc(typeid, size, zero=zero, coallocator=coallocator) result = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(TYPE)) - if self.gc.needs_zero_gc_pointers: + if not self.gc.malloc_zero_filled: gctypelayout.zero_gc_pointers(result) return result return self.malloc(TYPE, size, 'gc', zero) Modified: pypy/dist/pypy/translator/c/gc.py ============================================================================== --- pypy/dist/pypy/translator/c/gc.py (original) +++ pypy/dist/pypy/translator/c/gc.py Tue Feb 26 10:07:04 2008 @@ -40,7 +40,9 @@ return [] def pre_pre_gc_code(self): # code that goes before include g_prerequisite.h - return [] + gct = self.db.gctransformer + yield '/* using %s */' % (gct.__class__.__name__,) + yield '#define MALLOC_ZERO_FILLED %d' % (gct.malloc_zero_filled,) def pre_gc_code(self): return ['typedef void *GC_hidden_pointer;'] @@ -182,6 +184,8 @@ return ['gc'] def pre_pre_gc_code(self): + for line in BasicGcPolicy.pre_pre_gc_code(self): + yield line if sys.platform == "linux2": yield "#define _REENTRANT 1" yield "#define GC_LINUX_THREADS 1" @@ -247,9 +251,6 @@ gc_libraries = RefcountingGcPolicy.gc_libraries.im_func gc_startup_code = RefcountingGcPolicy.gc_startup_code.im_func - def pre_pre_gc_code(self): - yield '#define USING_NO_GC' - class FrameworkGcPolicy(BasicGcPolicy): transformerclass = framework.FrameworkGCTransformer @@ -277,9 +278,6 @@ def rtti_node_factory(self): return FrameworkGcRuntimeTypeInfo_OpaqueNode - def pre_pre_gc_code(self): - yield '#define USING_FRAMEWORK_GC' - def gc_startup_code(self): fnptr = self.db.gctransformer.frameworkgc_setup_ptr.value yield '%s();' % (self.db.get(fnptr),) Modified: pypy/dist/pypy/translator/c/src/mem.h ============================================================================== --- pypy/dist/pypy/translator/c/src/mem.h (original) +++ pypy/dist/pypy/translator/c/src/mem.h Tue Feb 26 10:07:04 2008 @@ -64,12 +64,6 @@ #define alloca _alloca #endif -#ifdef USING_FRAMEWORK_GC -#define MALLOC_ZERO_FILLED 0 -#else -#define MALLOC_ZERO_FILLED 1 -#endif - #define OP_STACK_MALLOC(size,r,restype) \ r = (restype) alloca(size); \ if (r != NULL) memset((void*) r, 0, size); From jared.grubb at codespeak.net Tue Feb 26 10:56:31 2008 From: jared.grubb at codespeak.net (jared.grubb at codespeak.net) Date: Tue, 26 Feb 2008 10:56:31 +0100 (CET) Subject: [pypy-svn] r51871 - in pypy/dist/pypy/rlib/parsing: . test Message-ID: <20080226095631.3F4FC1683D8@codespeak.net> Author: jared.grubb Date: Tue Feb 26 10:56:29 2008 New Revision: 51871 Added: pypy/dist/pypy/rlib/parsing/test/test_pcre_regtest.py (contents, props changed) Modified: pypy/dist/pypy/rlib/parsing/deterministic.py pypy/dist/pypy/rlib/parsing/regexparse.py pypy/dist/pypy/rlib/parsing/test/test_deterministic.py pypy/dist/pypy/rlib/parsing/test/test_regexparse.py Log: parsing/regex stuff: add support for {n,} '\cx' '\377'; also working on adapter class to let us run some PCRE regression tests (what better way to test our NFA's and DFA's than some vigorous RE's :) parsing/detrministic: make_nice_charset_repr now escapes the ] as well Modified: pypy/dist/pypy/rlib/parsing/deterministic.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/deterministic.py (original) +++ pypy/dist/pypy/rlib/parsing/deterministic.py Tue Feb 26 10:56:29 2008 @@ -27,13 +27,14 @@ # Change the above list into a list of sorted tuples real_result = [(c,l) for [c,l] in result] + # Sort longer runs first (hence -c), then alphabetically real_result.sort(key=lambda (l,c): (-c,l)) return real_result def make_nice_charset_repr(chars): # Compress the letters & digits letters = set(chars) & set("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") - therest = set(chars) - letters - set('-') + therest = set(chars) - letters - set(['-',']']) charranges = compress_char_set(letters) result = [] for a, num in charranges: @@ -45,8 +46,11 @@ else: result.append("%s-%s" % (repr(a)[1:-1], repr(chr(ord(a) + num - 1))[1:-1])) result += [repr(c)[1:-1] for c in therest] + # Handle the special chars that MUST get escaped if '-' in chars: result += ['\\-'] + if ']' in chars: + result += ['\\]'] return "".join(result) class LexerError(Exception): @@ -214,6 +218,9 @@ result.emit("i = 0") result.emit("state = 0") result.start_block("while 1:") + + # state_to_chars is a dict containing the sets of + # Ex: state_to_chars = { 0: set('a','b','c'), ...} state_to_chars = {} for (state, char), nextstate in self.transitions.iteritems(): state_to_chars.setdefault(state, {}).setdefault(nextstate, set()).add(char) Modified: pypy/dist/pypy/rlib/parsing/regexparse.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/regexparse.py (original) +++ pypy/dist/pypy/rlib/parsing/regexparse.py Tue Feb 26 10:56:29 2008 @@ -20,23 +20,38 @@ } for i in range(256): - # 'x' and numbers are reserved for hexadecimal/octal escapes - if chr(i) in 'x01234567': - continue - escaped = "\\" + chr(i) + if chr(i) not in 'x01234567': + # 'x' and numbers are reserved for hexadecimal/octal escapes + escaped = "\\" + chr(i) + if escaped not in ESCAPES: + ESCAPES[escaped] = chr(i) + + # Three digit octals + escaped = "\\%03o" % i if escaped not in ESCAPES: ESCAPES[escaped] = chr(i) + + if 0 <= i <= 077: + # Two digit octal digs are ok too + escaped = "\\%02o" % i + if escaped not in ESCAPES: + ESCAPES[escaped] = chr(i) + + # Add the ctrl-x types: + # Rule, according to PCRE: + # if x is a lower case letter, it is converted to upper case. + # Then bit 6 of the character (hex 40) is inverted. + # Thus, \cz => 0x1A, but \c{ => 0x3B, while \c; => 0x7B. + escaped = "\\c%s" % chr(i) + if escaped not in ESCAPES: + ESCAPES[escaped] = chr(ord(chr(i).upper()) ^ 0x40) + + for a in "0123456789ABCDEFabcdef": for b in "0123456789ABCDEFabcdef": escaped = "\\x%s%s" % (a, b) if escaped not in ESCAPES: ESCAPES[escaped] = chr(int("%s%s" % (a, b), 16)) -for a in "0123": - for b in "01234567": - for c in "01234567": - escaped = "\\x%s%s%s" % (a, b, c) - if escaped not in ESCAPES: - ESCAPES[escaped] = chr(int("%s%s%s" % (a, b, c), 8)) def unescape(s): result = [] @@ -90,11 +105,16 @@ | r1 = primary '?' return {regex.StringExpression("") | r1} - | r = primary + | r1 = primary + '{' + n = clippednumrange + '}' + return {r1 * n + r1.kleene()} + | r1 = primary '{' n = numrange '}' - return {r * n[0] + reduce(operator.or_, [r * i for i in range(n[1] - n[0] + 1)], regex.StringExpression(""))} + return {r1 * n[0] + reduce(operator.or_, [r1 * i for i in range(n[1] - n[0] + 1)], regex.StringExpression(""))} | primary; primary: @@ -112,7 +132,7 @@ return {c}; QUOTEDCHAR: - `(\\x[0-9a-fA-F]{2})|(\\.)`; + `(\\x[0-9a-fA-F]{2})|(\\[0-3]?[0-7][0-7])|(\\c.)|(\\.)`; CHAR: `[^\*\+\(\)\[\]\{\}\|\.\-\?\,\^]`; @@ -149,6 +169,11 @@ | n1 = NUM return {n1, n1}; +clippednumrange: + n1 = NUM + ',' + return {n1}; + NUM: c = `0|([1-9][0-9]*)` return {int(c)}; @@ -537,14 +562,14 @@ _call_status = self._primary() _result = _call_status.result _error = self._combine_errors(_error, _call_status.error) - r = _result + r1 = _result _result = self.__chars__('{') - _call_status = self._numrange() + _call_status = self._clippednumrange() _result = _call_status.result _error = self._combine_errors(_error, _call_status.error) n = _result _result = self.__chars__('}') - _result = (r * n[0] + reduce(operator.or_, [r * i for i in range(n[1] - n[0] + 1)], regex.StringExpression(""))) + _result = (r1 * n + r1.kleene()) break except BacktrackException, _exc: _error = self._combine_errors(_error, _exc.error) @@ -554,10 +579,27 @@ _call_status = self._primary() _result = _call_status.result _error = self._combine_errors(_error, _call_status.error) + r1 = _result + _result = self.__chars__('{') + _call_status = self._numrange() + _result = _call_status.result + _error = self._combine_errors(_error, _call_status.error) + n = _result + _result = self.__chars__('}') + _result = (r1 * n[0] + reduce(operator.or_, [r1 * i for i in range(n[1] - n[0] + 1)], regex.StringExpression(""))) break except BacktrackException, _exc: _error = self._combine_errors(_error, _exc.error) self._pos = _choice4 + _choice5 = self._pos + try: + _call_status = self._primary() + _result = _call_status.result + _error = self._combine_errors(_error, _call_status.error) + break + except BacktrackException, _exc: + _error = self._combine_errors(_error, _exc.error) + self._pos = _choice5 raise BacktrackException(_error) _call_status = self._primary() _result = _call_status.result @@ -787,7 +829,7 @@ try: _result = None _error = None - _result = self._regex1380912319() + _result = self._regex1192240515() assert _status.status != _status.LEFTRECURSION _status.status = _status.NORMAL _status.pos = self._pos @@ -1216,6 +1258,64 @@ _status.error = _error _status.status = _status.ERROR raise BacktrackException(_error) + def clippednumrange(self): + return self._clippednumrange().result + def _clippednumrange(self): + _key = self._pos + _status = self._dict_clippednumrange.get(_key, None) + if _status is None: + _status = self._dict_clippednumrange[_key] = Status() + else: + _statusstatus = _status.status + if _statusstatus == _status.NORMAL: + self._pos = _status.pos + return _status + elif _statusstatus == _status.ERROR: + raise BacktrackException(_status.error) + elif (_statusstatus == _status.INPROGRESS or + _statusstatus == _status.LEFTRECURSION): + _status.status = _status.LEFTRECURSION + if _status.result is not None: + self._pos = _status.pos + return _status + else: + raise BacktrackException(None) + elif _statusstatus == _status.SOMESOLUTIONS: + _status.status = _status.INPROGRESS + _startingpos = self._pos + try: + _result = None + _error = None + _call_status = self._NUM() + _result = _call_status.result + _error = _call_status.error + n1 = _result + _result = self.__chars__(',') + _result = (n1) + if _status.status == _status.LEFTRECURSION: + if _status.result is not None: + if _status.pos >= self._pos: + _status.status = _status.NORMAL + self._pos = _status.pos + return _status + _status.pos = self._pos + _status.status = _status.SOMESOLUTIONS + _status.result = _result + _status.error = _error + self._pos = _startingpos + return self._clippednumrange() + _status.status = _status.NORMAL + _status.pos = self._pos + _status.result = _result + _status.error = _error + return _status + except BacktrackException, _exc: + _status.pos = -1 + _status.result = None + _error = self._combine_errors(_error, _exc.error) + _status.error = _error + _status.status = _status.ERROR + raise BacktrackException(_error) def NUM(self): return self._NUM().result def _NUM(self): @@ -1265,6 +1365,7 @@ self._dict_subrange = {} self._dict_rangeelement = {} self._dict_numrange = {} + self._dict_clippednumrange = {} self._dict_NUM = {} self._pos = 0 self._inputstream = inputstream @@ -1282,10 +1383,10 @@ _result = self._inputstream[_pos: _upto] self._pos = _upto return _result - def _regex1323868075(self): + def _regex1192240515(self): _choice1 = self._pos _runner = self._Runner(self._inputstream, self._pos) - _i = _runner.recognize_1323868075(self._pos) + _i = _runner.recognize_1192240515(self._pos) if _runner.last_matched_state == -1: self._pos = _choice1 raise BacktrackException @@ -1296,10 +1397,10 @@ _result = self._inputstream[_pos: _upto] self._pos = _upto return _result - def _regex1380912319(self): + def _regex1323868075(self): _choice2 = self._pos _runner = self._Runner(self._inputstream, self._pos) - _i = _runner.recognize_1380912319(self._pos) + _i = _runner.recognize_1323868075(self._pos) if _runner.last_matched_state == -1: self._pos = _choice2 raise BacktrackException @@ -1360,7 +1461,7 @@ break runner.state = state return ~i - def recognize_1323868075(runner, i): + def recognize_1192240515(runner, i): #auto-generated code, don't edit assert i >= 0 input = runner.text @@ -1374,95 +1475,144 @@ runner.state = 0 return ~i if char == '\\': - state = 1 - elif '/' <= char <= '>': - state = 1 - elif '@' <= char <= 'Z': - state = 1 - elif '_' <= char <= 'z': - state = 1 - elif '\x00' <= char <= "'": - state = 1 - elif '~' <= char <= '\xff': - state = 1 + state = 6 else: break - runner.last_matched_state = state - runner.last_matched_index = i - 1 - runner.state = state - if i == len(input): - return i - else: - return ~i - break - runner.state = state - return ~i - def recognize_1380912319(runner, i): - #auto-generated code, don't edit - assert i >= 0 - input = runner.text - state = 0 - while 1: - if state == 0: + if state == 1: + runner.last_matched_index = i - 1 + runner.last_matched_state = state try: char = input[i] i += 1 except IndexError: - runner.state = 0 - return ~i - if char == '\\': + runner.state = 1 + return i + if '0' <= char <= '7': state = 4 else: break - if state == 1: + if state == 2: + runner.last_matched_index = i - 1 + runner.last_matched_state = state try: char = input[i] i += 1 except IndexError: - runner.state = 1 - return ~i - if 'A' <= char <= 'F': - state = 3 + runner.state = 2 + return i + if '0' <= char <= '9': + state = 5 + elif 'A' <= char <= 'F': + state = 5 elif 'a' <= char <= 'f': - state = 3 - elif '0' <= char <= '9': - state = 3 + state = 5 else: break - if state == 2: + if state == 3: runner.last_matched_index = i - 1 runner.last_matched_state = state try: char = input[i] i += 1 except IndexError: - runner.state = 2 + runner.state = 3 return i - if 'A' <= char <= 'F': - state = 1 - continue - elif 'a' <= char <= 'f': - state = 1 - continue - elif '0' <= char <= '9': - state = 1 - continue + if '\x00' <= char <= '\xff': + state = 7 else: break if state == 4: + runner.last_matched_index = i - 1 + runner.last_matched_state = state try: char = input[i] i += 1 except IndexError: runner.state = 4 + return i + if '0' <= char <= '7': + state = 7 + else: + break + if state == 5: + try: + char = input[i] + i += 1 + except IndexError: + runner.state = 5 + return ~i + if '0' <= char <= '9': + state = 7 + elif 'A' <= char <= 'F': + state = 7 + elif 'a' <= char <= 'f': + state = 7 + else: + break + if state == 6: + try: + char = input[i] + i += 1 + except IndexError: + runner.state = 6 return ~i - if char == 'x': + if '0' <= char <= '3': + state = 1 + continue + elif char == 'x': state = 2 continue - elif '\x00' <= char <= 'w': + elif char == 'c': state = 3 + continue + elif '4' <= char <= '7': + state = 4 + continue elif 'y' <= char <= '\xff': - state = 3 + state = 7 + elif '\x00' <= char <= '/': + state = 7 + elif '8' <= char <= 'b': + state = 7 + elif 'd' <= char <= 'w': + state = 7 + else: + break + runner.last_matched_state = state + runner.last_matched_index = i - 1 + runner.state = state + if i == len(input): + return i + else: + return ~i + break + runner.state = state + return ~i + def recognize_1323868075(runner, i): + #auto-generated code, don't edit + assert i >= 0 + input = runner.text + state = 0 + while 1: + if state == 0: + try: + char = input[i] + i += 1 + except IndexError: + runner.state = 0 + return ~i + if '~' <= char <= '\xff': + state = 1 + elif '\x00' <= char <= "'": + state = 1 + elif '_' <= char <= 'z': + state = 1 + elif '@' <= char <= 'Z': + state = 1 + elif '/' <= char <= '>': + state = 1 + elif char == '\\': + state = 1 else: break runner.last_matched_state = state @@ -1506,6 +1656,13 @@ + + + + + + + def test_generate(): f = py.magic.autopath() oldcontent = f.read() Modified: pypy/dist/pypy/rlib/parsing/test/test_deterministic.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/test/test_deterministic.py (original) +++ pypy/dist/pypy/rlib/parsing/test/test_deterministic.py Tue Feb 26 10:56:29 2008 @@ -170,4 +170,6 @@ assert make_nice_charset_repr("ABCabc") == 'A-Ca-c' assert make_nice_charset_repr("zycba") == 'a-cyz' assert make_nice_charset_repr(string.ascii_letters) == 'A-Za-z' - assert make_nice_charset_repr(string.printable) == 'A-Za-z0-9\\t\\x0b\\n\\r\\x0c! #"%$\'&)(+*,/.;:=@[]\\\\_^`{}|~\\-' + # this next one is ugly... need to clean it up (sometimes it fails because it's + # being generated from a dict, so the order is funky) + assert make_nice_charset_repr(string.printable) == 'A-Za-z0-9\\t\\x0b\\n\\r\\x0c! #"%$\'&)(+*,/.;:=@[\\\\_^`{}|~\\-\\]' \ No newline at end of file Added: pypy/dist/pypy/rlib/parsing/test/test_pcre_regtest.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rlib/parsing/test/test_pcre_regtest.py Tue Feb 26 10:56:29 2008 @@ -0,0 +1,172 @@ +# This file can (in progress) read and parse PCRE regression tests to try out +# on our regular expression library. +# +# To try this out, 'man pcretest' and then grab testinput1 and testoutput1 from +# the PCRE source code. (I need to look into whether we could distribute these +# files with pypy?) + +import py +from pypy.rlib.parsing.regexparse import make_runner, unescape, RegexParser +import string +import re + +py.test.skip("In Progress...") + +def get_simult_lines(tests, results, test_line_num=0): + """Returns a line from the input/output, ensuring that + we are sync'd up between the two.""" + test = tests.pop(0) + result = results.pop(0) + + test_line_num += 1 + + if test != result: + raise Exception("Lost sync between files at input line %d.\n INPUT: %s\n OUTPUT: %s" % (test_line_num, test, result)) + + return test + +def get_definition_line(tests, results): + """Gets a test definition line, formatted per the PCRE spec.""" + delim = None + test = '' + result = '' + + # A line is marked by a start-delimeter and an end-delimeter. + # The delimeter is non-alphanumeric + # If a backslash follows the delimiter, then the backslash should + # be appended to the end. (Otherwise, \ + delim would not be a + # delim anymore!) + while 1: + test += get_simult_lines(tests, results) + + if delim is None: + delim = test[0] + assert delim in (set(string.printable) - set(string.letters) - set(string.digits)) + test_re = re.compile(r'%(delim)s(([^%(delim)s]|\\%(delim)s)*([^\\]))%(delim)s(\\?)(.*)' % {'delim': delim}) + + matches = test_re.findall(test) + if matches: + break + + assert len(matches)==1 + test = matches[0][0] + + # Add the backslash, if we gotta + test += matches[0][-2] + flags = matches[0][-1] + + return test, flags + +def get_test_result(tests, results): + """Gets the expected return from the regular expression""" + # Second line is the test to run against the regex + # ' TEXT' + test = get_simult_lines(tests, results) + if not test: + return None, None + if not test.startswith(' '): + raise Exception("Input & output match, but I don't understand. (Got %r)" % test) + test = unescape(test[4:]) + + # Third line in the OUTPUT is the result, either: + # ' 0: ...' for a match + # 'No match' for no match + result = unescape(results.pop(0)) + if result == 'No match': + pass + elif result.startswith(' 0: '): + # Now we need to eat any further lines like: + # ' 1: ....' a subgroup match + while results[0]: + if results[0][2] == ':': + results.pop(0) + else: + break + else: + raise Exception("Lost sync in output.") + return test, result + +def test_file(): + """Open the PCRE tests and run them.""" + tests = [line.rstrip() for line in open('testinput1','r').readlines()] + results = [line.rstrip() for line in open('testoutput1','r').readlines()] + + regex_flag_mapping = { '': lambda s: s, + 'i': lambda s: s.upper() + } + + import pdb + while tests: + # First line is a test, in the form: + # '/regex expression/FLAGS' + regex, regex_flags = get_definition_line(tests, results) + + # Handle the flags: + try: + text_prepare = regex_flag_mapping[regex_flags] + except KeyError: + print "UNKNOWN FLAGS: %s" % regex_flags + continue + + print '%r' % regex + + skipped = any([op in regex for op in ['*?', '??', '+?', '}?']]) + if skipped: + print " SKIPPED (cant do non-greedy operators)" + # now burn all the tests for this regex + while 1: + test, result = get_test_result(tests, results) + if not test: + break # A blank line means we have nothing to do + continue + + regex_to_use = text_prepare(regex) + + anchor_left = regex_to_use.startswith('^') + anchor_right = regex_to_use.endswith('$') and not regex_to_use.endswith('\\$') + if anchor_left: + regex_to_use = regex_to_use[1:] # chop the ^ if it's there + if anchor_right: + regex_to_use = regex_to_use[:-1] # chop the $ if it's there + + # Finally, we make the pypy regex runner + runner = make_runner(regex_to_use) + + # Now run the test expressions against the Regex + while 1: + test, result = get_test_result(tests, results) + if not test: + break # A blank line means we have nothing to do + + # Create possible subsequences that we should test + if anchor_left: + subseq_gen = [0] + else: + subseq_gen = (start for start in range(0, len(test))) + + if anchor_right: + subseq_gen = ( (start, len(test)) for start in subseq_gen ) + else: + # Go backwards to simulate greediness + subseq_gen = ( (start, end) for start in subseq_gen for end in range(len(test)+1, start+1, -1) ) + + # Search the possibilities for a match... + for start, end in subseq_gen: + attempt = text_prepare(test[start:end]) + matched = runner.recognize(attempt) + if matched: + break + + # Did we get what we expected? + if result == 'No match': + if matched: + print " FALSE MATCH: regex==%r test==%r" % (regex, test) + else: + print " pass : regex==%r test==%r" % (regex, test) + elif result.startswith(' 0: '): + if not matched: + print " MISSED: regex==%r test==%r" % (regex, test) + elif not attempt==text_prepare(result[4:]): + print " BAD MATCH: regex==%r test==%r found==%r expect==%r" % (regex, test, attempt, result[4:]) + else: + print " pass : regex==%r test==%r" % (regex, test) Modified: pypy/dist/pypy/rlib/parsing/test/test_regexparse.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/test/test_regexparse.py (original) +++ pypy/dist/pypy/rlib/parsing/test/test_regexparse.py Tue Feb 26 10:56:29 2008 @@ -96,7 +96,7 @@ assert r.recognize("a" * 15) assert not r.recognize("a" * 14) assert not r.recognize("a" * 16) - assert not r.recognize("b" * 16) + assert not r.recognize("b" * 15) r = make_runner('a{2,10}') assert r.recognize("a" * 2) assert r.recognize("a" * 5) @@ -105,6 +105,14 @@ assert not r.recognize("a" + "b") assert not r.recognize("a" * 11) assert not r.recognize("a" * 12) + r = make_runner('a{3,}') + assert r.recognize("a" * 3) + assert r.recognize("a" * 5) + assert r.recognize("a" * 10) + assert r.recognize("a" * 12) + assert not r.recognize("a") + assert not r.recognize("a" + "b") + assert not r.recognize("a" * 2) def test_quotes(): r = make_runner('"[^\\"]*"') @@ -114,6 +122,13 @@ r = make_runner('\\n\\x0a') assert not r.recognize("n\n") assert r.recognize("\n\n") + r = make_runner('\\12\\012') + assert r.recognize("\n\n") + r = make_runner('\\377\\xff') + assert r.recognize("\xff\xff") + r = make_runner('\\?') + assert r.recognize("?") + assert not r.recognize("a") def test_comment(): r = make_runner("(/\\*[^\\*/]*\\*/)") From cfbolz at codespeak.net Tue Feb 26 11:27:36 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 26 Feb 2008 11:27:36 +0100 (CET) Subject: [pypy-svn] r51872 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080226102736.8D2951683D0@codespeak.net> Author: cfbolz Date: Tue Feb 26 11:27:34 2008 New Revision: 51872 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Log: get the green handling right when resuming. this passes the first dfa test Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Tue Feb 26 11:27:34 2008 @@ -69,7 +69,7 @@ interpreter.newjitstate(jitstate) interpreter.frame.pc = self.pc interpreter.frame.bytecode = self.bytecode - interpreter.frame.local_green = [] + interpreter.frame.local_green = jitstate.greens[:] jitstate.frame.dispatchqueue = dispatchqueue interpreter.bytecode_loop() finaljitstate = interpreter.jitstate @@ -449,6 +449,7 @@ @arguments() def opimpl_guard_global_merge(self): + rtimeshift.save_greens(self.jitstate, self.frame.local_green) rtimeshift.guard_global_merge(self.jitstate, self.frame.pc) return self.dispatch() Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Tue Feb 26 11:27:34 2008 @@ -138,7 +138,6 @@ self.check_insns(int_floordiv=1, int_mul=0) def test_dfa_compile(self): - py.test.skip("not working yet") from pypy.lang.automata.dfa import getautomaton, convertdfa, recognizetable a = getautomaton() dfatable, final_states = convertdfa(a) From antocuni at codespeak.net Tue Feb 26 13:53:01 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 26 Feb 2008 13:53:01 +0100 (CET) Subject: [pypy-svn] r51873 - pypy/extradoc/proposal Message-ID: <20080226125301.34237168048@codespeak.net> Author: antocuni Date: Tue Feb 26 13:52:58 2008 New Revision: 51873 Added: pypy/extradoc/proposal/ pypy/extradoc/proposal/openjdk-challenge.txt (contents, props changed) Log: first paragraphs of the proposal for the openjdk challenge Added: pypy/extradoc/proposal/openjdk-challenge.txt ============================================================================== --- (empty file) +++ pypy/extradoc/proposal/openjdk-challenge.txt Tue Feb 26 13:52:58 2008 @@ -0,0 +1,58 @@ +Porting the PyPy JIT to JVM and MLVM +==================================== + +PyPy and its JIT generator +-------------------------- + +PyPy_ is an open source research project that aims to produce a +flexible and fast implementation of the Python language (XXX: should +we mention other languages implementations?) + +PyPy is divided into two main parts: the Python interpreter, which +implements the Python language and its written in RPython_, and the +Translation Toolchain (TT), which transforms and translates programs +written in RPython into the final executables. RPython is a subset of +Python specifically designed to allow the TT to analyze RPython +programs and translate them into lower level, very efficient +executables. + +Currently, the TT of PyPy provides three complete backends that +generate C code, bytecode for CLI/.NET and bytecode for the JVM. By +using these backends, we can get Python implementations that run on a +standard C/Posix environment, on the CLI or on the JVM. + +It is important to underline that the job of the TT is not limited to +the translation into an efficient executable, but it actively +transform the source interpreter by adding new features and +translation aspects, such as garbage collection, stackless +capabilities, etc. (XXX: link). + +The most exciting feature of the TT is the ability to automatically +turn the interpreter into a JIT compiler that exploits partial +evaluation techniques to dynamically generate efficient code. + +Currently, the PyPy JIT works only in conjunction with the C backend; +early results are very good, since the resulting Python interpreter +can run numeric intensive computations at roughly the same speed of C +(XXX: link). + +XXX: should we mention the currently half-working CLI JIT backend? + + +Porting the JIT to the JVM +-------------------------- + +The goal of this proposal is to extend the PyPy JIT to work in +conjunction with the JVM backend. After the work has been completed, +it will be possibile to translate the interpreter into a Python +implementation that runs on top of the JVM and contains a JIT; the JIT +will dynamically translate part of Python programs into JVM bytecode, +which will then be executed by the underlying virtual machine. + + + + +.. _PyPy: http://codespeak.net/pypy +.. _RPython: http://codespeak.net/pypy/dist/pypy/doc/coding-guide.html#rpython + + From arigo at codespeak.net Tue Feb 26 17:59:00 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 26 Feb 2008 17:59:00 +0100 (CET) Subject: [pypy-svn] r51874 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080226165900.562D9168403@codespeak.net> Author: arigo Date: Tue Feb 26 17:58:57 2008 New Revision: 51874 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Log: This test passes with the same hack as in llinterp.py. Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Tue Feb 26 17:58:57 2008 @@ -1,5 +1,6 @@ from pypy.rlib.rarithmetic import intmask from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.objectmodel import we_are_translated, CDefinedIntSymbolic from pypy.jit.timeshifter import rtimeshift, rcontainer from pypy.jit.timeshifter.greenkey import empty_key, GreenKey, newgreendict from pypy.rpython.lltypesystem import lltype, llmemory @@ -686,6 +687,11 @@ genconst = self.get_greenarg() arg = genconst.revealconst(opdesc.ARGS[i]) args += (arg, ) + if not we_are_translated(): + if opdesc.opname == "int_is_true": + # special case for tests, as in llinterp.py + if type(args[1]) is CDefinedIntSymbolic: + args = (args[0], args[1].default) result = self.rgenop.genconst(opdesc.llop(*args)) self.green_result(result) implementation.argspec = ("green",) * len(list(numargs)) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Tue Feb 26 17:58:57 2008 @@ -233,7 +233,6 @@ self.check_insns(indirect_call=2) def test_method_call_promote(self): - py.test.skip("not working yet") class Base(object): pass class Int(Base): From antocuni at codespeak.net Tue Feb 26 20:30:49 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 26 Feb 2008 20:30:49 +0100 (CET) Subject: [pypy-svn] r51876 - pypy/extradoc/proposal Message-ID: <20080226193049.EFB6E1683F2@codespeak.net> Author: antocuni Date: Tue Feb 26 20:30:48 2008 New Revision: 51876 Modified: pypy/extradoc/proposal/openjdk-challenge.txt Log: this completes the first draft of the proposal Modified: pypy/extradoc/proposal/openjdk-challenge.txt ============================================================================== --- pypy/extradoc/proposal/openjdk-challenge.txt (original) +++ pypy/extradoc/proposal/openjdk-challenge.txt Tue Feb 26 20:30:48 2008 @@ -44,15 +44,140 @@ The goal of this proposal is to extend the PyPy JIT to work in conjunction with the JVM backend. After the work has been completed, -it will be possibile to translate the interpreter into a Python +it will be possible to translate the interpreter into a Python implementation that runs on top of the JVM and contains a JIT; the JIT will dynamically translate part of Python programs into JVM bytecode, which will then be executed by the underlying virtual machine. +XXX: should we explain in more detail what we are doing? +Porting the JIT to the MLVM +--------------------------- -.. _PyPy: http://codespeak.net/pypy -.. _RPython: http://codespeak.net/pypy/dist/pypy/doc/coding-guide.html#rpython +As stated above, PyPy JIT for JVM would work by dynamically emitting +and loading JVM bytecode at runtime. Even if this approach has been +tried in a couple of projects (XXX: link to JRuby for example), it has +to been said that the JVM was not originally designed for such +applications; for example, the process of loading a single method is +very expensive, since it involves the creation and loading of a +sorrounding class. + +The new Da Vinci Machine contains a lot of interesting features that +could be effectively exploited by the PyPy JIT to produce an even more +efficient implementation of the Python language, as `John Rose said`_ +after the talk with PyPy people. + +Features of the MLVM that could be exploited by PyPy JIT include but +are not limited to: dynamic invocation, lightweight bytecode loading, +tail calls, etc. + +Implementation wise, the JIT backends for the plain JVM and for the +MLVM could share most of the code, with the latter making use of the +special features when needed. + +Moreover, the experience of this project will help the MLVM team to +understand which features are really useful to implement dynamic +languages on top of the JVM and which one we still lack. + +Deliverables +------------ + +Due to the its strict dependency on PyPy, it will not possible to +release the result of the work as a separate and independent project. +In particular, to reach the goals of the proposal it will be necessary +to extensively modify parts of PyPy that are already there, as well as +write completely new code. + +If the project goes to completion, the code developed will be +integrated into the PyPy codebase; if Sun requires to release the code +under the SCA (thus sharing the copyright between the original author +and Sun itself), we will send to Sun a document in unified diff format +that extensively shows all and sole lines of code on which Sun will +have the copyright. + + +Project completion +------------------ + +PyPy JIT is still under heavy development; potentially, the resulting +JIT compiler will be able to optimize a large number of Python +programs, but at the moment it gives the best results only with +computational intensive functions that use only operations between +integers. + +We expect to get a pypy-jvm executable that can execute a function +with those characteristics at roughly the same speed as its equivalent +written in Java, excluding the costs of the JIT compilation itself, +which have not been optimized yet. + +For an example of a function with is highly optimized by the PyPy JIT, +look at the `function f1`_: when executed by a pypy-c compiled with +JIT support, it runs roughly at the same speed as its C equivalent +compiled with `gcc -O1`. (XXX: is there a link somewhere?) + +We also expect to find benchmarks in which the JIT that targets the +MLVM will perform better thant the JIT that targets the plain JVM, +though it is hard to speficy a precise commitment here without knowing +which features of the MLVM will be possible to use. + + +Relevance to the community +-------------------------- + +Recently the community showed a lot of interests in dynamic languages +on top of the JVM. Even if right now Jython is the only usable +implementation of Python for the JVM, PyPy can potentially become the +reference implementation in the future. + +To have a working JIT for the JVM is an important step to make PyPy +the fastest Python for the JVM ever. Moreover, due to the innovative +ideas implemented by PyPy, there are chances that Python could become +the fastest dynamic language that run on the top of the JVM. + +Finally, PyPy is not limited to Python: it is entirely possible to +write interpreters for languages other than Python and translate them +with the TT; as a proof of concept, PyPy already contains partial +implementations of Prolog, JavaScript, Scheme and Smalltalk. + +Since the JIT generator is independent of the Python languages, it +will be possible to automatically add a JIT compiler to every language +written using the PyPy TT; thus, PyPy could become a very attractive +platform to develop dynamic languages for the JVM. + + +Dependencies on Sun +------------------- + +There are no dependencies on Sun regarding the implementation of a JIT +compiler that targets the plain JVM. However, in order to implement a +JIT compiler that targets the new MLVM, we need the new features we +want to exploit to be implemented. + +Related work +------------ + +Dynamic generation of bytecode for object oriented virtual machine is +a hot topic: + + - `this paper`_ shows how this technique is exploited to write an + efficient implementation of EcmaScript which runs on top of the JVM; + + - JRuby also come with a JIT compiler that dynamically translates + ruby code into JVM bytecode; the difference with the PyPy approach + is that JRuby doesn't exploit type information that are known only + at runtime to produce specialized, efficient versions of the + function; moreover, while the JRuby JIT is hand-written, the whole + goal of the PyPy JIT generator is to generate it automatically + from the intepreter; + + - in the .NET world, IronPython also emit code dynamically to + optimize hot spots, but not in a pervasive way as JRuby or PyPy. + +.. _PyPy: http://codespeak.net/pypy +.. _RPython: http://codespeak.net/pypy/dist/pypy/doc/coding-guide.html#rpython +.. _`John Rose said`: http://blogs.sun.com/jrose/entry/a_day_with_pypy +.. _`function f1`: http://codespeak.net/svn/pypy/dist/demo/jit/f1.py +.. _`this paper`: http://www.ics.uci.edu/~franz/Site/pubs-pdf/ICS-TR-07-10.pdf From cfbolz at codespeak.net Tue Feb 26 23:51:12 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 26 Feb 2008 23:51:12 +0100 (CET) Subject: [pypy-svn] r51887 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080226225112.7437216842B@codespeak.net> Author: cfbolz Date: Tue Feb 26 23:51:10 2008 New Revision: 51887 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_promotion.py Log: more passing tests Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Tue Feb 26 23:51:10 2008 @@ -180,7 +180,6 @@ assert not res def test_dfa_compile3(self): - py.test.skip("not working yet") from pypy.lang.automata.dfa import getautomaton, recognize3 def main(gets): auto = getautomaton() Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_promotion.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_promotion.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_promotion.py Tue Feb 26 23:51:10 2008 @@ -306,7 +306,6 @@ assert res == 22 def test_raise_result_mixup(self): - py.test.skip("residual calls not supported") def w(x): pass class E(Exception): @@ -341,7 +340,6 @@ res == 1 def test_raise_result_mixup_some_more(self): - py.test.skip("residual calls not supported") def w(x): if x > 1000: return None From cfbolz at codespeak.net Wed Feb 27 00:00:11 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 27 Feb 2008 00:00:11 +0100 (CET) Subject: [pypy-svn] r51888 - in pypy/branch/jit-refactoring/pypy/jit: rainbow/test timeshifter/test Message-ID: <20080226230011.E0C2916842B@codespeak.net> Author: cfbolz Date: Wed Feb 27 00:00:09 2008 New Revision: 51888 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py Log: move over more passing tests Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Wed Feb 27 00:00:09 2008 @@ -268,3 +268,59 @@ res = self.timeshift_from_portal(ll_main, ll_function, [0], policy=P_NOVIRTUAL) assert res == ord('2') self.check_insns(indirect_call=0) + + def test_isinstance(self): + class Base(object): + pass + class Int(Base): + def __init__(self, n): + self.n = n + class Str(Base): + def __init__(self, s): + self.s = s + + def ll_main(n): + if n > 0: + o = Int(n) + else: + o = Str('123') + return ll_function(o) + + def ll_function(o): + hint(o, deepfreeze=True) + hint(o, concrete=True) + x = isinstance(o, Str) + return x + + + res = self.timeshift_from_portal(ll_main, ll_function, [5], policy=P_NOVIRTUAL) + assert not res + + def test_greenmethod_call_nonpromote(self): + class Base(object): + pass + class Int(Base): + def __init__(self, n): + self.n = n + def tag(self): + return 123 + class Str(Base): + def __init__(self, s): + self.s = s + def tag(self): + return 456 + + def ll_main(n): + if n > 0: + o = Int(n) + else: + o = Str('123') + return ll_function(o) + + def ll_function(o): + hint(None, global_merge_point=True) + return o.tag() + + res = self.timeshift_from_portal(ll_main, ll_function, [5], policy=P_NOVIRTUAL) + assert res == 123 + self.check_insns(indirect_call=1) Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py Wed Feb 27 00:00:09 2008 @@ -254,61 +254,6 @@ res = self.timeshift_from_portal(main, evaluate, [4, 7]) assert res == 11 - def test_isinstance(self): - class Base(object): - pass - class Int(Base): - def __init__(self, n): - self.n = n - class Str(Base): - def __init__(self, s): - self.s = s - - def ll_main(n): - if n > 0: - o = Int(n) - else: - o = Str('123') - return ll_function(o) - - def ll_function(o): - hint(o, deepfreeze=True) - hint(o, concrete=True) - x = isinstance(o, Str) - return x - - - res = self.timeshift_from_portal(ll_main, ll_function, [5], policy=P_NOVIRTUAL) - assert not res - - def test_greenmethod_call_nonpromote(self): - class Base(object): - pass - class Int(Base): - def __init__(self, n): - self.n = n - def tag(self): - return 123 - class Str(Base): - def __init__(self, s): - self.s = s - def tag(self): - return 456 - - def ll_main(n): - if n > 0: - o = Int(n) - else: - o = Str('123') - return ll_function(o) - - def ll_function(o): - hint(None, global_merge_point=True) - return o.tag() - - res = self.timeshift_from_portal(ll_main, ll_function, [5], policy=P_NOVIRTUAL) - assert res == 123 - self.check_insns(indirect_call=1) def test_residual_red_call_with_promoted_exc(self): def h(x): From cfbolz at codespeak.net Wed Feb 27 00:10:57 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 27 Feb 2008 00:10:57 +0100 (CET) Subject: [pypy-svn] r51890 - in pypy/branch/jit-refactoring/pypy/jit: rainbow/test timeshifter/test Message-ID: <20080226231057.51BE3168418@codespeak.net> Author: cfbolz Date: Wed Feb 27 00:10:56 2008 New Revision: 51890 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py Log: even more passing tests Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Wed Feb 27 00:10:56 2008 @@ -324,3 +324,55 @@ res = self.timeshift_from_portal(ll_main, ll_function, [5], policy=P_NOVIRTUAL) assert res == 123 self.check_insns(indirect_call=1) + + def test_cast_ptr_to_int(self): + GCS1 = lltype.GcStruct('s1', ('x', lltype.Signed)) + def g(p): + return lltype.cast_ptr_to_int(p) + def f(): + p = lltype.malloc(GCS1) + return g(p) - lltype.cast_ptr_to_int(p) + + res = self.timeshift_from_portal(f, g, [], policy=P_NOVIRTUAL) + assert res == 0 + + + def test_virt_obj_method_call_promote(self): + class Base(object): + pass + class Int(Base): + def __init__(self, n): + self.n = n + def double(self): + return Int(self.n * 2) + def get(self): + return self.n + class Str(Base): + def __init__(self, s): + self.s = s + def double(self): + return Str(self.s + self.s) + def get(self): + return ord(self.s[4]) + + def ll_make(n): + if n > 0: + return Int(n) + else: + return Str('123') + + def ll_function(n): + hint(None, global_merge_point=True) + o = ll_make(n) + hint(o.__class__, promote=True) + return o.double().get() + + res = self.timeshift_from_portal(ll_function, ll_function, [5], + policy=StopAtXPolicy(ll_make)) + assert res == 10 + self.check_insns(indirect_call=0, malloc=0) + + res = self.timeshift_from_portal(ll_function, ll_function, [0], + policy=StopAtXPolicy(ll_make)) + assert res == ord('2') + self.check_insns(indirect_call=0, malloc=0) Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py Wed Feb 27 00:10:56 2008 @@ -144,45 +144,6 @@ - def test_virt_obj_method_call_promote(self): - class Base(object): - pass - class Int(Base): - def __init__(self, n): - self.n = n - def double(self): - return Int(self.n * 2) - def get(self): - return self.n - class Str(Base): - def __init__(self, s): - self.s = s - def double(self): - return Str(self.s + self.s) - def get(self): - return ord(self.s[4]) - - def ll_make(n): - if n > 0: - return Int(n) - else: - return Str('123') - - def ll_function(n): - hint(None, global_merge_point=True) - o = ll_make(n) - hint(o.__class__, promote=True) - return o.double().get() - - res = self.timeshift_from_portal(ll_function, ll_function, [5], - policy=StopAtXPolicy(ll_make)) - assert res == 10 - self.check_insns(indirect_call=0, malloc=0) - - res = self.timeshift_from_portal(ll_function, ll_function, [0], - policy=StopAtXPolicy(ll_make)) - assert res == ord('2') - self.check_insns(indirect_call=0, malloc=0) def test_simple_recursive_portal_call(self): @@ -301,17 +262,6 @@ res = self.timeshift_from_portal(f, f, [15], policy=P_OOPSPEC) assert res == 15 - def test_cast_ptr_to_int(self): - GCS1 = lltype.GcStruct('s1', ('x', lltype.Signed)) - def g(p): - return lltype.cast_ptr_to_int(p) - def f(): - p = lltype.malloc(GCS1) - return g(p) - lltype.cast_ptr_to_int(p) - - res = self.timeshift_from_portal(f, g, [], policy=P_NOVIRTUAL) - assert res == 0 - def test_portal_returns_none(self): py.test.skip("portal returning None is not supported") def g(x): From pypy-svn at codespeak.net Wed Feb 27 05:43:45 2008 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Wed, 27 Feb 2008 05:43:45 +0100 (CET) Subject: [pypy-svn] February 72% OFF Message-ID: <20080227103924.5074.qmail@adsl-pool243-165.vologda.ru> An HTML attachment was scrubbed... URL: From arigo at codespeak.net Wed Feb 27 09:16:23 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 27 Feb 2008 09:16:23 +0100 (CET) Subject: [pypy-svn] r51891 - pypy/extradoc/proposal Message-ID: <20080227081623.38AE71683E0@codespeak.net> Author: arigo Date: Wed Feb 27 09:16:21 2008 New Revision: 51891 Modified: pypy/extradoc/proposal/openjdk-challenge.txt Log: Comments. Modified: pypy/extradoc/proposal/openjdk-challenge.txt ============================================================================== --- pypy/extradoc/proposal/openjdk-challenge.txt (original) +++ pypy/extradoc/proposal/openjdk-challenge.txt Wed Feb 27 09:16:21 2008 @@ -115,6 +115,7 @@ look at the `function f1`_: when executed by a pypy-c compiled with JIT support, it runs roughly at the same speed as its C equivalent compiled with `gcc -O1`. (XXX: is there a link somewhere?) +(XXX it's only gcc -O0, not -O1) We also expect to find benchmarks in which the JIT that targets the MLVM will perform better thant the JIT that targets the plain JVM, @@ -174,6 +175,12 @@ - in the .NET world, IronPython also emit code dynamically to optimize hot spots, but not in a pervasive way as JRuby or PyPy. + XXX (arigo) I'm confused by the previous sentence. I thought + that IronPython and Jython both emitted .NET/JVM bytecode as their + only way to compile Python source. I also imagined that JRuby + did exactly the same. I.e. all of IronPython, Jython and JRuby + work by turning each source unit to native bytecode by a direct + translation - no? .. _PyPy: http://codespeak.net/pypy From jared.grubb at codespeak.net Wed Feb 27 09:31:36 2008 From: jared.grubb at codespeak.net (jared.grubb at codespeak.net) Date: Wed, 27 Feb 2008 09:31:36 +0100 (CET) Subject: [pypy-svn] r51892 - in pypy/dist/pypy/rlib/parsing: . test Message-ID: <20080227083136.D9DB61683E0@codespeak.net> Author: jared.grubb Date: Wed Feb 27 09:31:36 2008 New Revision: 51892 Modified: pypy/dist/pypy/rlib/parsing/deterministic.py pypy/dist/pypy/rlib/parsing/regexparse.py pypy/dist/pypy/rlib/parsing/test/test_pcre_regtest.py pypy/dist/pypy/rlib/parsing/test/test_regexparse.py Log: rlib.parsing.regexparse: add \d\D\w\W\s\S char classes; make []x] a valid char class that tests for ']' or 'x'; add \e escape option rlib.parsing.deterministic: remove escaping of ] that i put in last time rlib.parsing.test.test_regexparse: add tests for new regex features rlib.parsing.test.test_pcre_regtest: supports testing MANY more of the PCRE tests... still not 100%, but getting there! Modified: pypy/dist/pypy/rlib/parsing/deterministic.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/deterministic.py (original) +++ pypy/dist/pypy/rlib/parsing/deterministic.py Wed Feb 27 09:31:36 2008 @@ -34,7 +34,7 @@ def make_nice_charset_repr(chars): # Compress the letters & digits letters = set(chars) & set("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") - therest = set(chars) - letters - set(['-',']']) + therest = set(chars) - letters - set(['-']) charranges = compress_char_set(letters) result = [] for a, num in charranges: @@ -49,8 +49,6 @@ # Handle the special chars that MUST get escaped if '-' in chars: result += ['\\-'] - if ']' in chars: - result += ['\\]'] return "".join(result) class LexerError(Exception): Modified: pypy/dist/pypy/rlib/parsing/regexparse.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/regexparse.py (original) +++ pypy/dist/pypy/rlib/parsing/regexparse.py Wed Feb 27 09:31:36 2008 @@ -11,6 +11,7 @@ ESCAPES = { "\\a": "\a", "\\b": "\b", + "\\e": "\x1b", "\\f": "\f", "\\n": "\n", "\\r": "\r", @@ -20,7 +21,7 @@ } for i in range(256): - if chr(i) not in 'x01234567': + if chr(i) not in 'x01234567sSwWdD': # 'x' and numbers are reserved for hexadecimal/octal escapes escaped = "\\" + chr(i) if escaped not in ESCAPES: @@ -120,6 +121,8 @@ primary: ['('] regex [')'] | range + | cc = charclass + return {reduce(operator.or_, [regex.RangeExpression(a, chr(ord(a) + b - 1)) for a, b in compress_char_set(cc)])} | c = char return {regex.StringExpression(c)} | '.' @@ -133,7 +136,7 @@ QUOTEDCHAR: `(\\x[0-9a-fA-F]{2})|(\\[0-3]?[0-7][0-7])|(\\c.)|(\\.)`; - + CHAR: `[^\*\+\(\)\[\]\{\}\|\.\-\?\,\^]`; @@ -150,11 +153,15 @@ | subrange; subrange: + ']' l = rangeelement+ + return {reduce(operator.or_, [set(["]"])] + l)} + | l = rangeelement+ return {reduce(operator.or_, l)}; rangeelement: - c1 = char + charclass + | c1 = char '-' c2 = char return {set([chr(i) for i in range(ord(c1), ord(c2) + 1)])} @@ -174,6 +181,25 @@ ',' return {n1}; +charclass: + '\' 'd' + return { set([chr(c) for c in range(ord('0'), ord('9')+1)]) } + | '\' + 's' + return { set(['\t', '\n', '\f', '\r', ' ']) } + | '\' + 'w' + return { set([chr(c) for c in range(ord('a'), ord('z')+1)] + [chr(c) for c in range(ord('A'), ord('Z')+1)] + [chr(c) for c in range(ord('0'), ord('9')+1)] + ['_']) } + | '\' + 'D' + return { set([chr(c) for c in range(256)]) - set([chr(c) for c in range(ord('0'), ord('9')+1)]) } + | '\' + 'S' + return { set([chr(c) for c in range(256)]) - set(['\t', '\n', '\f', '\r', ' ']) } + | '\' + 'W' + return { set([chr(c) for c in range(256)]) - set([chr(c) for c in range(ord('a'), ord('z')+1)] + [chr(c) for c in range(ord('A'), ord('Z')+1)] + [chr(c) for c in range(ord('0'), ord('9')+1)] + ['_'])}; + NUM: c = `0|([1-9][0-9]*)` return {int(c)}; @@ -684,6 +710,17 @@ self._pos = _choice3 _choice4 = self._pos try: + _call_status = self._charclass() + _result = _call_status.result + _error = self._combine_errors(_error, _call_status.error) + cc = _result + _result = (reduce(operator.or_, [regex.RangeExpression(a, chr(ord(a) + b - 1)) for a, b in compress_char_set(cc)])) + break + except BacktrackException, _exc: + _error = self._combine_errors(_error, _exc.error) + self._pos = _choice4 + _choice5 = self._pos + try: _call_status = self._char() _result = _call_status.result _error = self._combine_errors(_error, _call_status.error) @@ -692,15 +729,15 @@ break except BacktrackException, _exc: _error = self._combine_errors(_error, _exc.error) - self._pos = _choice4 - _choice5 = self._pos + self._pos = _choice5 + _choice6 = self._pos try: _result = self.__chars__('.') _result = (regex.RangeExpression(chr(0), chr(255))) break except BacktrackException, _exc: _error = self._combine_errors(_error, _exc.error) - self._pos = _choice5 + self._pos = _choice6 raise BacktrackException(_error) _result = self.__chars__('.') _result = (regex.RangeExpression(chr(0), chr(255))) @@ -1041,25 +1078,79 @@ try: _result = None _error = None - _all0 = [] - _call_status = self._rangeelement() - _result = _call_status.result - _error = _call_status.error - _all0.append(_result) while 1: - _choice1 = self._pos + _choice0 = self._pos try: + _result = self.__chars__(']') + _all1 = [] _call_status = self._rangeelement() _result = _call_status.result - _error = self._combine_errors(_error, _call_status.error) - _all0.append(_result) + _error = _call_status.error + _all1.append(_result) + while 1: + _choice2 = self._pos + try: + _call_status = self._rangeelement() + _result = _call_status.result + _error = self._combine_errors(_error, _call_status.error) + _all1.append(_result) + except BacktrackException, _exc: + _error = self._combine_errors(_error, _exc.error) + self._pos = _choice2 + break + _result = _all1 + l = _result + _result = (reduce(operator.or_, [set(["]"])] + l)) + break except BacktrackException, _exc: _error = self._combine_errors(_error, _exc.error) - self._pos = _choice1 + self._pos = _choice0 + _choice3 = self._pos + try: + _all4 = [] + _call_status = self._rangeelement() + _result = _call_status.result + _error = self._combine_errors(_error, _call_status.error) + _all4.append(_result) + while 1: + _choice5 = self._pos + try: + _call_status = self._rangeelement() + _result = _call_status.result + _error = self._combine_errors(_error, _call_status.error) + _all4.append(_result) + except BacktrackException, _exc: + _error = self._combine_errors(_error, _exc.error) + self._pos = _choice5 + break + _result = _all4 + l = _result + _result = (reduce(operator.or_, l)) break - _result = _all0 - l = _result - _result = (reduce(operator.or_, l)) + except BacktrackException, _exc: + _error = self._combine_errors(_error, _exc.error) + self._pos = _choice3 + raise BacktrackException(_error) + _all6 = [] + _call_status = self._rangeelement() + _result = _call_status.result + _error = self._combine_errors(_error, _call_status.error) + _all6.append(_result) + while 1: + _choice7 = self._pos + try: + _call_status = self._rangeelement() + _result = _call_status.result + _error = self._combine_errors(_error, _call_status.error) + _all6.append(_result) + except BacktrackException, _exc: + _error = self._combine_errors(_error, _exc.error) + self._pos = _choice7 + break + _result = _all6 + l = _result + _result = (reduce(operator.or_, l)) + break if _status.status == _status.LEFTRECURSION: if _status.result is not None: if _status.pos >= self._pos: @@ -1115,9 +1206,18 @@ while 1: _choice0 = self._pos try: - _call_status = self._char() + _call_status = self._charclass() _result = _call_status.result _error = _call_status.error + break + except BacktrackException, _exc: + _error = self._combine_errors(_error, _exc.error) + self._pos = _choice0 + _choice1 = self._pos + try: + _call_status = self._char() + _result = _call_status.result + _error = self._combine_errors(_error, _call_status.error) c1 = _result _result = self.__chars__('-') _call_status = self._char() @@ -1128,8 +1228,8 @@ break except BacktrackException, _exc: _error = self._combine_errors(_error, _exc.error) - self._pos = _choice0 - _choice1 = self._pos + self._pos = _choice1 + _choice2 = self._pos try: _call_status = self._char() _result = _call_status.result @@ -1139,7 +1239,7 @@ break except BacktrackException, _exc: _error = self._combine_errors(_error, _exc.error) - self._pos = _choice1 + self._pos = _choice2 raise BacktrackException(_error) _call_status = self._char() _result = _call_status.result @@ -1316,6 +1416,97 @@ _status.error = _error _status.status = _status.ERROR raise BacktrackException(_error) + def charclass(self): + return self._charclass().result + def _charclass(self): + _key = self._pos + _status = self._dict_charclass.get(_key, None) + if _status is None: + _status = self._dict_charclass[_key] = Status() + else: + _statusstatus = _status.status + if _statusstatus == _status.NORMAL: + self._pos = _status.pos + return _status + elif _statusstatus == _status.ERROR: + raise BacktrackException(_status.error) + _startingpos = self._pos + try: + _result = None + _error = None + while 1: + _choice0 = self._pos + try: + _result = self.__chars__('\\') + _result = self.__chars__('d') + _result = ( set([chr(c) for c in range(ord('0'), ord('9')+1)]) ) + break + except BacktrackException, _exc: + _error = _exc.error + self._pos = _choice0 + _choice1 = self._pos + try: + _result = self.__chars__('\\') + _result = self.__chars__('s') + _result = ( set(['\t', '\n', '\f', '\r', ' ']) ) + break + except BacktrackException, _exc: + _error = self._combine_errors(_error, _exc.error) + self._pos = _choice1 + _choice2 = self._pos + try: + _result = self.__chars__('\\') + _result = self.__chars__('w') + _result = ( set([chr(c) for c in range(ord('a'), ord('z')+1)] + [chr(c) for c in range(ord('A'), ord('Z')+1)] + [chr(c) for c in range(ord('0'), ord('9')+1)] + ['_']) ) + break + except BacktrackException, _exc: + _error = self._combine_errors(_error, _exc.error) + self._pos = _choice2 + _choice3 = self._pos + try: + _result = self.__chars__('\\') + _result = self.__chars__('D') + _result = ( set([chr(c) for c in range(256)]) - set([chr(c) for c in range(ord('0'), ord('9')+1)]) ) + break + except BacktrackException, _exc: + _error = self._combine_errors(_error, _exc.error) + self._pos = _choice3 + _choice4 = self._pos + try: + _result = self.__chars__('\\') + _result = self.__chars__('S') + _result = ( set([chr(c) for c in range(256)]) - set(['\t', '\n', '\f', '\r', ' ']) ) + break + except BacktrackException, _exc: + _error = self._combine_errors(_error, _exc.error) + self._pos = _choice4 + _choice5 = self._pos + try: + _result = self.__chars__('\\') + _result = self.__chars__('W') + _result = ( set([chr(c) for c in range(256)]) - set([chr(c) for c in range(ord('a'), ord('z')+1)] + [chr(c) for c in range(ord('A'), ord('Z')+1)] + [chr(c) for c in range(ord('0'), ord('9')+1)] + ['_'])) + break + except BacktrackException, _exc: + _error = self._combine_errors(_error, _exc.error) + self._pos = _choice5 + raise BacktrackException(_error) + _result = self.__chars__('\\') + _result = self.__chars__('W') + _result = ( set([chr(c) for c in range(256)]) - set([chr(c) for c in range(ord('a'), ord('z')+1)] + [chr(c) for c in range(ord('A'), ord('Z')+1)] + [chr(c) for c in range(ord('0'), ord('9')+1)] + ['_'])) + break + assert _status.status != _status.LEFTRECURSION + _status.status = _status.NORMAL + _status.pos = self._pos + _status.result = _result + _status.error = _error + return _status + except BacktrackException, _exc: + _status.pos = -1 + _status.result = None + _error = self._combine_errors(_error, _exc.error) + _status.error = _error + _status.status = _status.ERROR + raise BacktrackException(_error) def NUM(self): return self._NUM().result def _NUM(self): @@ -1366,6 +1557,7 @@ self._dict_rangeelement = {} self._dict_numrange = {} self._dict_clippednumrange = {} + self._dict_charclass = {} self._dict_NUM = {} self._pos = 0 self._inputstream = inputstream @@ -1663,6 +1855,13 @@ + + + + + + + def test_generate(): f = py.magic.autopath() oldcontent = f.read() Modified: pypy/dist/pypy/rlib/parsing/test/test_pcre_regtest.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/test/test_pcre_regtest.py (original) +++ pypy/dist/pypy/rlib/parsing/test/test_pcre_regtest.py Wed Feb 27 09:31:36 2008 @@ -6,7 +6,7 @@ # files with pypy?) import py -from pypy.rlib.parsing.regexparse import make_runner, unescape, RegexParser +from pypy.rlib.parsing.regexparse import make_runner, unescape import string import re @@ -25,66 +25,75 @@ return test -def get_definition_line(tests, results): - """Gets a test definition line, formatted per the PCRE spec.""" - delim = None - test = '' - result = '' - - # A line is marked by a start-delimeter and an end-delimeter. - # The delimeter is non-alphanumeric - # If a backslash follows the delimiter, then the backslash should - # be appended to the end. (Otherwise, \ + delim would not be a - # delim anymore!) - while 1: - test += get_simult_lines(tests, results) +def create_regex_iterator(tests, results): + """Gets a test definition line, formatted per the PCRE spec. This is a + generator that returns each regex test.""" + while tests: + delim = None + regex = '' - if delim is None: - delim = test[0] - assert delim in (set(string.printable) - set(string.letters) - set(string.digits)) - test_re = re.compile(r'%(delim)s(([^%(delim)s]|\\%(delim)s)*([^\\]))%(delim)s(\\?)(.*)' % {'delim': delim}) - - matches = test_re.findall(test) - if matches: - break - - assert len(matches)==1 - test = matches[0][0] + # A line is marked by a start-delimeter and an end-delimeter. + # The delimeter is non-alphanumeric + # If a backslash follows the delimiter, then the backslash should + # be appended to the end. (Otherwise, \ + delim would not be a + # delim anymore!) + while 1: + regex += get_simult_lines(tests, results) - # Add the backslash, if we gotta - test += matches[0][-2] - flags = matches[0][-1] + if delim is None: + delim = regex[0] + assert delim in (set(string.printable) - set(string.letters) - set(string.digits)) + test_re = re.compile(r'%(delim)s(([^%(delim)s]|\\%(delim)s)*([^\\]))%(delim)s(\\?)(.*)' % {'delim': delim}) + # last two groups are an optional backslash and optional flags + + matches = test_re.findall(regex) + if matches: + break - return test, flags + assert len(matches)==1 -def get_test_result(tests, results): - """Gets the expected return from the regular expression""" + regex = matches[0][0] + regex += matches[0][-2] # Add the backslash, if we gotta + flags = matches[0][-1] # Get the flags for the regex + + yield regex, flags + +def create_result_iterator(tests, results): + """Gets the expected return sets for each regular expression.""" # Second line is the test to run against the regex # ' TEXT' - test = get_simult_lines(tests, results) - if not test: - return None, None - if not test.startswith(' '): - raise Exception("Input & output match, but I don't understand. (Got %r)" % test) - test = unescape(test[4:]) - - # Third line in the OUTPUT is the result, either: - # ' 0: ...' for a match - # 'No match' for no match - result = unescape(results.pop(0)) - if result == 'No match': - pass - elif result.startswith(' 0: '): - # Now we need to eat any further lines like: - # ' 1: ....' a subgroup match - while results[0]: - if results[0][2] == ':': - results.pop(0) - else: - break - else: - raise Exception("Lost sync in output.") - return test, result + while 1: + test = get_simult_lines(tests, results) + if not test: + raise StopIteration + if not test.startswith(' '): + raise Exception("Input & output match, but I don't understand. (Got %r)" % test) + if test.endswith('\\'): # Tests that end in \ expect the \ to be chopped off + assert not test.endswith('\\\\') # make sure there are no \\ at end + test = test[:-1] + test = unescape(test[4:]) + + # Third line in the OUTPUT is the result, either: + # ' 0: ...' for a match (but this is ONLY escaped by \x__ types) + # 'No match' for no match + result = results.pop(0) + result = re.sub(r'\\x([0-9a-fA-F]{2})', lambda m: chr(int(m.group(1),16)), result) + if result == 'No match': + pass + elif result.startswith(' 0:'): + # Now we need to eat any further lines like: + # ' 1: ....' a subgroup match + while results[0]: + if results[0][2] == ':': + results.pop(0) + else: + break + else: + raise Exception("Lost sync in output.") + yield test, result + +class SkipException(Exception): + pass def test_file(): """Open the PCRE tests and run them.""" @@ -95,60 +104,62 @@ 'i': lambda s: s.upper() } + regex_set = create_regex_iterator(tests, results) import pdb - while tests: - # First line is a test, in the form: - # '/regex expression/FLAGS' - regex, regex_flags = get_definition_line(tests, results) - - # Handle the flags: + for regex, regex_flags in regex_set: try: - text_prepare = regex_flag_mapping[regex_flags] - except KeyError: - print "UNKNOWN FLAGS: %s" % regex_flags - continue - - print '%r' % regex + print '%r' % regex - skipped = any([op in regex for op in ['*?', '??', '+?', '}?']]) - if skipped: - print " SKIPPED (cant do non-greedy operators)" - # now burn all the tests for this regex - while 1: - test, result = get_test_result(tests, results) - if not test: - break # A blank line means we have nothing to do - continue + # Create an iterator to grab the test/results for this regex + result_set = create_result_iterator(tests, results) + + # Handle the flags: + if regex_flags in regex_flag_mapping: + text_prepare = regex_flag_mapping[regex_flags] + elif 'x' in regex_flags: + raise SkipException("Cant do extended PRCE expressions") + else: + print "UNKNOWN FLAGS: %s" % regex_flags + continue + + skipped = any([op in regex for op in ['*?', '??', '+?', '}?', '(?']]) + if skipped: + raise SkipException("Cant do non-greedy operators or '(?' constructions)") - regex_to_use = text_prepare(regex) + regex_to_use = text_prepare(regex) - anchor_left = regex_to_use.startswith('^') - anchor_right = regex_to_use.endswith('$') and not regex_to_use.endswith('\\$') - if anchor_left: - regex_to_use = regex_to_use[1:] # chop the ^ if it's there - if anchor_right: - regex_to_use = regex_to_use[:-1] # chop the $ if it's there + anchor_left = regex_to_use.startswith('^') + anchor_right = regex_to_use.endswith('$') and not regex_to_use.endswith('\\$') + if anchor_left: + regex_to_use = regex_to_use[1:] # chop the ^ if it's there + if anchor_right: + regex_to_use = regex_to_use[:-1] # chop the $ if it's there + if not regex_to_use: + raise SkipException("Cant do blank regex") + except SkipException, e: + print " SKIPPED (%s)" % e.message + # now burn all the tests for this regex + for _ in result_set: + pass + continue + # Finally, we make the pypy regex runner runner = make_runner(regex_to_use) - + # Now run the test expressions against the Regex - while 1: - test, result = get_test_result(tests, results) - if not test: - break # A blank line means we have nothing to do - + for test, result in result_set: # Create possible subsequences that we should test if anchor_left: - subseq_gen = [0] + start_range = [0] else: - subseq_gen = (start for start in range(0, len(test))) + start_range = range(0, len(test)) if anchor_right: - subseq_gen = ( (start, len(test)) for start in subseq_gen ) + subseq_gen = ( (start, len(test)) for start in start_range ) else: # Go backwards to simulate greediness - subseq_gen = ( (start, end) for start in subseq_gen for end in range(len(test)+1, start+1, -1) ) + subseq_gen = ( (start, end) for start in start_range for end in range(len(test)+1, start, -1) ) # Search the possibilities for a match... for start, end in subseq_gen: @@ -162,11 +173,11 @@ if matched: print " FALSE MATCH: regex==%r test==%r" % (regex, test) else: - print " pass : regex==%r test==%r" % (regex, test) + print " pass: regex==%r test==%r" % (regex, test) elif result.startswith(' 0: '): if not matched: print " MISSED: regex==%r test==%r" % (regex, test) elif not attempt==text_prepare(result[4:]): print " BAD MATCH: regex==%r test==%r found==%r expect==%r" % (regex, test, attempt, result[4:]) else: - print " pass : regex==%r test==%r" % (regex, test) + print " pass: regex==%r test==%r" % (regex, test) Modified: pypy/dist/pypy/rlib/parsing/test/test_regexparse.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/test/test_regexparse.py (original) +++ pypy/dist/pypy/rlib/parsing/test/test_regexparse.py Wed Feb 27 09:31:36 2008 @@ -33,6 +33,11 @@ assert r.recognize("aaaaaa") assert not r.recognize("a") assert not r.recognize("aabb") + r = make_runner("(\\x61a)*") + assert r.recognize("aa") + assert r.recognize("aaaaaa") + assert not r.recognize("a") + assert not r.recognize("aabb") def test_range(): r = make_runner("[A-Z]") @@ -165,3 +170,64 @@ assert r.recognize("-0.912E+0001") assert not r.recognize("-0.a912E+0001") assert r.recognize("5") + +def test_charclass(): + r = make_runner(r"\d") + assert r.recognize('0') + assert r.recognize('5') + assert r.recognize('9') + assert not r.recognize('d') + r = make_runner(r"\d{2,}") + assert r.recognize('09') + assert r.recognize('158') + assert not r.recognize('1') + r = make_runner(r"\D") + assert r.recognize('d') + assert r.recognize('\n') + assert not r.recognize('0') + assert not r.recognize('1234') + r = make_runner(r"\s\S") + assert r.recognize(' d') + assert r.recognize('\t9') + assert not r.recognize('d ') + assert not r.recognize('99') + assert not r.recognize('\r\r') + r = make_runner(r"\w+") + assert r.recognize('word') + assert r.recognize('variable_name') + assert r.recognize('abc123') + assert not r.recognize('word\n') + assert not r.recognize('hey hey') + r = make_runner(r"\w\W\w") + assert r.recognize('9 9') + assert r.recognize('_\fx') + assert not r.recognize('\n\r\t') + +def test_charclass_in_range(): + r = make_runner(r"[\de]") + assert r.recognize('0') + assert r.recognize('5') + assert r.recognize('9') + assert r.recognize('e') + assert not r.recognize('d') + r = make_runner(r"[\de]{2,}") + assert r.recognize('09') + assert r.recognize('158') + assert r.recognize('3eee') + assert not r.recognize('1') + assert not r.recognize('ddee') + r = make_runner(r"[\D5]") + assert r.recognize('d') + assert r.recognize('\n') + assert r.recognize('5') + assert not r.recognize('0') + r = make_runner(r"[\s][\S]") + assert r.recognize(' d') + assert r.recognize('\t9') + assert not r.recognize('d ') + assert not r.recognize('99') + assert not r.recognize('\r\r') + r = make_runner(r"[\w]+\W[\w]+") + assert r.recognize('hey hey') + assert not r.recognize('word') + assert not r.recognize('variable_name') From arigo at codespeak.net Wed Feb 27 10:14:32 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 27 Feb 2008 10:14:32 +0100 (CET) Subject: [pypy-svn] r51893 - pypy/dist/pypy/rpython/memory/gc Message-ID: <20080227091432.C1E001683D9@codespeak.net> Author: arigo Date: Wed Feb 27 10:14:30 2008 New Revision: 51893 Modified: pypy/dist/pypy/rpython/memory/gc/generation.py Log: A tweak to remove a (fast but) pointless run-time check in all mallocs. Modified: pypy/dist/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gc/generation.py (original) +++ pypy/dist/pypy/rpython/memory/gc/generation.py Wed Feb 27 10:14:30 2008 @@ -52,6 +52,15 @@ self.young_objects_with_weakrefs = self.AddressStack() self.reset_nursery() + # compute the constant lower bounds for the attributes + # largest_young_fixedsize and largest_young_var_basesize. + # It is expected that most (or all) objects have a fixedsize + # that is much lower anyway. + sz = self.get_young_fixedsize(self.min_nursery_size) + self.lb_young_fixedsize = sz + sz = self.get_young_var_basesize(self.min_nursery_size) + self.lb_young_var_basesize = sz + def setup(self): SemiSpaceGC.setup(self) self.set_nursery_size(self.initial_nursery_size) @@ -77,8 +86,8 @@ # Compute the new bounds for how large young objects can be # (larger objects are allocated directly old). XXX adjust self.nursery_size = newsize - self.largest_young_fixedsize = newsize // 2 - 1 - self.largest_young_var_basesize = newsize // 4 - 1 + self.largest_young_fixedsize = self.get_young_fixedsize(newsize) + self.largest_young_var_basesize = self.get_young_var_basesize(newsize) scale = 0 while (self.min_nursery_size << (scale+1)) <= newsize: scale += 1 @@ -99,13 +108,24 @@ # a new nursery (e.g. if it invokes finalizers). self.semispace_collect() + def get_young_fixedsize(self, nursery_size): + return nursery_size // 2 - 1 + + def get_young_var_basesize(self, nursery_size): + return nursery_size // 4 - 1 + def is_in_nursery(self, addr): return self.nursery <= addr < self.nursery_top def malloc_fixedsize_clear(self, typeid, size, can_collect, has_finalizer=False, contains_weakptr=False): if (has_finalizer or not can_collect or - raw_malloc_usage(size) > self.largest_young_fixedsize): + (raw_malloc_usage(size) > self.lb_young_var_basesize and + raw_malloc_usage(size) > self.largest_young_fixedsize)): + # ^^^ we do two size comparisons; the first one appears redundant, + # but it can be constant-folded if 'size' is a constant; then + # it almost always folds down to False, which kills the + # second comparison as well. ll_assert(not contains_weakptr, "wrong case for mallocing weakref") # "non-simple" case or object too big: don't use the nursery return SemiSpaceGC.malloc_fixedsize_clear(self, typeid, size, @@ -160,7 +180,12 @@ if (has_finalizer or not can_collect or too_many_items or - raw_malloc_usage(size) > self.largest_young_var_basesize): + (raw_malloc_usage(size) > self.lb_young_var_basesize and + raw_malloc_usage(size) > self.largest_young_var_basesize)): + # ^^^ we do two size comparisons; the first one appears redundant, + # but it can be constant-folded if 'size' is a constant; then + # it almost always folds down to False, which kills the + # second comparison as well. return SemiSpaceGC.malloc_varsize_clear(self, typeid, length, size, itemsize, offset_to_length, can_collect, has_finalizer) From antocuni at codespeak.net Wed Feb 27 10:15:05 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 27 Feb 2008 10:15:05 +0100 (CET) Subject: [pypy-svn] r51894 - pypy/dist/pypy/jit/codegen/cli/test Message-ID: <20080227091505.9E4CF1683D9@codespeak.net> Author: antocuni Date: Wed Feb 27 10:15:05 2008 New Revision: 51894 Modified: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Log: explicitly skip all the missing tests Modified: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Wed Feb 27 10:15:05 2008 @@ -4,53 +4,6 @@ from pypy.jit.codegen.test.rgenop_tests import AbstractRGenOpTests, OOType from pypy.translator.cli.test.runtest import compile_function -passing = set() -def fn(): - prefixes = [ - 'test_adder', - 'test_dummy', - 'test_hide_and_reveal', - 'test_hide_and_reveal_p', - 'test_largedummy_direct', # _compile works if we set a higher maxstack - 'test_branching', - 'test_goto', - 'test_if', - # 'test_switch', # no promotion/flexswitch for now please :-) - # 'test_defaultonly_switch', # the same - 'test_fact', - 'test_calling_pause', - 'test_longwinded_and', - 'test_condition_result_cross_link_direct', - 'test_multiple_cmps', - 'test_flipped_cmp_with_immediate', - 'test_tight_loop', - 'test_jump_to_block_with_many_vars', - 'test_same_as', - 'test_pause_and_resume', - 'test_like_residual_red_call_with_exc', - 'test_call_functions_with_different_signatures', - 'test_bool_not_direct', - # 'test_read_frame_var', # not for now - # 'test_write_frame_place', - # 'test_write_lots_of_frame_places_direct', - # 'test_read_frame_place_direct', - # 'test_read_frame_place_compile' - # 'test_frame_vars_like_the_frontend_direct', - 'test_unaliasing_variables_direct', - # 'test_from_random_direct', # mono crashes - 'test_from_random_2_direct', - # 'test_from_random_3_direct', # we need yet another delegate type - 'test_from_random_4_direct', - # 'test_from_random_5_direct', # we need yet another delegate type - ] - - for p in prefixes: - passing.add(p) - passing.add(p + '_direct') - passing.add(p + '_compile') -fn() -del fn - class TestRCliGenop(AbstractRGenOpTests): RGenOp = RCliGenOp T = OOType @@ -72,10 +25,80 @@ def directtesthelper(self, FUNCTYPE, func): py.test.skip('???') - def __getattribute__(self, name): - if name.startswith('test_') and name not in passing: - def fn(): - py.test.skip("doesn't work yet") - return fn - else: - return object.__getattribute__(self, name) + def test_largedummy_compile(self): + py.test.skip('it works only if we increase .maxstack') + + def test_switch_direct(self): + py.test.skip('no promotion/flexswitch for now please :-)') + + def test_switch_compile(self): + py.test.skip('no promotion/flexswitch for now please :-)') + + def test_large_switch_direct(self): + py.test.skip('no promotion/flexswitch for now please :-)') + + def test_large_switch_compile(self): + py.test.skip('no promotion/flexswitch for now please :-)') + + def test_defaultonly_switch(self): + py.test.skip('no promotion/flexswitch for now please :-)') + + def test_read_frame_var_direct(self): + py.test.skip('fixme: add support for frames') + + def test_read_frame_var_compile(self): + py.test.skip('fixme: add support for frames') + + def test_write_frame_place_direct(self): + py.test.skip('fixme: add support for frames') + + def test_write_frame_place_compile(self): + py.test.skip('fixme: add support for frames') + + def test_write_lots_of_frame_places_direct(self): + py.test.skip('fixme: add support for frames') + + def test_read_frame_place_direct(self): + py.test.skip('fixme: add support for frames') + + def test_read_frame_place_compile(self): + py.test.skip('fixme: add support for frames') + + def test_frame_vars_like_the_frontend_direct(self): + py.test.skip('fixme: add support for frames') + + def test_from_random_direct(self): + py.test.skip('mono crashes') + + def test_from_random_3_direct(self): + py.test.skip('we need yet another delegate type') + + def test_from_random_5_direct(self): + py.test.skip('we need yet another delegate type') + + def test_genzeroconst(self): + py.test.skip('fixme') + + def test_ovfcheck_adder_direct(self): + py.test.skip('fixme') + + def test_ovfcheck_adder_compile(self): + py.test.skip('fixme') + + def test_ovfcheck1_direct(self): + py.test.skip('fixme') + + def test_ovfcheck1_compile(self): + py.test.skip('fixme') + + def test_ovfcheck2_direct(self): + py.test.skip('fixme') + + def test_cast_direct(self): + py.test.skip('fixme') + + def test_array_of_ints(self): + py.test.skip('fixme') + + def test_interior_access(self): + py.test.skip('fixme') From arigo at codespeak.net Wed Feb 27 10:16:39 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 27 Feb 2008 10:16:39 +0100 (CET) Subject: [pypy-svn] r51895 - in pypy/dist/pypy: rpython/lltypesystem rpython/memory rpython/memory/gctransform translator/c translator/js/test Message-ID: <20080227091639.154111683E5@codespeak.net> Author: arigo Date: Wed Feb 27 10:16:38 2008 New Revision: 51895 Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py pypy/dist/pypy/rpython/lltypesystem/lltype.py pypy/dist/pypy/rpython/lltypesystem/rstr.py pypy/dist/pypy/rpython/memory/gctransform/transform.py pypy/dist/pypy/rpython/memory/gctypelayout.py pypy/dist/pypy/translator/c/node.py pypy/dist/pypy/translator/js/test/test_typed.py Log: issue356 resolved kill the extra null character at the end of rpystrings. Modified: pypy/dist/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/llmemory.py Wed Feb 27 10:16:38 2008 @@ -237,8 +237,6 @@ count = rest[0].repeat else: count = 0 - if self.TYPE._hints.get('isrpystring'): - count -= 1 # because malloc() will give us the extra char for free p = lltype.malloc(parenttype or self.TYPE, count, immortal = self.TYPE._gckind == 'raw', zero = zero) Modified: pypy/dist/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/lltype.py Wed Feb 27 10:16:38 2008 @@ -1068,15 +1068,6 @@ else: return None - def __iter__(self): - # this is a work-around for the 'isrpystring' hack in __getitem__, - # which otherwise causes list(p) to include the extra \x00 character. - if isinstance(self._T, (Array, FixedSizeArray)): - for i in range(self._obj.getlength()): - yield self[i] - else: - raise TypeError("%r instance is not an array" % (self._T,)) - def __repr__(self): return '<%s>' % (self,) @@ -1497,37 +1488,16 @@ def getbounds(self): stop = len(self.items) - if self._TYPE._hints.get('isrpystring', False): - # special hack for the null terminator - assert self._TYPE.OF == Char - stop += 1 return 0, stop def getitem(self, index, uninitialized_ok=False): - try: - v = self.items[index] - if isinstance(v, _uninitialized) and not uninitialized_ok: - raise UninitializedMemoryAccess("%r[%s]"%(self, index)) - return v - except IndexError: - if (self._TYPE._hints.get('isrpystring', False) and - index == len(self.items)): - # special hack for the null terminator - assert self._TYPE.OF == Char - return '\x00' - raise + v = self.items[index] + if isinstance(v, _uninitialized) and not uninitialized_ok: + raise UninitializedMemoryAccess("%r[%s]"%(self, index)) + return v def setitem(self, index, value): - try: - self.items[index] = value - except IndexError: - if (self._TYPE._hints.get('isrpystring', False) and - index == len(self.items)): - # special hack for the null terminator: can overwrite it - # with another null - assert value == '\x00' - return - raise + self.items[index] = value assert not '__dict__' in dir(_array) assert not '__dict__' in dir(_struct) Modified: pypy/dist/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rstr.py Wed Feb 27 10:16:38 2008 @@ -48,8 +48,7 @@ return emptyunicode STR.become(GcStruct('rpy_string', ('hash', Signed), - ('chars', Array(Char, hints={'immutable': True, - 'isrpystring': True})), + ('chars', Array(Char, hints={'immutable': True})), adtmeths={'malloc' : staticAdtMethod(mallocstr), 'empty' : staticAdtMethod(emptystrfun)})) UNICODE.become(GcStruct('rpy_unicode', ('hash', Signed), Modified: pypy/dist/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/dist/pypy/rpython/memory/gctransform/transform.py Wed Feb 27 10:16:38 2008 @@ -559,10 +559,7 @@ else: ARRAY = TYPE assert isinstance(ARRAY, lltype.Array) - if ARRAY._hints.get('isrpystring', False): - c_const_size = intconst(llmemory.sizeof(TYPE, 1)) - else: - c_const_size = intconst(llmemory.sizeof(TYPE, 0)) + c_const_size = intconst(llmemory.sizeof(TYPE, 0)) c_item_size = intconst(llmemory.sizeof(ARRAY.OF)) if ARRAY._hints.get("nolength", False): Modified: pypy/dist/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/dist/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/dist/pypy/rpython/memory/gctypelayout.py Wed Feb 27 10:16:38 2008 @@ -160,9 +160,6 @@ ofs1 = llmemory.offsetof(TYPE, TYPE._arrayfld) info.ofstolength = ofs1 + llmemory.ArrayLengthOffset(ARRAY) info.ofstovar = ofs1 + llmemory.itemoffsetof(ARRAY, 0) - # XXX we probably don't need isrpystring any more - if ARRAY._hints.get('isrpystring'): - info.fixedsize = llmemory.sizeof(TYPE, 1) else: ARRAY = TYPE info.ofstolength = llmemory.ArrayLengthOffset(ARRAY) Modified: pypy/dist/pypy/translator/c/node.py ============================================================================== --- pypy/dist/pypy/translator/c/node.py (original) +++ pypy/dist/pypy/translator/c/node.py Wed Feb 27 10:16:38 2008 @@ -200,9 +200,6 @@ self.LLTYPE = ARRAY original_varlength = varlength self.gcfields = [] - - if ARRAY._hints.get('isrpystring'): - varlength += 1 # for the NUL char terminator at the end of the string self.varlength = varlength if original_varlength == 1: basename = 'array' @@ -597,8 +594,6 @@ yield '}' elif self.T.OF == Char: s = ''.join(self.obj.items) - if self.T._hints.get('isrpystring', False): - s += '\x00' yield '\t%s%s' % (length, c_char_array_constant(s)) yield '}' else: Modified: pypy/dist/pypy/translator/js/test/test_typed.py ============================================================================== --- pypy/dist/pypy/translator/js/test/test_typed.py (original) +++ pypy/dist/pypy/translator/js/test/test_typed.py Wed Feb 27 10:16:38 2008 @@ -158,7 +158,6 @@ assert res == testfn_endswith(i, j) def test_str_join(): - #py.test.skip("issue with malloc_varsize of varsized struct (rpystring here)") #py.test.skip("stringBuilder support") def testfn(i, j): s1 = [ '', ',', ' and '] From lac at codespeak.net Wed Feb 27 10:25:03 2008 From: lac at codespeak.net (lac at codespeak.net) Date: Wed, 27 Feb 2008 10:25:03 +0100 (CET) Subject: [pypy-svn] r51896 - pypy/extradoc/proposal Message-ID: <20080227092503.7EDE91683CF@codespeak.net> Author: lac Date: Wed Feb 27 10:25:02 2008 New Revision: 51896 Modified: pypy/extradoc/proposal/openjdk-challenge.txt Log: Fix typos and minor grammatical errors; whack one extremely clumsy sentence; and add a few XXX:s for ideas I have for improvement Modified: pypy/extradoc/proposal/openjdk-challenge.txt ============================================================================== --- pypy/extradoc/proposal/openjdk-challenge.txt (original) +++ pypy/extradoc/proposal/openjdk-challenge.txt Wed Feb 27 10:25:02 2008 @@ -9,8 +9,8 @@ we mention other languages implementations?) PyPy is divided into two main parts: the Python interpreter, which -implements the Python language and its written in RPython_, and the -Translation Toolchain (TT), which transforms and translates programs +implements the Python language and is written in RPython_, and the +Translation Toolchain (TT), (XXX: written in Python?) which transforms and translates programs written in RPython into the final executables. RPython is a subset of Python specifically designed to allow the TT to analyze RPython programs and translate them into lower level, very efficient @@ -22,17 +22,18 @@ standard C/Posix environment, on the CLI or on the JVM. It is important to underline that the job of the TT is not limited to -the translation into an efficient executable, but it actively -transform the source interpreter by adding new features and +translation into an efficient executable, but it actively +transforms the source interpreter by adding new features and translation aspects, such as garbage collection, stackless -capabilities, etc. (XXX: link). +capabilities, etc. (XXX: link) (XXX: not sure stackless is well known, +say microthreading instead?? mention proxying or security aspects??). The most exciting feature of the TT is the ability to automatically turn the interpreter into a JIT compiler that exploits partial evaluation techniques to dynamically generate efficient code. Currently, the PyPy JIT works only in conjunction with the C backend; -early results are very good, since the resulting Python interpreter +early results are very good, the resulting Python interpreter can run numeric intensive computations at roughly the same speed of C (XXX: link). @@ -90,12 +91,16 @@ write completely new code. If the project goes to completion, the code developed will be -integrated into the PyPy codebase; if Sun requires to release the code +integrated into the PyPy codebase; if Sun requires us to release the code under the SCA (thus sharing the copyright between the original author and Sun itself), we will send to Sun a document in unified diff format that extensively shows all and sole lines of code on which Sun will have the copyright. +PyPy is already licensed under the extremely permissive MIT license, +so there are no legal copyright barriers preventing us from sharing +code in such a way. + Project completion ------------------ @@ -126,20 +131,22 @@ Relevance to the community -------------------------- -Recently the community showed a lot of interests in dynamic languages -on top of the JVM. Even if right now Jython is the only usable -implementation of Python for the JVM, PyPy can potentially become the +Recently the community has shown a lot of interest in dynamic languages +which run +on top of the JVM. Even if currently Jython(XXX: link) is the only usable +implementation of Python for the JVM, PyPy has the potential to become the reference implementation in the future. -To have a working JIT for the JVM is an important step to make PyPy -the fastest Python for the JVM ever. Moreover, due to the innovative -ideas implemented by PyPy, there are chances that Python could become -the fastest dynamic language that run on the top of the JVM. +To have a working JIT for the JVM is an important step towards making PyPy +the fastest Python for the JVM, ever. Moreover, due to the innovative +ideas implemented by PyPy, it is possible (XXX: likely?) that Python could become +the fastest dynamic language that runs on the top of the JVM. Finally, PyPy is not limited to Python: it is entirely possible to write interpreters for languages other than Python and translate them -with the TT; as a proof of concept, PyPy already contains partial -implementations of Prolog, JavaScript, Scheme and Smalltalk. +with the TT; as a proof of concept, PyPy already contains +fairly complete implementations of Prolog and Smalltalk, as well +as partial implementations of JavaScript and Scheme. (XXX: did I overstate things?) Since the JIT generator is independent of the Python languages, it will be possible to automatically add a JIT compiler to every language @@ -164,15 +171,15 @@ - `this paper`_ shows how this technique is exploited to write an efficient implementation of EcmaScript which runs on top of the JVM; - - JRuby also come with a JIT compiler that dynamically translates - ruby code into JVM bytecode; the difference with the PyPy approach - is that JRuby doesn't exploit type information that are known only + - JRuby also comes with a JIT compiler that dynamically translates + Ruby code into JVM bytecode, however, unlike PyPy + JRuby doesn't exploit type information that is known only at runtime to produce specialized, efficient versions of the function; moreover, while the JRuby JIT is hand-written, the whole goal of the PyPy JIT generator is to generate it automatically from the intepreter; - - in the .NET world, IronPython also emit code dynamically to + - in the .NET world, IronPython also emits code dynamically to optimize hot spots, but not in a pervasive way as JRuby or PyPy. XXX (arigo) I'm confused by the previous sentence. I thought From antocuni at codespeak.net Wed Feb 27 10:38:29 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 27 Feb 2008 10:38:29 +0100 (CET) Subject: [pypy-svn] r51897 - in pypy/dist/pypy: jit/codegen/cli jit/codegen/cli/test translator/cli translator/cli/src Message-ID: <20080227093829.7AB711683E0@codespeak.net> Author: antocuni Date: Wed Feb 27 10:38:27 2008 New Revision: 51897 Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/src/pypylib.cs Log: add support for FUNC5 and FUNC27, but the corresponding tests still crash for other reasons :-( Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/rgenop.py Wed Feb 27 10:38:27 2008 @@ -17,6 +17,8 @@ SM_INT__INT_1 = ootype.StaticMethod([ootype.Signed], ootype.Signed) SM_INT__INT_2 = ootype.StaticMethod([ootype.Signed] * 2, ootype.Signed) SM_INT__INT_3 = ootype.StaticMethod([ootype.Signed] * 3, ootype.Signed) +SM_INT__INT_5 = ootype.StaticMethod([ootype.Signed] * 5, ootype.Signed) +SM_INT__INT_27 = ootype.StaticMethod([ootype.Signed] * 27, ootype.Signed) SM_INT__INT_100 = ootype.StaticMethod([ootype.Signed] * 100, ootype.Signed) def token2clitype(tok): @@ -32,6 +34,10 @@ return typeof(SM_INT__INT_2) elif tok == ([''] * 3, ''): return typeof(SM_INT__INT_3) + elif tok == ([''] * 5, ''): + return typeof(SM_INT__INT_5) + elif tok == ([''] * 27, ''): + return typeof(SM_INT__INT_27) elif tok == ([''] * 100, ''): return typeof(SM_INT__INT_100) else: Modified: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Wed Feb 27 10:38:27 2008 @@ -71,10 +71,10 @@ py.test.skip('mono crashes') def test_from_random_3_direct(self): - py.test.skip('we need yet another delegate type') + py.test.skip('infinite loop') def test_from_random_5_direct(self): - py.test.skip('we need yet another delegate type') + py.test.skip('mono crash') def test_genzeroconst(self): py.test.skip('fixme') Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Wed Feb 27 10:38:27 2008 @@ -642,5 +642,7 @@ ootype.StaticMethod([ootype.Signed], ootype.Signed): CLR.pypy.test.DelegateType_int__int_1, ootype.StaticMethod([ootype.Signed] * 2, ootype.Signed): CLR.pypy.test.DelegateType_int__int_2, ootype.StaticMethod([ootype.Signed] * 3, ootype.Signed): CLR.pypy.test.DelegateType_int__int_3, + ootype.StaticMethod([ootype.Signed] * 5, ootype.Signed): CLR.pypy.test.DelegateType_int__int_5, + ootype.StaticMethod([ootype.Signed] * 27, ootype.Signed): CLR.pypy.test.DelegateType_int__int_27, ootype.StaticMethod([ootype.Signed] * 100, ootype.Signed): CLR.pypy.test.DelegateType_int__int_100 } Modified: pypy/dist/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/dist/pypy/translator/cli/src/pypylib.cs Wed Feb 27 10:38:27 2008 @@ -53,6 +53,8 @@ public delegate int DelegateType_int__int_1(int a); public delegate int DelegateType_int__int_2(int a, int b); public delegate int DelegateType_int__int_3(int a, int b, int c); + public delegate int DelegateType_int__int_5(int a, int b, int c, int d, int e); + public delegate int DelegateType_int__int_27(int a0, int a1, int a2, int a3, int a4, int a5, int a6, int a7, int a8, int a9, int a10, int a11, int a12, int a13, int a14, int a15, int a16, int a17, int a18, int a19, int a20, int a21, int a22, int a23, int a24, int a25, int a26); public delegate int DelegateType_int__int_100(int a0, int a1, int a2, int a3, int a4, int a5, int a6, int a7, int a8, int a9, int a10, int a11, int a12, int a13, int a14, int a15, int a16, int a17, int a18, int a19, int a20, int a21, int a22, int a23, int a24, int a25, int a26, int a27, int a28, int a29, int a30, int a31, int a32, int a33, int a34, int a35, int a36, int a37, int a38, int a39, int a40, int a41, int a42, int a43, int a44, int a45, int a46, int a47, int a48, int a49, int a50, int a51, int a52, int a53, int a54, int a55, int a56, int a57, int a58, int a59, int a60, int a61, int a62, int a63, int a64, int a65, int a66, int a67, int a68, int a69, int a70, int a71, int a72, int a73, int a74, int a75, int a76, int a77, int a78, int a79, int a80, int a81, int a82, int a83, int a84, int a85, int a86, int a87, int a88, int a89, int a90, int a91, int a92, int a93, int a94, int a95, int a96, int a97, int a98, int a99); } From arigo at codespeak.net Wed Feb 27 11:14:15 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 27 Feb 2008 11:14:15 +0100 (CET) Subject: [pypy-svn] r51898 - pypy/dist/pypy/annotation Message-ID: <20080227101415.EA91D1683EC@codespeak.net> Author: arigo Date: Wed Feb 27 11:14:14 2008 New Revision: 51898 Modified: pypy/dist/pypy/annotation/binaryop.py Log: Fix error message. Modified: pypy/dist/pypy/annotation/binaryop.py ============================================================================== --- pypy/dist/pypy/annotation/binaryop.py (original) +++ pypy/dist/pypy/annotation/binaryop.py Wed Feb 27 11:14:14 2008 @@ -625,10 +625,10 @@ def check_negative_slice(s_slice): if isinstance(s_slice.start, SomeInteger) and not s_slice.start.nonneg: - raise TypeError("%s not proven to have negative start" % s_slice) + raise TypeError("%s not proven to have non-negative start" % s_slice) if isinstance(s_slice.stop, SomeInteger) and not s_slice.stop.nonneg and\ getattr(s_slice.stop, 'const', 0) != -1: - raise TypeError("%s not proven to have negative stop" % s_slice) + raise TypeError("%s not proven to have non-negative stop" % s_slice) class __extend__(pairtype(SomeList, SomeSlice)): From arigo at codespeak.net Wed Feb 27 14:08:49 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 27 Feb 2008 14:08:49 +0100 (CET) Subject: [pypy-svn] r51899 - pypy/dist/pypy/rpython/module Message-ID: <20080227130849.B0A601683DE@codespeak.net> Author: arigo Date: Wed Feb 27 14:08:48 2008 New Revision: 51899 Modified: pypy/dist/pypy/rpython/module/ll_os.py pypy/dist/pypy/rpython/module/ll_os_stat.py Log: Missing #include. It's left as an exercise to the reader to understand why this causes large file support to fail. On the other hand, the _FILE_OFFSET_BITS macro doesn't need to be set explicitly because pyconfig.h sets it too, and it's confusing that only the stat() functions cause it to be explicitly set -- other os functions would need it too. Modified: pypy/dist/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os.py (original) +++ pypy/dist/pypy/rpython/module/ll_os.py Wed Feb 27 14:08:48 2008 @@ -32,7 +32,7 @@ # XXX many of these includes are not portable at all includes += ['dirent.h', 'sys/stat.h', 'sys/times.h', 'utime.h', 'sys/types.h', 'unistd.h', - 'signal.h', 'sys/wait.h'] + 'signal.h', 'sys/wait.h', 'fcntl.h'] else: includes += ['sys/utime.h'] Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_stat.py Wed Feb 27 14:08:48 2008 @@ -146,7 +146,6 @@ INCLUDES = ['sys/types.h', 'sys/stat.h', 'unistd.h'] compilation_info = ExternalCompilationInfo( - pre_include_lines = ['#define _FILE_OFFSET_BITS 64'], includes = INCLUDES ) From pedronis at codespeak.net Wed Feb 27 14:38:36 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 27 Feb 2008 14:38:36 +0100 (CET) Subject: [pypy-svn] r51900 - in pypy/dist/pypy: lib/_ctypes lib/app_test/ctypes module/_rawffi module/_rawffi/test rlib/test Message-ID: <20080227133836.B573F1683E0@codespeak.net> Author: pedronis Date: Wed Feb 27 14:38:34 2008 New Revision: 51900 Modified: pypy/dist/pypy/lib/_ctypes/array.py pypy/dist/pypy/lib/_ctypes/basics.py pypy/dist/pypy/lib/_ctypes/function.py pypy/dist/pypy/lib/_ctypes/structure.py pypy/dist/pypy/lib/app_test/ctypes/test_as_parameter.py pypy/dist/pypy/lib/app_test/ctypes/test_functions.py pypy/dist/pypy/module/_rawffi/array.py pypy/dist/pypy/module/_rawffi/interp_rawffi.py pypy/dist/pypy/module/_rawffi/structure.py pypy/dist/pypy/module/_rawffi/test/test__rawffi.py pypy/dist/pypy/rlib/test/test_libffi.py Log: Support for functions returning structure values. Now though the mess gettypecode/get_ffi_type/get_arg_type... needs to be refactored and cleaned up. See the new xxxs. Modified: pypy/dist/pypy/lib/_ctypes/array.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/array.py (original) +++ pypy/dist/pypy/lib/_ctypes/array.py Wed Feb 27 14:38:34 2008 @@ -85,6 +85,9 @@ res._index = index return res.__ctypes_from_outparam__() + def _CData_retval(self, resbuffer): + raise NotImplementedError + def _CData_value(self, value): # array accepts very strange parameters as part of structure # or function argument... Modified: pypy/dist/pypy/lib/_ctypes/basics.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/basics.py (original) +++ pypy/dist/pypy/lib/_ctypes/basics.py Wed Feb 27 14:38:34 2008 @@ -47,19 +47,22 @@ # interested only in value anyway return cobj._get_buffer_value() - def _CData_output(self, resarray, base=None, index=-1, needs_free=False): - assert isinstance(resarray, _rawffi.ArrayInstance) + def _CData_output(self, resbuffer, base=None, index=-1, needs_free=False): + assert isinstance(resbuffer, _rawffi.ArrayInstance) """Used when data exits ctypes and goes into user code. - 'resarray' is a _rawffi array of length 1 containing the value, + 'resbuffer' is a _rawffi array of length 1 containing the value, and this returns a general Python object that corresponds. """ res = self.__new__(self) - res.__dict__['_buffer'] = resarray + res.__dict__['_buffer'] = resbuffer res.__dict__['_base'] = base res.__dict__['_index'] = index res.__dict__['_needs_free'] = needs_free return res.__ctypes_from_outparam__() + def _CData_retval(self, resbuffer): + return self._CData_output(resbuffer, needs_free=True) + def __mul__(self, other): from _ctypes.array import create_array_type return create_array_type(self, other) Modified: pypy/dist/pypy/lib/_ctypes/function.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/function.py (original) +++ pypy/dist/pypy/lib/_ctypes/function.py Wed Feb 27 14:38:34 2008 @@ -80,18 +80,22 @@ restype = self._restype_ funcptr = self._getfuncptr(argtypes, restype) args = self._wrap_args(argtypes, args) - resarray = funcptr(*[arg._buffer for obj, arg in args]) + resbuffer = funcptr(*[arg._buffer for obj, arg in args]) if restype is not None: - return restype._CData_output(resarray, needs_free=True) + return restype._CData_retval(resbuffer) else: - resarray.free() + resbuffer.free() def _getfuncptr(self, argtypes, restype): if restype is None: import ctypes restype = ctypes.c_int argshapes = [arg._ffiargshape for arg in argtypes] - return self.dll._handle.ptr(self.name, argshapes, restype._ffiletter) + if isinstance(restype._ffiargshape, str): # xxx refactor + resshape = restype._ffiargshape + else: + resshape = restype._ffistruct + return self.dll._handle.ptr(self.name, argshapes, resshape) def _guess_argtypes(self, args): from _ctypes import _CData Modified: pypy/dist/pypy/lib/_ctypes/structure.py ============================================================================== --- pypy/dist/pypy/lib/_ctypes/structure.py (original) +++ pypy/dist/pypy/lib/_ctypes/structure.py Wed Feb 27 14:38:34 2008 @@ -155,6 +155,14 @@ res.__dict__['_base'] = base res.__dict__['_index'] = index return res.__ctypes_from_outparam__() + + def _CData_retval(self, resbuffer): + res = self.__new__(self) + res.__dict__['_buffer'] = resbuffer + res.__dict__['_base'] = None + res.__dict__['_index'] = -1 + res.__dict__['_needs_free'] = True + return res.__ctypes_from_outparam__() class Structure(_CData): __metaclass__ = StructureMeta Modified: pypy/dist/pypy/lib/app_test/ctypes/test_as_parameter.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_as_parameter.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_as_parameter.py Wed Feb 27 14:38:34 2008 @@ -164,7 +164,6 @@ assert got == expected def test_struct_return_2H(self): - py.test.skip("Structure by value") class S2H(Structure): _fields_ = [("x", c_short), ("y", c_short)] @@ -175,7 +174,6 @@ assert (s2h.x, s2h.y) == (99*2, 88*3) def test_struct_return_8H(self): - py.test.skip("Structure by value") class S8I(Structure): _fields_ = [("a", c_int), ("b", c_int), Modified: pypy/dist/pypy/lib/app_test/ctypes/test_functions.py ============================================================================== --- pypy/dist/pypy/lib/app_test/ctypes/test_functions.py (original) +++ pypy/dist/pypy/lib/app_test/ctypes/test_functions.py Wed Feb 27 14:38:34 2008 @@ -330,7 +330,6 @@ assert got == expected def test_struct_return_2H(self): - py.test.skip("Raw structure return") class S2H(Structure): _fields_ = [("x", c_short), ("y", c_short)] @@ -352,7 +351,6 @@ assert (s2h.x, s2h.y) == (99*2, 88*3) def test_struct_return_8H(self): - py.test.skip("Raw structure return") class S8I(Structure): _fields_ = [("a", c_int), ("b", c_int), Modified: pypy/dist/pypy/module/_rawffi/array.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/array.py (original) +++ pypy/dist/pypy/module/_rawffi/array.py Wed Feb 27 14:38:34 2008 @@ -10,7 +10,7 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.interpreter.error import OperationError, wrap_oserror from pypy.module._rawffi.interp_rawffi import segfault_exception -from pypy.module._rawffi.interp_rawffi import W_DataInstance +from pypy.module._rawffi.interp_rawffi import W_DataShape, W_DataInstance from pypy.module._rawffi.interp_rawffi import unwrap_value, wrap_value from pypy.module._rawffi.interp_rawffi import unpack_typecode, letter2tp from pypy.rlib.rarithmetic import intmask, r_uint @@ -26,7 +26,7 @@ return ll_array[pos] get_elem._annspecialcase_ = 'specialize:arg(2)' -class W_Array(Wrappable): +class W_Array(W_DataShape): def __init__(self, space, itemtp): assert isinstance(itemtp, tuple) self.space = space Modified: pypy/dist/pypy/module/_rawffi/interp_rawffi.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/interp_rawffi.py (original) +++ pypy/dist/pypy/module/_rawffi/interp_rawffi.py Wed Feb 27 14:38:34 2008 @@ -108,6 +108,7 @@ self.w_cache = space.newdict() self.space = space + # xxx refactor away ! def get_arg_type(self, letter, argsize, argalignment): space = self.space if letter == 'V': # xxx leaks @@ -123,15 +124,28 @@ """ Get a pointer for function name with provided argtypes and restype """ + # xxx refactor if space.is_w(w_restype, space.w_None): - restype = 'v' + resshape = None ffi_restype = ffi_type_void + elif space.is_true(space.isinstance(w_restype, space.w_str)): + tp_letter = space.str_w(w_restype) + if tp_letter == 'v': + resshape = None + ffi_restype = ffi_type_void + else: + from pypy.module._rawffi.array import get_array_cache + cache = get_array_cache(space) + resshape = cache.get_array_type(letter2tp(space, tp_letter)) + ffi_restype = self.get_type(tp_letter) else: - restype = space.str_w(w_restype) - ffi_restype = self.get_type(restype) + from pypy.module._rawffi.structure import W_Structure + resshape = space.interp_w(W_Structure, w_restype) + ffi_restype = resshape.get_ffi_type() + w = space.wrap w_argtypes = space.newtuple(space.unpackiterable(w_argtypes)) - w_key = space.newtuple([w(name), w_argtypes, w(restype)]) + w_key = space.newtuple([w(name), w_argtypes, w(resshape)]) try: return space.getitem(self.w_cache, w_key) except OperationError, e: @@ -146,7 +160,7 @@ in argtypes] try: ptr = self.cdll.getrawpointer(name, ffi_argtypes, ffi_restype) - w_funcptr = W_FuncPtr(space, ptr, argtypes, restype) + w_funcptr = W_FuncPtr(space, ptr, argtypes, resshape) space.setitem(self.w_cache, w_key, w_funcptr) return w_funcptr except KeyError: @@ -187,7 +201,7 @@ where argtype_list is a list of single characters and restype is a single character. The character meanings are more or less the same as in the struct module, except that s has trailing \x00 added, while p is considered a raw -buffer.""" +buffer.""" # xxx fix doc ) unroll_letters_for_numbers = unrolling_iterable("bBhHiIlLqQ") @@ -197,6 +211,10 @@ w_exception = space.getattr(w_mod, space.wrap("SegfaultException")) return OperationError(w_exception, space.wrap(reason)) +class W_DataShape(Wrappable): + + def allocate(self, space, length, autofree=False): + raise NotImplementedError class W_DataInstance(Wrappable): def __init__(self, space, size, address=r_uint(0)): @@ -305,14 +323,9 @@ wrap_value._annspecialcase_ = 'specialize:arg(1)' class W_FuncPtr(Wrappable): - def __init__(self, space, ptr, argtypes, restype): - from pypy.module._rawffi.array import get_array_cache + def __init__(self, space, ptr, argtypes, resshape): self.ptr = ptr - if restype != 'v': - cache = get_array_cache(space) - self.resarray = cache.get_array_type(letter2tp(space, restype)) - else: - self.resarray = None + self.resshape = resshape self.argtypes = argtypes def call(self, space, args_w): @@ -352,8 +365,8 @@ raise OperationError(space.w_TypeError, space.wrap(msg)) args_ll.append(arg.ll_buffer) # XXX we could avoid the intermediate list args_ll - if self.resarray is not None: - result = self.resarray.allocate(space, 1) + if self.resshape is not None: + result = self.resshape.allocate(space, 1) self.ptr.call(args_ll, result.ll_buffer) return space.wrap(result) else: Modified: pypy/dist/pypy/module/_rawffi/structure.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/structure.py (original) +++ pypy/dist/pypy/module/_rawffi/structure.py Wed Feb 27 14:38:34 2008 @@ -11,9 +11,10 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.interpreter.error import OperationError, wrap_oserror from pypy.module._rawffi.interp_rawffi import segfault_exception -from pypy.module._rawffi.interp_rawffi import W_DataInstance +from pypy.module._rawffi.interp_rawffi import W_DataShape, W_DataInstance from pypy.module._rawffi.interp_rawffi import wrap_value, unwrap_value from pypy.module._rawffi.interp_rawffi import unpack_typecode +from pypy.rlib import libffi from pypy.rlib.rarithmetic import intmask, r_uint def unpack_fields(space, w_fields): @@ -45,7 +46,7 @@ return size, alignment, pos -class W_Structure(Wrappable): +class W_Structure(W_DataShape): def __init__(self, space, w_fields): fields = unpack_fields(space, w_fields) name_to_index = {} @@ -62,6 +63,12 @@ self.fields = fields self.name_to_index = name_to_index + def allocate(self, space, length, autofree=False): + # length is ignored! + if autofree: + return W_StructureInstanceAutoFree(space, self) + return W_StructureInstance(space, self, 0) + def getindex(self, space, attr): try: return self.name_to_index[attr] @@ -70,9 +77,7 @@ "C Structure has no attribute %s" % attr)) def descr_call(self, space, autofree=False): - if autofree: - return space.wrap(W_StructureInstanceAutoFree(space, self)) - return space.wrap(W_StructureInstance(space, self, 0)) + return space.wrap(self.allocate(space, 1, autofree)) descr_call.unwrap_spec = ['self', ObjSpace, int] def descr_repr(self, space): @@ -96,6 +101,21 @@ space.wrap(self.alignment)]) descr_gettypecode.unwrap_spec = ['self', ObjSpace] + # get the corresponding ffi_type + ffi_type = lltype.nullptr(libffi.FFI_TYPE_P.TO) + + def get_ffi_type(self): + if not self.ffi_type: + self.ffi_type = libffi.make_struct_ffitype(self.size, + self.alignment) + return self.ffi_type + + def __del__(self): + if self.ffi_type: + lltype.free(self.ffi_type, flavor='raw') + + + def descr_new_structure(space, w_type, w_fields): return space.wrap(W_Structure(space, w_fields)) Modified: pypy/dist/pypy/module/_rawffi/test/test__rawffi.py ============================================================================== --- pypy/dist/pypy/module/_rawffi/test/test__rawffi.py (original) +++ pypy/dist/pypy/module/_rawffi/test/test__rawffi.py Wed Feb 27 14:38:34 2008 @@ -120,6 +120,24 @@ long sum_x_y(struct x_y s) { return s.x + s.y; } + + struct s2h { + short x; + short y; + }; + + struct s2h give(short x, short y) { + struct s2h out; + out.x = x; + out.y = y; + return out; + } + + struct s2h perturb(struct s2h inp) { + inp.x *= 2; + inp.y *= 3; + return inp; + } ''')) return compile_c_module([c_file], 'x', ExternalCompilationInfo()) @@ -632,6 +650,40 @@ res = sum_x_y(x_y) assert res[0] == 420 x_y.free() + res.free() + + def test_ret_struct(self): + import _rawffi + S2H = _rawffi.Structure([('x', 'h'), ('y', 'h')]) + s2h = S2H() + lib = _rawffi.CDLL(self.lib_name) + give = lib.ptr('give', ['h', 'h'], S2H) + a1 = _rawffi.Array('h')(1) + a2 = _rawffi.Array('h')(1) + a1[0] = 13 + a2[0] = 17 + res = give(a1, a2) + assert isinstance(res, _rawffi.StructureInstance) + assert res.shape is S2H + assert res.x == 13 + assert res.y == 17 + res.free() + a1.free() + a2.free() + + s2h.x = 7 + s2h.y = 11 + perturb = lib.ptr('perturb', [S2H.gettypecode()], S2H) + res = perturb(s2h) + assert isinstance(res, _rawffi.StructureInstance) + assert res.shape is S2H + assert res.x == 14 + assert res.y == 33 + assert s2h.x == 7 + assert s2h.y == 11 + res.free() + + s2h.free() class AppTestAutoFree: Modified: pypy/dist/pypy/rlib/test/test_libffi.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_libffi.py (original) +++ pypy/dist/pypy/rlib/test/test_libffi.py Wed Feb 27 14:38:34 2008 @@ -231,3 +231,78 @@ del lib assert not ALLOCATED + + def test_ret_struct_val(self): + from pypy.translator.tool.cbuild import compile_c_module, \ + ExternalCompilationInfo + from pypy.tool.udir import udir + + c_file = udir.ensure("test_libffi", dir=1).join("xlib.c") + c_file.write(py.code.Source(''' + #include + #include + + struct s2h { + short x; + short y; + }; + + struct s2h give(short x, short y) { + struct s2h out; + out.x = x; + out.y = y; + return out; + } + + struct s2h perturb(struct s2h inp) { + inp.x *= 2; + inp.y *= 3; + return inp; + } + + ''')) + lib_name = compile_c_module([c_file], 'x', ExternalCompilationInfo()) + + lib = CDLL(lib_name) + + size = ffi_type_sshort.c_size*2 + alignment = ffi_type_sshort.c_alignment + tp = make_struct_ffitype(size, alignment) + + give = lib.getrawpointer('give', [ffi_type_sshort, ffi_type_sshort], + tp) + inbuffer = lltype.malloc(rffi.SHORTP.TO, 2, flavor='raw') + inbuffer[0] = rffi.cast(rffi.SHORT, 40) + inbuffer[1] = rffi.cast(rffi.SHORT, 72) + + outbuffer = lltype.malloc(rffi.SHORTP.TO, 2, flavor='raw') + + give.call([rffi.cast(rffi.VOIDP, inbuffer), + rffi.cast(rffi.VOIDP, rffi.ptradd(inbuffer, 1))], + rffi.cast(rffi.VOIDP, outbuffer)) + + assert outbuffer[0] == 40 + assert outbuffer[1] == 72 + + perturb = lib.getrawpointer('perturb', [tp], tp) + + inbuffer[0] = rffi.cast(rffi.SHORT, 7) + inbuffer[1] = rffi.cast(rffi.SHORT, 11) + + perturb.call([rffi.cast(rffi.VOIDP, inbuffer)], + rffi.cast(rffi.VOIDP, outbuffer)) + + assert inbuffer[0] == 7 + assert inbuffer[1] == 11 + + assert outbuffer[0] == 14 + assert outbuffer[1] == 33 + + lltype.free(outbuffer, flavor='raw') + lltype.free(inbuffer, flavor='raw') + del give + del perturb + lltype.free(tp, flavor='raw') + del lib + + assert not ALLOCATED From arigo at codespeak.net Wed Feb 27 14:54:40 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 27 Feb 2008 14:54:40 +0100 (CET) Subject: [pypy-svn] r51901 - in pypy/dist/pypy: module/_file module/_file/test module/posix/test rlib rpython/module translator/c/test Message-ID: <20080227135440.784841683FF@codespeak.net> Author: arigo Date: Wed Feb 27 14:54:38 2008 New Revision: 51901 Added: pypy/dist/pypy/module/_file/test/test_large_file.py pypy/dist/pypy/module/posix/test/__init__.py Modified: pypy/dist/pypy/module/_file/interp_file.py pypy/dist/pypy/module/posix/test/test_posix2.py pypy/dist/pypy/rlib/streamio.py pypy/dist/pypy/rpython/module/ll_os_stat.py pypy/dist/pypy/translator/c/test/test_extfunc.py pypy/dist/pypy/translator/c/test/test_standalone.py Log: issue357 testing Support for 64-bit offsets in 'file' objects. Also includes a test for r51899 in test_standalone. (Note that r51899 broke test_extfunc in an obscure way) Modified: pypy/dist/pypy/module/_file/interp_file.py ============================================================================== --- pypy/dist/pypy/module/_file/interp_file.py (original) +++ pypy/dist/pypy/module/_file/interp_file.py Wed Feb 27 14:54:38 2008 @@ -1,6 +1,7 @@ import py import os from pypy.rlib import streamio +from pypy.rlib.rarithmetic import r_longlong from pypy.module._file.interp_stream import W_AbstractStream from pypy.module._file.interp_stream import StreamErrors, wrap_streamerror from pypy.interpreter.error import OperationError @@ -177,7 +178,7 @@ if w_size is None or space.is_w(w_size, space.w_None): size = stream.tell() else: - size = space.int_w(w_size) + size = space.r_longlong_w(w_size) stream.truncate(size) def direct_write(self, data): @@ -309,7 +310,7 @@ total number of bytes in the lines returned.""", wrapresult = "wrap_list_of_str(space, result)") - _decl(locals(), "seek", ['self', int, int], + _decl(locals(), "seek", ['self', r_longlong, int], """seek(offset[, whence]) -> None. Move to new file position. Argument offset is a byte count. Optional argument whence defaults to Added: pypy/dist/pypy/module/_file/test/test_large_file.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/module/_file/test/test_large_file.py Wed Feb 27 14:54:38 2008 @@ -0,0 +1,69 @@ +import py + +from pypy.conftest import gettestobjspace + +class AppTestLargeFile(object): + def setup_class(cls): + cls.space = gettestobjspace(usemodules=("_file", )) + cls.w_temppath = cls.space.wrap( + str(py.test.ensuretemp("fileimpl").join("large.data"))) + + def setup_method(self, meth): + if getattr(meth, 'need_sparse_files', False): + from pypy.module.posix.test.test_posix2 import need_sparse_files + need_sparse_files() + + def test_large_seek_offsets(self): + import _file + FAR = 0x122223333 + f = _file.file(self.temppath, "w+b") + f.write("hello world") + f.seek(FAR) + assert f.tell() == FAR + f.seek(-10, 1) + assert f.tell() == FAR - 10 + f.seek(0x123456789, 1) + assert f.tell() == FAR - 10 + 0x123456789 + f.seek(-FAR, 1) + assert f.tell() == -10 + 0x123456789 + f.seek(FAR, 2) + assert f.tell() == len("hello world") + FAR + f.close() + + def test_large_sparse(self): + import _file + FAR = 0x122223333 + f = _file.file(self.temppath, "w+b") + f.seek(FAR) + f.write('end') + f.seek(0) + data = f.read(1234) + assert data == 1234 * '\x00' + f.seek(FAR-2-1234, 1) + data = f.read(4321) + assert data == '\x00\x00end' + f.seek(-1, 2) + assert f.tell() == FAR + 2 + f.truncate() + f.seek(0, 2) + assert f.tell() == FAR + 2 + f.truncate(FAR + 1) + f.seek(FAR-2, 0) + data = f.read(1) + assert data == '\x00' + assert f.tell() == FAR - 1 + data = f.read(1) + assert data == '\x00' + data = f.read(1) + assert data == 'e' + data = f.read(1) + assert data == '' + assert f.tell() == FAR + 1 + import sys + if FAR > sys.maxint: + f.seek(0) + raises(OverflowError, f.read, FAR) + raises(OverflowError, f.readline, FAR) + raises(OverflowError, f.readlines, FAR) + f.close() + test_large_sparse.need_sparse_files = True Added: pypy/dist/pypy/module/posix/test/__init__.py ============================================================================== Modified: pypy/dist/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/dist/pypy/module/posix/test/test_posix2.py (original) +++ pypy/dist/pypy/module/posix/test/test_posix2.py Wed Feb 27 14:54:38 2008 @@ -19,6 +19,12 @@ pdir.join('another_longer_file_name').write("test3") mod.pdir = pdir +def need_sparse_files(): + if sys.platform == 'darwin': + py.test.skip("no sparse files on default Mac OS X file system") + if os.name == 'nt': + py.test.skip("no sparse files on Windows") + class AppTestPosix: def setup_class(cls): cls.space = space @@ -31,6 +37,10 @@ cls.w_geteuid = space.wrap(os.geteuid()) if hasattr(os, 'getgid'): cls.w_getgid = space.wrap(os.getgid()) + + def setup_method(self, meth): + if getattr(meth, 'need_sparse_files', False): + need_sparse_files() def test_posix_is_pypy_s(self): assert self.posix.__file__ @@ -296,11 +306,6 @@ def test_largefile(self): os = self.posix - import sys - if sys.platform == 'darwin': - skip("no sparse files on default Mac OS X file system") - if os.__name__ == 'nt': - skip("no sparse files on Windows") fd = os.open(self.path2, os.O_RDWR | os.O_CREAT, 0666) os.ftruncate(fd, 10000000000L) res = os.lseek(fd, 9900000000L, 0) @@ -313,6 +318,7 @@ st = os.stat(self.path2) assert st.st_size == 10000000000L + test_largefile.need_sparse_files = True class AppTestEnvironment(object): def setup_class(cls): Modified: pypy/dist/pypy/rlib/streamio.py ============================================================================== --- pypy/dist/pypy/rlib/streamio.py (original) +++ pypy/dist/pypy/rlib/streamio.py Wed Feb 27 14:54:38 2008 @@ -29,7 +29,16 @@ """ +# +# File offsets are all 'r_longlong', but a single read or write cannot +# transfer more data that fits in an RPython 'int' (because that would not +# fit in a single string anyway). This module needs to be careful about +# where r_longlong values end up: as argument to seek() and truncate() and +# return value of tell(), but not as argument to read(). +# + import os, sys +from pypy.rlib.rarithmetic import r_longlong, intmask from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC O_BINARY = getattr(os, "O_BINARY", 0) @@ -226,10 +235,10 @@ os.lseek(self.fd, offset, whence) def tell(self): - #XXX we really want r_longlong - return int(os.lseek(self.fd, 0, 1)) + return os.lseek(self.fd, 0, 1) def read(self, n): + assert isinstance(n, int) return os.read(self.fd, n) def write(self, data): @@ -298,6 +307,7 @@ return data def read(self, n): + assert isinstance(n, int) end = self.pos + n data = self.mm[self.pos:end] if not data: @@ -356,10 +366,10 @@ ("read", [int]), ("write", [str]), ("tell", []), - ("seek", ["index", int]), + ("seek", [r_longlong, int]), ("readall", []), ("readline", []), - ("truncate", [int]), + ("truncate", [r_longlong]), ("flush", []), ("flushable", []), ("close", []), @@ -389,6 +399,14 @@ return d[meth_name] +def offset2int(offset): + intoffset = intmask(offset) + if intoffset != offset: + raise StreamError("seek() from a non-seekable source:" + " this would read and discard more" + " than sys.maxint bytes") + return intoffset + class BufferingInputStream(Stream): """Standard buffering input stream. @@ -450,22 +468,25 @@ while self.lines: line = self.lines[0] if offset <= len(line): - assert offset >= 0 - self.lines[0] = line[offset:] + intoffset = intmask(offset) + assert intoffset >= 0 + self.lines[0] = line[intoffset:] return offset -= len(self.lines[0]) - 1 del self.lines[0] assert not self.lines if offset <= len(self.buf): - assert offset >= 0 - self.buf = self.buf[offset:] + intoffset = intmask(offset) + assert intoffset >= 0 + self.buf = self.buf[intoffset:] return offset -= len(self.buf) self.buf = "" try: self.do_seek(offset, 1) except NotImplementedError: - self.read(offset) + intoffset = offset2int(offset) + self.read(intoffset) return if whence == 2: try: @@ -478,6 +499,7 @@ return # Skip relative to EOF by reading and saving only just as # much as needed + intoffset = offset2int(offset) data = "\n".join(self.lines + [self.buf]) total = len(data) buffers = [data] @@ -489,10 +511,10 @@ break buffers.append(data) total += len(data) - while buffers and total >= len(buffers[0]) - offset: + while buffers and total >= len(buffers[0]) - intoffset: total -= len(buffers[0]) del buffers[0] - cutoff = total + offset + cutoff = total + intoffset if cutoff < 0: raise StreamError("cannot seek back") if buffers: @@ -517,6 +539,7 @@ return "".join(more) def read(self, n): + assert isinstance(n, int) assert n >= 0 if self.lines: # See if this can be satisfied from self.lines[0] Modified: pypy/dist/pypy/rpython/module/ll_os_stat.py ============================================================================== --- pypy/dist/pypy/rpython/module/ll_os_stat.py (original) +++ pypy/dist/pypy/rpython/module/ll_os_stat.py Wed Feb 27 14:54:38 2008 @@ -146,6 +146,7 @@ INCLUDES = ['sys/types.h', 'sys/stat.h', 'unistd.h'] compilation_info = ExternalCompilationInfo( + pre_include_lines = ['#define _FILE_OFFSET_BITS 64'], includes = INCLUDES ) Modified: pypy/dist/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/dist/pypy/translator/c/test/test_extfunc.py Wed Feb 27 14:54:38 2008 @@ -111,12 +111,10 @@ os.unlink(filename) def test_largefile(): - if sys.platform == 'darwin': - skip("no sparse files on default Mac OS X file system") if not hasattr(os, 'ftruncate'): py.test.skip("this os has no ftruncate :-(") - if os.name == 'nt': - py.test.skip("no sparse files on Windows") + from pypy.module.posix.test.test_posix2 import need_sparse_files + need_sparse_files() filename = str(udir.join('test_largefile')) r4800000000 = r_longlong(4800000000L) r4900000000 = r_longlong(4900000000L) Modified: pypy/dist/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_standalone.py (original) +++ pypy/dist/pypy/translator/c/test/test_standalone.py Wed Feb 27 14:54:38 2008 @@ -1,6 +1,7 @@ import py import sys, os +from pypy.rlib.rarithmetic import r_longlong from pypy.translator.translator import TranslationContext from pypy.translator.backendopt import all from pypy.translator.c.genc import CStandaloneBuilder @@ -164,3 +165,28 @@ exe = t.compile() out = py.process.cmdexec("%s 500" % exe) assert int(out) == 500*501/2 + +def test_standalone_large_files(): + from pypy.module.posix.test.test_posix2 import need_sparse_files + need_sparse_files() + filename = str(udir.join('test_standalone_largefile')) + r4800000000 = r_longlong(4800000000L) + def entry_point(argv): + fd = os.open(filename, os.O_RDWR | os.O_CREAT, 0644) + os.lseek(fd, r4800000000, 0) + os.write(fd, "$") + newpos = os.lseek(fd, 0, 1) + if newpos == r4800000000 + 1: + print "OK" + else: + print "BAD POS" + os.close(fd) + return 0 + t = TranslationContext() + t.buildannotator().build_types(entry_point, [s_list_of_strings]) + t.buildrtyper().specialize() + cbuilder = CStandaloneBuilder(t, entry_point, t.config) + cbuilder.generate_source() + cbuilder.compile() + data = cbuilder.cmdexec('hi there') + assert data.strip() == "OK" From pedronis at codespeak.net Wed Feb 27 15:31:34 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 27 Feb 2008 15:31:34 +0100 (CET) Subject: [pypy-svn] r51902 - pypy/branch/rawffi-shape-cleanup Message-ID: <20080227143134.24FDF168430@codespeak.net> Author: pedronis Date: Wed Feb 27 15:31:34 2008 New Revision: 51902 Added: pypy/branch/rawffi-shape-cleanup/ - copied from r51901, pypy/dist/ Log: branch to unify the various kind of shapes passed back in into rawffi using rawffi types directly insteand of some indirect information. From pedronis at codespeak.net Wed Feb 27 15:41:47 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 27 Feb 2008 15:41:47 +0100 (CET) Subject: [pypy-svn] r51903 - pypy/branch/rawffi-shape-cleanup/pypy/module/_rawffi/test Message-ID: <20080227144147.25696168449@codespeak.net> Author: pedronis Date: Wed Feb 27 15:41:44 2008 New Revision: 51903 Modified: pypy/branch/rawffi-shape-cleanup/pypy/module/_rawffi/test/test_nested.py Log: xxx before I forget Modified: pypy/branch/rawffi-shape-cleanup/pypy/module/_rawffi/test/test_nested.py ============================================================================== --- pypy/branch/rawffi-shape-cleanup/pypy/module/_rawffi/test/test_nested.py (original) +++ pypy/branch/rawffi-shape-cleanup/pypy/module/_rawffi/test/test_nested.py Wed Feb 27 15:41:44 2008 @@ -83,3 +83,5 @@ assert rawbuf[4] == 13 assert rawbuf[7] == 17 a.free() + + # xxx missing array in structure From pedronis at codespeak.net Wed Feb 27 16:37:15 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 27 Feb 2008 16:37:15 +0100 (CET) Subject: [pypy-svn] r51904 - in pypy/branch/rawffi-shape-cleanup/pypy/module/_rawffi: . test Message-ID: <20080227153715.2DA281683DE@codespeak.net> Author: pedronis Date: Wed Feb 27 16:37:13 2008 New Revision: 51904 Modified: pypy/branch/rawffi-shape-cleanup/pypy/module/_rawffi/interp_rawffi.py pypy/branch/rawffi-shape-cleanup/pypy/module/_rawffi/test/test__rawffi.py Log: started refactoring, now CDLL.ptr takes argtypes of the form: letter or (W_Structure, 1) changes test that pass accordingly. ctypes is broken by this obviously! Modified: pypy/branch/rawffi-shape-cleanup/pypy/module/_rawffi/interp_rawffi.py ============================================================================== --- pypy/branch/rawffi-shape-cleanup/pypy/module/_rawffi/interp_rawffi.py (original) +++ pypy/branch/rawffi-shape-cleanup/pypy/module/_rawffi/interp_rawffi.py Wed Feb 27 16:37:13 2008 @@ -51,9 +51,10 @@ } TYPEMAP_PTR_LETTERS = "POszZ" -UNPACKED_TYPECODES = dict([(code, (code, - intmask(field_desc.c_size), - intmask(field_desc.c_alignment))) +def size_alignment(ffi_type): + return intmask(ffi_type.c_size), intmask(ffi_type.c_alignment) + +UNPACKED_TYPECODES = dict([(code, (code,) + size_alignment(field_desc)) for code, field_desc in TYPEMAP.items()]) LL_TYPEMAP = { @@ -100,6 +101,26 @@ except KeyError: raise OperationError(space.w_ValueError, space.wrap( "Unknown type letter %s" % (key,))) + +def unpack_shape(space, w_shape, allow_void=False, shape=False): + resshape = None + if space.is_true(space.isinstance(w_shape, space.w_str)): + letter = space.str_w(w_shape) + if allow_void and letter == 'v': + return 'v', ffi_type_void, None + ffi_type = _get_type_(space, letter) + if shape: + from pypy.module._rawffi.array import get_array_cache + cache = get_array_cache(space) + resshape = cache.get_array_type(letter2tp(space, letter)) + else: + letter = 'V' + w_shapetype, w_length = space.unpacktuple(w_shape, expected_length=2) + from pypy.module._rawffi.structure import W_Structure + resshape = space.interp_w(W_Structure, w_shapetype) + ffi_type = resshape.get_ffi_type() + return letter, ffi_type, resshape + class W_CDLL(Wrappable): def __init__(self, space, name): @@ -128,23 +149,13 @@ if space.is_w(w_restype, space.w_None): resshape = None ffi_restype = ffi_type_void - elif space.is_true(space.isinstance(w_restype, space.w_str)): - tp_letter = space.str_w(w_restype) - if tp_letter == 'v': - resshape = None - ffi_restype = ffi_type_void - else: - from pypy.module._rawffi.array import get_array_cache - cache = get_array_cache(space) - resshape = cache.get_array_type(letter2tp(space, tp_letter)) - ffi_restype = self.get_type(tp_letter) else: - from pypy.module._rawffi.structure import W_Structure - resshape = space.interp_w(W_Structure, w_restype) - ffi_restype = resshape.get_ffi_type() - + tp_letter, ffi_restype, resshape = unpack_shape(space, w_restype, + allow_void=True, + shape=True) w = space.wrap - w_argtypes = space.newtuple(space.unpackiterable(w_argtypes)) + argtypes_w = space.unpackiterable(w_argtypes) + w_argtypes = space.newtuple(argtypes_w) w_key = space.newtuple([w(name), w_argtypes, w(resshape)]) try: return space.getitem(self.w_cache, w_key) @@ -152,15 +163,17 @@ if e.match(space, space.w_KeyError): pass else: - raise - argtypes_w = space.unpackiterable(w_argtypes) - argtypes = [unpack_typecode(space, w_arg) for w_arg in argtypes_w] - ffi_argtypes = [self.get_arg_type(letter, argsize, argalignment) - for letter, argsize, argalignment - in argtypes] + rais + argletters = [] + ffi_argtypes = [] + for w_arg in argtypes_w: + argletter, ffi_argtype, _ = unpack_shape(space, w_arg) + argletters.append(argletter) + ffi_argtypes.append(ffi_argtype) + try: ptr = self.cdll.getrawpointer(name, ffi_argtypes, ffi_restype) - w_funcptr = W_FuncPtr(space, ptr, argtypes, resshape) + w_funcptr = W_FuncPtr(space, ptr, argletters, resshape) space.setitem(self.w_cache, w_key, w_funcptr) return w_funcptr except KeyError: @@ -323,31 +336,32 @@ wrap_value._annspecialcase_ = 'specialize:arg(1)' class W_FuncPtr(Wrappable): - def __init__(self, space, ptr, argtypes, resshape): + def __init__(self, space, ptr, argletters, resshape): self.ptr = ptr + self.argletters = argletters self.resshape = resshape - self.argtypes = argtypes def call(self, space, args_w): from pypy.module._rawffi.array import W_ArrayInstance from pypy.module._rawffi.structure import W_StructureInstance argnum = len(args_w) - if argnum != len(self.argtypes): + if argnum != len(self.argletters): msg = "Wrong number of argument: expected %d, got %d" % ( - len(self.argtypes), argnum) + len(self.argletters), argnum) raise OperationError(space.w_TypeError, space.wrap(msg)) args_ll = [] for i in range(argnum): - argtype_letter, argtype_size, argtype_alignment = self.argtypes[i] + argletter = self.argletters[i] w_arg = args_w[i] - if argtype_letter == 'V': # by value object + if argletter == 'V': # by value object arg = space.interp_w(W_StructureInstance, w_arg) - if (arg.shape.size != argtype_size or - arg.shape.alignment != argtype_alignment): + xsize, xalignment = size_alignment(self.ptr.argtypes[i]) + if (arg.shape.size != xsize or + arg.shape.alignment != xalignment): msg = ("Argument %d should be a structure of size %d and " "alignment %d, " "got instead size %d and alignment %d" % - (i+1, argtype_size, argtype_alignment, + (i+1, xsize, xalignment, arg.shape.size, arg.shape.alignment)) raise OperationError(space.w_TypeError, space.wrap(msg)) else: @@ -357,11 +371,11 @@ "got length %d" % (i+1, arg.length)) raise OperationError(space.w_TypeError, space.wrap(msg)) letter = arg.shape.itemtp[0] - if letter != argtype_letter: - if not (argtype_letter in TYPEMAP_PTR_LETTERS and + if letter != argletter: + if not (argletter in TYPEMAP_PTR_LETTERS and letter in TYPEMAP_PTR_LETTERS): msg = "Argument %d should be typecode %s, got %s" % ( - i+1, argtype_letter, letter) + i+1, argletter, letter) raise OperationError(space.w_TypeError, space.wrap(msg)) args_ll.append(arg.ll_buffer) # XXX we could avoid the intermediate list args_ll Modified: pypy/branch/rawffi-shape-cleanup/pypy/module/_rawffi/test/test__rawffi.py ============================================================================== --- pypy/branch/rawffi-shape-cleanup/pypy/module/_rawffi/test/test__rawffi.py (original) +++ pypy/branch/rawffi-shape-cleanup/pypy/module/_rawffi/test/test__rawffi.py Wed Feb 27 16:37:13 2008 @@ -644,7 +644,7 @@ X_Y = _rawffi.Structure([('x', 'l'), ('y', 'l')]) x_y = X_Y() lib = _rawffi.CDLL(self.lib_name) - sum_x_y = lib.ptr('sum_x_y', [X_Y.gettypecode()], 'l') + sum_x_y = lib.ptr('sum_x_y', [(X_Y, 1)], 'l') x_y.x = 200 x_y.y = 220 res = sum_x_y(x_y) @@ -657,7 +657,7 @@ S2H = _rawffi.Structure([('x', 'h'), ('y', 'h')]) s2h = S2H() lib = _rawffi.CDLL(self.lib_name) - give = lib.ptr('give', ['h', 'h'], S2H) + give = lib.ptr('give', ['h', 'h'], (S2H, 1)) a1 = _rawffi.Array('h')(1) a2 = _rawffi.Array('h')(1) a1[0] = 13 @@ -673,7 +673,7 @@ s2h.x = 7 s2h.y = 11 - perturb = lib.ptr('perturb', [S2H.gettypecode()], S2H) + perturb = lib.ptr('perturb', [(S2H, 1)], (S2H, 1)) res = perturb(s2h) assert isinstance(res, _rawffi.StructureInstance) assert res.shape is S2H From antocuni at codespeak.net Wed Feb 27 17:24:07 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 27 Feb 2008 17:24:07 +0100 (CET) Subject: [pypy-svn] r51905 - pypy/extradoc/proposal Message-ID: <20080227162407.D540C16841B@codespeak.net> Author: antocuni Date: Wed Feb 27 17:24:06 2008 New Revision: 51905 Modified: pypy/extradoc/proposal/openjdk-challenge.txt Log: expand the proposal with more infos Modified: pypy/extradoc/proposal/openjdk-challenge.txt ============================================================================== --- pypy/extradoc/proposal/openjdk-challenge.txt (original) +++ pypy/extradoc/proposal/openjdk-challenge.txt Wed Feb 27 17:24:06 2008 @@ -5,16 +5,15 @@ -------------------------- PyPy_ is an open source research project that aims to produce a -flexible and fast implementation of the Python language (XXX: should -we mention other languages implementations?) +flexible and fast implementation of the Python language. PyPy is divided into two main parts: the Python interpreter, which implements the Python language and is written in RPython_, and the -Translation Toolchain (TT), (XXX: written in Python?) which transforms and translates programs -written in RPython into the final executables. RPython is a subset of -Python specifically designed to allow the TT to analyze RPython -programs and translate them into lower level, very efficient -executables. +Translation Toolchain (TT), written in Python, which transforms and +translates programs written in RPython into the final executables. +RPython is a subset of Python specifically designed to allow the TT to +analyze RPython programs and translate them into lower level, very +efficient executables. Currently, the TT of PyPy provides three complete backends that generate C code, bytecode for CLI/.NET and bytecode for the JVM. By @@ -22,11 +21,10 @@ standard C/Posix environment, on the CLI or on the JVM. It is important to underline that the job of the TT is not limited to -translation into an efficient executable, but it actively -transforms the source interpreter by adding new features and -translation aspects, such as garbage collection, stackless -capabilities, etc. (XXX: link) (XXX: not sure stackless is well known, -say microthreading instead?? mention proxying or security aspects??). +translation into an efficient executable, but it actively transforms +the source interpreter by adding new features and translation aspects, +such as garbage collection, microthreading (like `Stackless Python`_), +etc. The most exciting feature of the TT is the ability to automatically turn the interpreter into a JIT compiler that exploits partial @@ -34,10 +32,13 @@ Currently, the PyPy JIT works only in conjunction with the C backend; early results are very good, the resulting Python interpreter -can run numeric intensive computations at roughly the same speed of C -(XXX: link). +can run numeric intensive computations at roughly the same speed of C, +as shown by the `technical report`_ on the JIT. -XXX: should we mention the currently half-working CLI JIT backend? +Moreover, there is an experimental JIT backend that emits code for the +CLI; it is still work in progress and very incomplete, but it shows +that the it is possible to adapt the PyPy JIT to emit code for object +oriented virtual machines. Porting the JIT to the JVM @@ -50,15 +51,13 @@ will dynamically translate part of Python programs into JVM bytecode, which will then be executed by the underlying virtual machine. -XXX: should we explain in more detail what we are doing? - Porting the JIT to the MLVM --------------------------- As stated above, PyPy JIT for JVM would work by dynamically emitting and loading JVM bytecode at runtime. Even if this approach has been -tried in a couple of projects (XXX: link to JRuby for example), it has +tried in a couple of projects (see the "Related Work" section), it has to been said that the JVM was not originally designed for such applications; for example, the process of loading a single method is very expensive, since it involves the creation and loading of a @@ -81,6 +80,7 @@ understand which features are really useful to implement dynamic languages on top of the JVM and which one we still lack. + Deliverables ------------ @@ -119,8 +119,13 @@ For an example of a function with is highly optimized by the PyPy JIT, look at the `function f1`_: when executed by a pypy-c compiled with JIT support, it runs roughly at the same speed as its C equivalent -compiled with `gcc -O1`. (XXX: is there a link somewhere?) -(XXX it's only gcc -O0, not -O1) +compiled with `gcc -O0`. + +Making the Python interpreter to exploit the full potential of the JIT +is a separate task and it is out of the scope of this proposal; it is +important to underline that once the JVM backend for the JIT is +complete, the resulting pypy-jvm will automatically take advantage of +all the optimizations written for the others backend. We also expect to find benchmarks in which the JIT that targets the MLVM will perform better thant the JIT that targets the plain JVM, @@ -131,22 +136,21 @@ Relevance to the community -------------------------- -Recently the community has shown a lot of interest in dynamic languages -which run -on top of the JVM. Even if currently Jython(XXX: link) is the only usable -implementation of Python for the JVM, PyPy has the potential to become the -reference implementation in the future. +Recently the community has shown a lot of interest in dynamic +languages which run on top of the JVM. Even if currently Jython_ is +the only usable implementation of Python for the JVM, PyPy has the +potential to become the reference implementation in the future. To have a working JIT for the JVM is an important step towards making PyPy the fastest Python for the JVM, ever. Moreover, due to the innovative -ideas implemented by PyPy, it is possible (XXX: likely?) that Python could become +ideas implemented by PyPy, it is likely that Python could become the fastest dynamic language that runs on the top of the JVM. Finally, PyPy is not limited to Python: it is entirely possible to write interpreters for languages other than Python and translate them -with the TT; as a proof of concept, PyPy already contains -fairly complete implementations of Prolog and Smalltalk, as well -as partial implementations of JavaScript and Scheme. (XXX: did I overstate things?) +with the TT; as a proof of concept, PyPy already contains +implementations of Prolog, Smalltalk, JavaScript and Scheme, with +various degrees of completeness. Since the JIT generator is independent of the Python languages, it will be possible to automatically add a JIT compiler to every language @@ -171,27 +175,66 @@ - `this paper`_ shows how this technique is exploited to write an efficient implementation of EcmaScript which runs on top of the JVM; - - JRuby also comes with a JIT compiler that dynamically translates - Ruby code into JVM bytecode, however, unlike PyPy - JRuby doesn't exploit type information that is known only - at runtime to produce specialized, efficient versions of the - function; moreover, while the JRuby JIT is hand-written, the whole - goal of the PyPy JIT generator is to generate it automatically - from the intepreter; - - - in the .NET world, IronPython also emits code dynamically to - optimize hot spots, but not in a pervasive way as JRuby or PyPy. - - XXX (arigo) I'm confused by the previous sentence. I thought - that IronPython and Jython both emitted .NET/JVM bytecode as their - only way to compile Python source. I also imagined that JRuby - did exactly the same. I.e. all of IronPython, Jython and JRuby - work by turning each source unit to native bytecode by a direct - translation - no? + - Jython compiles Python source code to JVM bytecode; however, + unlike most compilers, the compilation phase occours when the JVM + has already been started, by generating and loading bytecode on + the fly; despite emitting code at runtime, this kind of compiler + really works ahead of time (AOT), because the code is fully + emitted before the program starts, and it doesn't exploit + additional informations that would be available only at runtime + (e.g., informations about the types that each variable can + assume); + + - JRuby supports interpretation, AOT compilation and JIT + compilation; when the JIT compilation is enabled, JRuby interprets + methods until a call threshold is reached, then it compiles the + method body to JVM bytecode to be executed from that point on; + however, even if the compilation is truly just in time, JRuby + doesn't exploit type informations that are known only at runtime to + produce specialized, efficient versions of the function; + + - in the .NET world, IronPython works more or less as Jython; + additionally, it exploits dynamic code generation to implement + `Polymorphic Inline Caches`_. + +PyPy JIT is different of all of these, because runtime and compile +time are continuously intermixed; by waiting until the very last +possible moment to emit code, the JIT compiler is able to exploit all +the runtime informations that wouldn't be available before, e.g. the +exact type of all the variables involved; thus, it can generate many +specialized, fast versions of each function, which in theory could run +at the same speed of manually written Java code. + +Moreover, the JIT compiler is automatically generated by the TT: we +believe, based on previous experiences as Psyco_, that manually +writing a JIT compiler of that kind is hard and error prone, +especially when the source language is as complex as Python; by +writing a JIT compiler generator, we get JIT compilers that are +correct by design for all languages implemented through the TT for +free. + + +Developer +--------- + +Antonio Cuni is one of the core developers of PyPy; he is the main +author of the CLI backend, and the coauthor of the JVM backend; +recently, it began working on the experimental CLI backend for the +JIT. + +Currently, he is a PhD student at Univerist? degli Studi di Genova, +doing research in the area of implementation of dynamic languages on +top of object oriented virtual machines. .. _PyPy: http://codespeak.net/pypy .. _RPython: http://codespeak.net/pypy/dist/pypy/doc/coding-guide.html#rpython +.. _`Stackless Python`: http://www.stackless.com/ +.. _`technical report`: http://codespeak.net/pypy/extradoc/eu-report/D08.2_JIT_Compiler_Architecture-2007-05-01.pdf .. _`John Rose said`: http://blogs.sun.com/jrose/entry/a_day_with_pypy +.. _Jython: http://www.jython.org .. _`function f1`: http://codespeak.net/svn/pypy/dist/demo/jit/f1.py .. _`this paper`: http://www.ics.uci.edu/~franz/Site/pubs-pdf/ICS-TR-07-10.pdf +.. _`Polymorphic Inline Caches`: http://www.cs.ucsb.edu/~urs/oocsb/papers/ecoop91.pdf +.. _Psyco: http://psyco.sourceforge.net/ + From antocuni at codespeak.net Wed Feb 27 17:59:10 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 27 Feb 2008 17:59:10 +0100 (CET) Subject: [pypy-svn] r51906 - pypy/extradoc/proposal Message-ID: <20080227165910.30157168404@codespeak.net> Author: antocuni Date: Wed Feb 27 17:59:07 2008 New Revision: 51906 Modified: pypy/extradoc/proposal/openjdk-challenge.txt Log: insert an "advertisement-like" sentence Modified: pypy/extradoc/proposal/openjdk-challenge.txt ============================================================================== --- pypy/extradoc/proposal/openjdk-challenge.txt (original) +++ pypy/extradoc/proposal/openjdk-challenge.txt Wed Feb 27 17:59:07 2008 @@ -28,7 +28,11 @@ The most exciting feature of the TT is the ability to automatically turn the interpreter into a JIT compiler that exploits partial -evaluation techniques to dynamically generate efficient code. +evaluation techniques to dynamically generate efficient code. The +novel idea behind PyPy JIT is to delay the compilation until we know +all the informations useful for emitting optimized code, thus +being potentially much more efficient than all the current other +alternatives (see the "Related Work" section). Currently, the PyPy JIT works only in conjunction with the C backend; early results are very good, the resulting Python interpreter From antocuni at codespeak.net Wed Feb 27 18:13:28 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 27 Feb 2008 18:13:28 +0100 (CET) Subject: [pypy-svn] r51907 - pypy/extradoc/proposal Message-ID: <20080227171328.7381F168430@codespeak.net> Author: antocuni Date: Wed Feb 27 18:13:27 2008 New Revision: 51907 Modified: pypy/extradoc/proposal/openjdk-challenge.txt Log: typos Modified: pypy/extradoc/proposal/openjdk-challenge.txt ============================================================================== --- pypy/extradoc/proposal/openjdk-challenge.txt (original) +++ pypy/extradoc/proposal/openjdk-challenge.txt Wed Feb 27 18:13:27 2008 @@ -65,7 +65,7 @@ to been said that the JVM was not originally designed for such applications; for example, the process of loading a single method is very expensive, since it involves the creation and loading of a -sorrounding class. +surrounding class. The new Da Vinci Machine contains a lot of interesting features that could be effectively exploited by the PyPy JIT to produce an even more @@ -132,8 +132,8 @@ all the optimizations written for the others backend. We also expect to find benchmarks in which the JIT that targets the -MLVM will perform better thant the JIT that targets the plain JVM, -though it is hard to speficy a precise commitment here without knowing +MLVM will perform better than the JIT that targets the plain JVM, +though it is hard to specify a precise commitment here without knowing which features of the MLVM will be possible to use. @@ -180,7 +180,7 @@ efficient implementation of EcmaScript which runs on top of the JVM; - Jython compiles Python source code to JVM bytecode; however, - unlike most compilers, the compilation phase occours when the JVM + unlike most compilers, the compilation phase occurs when the JVM has already been started, by generating and loading bytecode on the fly; despite emitting code at runtime, this kind of compiler really works ahead of time (AOT), because the code is fully From pedronis at codespeak.net Wed Feb 27 22:39:48 2008 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 27 Feb 2008 22:39:48 +0100 (CET) Subject: [pypy-svn] r51908 - pypy/dist/pypy/doc/discussion Message-ID: <20080227213948.BE3EA1683CF@codespeak.net> Author: pedronis Date: Wed Feb 27 22:39:45 2008 New Revision: 51908 Modified: pypy/dist/pypy/doc/discussion/ctypes_todo.txt Log: update Modified: pypy/dist/pypy/doc/discussion/ctypes_todo.txt ============================================================================== --- pypy/dist/pypy/doc/discussion/ctypes_todo.txt (original) +++ pypy/dist/pypy/doc/discussion/ctypes_todo.txt Wed Feb 27 22:39:45 2008 @@ -15,13 +15,12 @@ - there are features, which we don't support like buffer() and array() protocols. - - decide who should create own and free the needed ffitypes for structures by value args, - right now we simple leak them. Structure types etc seem the right place for this. + - started refactoring (on the rawffi-shape-cleanup) to use rawffi types + instead of size, aligment tuples to describe members and arguments. + Rawffi composite types should be able to create corresponding ffi_types. - no tests for passing a union by value - - returning structures by value is still not implemented - - for some ABIs we will need completely filled ffitypes to do the right thing for passing structures by value From pypy-svn at codespeak.net Wed Feb 27 23:55:35 2008 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Wed, 27 Feb 2008 23:55:35 +0100 (CET) Subject: [pypy-svn] February 74% OFF Message-ID: <20080227125503.35375.qmail@catv-5985e861.catv.broadband.hu> An HTML attachment was scrubbed... URL: From cfbolz at codespeak.net Thu Feb 28 14:56:01 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 28 Feb 2008 14:56:01 +0100 (CET) Subject: [pypy-svn] r51914 - in pypy/branch/jit-refactoring/pypy/jit: codegen/llgraph rainbow/test Message-ID: <20080228135601.05D2A168458@codespeak.net> Author: cfbolz Date: Thu Feb 28 14:55:59 2008 New Revision: 51914 Modified: pypy/branch/jit-refactoring/pypy/jit/codegen/llgraph/llimpl.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Log: fix a problem with sharing blocks between flow graphs Modified: pypy/branch/jit-refactoring/pypy/jit/codegen/llgraph/llimpl.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/codegen/llgraph/llimpl.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/codegen/llgraph/llimpl.py Thu Feb 28 14:55:59 2008 @@ -599,11 +599,26 @@ block.renamevariables(mapping) done[block] = True +def _find_and_set_return_block(graph): + returnblocks = {} + for block in graph.iterblocks(): + if block.exits == () and len(block.inputargs) == 1: + returnblocks[block] = None + assert len(returnblocks) == 1, "ambiguous return block" + graph.returnblock = iter(returnblocks).next() + + def _buildgraph(graph): assert graph.startblock.operations[0].opname == 'debug_assert' del graph.startblock.operations[0] # rgenop makes graphs that use the same variable in several blocks, fixduplicatevars(graph) # fix this now + # through caching it is possible that parts of another graph are reused + # make a copy of the whole graph to no longer share data + newgraph = flowmodel.copygraph(graph) + graph.__dict__.update(newgraph.__dict__) + _find_and_set_return_block(graph) + flowmodel.checkgraph(graph) eliminate_empty_blocks(graph) # we cannot call join_blocks(graph) here! It has a subtle problem: Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Thu Feb 28 14:55:59 2008 @@ -158,7 +158,6 @@ assert not res def test_dfa_compile2(self): - py.test.skip("not working yet") from pypy.lang.automata.dfa import getautomaton, convertagain, recognizeparts more = [convertagain(getautomaton()), convertagain(getautomaton())] def main(gets, gets2): From antocuni at codespeak.net Thu Feb 28 15:26:17 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 28 Feb 2008 15:26:17 +0100 (CET) Subject: [pypy-svn] r51915 - in pypy/dist/pypy/jit/codegen/cli: . test Message-ID: <20080228142617.4A1B6168425@codespeak.net> Author: antocuni Date: Thu Feb 28 15:26:16 2008 New Revision: 51915 Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Log: test_genzeroconst passes. Not sure it's correct, though Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/rgenop.py Thu Feb 28 15:26:16 2008 @@ -206,6 +206,13 @@ assert False, "XXX not implemented" @staticmethod + def genzeroconst(kind): + if kind == '': + return IntConst(0) + else: + return zero_const # ??? + + @staticmethod @specialize.memo() def sigToken(FUNCTYPE): """Return a token describing the signature of FUNCTYPE.""" @@ -406,3 +413,4 @@ global_rgenop = RCliGenOp() RCliGenOp.constPrebuiltGlobal = global_rgenop.genconst +zero_const = ObjectConst(ootype.null(ootype.ROOT)) Modified: pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/test/test_rgenop.py Thu Feb 28 15:26:16 2008 @@ -76,9 +76,6 @@ def test_from_random_5_direct(self): py.test.skip('mono crash') - def test_genzeroconst(self): - py.test.skip('fixme') - def test_ovfcheck_adder_direct(self): py.test.skip('fixme') From cfbolz at codespeak.net Thu Feb 28 15:55:29 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 28 Feb 2008 15:55:29 +0100 (CET) Subject: [pypy-svn] r51916 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter/test Message-ID: <20080228145529.300C2168440@codespeak.net> Author: cfbolz Date: Thu Feb 28 15:55:28 2008 New Revision: 51916 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py Log: move over more tests and fix one of them. Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py Thu Feb 28 15:55:28 2008 @@ -25,7 +25,8 @@ self.PortalState = make_state_class( self.args_specification, self.RESIDUAL_FUNCTYPE, self.sigtoken, self.codewriter.all_graphs[self.portalgraph], - self.rtyper) + self.rtyper, + self.codewriter) self.make_state_instance() self.mutate_origportalgraph() @@ -97,7 +98,7 @@ def make_state_class(args_specification, RESIDUAL_FUNCTYPE, sigtoken, - portal_jitcode, rtyper): + portal_jitcode, rtyper, codewriter): args_specification = unrolling_iterable(args_specification) class PortalState(object): def __init__(self, interpreter, portalbytecode): @@ -154,7 +155,8 @@ fn = gv_generated.revealconst(lltype.Ptr(RESIDUAL_FUNCTYPE)) if not we_are_translated(): # run the generated code on top of the llinterp for testing - llinterp = LLInterpreter(rtyper) + exc_data_ptr = codewriter.exceptiondesc.exc_data_ptr + llinterp = LLInterpreter(rtyper, exc_data_ptr=exc_data_ptr) res = llinterp.eval_graph(fn._obj.graph, residualargs) return res else: Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Thu Feb 28 15:55:28 2008 @@ -59,12 +59,13 @@ backendoptimize=backendoptimize) self.main_args = main_args self.main_is_portal = main is portal - llinterp = LLInterpreter(self.rtyper) + llinterp = LLInterpreter(self.rtyper, + exc_data_ptr= + self.writer.exceptiondesc.exc_data_ptr) res = llinterp.eval_graph(self.maingraph, main_args) return res def get_residual_graph(self): - llinterp = LLInterpreter(self.rtyper) portalstate = self.rewriter.state if self.main_is_portal: residual_graph = portalstate.readportal(*self.main_args)._obj.graph @@ -375,3 +376,52 @@ policy=StopAtXPolicy(ll_make)) assert res == ord('2') self.check_insns(indirect_call=0, malloc=0) + + + def test_residual_red_call_with_promoted_exc(self): + def h(x): + if x > 0: + return x+1 + else: + raise ValueError + + def g(x): + return 2*h(x) + + def f(x): + hint(None, global_merge_point=True) + try: + return g(x) + except ValueError: + return 7 + + stop_at_h = StopAtXPolicy(h) + res = self.timeshift_from_portal(f, f, [20], policy=stop_at_h) + assert res == 42 + self.check_insns(int_add=0) + + res = self.timeshift_from_portal(f, f, [-20], policy=stop_at_h) + assert res == 7 + self.check_insns(int_add=0) + + def test_residual_oop_raising(self): + py.test.skip("not working yet") + def g(x): + lst = [] + if x > 10: + lst.append(x) + return lst + def f(x): + hint(None, global_merge_point=True) + lst = g(x) + try: + return lst[0] + except IndexError: + return -42 + + res = self.timeshift_from_portal(f, f, [5], policy=P_OOPSPEC) + assert res == -42 + + res = self.timeshift_from_portal(f, f, [15], policy=P_OOPSPEC) + assert res == 15 + Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py Thu Feb 28 15:55:28 2008 @@ -141,10 +141,6 @@ return calls class TestPortal(PortalTest): - - - - def test_simple_recursive_portal_call(self): def main(code, x): @@ -216,52 +212,6 @@ assert res == 11 - def test_residual_red_call_with_promoted_exc(self): - def h(x): - if x > 0: - return x+1 - else: - raise ValueError - - def g(x): - return 2*h(x) - - def f(x): - hint(None, global_merge_point=True) - try: - return g(x) - except ValueError: - return 7 - - stop_at_h = StopAtXPolicy(h) - res = self.timeshift_from_portal(f, f, [20], policy=stop_at_h) - assert res == 42 - self.check_insns(int_add=0) - - res = self.timeshift_from_portal(f, f, [-20], policy=stop_at_h) - assert res == 7 - self.check_insns(int_add=0) - - def test_residual_oop_raising(self): - def g(x): - lst = [] - if x > 10: - lst.append(x) - return lst - def f(x): - hint(None, global_merge_point=True) - lst = g(x) - try: - return lst[0] - except IndexError: - return -42 - - res = self.timeshift_from_portal(f, f, [5], policy=P_OOPSPEC) - assert res == -42 - - res = self.timeshift_from_portal(f, f, [15], policy=P_OOPSPEC) - assert res == 15 - def test_portal_returns_none(self): py.test.skip("portal returning None is not supported") def g(x): From cfbolz at codespeak.net Thu Feb 28 16:03:14 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 28 Feb 2008 16:03:14 +0100 (CET) Subject: [pypy-svn] r51917 - pypy/branch/jit-refactoring/pypy/jit/codegen/llgraph Message-ID: <20080228150314.62C77168440@codespeak.net> Author: cfbolz Date: Thu Feb 28 16:03:13 2008 New Revision: 51917 Modified: pypy/branch/jit-refactoring/pypy/jit/codegen/llgraph/llimpl.py Log: oops. of course graphs can have no returnblock at all Modified: pypy/branch/jit-refactoring/pypy/jit/codegen/llgraph/llimpl.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/codegen/llgraph/llimpl.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/codegen/llgraph/llimpl.py Thu Feb 28 16:03:13 2008 @@ -604,8 +604,9 @@ for block in graph.iterblocks(): if block.exits == () and len(block.inputargs) == 1: returnblocks[block] = None - assert len(returnblocks) == 1, "ambiguous return block" - graph.returnblock = iter(returnblocks).next() + if returnblocks: + assert len(returnblocks) == 1, "ambiguous return block" + graph.returnblock = iter(returnblocks).next() def _buildgraph(graph): From cfbolz at codespeak.net Thu Feb 28 16:18:48 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 28 Feb 2008 16:18:48 +0100 (CET) Subject: [pypy-svn] r51918 - in pypy/branch/jit-refactoring/pypy/jit: rainbow timeshifter Message-ID: <20080228151848.D9B06168461@codespeak.net> Author: cfbolz Date: Thu Feb 28 16:18:47 2008 New Revision: 51918 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/oop.py Log: make oop handlers a bit more rpython Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Thu Feb 28 16:18:47 2008 @@ -522,19 +522,19 @@ @arguments("oopspec", "bool", returns="red") def opimpl_red_oopspec_call_0(self, oopspec, deepfrozen): - return oopspec.ll_handler(self.jitstate, oopspec, deepfrozen) + return oopspec.ll_handler_0(self.jitstate, oopspec, deepfrozen) @arguments("oopspec", "bool", "red", returns="red") def opimpl_red_oopspec_call_1(self, oopspec, deepfrozen, arg1): - return oopspec.ll_handler(self.jitstate, oopspec, deepfrozen, arg1) + return oopspec.ll_handler_1(self.jitstate, oopspec, deepfrozen, arg1) @arguments("oopspec", "bool", "red", "red", returns="red") def opimpl_red_oopspec_call_2(self, oopspec, deepfrozen, arg1, arg2): - return oopspec.ll_handler(self.jitstate, oopspec, deepfrozen, arg1, arg2) + return oopspec.ll_handler_2(self.jitstate, oopspec, deepfrozen, arg1, arg2) @arguments("oopspec", "bool", "red", "red", "red", returns="red") def opimpl_red_oopspec_call_3(self, oopspec, deepfrozen, arg1, arg2, arg3): - return oopspec.ll_handler(self.jitstate, oopspec, deepfrozen, arg1, arg2, arg3) + return oopspec.ll_handler_3(self.jitstate, oopspec, deepfrozen, arg1, arg2, arg3) @arguments("red", "calldesc", "bool", "red_varargs", "promotiondesc") def opimpl_red_residual_call(self, funcbox, calldesc, withexc, Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/oop.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/oop.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/oop.py Thu Feb 28 16:18:47 2008 @@ -86,8 +86,11 @@ vmodule = __import__('pypy.jit.timeshifter.v%s' % (typename,), None, None, [method]) self.typedesc = vmodule.TypeDesc(RGenOp, rtyper, SELFTYPE) - self.ll_handler = getattr(vmodule, method) - self.couldfold = getattr(self.ll_handler, 'couldfold', False) + handler = getattr(vmodule, method) + setattr(self, "ll_handler_%s" % (len(OOPARGTYPES), ), handler) + + self.ll_handler = handler + self.couldfold = getattr(handler, 'couldfold', False) if self.couldfold: oopargcheck = ll_func.oopargcheck # required if couldfold=True From lac at codespeak.net Thu Feb 28 16:29:12 2008 From: lac at codespeak.net (lac at codespeak.net) Date: Thu, 28 Feb 2008 16:29:12 +0100 (CET) Subject: [pypy-svn] r51919 - pypy/extradoc/proposal Message-ID: <20080228152912.1B09016846C@codespeak.net> Author: lac Date: Thu Feb 28 16:29:11 2008 New Revision: 51919 Modified: pypy/extradoc/proposal/openjdk-challenge.txt Log: Fix grammatical errors/typos. Rewrite one clunky sentence. Add one XXX for a sentence that wasn't clear to me. Modified: pypy/extradoc/proposal/openjdk-challenge.txt ============================================================================== --- pypy/extradoc/proposal/openjdk-challenge.txt (original) +++ pypy/extradoc/proposal/openjdk-challenge.txt Thu Feb 28 16:29:11 2008 @@ -16,7 +16,7 @@ efficient executables. Currently, the TT of PyPy provides three complete backends that -generate C code, bytecode for CLI/.NET and bytecode for the JVM. By +generate C code, bytecode for CLI/.NET or bytecode for the JVM. By using these backends, we can get Python implementations that run on a standard C/Posix environment, on the CLI or on the JVM. @@ -34,14 +34,14 @@ being potentially much more efficient than all the current other alternatives (see the "Related Work" section). -Currently, the PyPy JIT works only in conjunction with the C backend; -early results are very good, the resulting Python interpreter -can run numeric intensive computations at roughly the same speed of C, +Currently, the PyPy JIT works only in conjunction with the C backend. +Early results are very good. The resulting Python interpreter +can run numerically intensive computations at roughly the same speed of C, as shown by the `technical report`_ on the JIT. Moreover, there is an experimental JIT backend that emits code for the -CLI; it is still work in progress and very incomplete, but it shows -that the it is possible to adapt the PyPy JIT to emit code for object +CLI; it is still a work in progress and very incomplete, but it shows +that the is possible to adapt the PyPy JIT to emit code for object oriented virtual machines. @@ -129,7 +129,8 @@ is a separate task and it is out of the scope of this proposal; it is important to underline that once the JVM backend for the JIT is complete, the resulting pypy-jvm will automatically take advantage of -all the optimizations written for the others backend. +all the optimizations written for the other backends. XXX expand +this by one sentence because this is unclear as written We also expect to find benchmarks in which the JIT that targets the MLVM will perform better than the JIT that targets the plain JVM, @@ -194,17 +195,18 @@ methods until a call threshold is reached, then it compiles the method body to JVM bytecode to be executed from that point on; however, even if the compilation is truly just in time, JRuby - doesn't exploit type informations that are known only at runtime to + doesn't exploit type information that is known only at runtime to produce specialized, efficient versions of the function; - in the .NET world, IronPython works more or less as Jython; additionally, it exploits dynamic code generation to implement `Polymorphic Inline Caches`_. -PyPy JIT is different of all of these, because runtime and compile +The PyPy JIT is different from all of these, because runtime and compile time are continuously intermixed; by waiting until the very last possible moment to emit code, the JIT compiler is able to exploit all -the runtime informations that wouldn't be available before, e.g. the +the runtime information, including that which is only available late +in the process, e.g. the exact type of all the variables involved; thus, it can generate many specialized, fast versions of each function, which in theory could run at the same speed of manually written Java code. From pypy-svn at codespeak.net Thu Feb 28 04:42:58 2008 From: pypy-svn at codespeak.net (pypy-svn at codespeak.net) Date: Thu, 28 Feb 2008 05:42:58 +0200 Subject: [pypy-svn] Sale 79% OFF Message-ID: <20080228074258.3135.qmail@dsl85-106-43896.ttnet.net.tr> An HTML attachment was scrubbed... URL: From cfbolz at codespeak.net Thu Feb 28 17:34:09 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 28 Feb 2008 17:34:09 +0100 (CET) Subject: [pypy-svn] r51920 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080228163409.3B563168458@codespeak.net> Author: cfbolz Date: Thu Feb 28 17:34:07 2008 New Revision: 51920 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Log: support for oopspec calls that are residual and raise Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Thu Feb 28 17:34:07 2008 @@ -185,7 +185,7 @@ bytecode._source = self.assembler bytecode._interpreter = self.interpreter bytecode._labelpos = labelpos - #bytecode.dump() + bytecode.dump() if is_portal: self.finish_all_graphs() self.interpreter.set_num_global_mergepoints( @@ -740,11 +740,21 @@ else: deepfrozen = False - self.emit("red_oopspec_call_%s" % (len(args), )) + hasresult = op.result.concretetype != lltype.Void + self.emit("red_oopspec_call%s_%s" % ("_noresult" * (not hasresult), + len(args))) self.emit(oopspecdescindex) self.emit(deepfrozen) self.emit(*args) - self.register_redvar(op.result) + if hasresult: + self.register_redvar(op.result) + + if withexc: + self.emit("goto_if_oopcall_was_virtual", tlabel(("oop_call", op))) + self.emit("after_oop_residual_call") + self.emit(self.promotiondesc_position(lltype.Signed)) + + self.emit(label(("oop_call", op))) def handle_green_call(self, op, withexc): voidargs = [const.value for const in op.args[1:] Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Thu Feb 28 17:34:07 2008 @@ -227,6 +227,7 @@ bytecode = self.load_2byte() assert bytecode >= 0 result = self.opcode_implementations[bytecode](self) + #assert self.frame.local_boxes[-1] is not None if result is STOP: return else: @@ -378,6 +379,11 @@ if valuebox.is_constant(): self.frame.pc = target + @arguments("jumptarget") + def opimpl_goto_if_oopcall_was_virtual(self, target): + if not rtimeshift.oopspec_was_residual(self.jitstate): + self.frame.pc = target + @arguments("red", returns="red") def opimpl_red_ptr_nonzero(self, ptrbox): return rtimeshift.genptrnonzero(self.jitstate, ptrbox, False) @@ -536,6 +542,36 @@ def opimpl_red_oopspec_call_3(self, oopspec, deepfrozen, arg1, arg2, arg3): return oopspec.ll_handler_3(self.jitstate, oopspec, deepfrozen, arg1, arg2, arg3) + @arguments("oopspec", "bool") + def opimpl_red_oopspec_call_noresult_0(self, oopspec, deepfrozen): + oopspec.ll_handler_0(self.jitstate, oopspec, deepfrozen) + + @arguments("oopspec", "bool", "red") + def opimpl_red_oopspec_call_noresult_1(self, oopspec, deepfrozen, arg1): + oopspec.ll_handler_1(self.jitstate, oopspec, deepfrozen, arg1) + + @arguments("oopspec", "bool", "red", "red") + def opimpl_red_oopspec_call_noresult_2(self, oopspec, deepfrozen, arg1, arg2): + oopspec.ll_handler_2(self.jitstate, oopspec, deepfrozen, arg1, arg2) + + @arguments("oopspec", "bool", "red", "red", "red") + def opimpl_red_oopspec_call_noresult_3(self, oopspec, deepfrozen, arg1, arg2, arg3): + oopspec.ll_handler_3(self.jitstate, oopspec, deepfrozen, arg1, arg2, arg3) + + @arguments("promotiondesc") + def opimpl_after_oop_residual_call(self, promotiondesc): + exceptiondesc = self.exceptiondesc + check_forced = False + flagbox = rtimeshift.after_residual_call(self.jitstate, + exceptiondesc, check_forced) + done = rtimeshift.promote(self.jitstate, flagbox, promotiondesc) + if done: + return self.dispatch() + gv_flag = flagbox.getgenvar(self.jitstate) + assert gv_flag.is_const + rtimeshift.residual_fetch(self.jitstate, self.exceptiondesc, + check_forced, flagbox) + @arguments("red", "calldesc", "bool", "red_varargs", "promotiondesc") def opimpl_red_residual_call(self, funcbox, calldesc, withexc, redargs, promotiondesc): Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Thu Feb 28 17:34:07 2008 @@ -70,6 +70,7 @@ class InterpretationTest(object): RGenOp = LLRGenOp + small = False def setup_class(cls): cls.on_llgraph = cls.RGenOp is LLRGenOp Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Thu Feb 28 17:34:07 2008 @@ -405,7 +405,6 @@ self.check_insns(int_add=0) def test_residual_oop_raising(self): - py.test.skip("not working yet") def g(x): lst = [] if x > 10: From antocuni at codespeak.net Thu Feb 28 18:39:17 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 28 Feb 2008 18:39:17 +0100 (CET) Subject: [pypy-svn] r51926 - in pypy/dist/pypy/translator/cli: . test Message-ID: <20080228173917.3E2E2168424@codespeak.net> Author: antocuni Date: Thu Feb 28 18:39:16 2008 New Revision: 51926 Modified: pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: add a way to store instances of System.Type as PBCs. It is a bit convoluted because it converts them into values of type ootype.Class, then it does a cast at runtime via box/clidowncast, but it's the best we can do until we refactor the way .NET is accessed within RPython. Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Thu Feb 28 18:39:16 2008 @@ -360,6 +360,11 @@ return CLR.System.Char(x) else: return CLR.System.String(x) + elif isinstance(x, ootype._class): + name = '%s.%s' % (x._INSTANCE._namespace, x._INSTANCE._classname) + t = CLR.System.Type.GetType(name) + assert t is not None + return t elif isinstance(x, PythonNet.System.Object): return x elif x is None: @@ -555,6 +560,12 @@ TYPE = cliClass._INSTANCE return PythonNet.System.Type.GetType(TYPE._assembly_qualified_name) + +def classof(cliClass): + assert isinstance(cliClass, CliClass) + TYPE = cliClass._INSTANCE + return ootype._class(TYPE) + class Entry(ExtRegistryEntry): _about_ = typeof Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Thu Feb 28 18:39:16 2008 @@ -8,7 +8,7 @@ from pypy.translator.cli.dotnet import SomeCliClass, SomeCliStaticMethod,\ NativeInstance, CLR, box, unbox, OverloadingResolver, NativeException,\ native_exc, new_array, init_array, typeof, eventhandler, clidowncast,\ - fieldinfo_for_const + fieldinfo_for_const, classof System = CLR.System ArrayList = CLR.System.Collections.ArrayList @@ -604,6 +604,14 @@ res = self.interpret(fn, []) assert res == 42 + def test_classof(self): + int32_class = classof(System.Int32) + def fn(): + int32_obj = box(int32_class) + int32_type = clidowncast(int32_obj, System.Type) + return int32_type.get_Name() + assert self.interpret(fn, []) == 'Int32' + class TestPythonnet(TestDotnetRtyping): # don't interpreter functions but execute them directly through pythonnet def interpret(self, f, args, backendopt='ignored'): From antocuni at codespeak.net Thu Feb 28 20:40:49 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 28 Feb 2008 20:40:49 +0100 (CET) Subject: [pypy-svn] r51927 - in pypy/dist/pypy/translator/cli: . test Message-ID: <20080228194049.383101684C2@codespeak.net> Author: antocuni Date: Thu Feb 28 20:40:46 2008 New Revision: 51927 Modified: pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: make sure we can compare the result of classof() Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Thu Feb 28 20:40:46 2008 @@ -564,7 +564,7 @@ def classof(cliClass): assert isinstance(cliClass, CliClass) TYPE = cliClass._INSTANCE - return ootype._class(TYPE) + return ootype.runtimeClass(TYPE) class Entry(ExtRegistryEntry): _about_ = typeof Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Thu Feb 28 20:40:46 2008 @@ -612,6 +612,13 @@ return int32_type.get_Name() assert self.interpret(fn, []) == 'Int32' + def test_classof_compare(self): + int32_a = classof(System.Int32) + int32_b = classof(System.Int32) + def fn(): + return int32_a is int32_b + assert self.interpret(fn, []) + class TestPythonnet(TestDotnetRtyping): # don't interpreter functions but execute them directly through pythonnet def interpret(self, f, args, backendopt='ignored'): From antocuni at codespeak.net Fri Feb 29 10:47:11 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 29 Feb 2008 10:47:11 +0100 (CET) Subject: [pypy-svn] r51935 - in pypy/dist/pypy/translator/cli: . test Message-ID: <20080229094711.338F31684F7@codespeak.net> Author: antocuni Date: Fri Feb 29 10:47:10 2008 New Revision: 51935 Modified: pypy/dist/pypy/translator/cli/constant.py pypy/dist/pypy/translator/cli/database.py pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: another horrible hack to allow classof(FUNCTYPE). That's needed because in the JIT we really need to know the .NET type of StaticMethods. Modified: pypy/dist/pypy/translator/cli/constant.py ============================================================================== --- pypy/dist/pypy/translator/cli/constant.py (original) +++ pypy/dist/pypy/translator/cli/constant.py Fri Feb 29 10:47:10 2008 @@ -341,7 +341,7 @@ def push_inline(self, gen, EXPECTED_TYPE): if not self.is_null(): INSTANCE = self.value._INSTANCE - gen.ilasm.opcode('ldtoken', self.db.class_name(INSTANCE)) + gen.ilasm.opcode('ldtoken', self.db.class_or_delegate_name(INSTANCE)) gen.ilasm.call('class [mscorlib]System.Type class [mscorlib]System.Type::GetTypeFromHandle(valuetype [mscorlib]System.RuntimeTypeHandle)') return super(CLIClassConst, self).push_inline(gen, EXPECTED_TYPE) Modified: pypy/dist/pypy/translator/cli/database.py ============================================================================== --- pypy/dist/pypy/translator/cli/database.py (original) +++ pypy/dist/pypy/translator/cli/database.py Fri Feb 29 10:47:10 2008 @@ -150,3 +150,9 @@ self.delegates[TYPE] = name self.pending_node(Delegate(self, TYPE, name)) return name + + def class_or_delegate_name(self, TYPE): + if isinstance(TYPE, ootype.StaticMethod): + return self.record_delegate(TYPE) + else: + return self.class_name(TYPE) Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Fri Feb 29 10:47:10 2008 @@ -361,6 +361,9 @@ else: return CLR.System.String(x) elif isinstance(x, ootype._class): + TYPE = x._INSTANCE + if isinstance(TYPE, ootype.StaticMethod): + return typeof(TYPE) name = '%s.%s' % (x._INSTANCE._namespace, x._INSTANCE._classname) t = CLR.System.Type.GetType(name) assert t is not None @@ -560,11 +563,19 @@ TYPE = cliClass._INSTANCE return PythonNet.System.Type.GetType(TYPE._assembly_qualified_name) - -def classof(cliClass): - assert isinstance(cliClass, CliClass) - TYPE = cliClass._INSTANCE - return ootype.runtimeClass(TYPE) +def classof(cliClass_or_type): + if isinstance(cliClass_or_type, ootype.StaticMethod): + try: + FUNC = cliClass_or_type + return known_delegates_class[FUNC] + except KeyError: + cls = ootype._class(FUNC) + known_delegates_class[FUNC] = cls + return cls + else: + assert isinstance(cliClass_or_type, CliClass) + TYPE = cliClass_or_type._INSTANCE + return ootype.runtimeClass(TYPE) class Entry(ExtRegistryEntry): _about_ = typeof @@ -657,3 +668,5 @@ ootype.StaticMethod([ootype.Signed] * 27, ootype.Signed): CLR.pypy.test.DelegateType_int__int_27, ootype.StaticMethod([ootype.Signed] * 100, ootype.Signed): CLR.pypy.test.DelegateType_int__int_100 } + +known_delegates_class = {} Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Fri Feb 29 10:47:10 2008 @@ -619,6 +619,17 @@ return int32_a is int32_b assert self.interpret(fn, []) + def test_classof_functype(self): + # this test is overridden in TestPythonnet + c = classof(FUNCTYPE) + def fn(): + obj = box(c) + t = clidowncast(obj, System.Type) + return t.get_Name() + res = self.interpret(fn, []) + assert res.startswith('StaticMethod__') + + class TestPythonnet(TestDotnetRtyping): # don't interpreter functions but execute them directly through pythonnet def interpret(self, f, args, backendopt='ignored'): @@ -637,6 +648,16 @@ res = self.interpret(fn, []) assert res == 'DelegateType_int__int_2' + def test_classof_functype(self): + # this test is overridden in TestPythonnet + c = classof(FUNCTYPE) + def fn(): + obj = box(c) + t = clidowncast(obj, System.Type) + return t.get_Name() + res = self.interpret(fn, []) + assert res == 'DelegateType_int__int_2' + def test_fieldinfo_for_const(self): pass # it makes sense only during translation From cfbolz at codespeak.net Fri Feb 29 11:17:41 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 29 Feb 2008 11:17:41 +0100 (CET) Subject: [pypy-svn] r51937 - pypy/branch/jit-refactoring/pypy/jit/timeshifter Message-ID: <20080229101741.2DB7616850C@codespeak.net> Author: cfbolz Date: Fri Feb 29 11:17:40 2008 New Revision: 51937 Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Log: clean up some things that are no longer needed Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/rtimeshift.py Fri Feb 29 11:17:40 2008 @@ -344,8 +344,6 @@ start_new_block(states_dic, jitstate, key, global_resumer) return False -retrieve_jitstate_for_merge._annspecialcase_ = "specialize:arglltype(2)" - def cleanup_partial_data(partialdatamatch): # remove entries from PartialDataStruct unless they matched # their frozen equivalent @@ -1225,16 +1223,6 @@ return jitstate -def replayable_ensure_queue(jitstate, DispatchQueueClass): - if jitstate.frame is None: # common case - return DispatchQueueClass() - else: - # replaying - dispatchqueue = jitstate.frame.dispatchqueue - assert isinstance(dispatchqueue, DispatchQueueClass) - return dispatchqueue -replayable_ensure_queue._annspecialcase_ = 'specialize:arg(1)' - def enter_frame(jitstate, dispatchqueue): if jitstate.frame: resuming = jitstate.get_resuming() From cfbolz at codespeak.net Fri Feb 29 11:19:22 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 29 Feb 2008 11:19:22 +0100 (CET) Subject: [pypy-svn] r51938 - in pypy/branch/jit-refactoring/pypy/jit: rainbow rainbow/test timeshifter/test Message-ID: <20080229101922.CB2B0168465@codespeak.net> Author: cfbolz Date: Fri Feb 29 11:19:22 2008 New Revision: 51938 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py Log: some support for recursive portal calls Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Fri Feb 29 11:19:22 2008 @@ -103,7 +103,9 @@ def make_bytecode(self, graph, is_portal=True): remove_same_as(graph) if is_portal: - self.all_graphs[graph] = JitCode.__new__(JitCode) + bytecode = JitCode.__new__(JitCode) + bytecode.is_portal = True + self.all_graphs[graph] = bytecode self.seen_blocks = {} self.assembler = [] self.constants = [] @@ -795,11 +797,17 @@ assert len(targets) == 1 targetgraph, = targets.values() graphindex = self.graph_position(targetgraph) + bytecode = self.all_graphs[targetgraph] args = targetgraph.getargs() emitted_args = self.args_of_call(op.args[1:], args) - self.emit("red_direct_call") - self.emit(*emitted_args) - self.emit(graphindex) + + if bytecode.is_portal: + self.emit("portal_call", *emitted_args) + else: + self.emit("red_direct_call") + self.emit(*emitted_args) + self.emit(graphindex) + if kind == "red": self.register_redvar(op.result) self.emit("red_after_direct_call") Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Fri Feb 29 11:19:22 2008 @@ -17,6 +17,7 @@ green vars are positive indexes green consts are negative indexes """ + is_portal = False def __init__(self, name, code, constants, typekinds, redboxclasses, keydescs, structtypedescs, fielddescs, arrayfielddescs, @@ -487,6 +488,10 @@ self.frame.local_green) assert newjitstate is self.jitstate + @arguments("green_varargs", "red_varargs") + def opimpl_portal_call(self, greenargs, redargs): + self.portalstate.portal_reentry(greenargs, redargs) + @arguments("green", "calldesc", "green_varargs") def opimpl_green_direct_call(self, fnptr_gv, calldesc, greenargs): calldesc.green_call(self, fnptr_gv, greenargs) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/portal.py Fri Feb 29 11:19:22 2008 @@ -42,11 +42,11 @@ concretetype = originalconcretetype(binding) if binding.is_green(): ORIGARGS.append(concretetype) - arg_spec = "green", None, None + arg_spec = "green", None, None, concretetype else: argdesc = self.getportalargdesc(concretetype) arg_spec = ("red", argdesc.residual_args_collector(), - argdesc.arg_redbox_maker()) + argdesc.arg_redbox_maker(), concretetype) ARGS.extend(argdesc.residual_argtypes()) args_specification.append(arg_spec) self.args_specification = args_specification @@ -124,7 +124,7 @@ def make_key(self, *args): key = () i = 0 - for color, collect_residual_args, _ in args_specification: + for color, collect_residual_args, _, _ in args_specification: if color == "green": x = args[i] if isinstance(lltype.typeOf(x), lltype.Ptr): @@ -133,10 +133,26 @@ i = i + 1 return key + def make_key_from_genconsts(self, green_gv): + key = () + i = 0 + j = 0 + for color, collect_residual_args, _, TYPE in args_specification: + if color == "green": + genconst = green_gv[j] + x = genconst.revealconst(TYPE) + if isinstance(TYPE, lltype.Ptr): + x = llmemory.cast_ptr_to_adr(x) + key = key + (x,) + j += 1 + i = i + 1 + return key + + def make_residualargs(self, *args): residualargs = () i = 0 - for color, collect_residual_args, _ in args_specification: + for color, collect_residual_args, _, _ in args_specification: if color != "green": residualargs = residualargs + collect_residual_args(args[i]) i = i + 1 @@ -162,6 +178,48 @@ else: return fn(*residualargs) + def portal_reentry(self, greenargs, redargs): + jitstate = self.interpreter.jitstate + curbuilder = jitstate.curbuilder + rgenop = self.interpreter.rgenop + i = 0 + cache = self.cache + key = self.make_key_from_genconsts(greenargs) + try: + gv_generated = cache[key] + except KeyError: + builder, gv_generated, inputargs_gv = rgenop.newgraph(sigtoken, + "generated") + self.cache[key] = gv_generated + top_jitstate = self.interpreter.fresh_jitstate(builder) + newredargs = () + red_i = 0 + for color, _, make_arg_redbox, _ in args_specification: + if color == "red": + box = make_arg_redbox(top_jitstate, inputargs_gv, red_i) + red_i += make_arg_redbox.consumes + newredargs += (box,) + newredargs = list(newredargs) + + self.graph_compilation_queue.append( + (top_jitstate, greenargs, newredargs)) + residualargs_gv = [box.getgenvar(jitstate) for box in redargs] + + gv_res = curbuilder.genop_call(sigtoken, gv_generated, + residualargs_gv) + self.interpreter.exceptiondesc.fetch_global_excdata(jitstate) + + RESTYPE = RESIDUAL_FUNCTYPE.RESULT + reskind = rgenop.kindToken(RESTYPE) + boxbuilder = rvalue.ll_redboxbuilder(RESTYPE) + + if RESTYPE == lltype.Void: + retbox = None + else: + retbox = boxbuilder(reskind, gv_res) + jitstate.returnbox = retbox + assert jitstate.next is None + def compile(self, key, *args): portal_ts_args = () @@ -173,14 +231,12 @@ greenargs = () redargs = () red_i = 0 - for color, _, make_arg_redbox in args_specification: + for color, _, make_arg_redbox, _ in args_specification: + llvalue = args[0] + args = args[1:] if color == "green": - llvalue = args[0] - args = args[1:] greenargs += (rgenop.genconst(llvalue),) else: - llvalue = args[0] - args = args[1:] box = make_arg_redbox(top_jitstate, inputargs_gv, red_i) red_i += make_arg_redbox.consumes redargs += (box,) @@ -196,7 +252,7 @@ def readportal(self, *args): i = 0 key = () - for color, _, _ in args_specification: + for color, _, _, _ in args_specification: if color == "green": x = args[i] if isinstance(lltype.typeOf(x), lltype.Ptr): Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Fri Feb 29 11:19:22 2008 @@ -554,12 +554,14 @@ self.check_insns({'int_gt': 1}) def test_recursive_call(self): + def indirection(n, fudge): + return ll_pseudo_factorial(n, fudge) def ll_pseudo_factorial(n, fudge): k = hint(n, concrete=True) if n <= 0: return 1 return n * ll_pseudo_factorial(n - 1, fudge + n) - fudge - res = self.interpret(ll_pseudo_factorial, [4, 2], [0]) + res = self.interpret(indirection, [4, 2], [0]) expected = ll_pseudo_factorial(4, 2) assert res == expected Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_portal.py Fri Feb 29 11:19:22 2008 @@ -424,3 +424,119 @@ res = self.timeshift_from_portal(f, f, [15], policy=P_OOPSPEC) assert res == 15 + + def test_simple_recursive_portal_call(self): + + def main(code, x): + return evaluate(code, x) + + def evaluate(y, x): + hint(y, concrete=True) + if y <= 0: + return x + z = 1 + evaluate(y - 1, x) + return z + + res = self.timeshift_from_portal(main, evaluate, [3, 2]) + assert res == 5 + + res = self.timeshift_from_portal(main, evaluate, [3, 5]) + assert res == 8 + + res = self.timeshift_from_portal(main, evaluate, [4, 7]) + assert res == 11 + + + def test_simple_recursive_portal_call2(self): + + def main(code, x): + return evaluate(code, x) + + def evaluate(y, x): + hint(y, concrete=True) + if x <= 0: + return y + z = evaluate(y, x - 1) + 1 + return z + + res = self.timeshift_from_portal(main, evaluate, [3, 2]) + assert res == 5 + + res = self.timeshift_from_portal(main, evaluate, [3, 5]) + assert res == 8 + + res = self.timeshift_from_portal(main, evaluate, [4, 7]) + assert res == 11 + + def test_simple_recursive_portal_call_with_exc(self): + + def main(code, x): + return evaluate(code, x) + + class Bottom(Exception): + pass + + def evaluate(y, x): + hint(y, concrete=True) + if y <= 0: + raise Bottom + try: + z = 1 + evaluate(y - 1, x) + except Bottom: + z = 1 + x + return z + + res = self.timeshift_from_portal(main, evaluate, [3, 2]) + assert res == 5 + + res = self.timeshift_from_portal(main, evaluate, [3, 5]) + assert res == 8 + + res = self.timeshift_from_portal(main, evaluate, [4, 7]) + assert res == 11 + + + def test_portal_returns_none(self): + py.test.skip("portal returning None is not supported") + def g(x): + x = hint(x, promote=True) + if x == 42: + return None + def f(x): + return g(x) + + res = self.timeshift_from_portal(f, g, [42], policy=P_NOVIRTUAL) + + def test_portal_returns_none_with_origins(self): + py.test.skip("portal returning None is not supported") + def returnNone(): + pass + def returnNone2(): + pass + def g(x): + x = hint(x, promote=True) + if x == 42: + return returnNone() + return returnNone2() + def f(x): + return g(x) + + res = self.timeshift_from_portal(f, g, [42], policy=P_NOVIRTUAL) + + def test_recursive_portal_call(self): + py.test.skip("not working yet") + def indirection(green, red): + newgreen = hint((green + red) % 100, promote=True) + return portal(newgreen, red + 1) + def portal(green, red): + hint(None, global_merge_point=True) + green = abs(green) + red = abs(red) + hint(green, concrete=True) + if green > 42: + return 0 + if red > 42: + return 1 + return indirection(green, red) + res = self.timeshift_from_portal(portal, portal, [41, 1], policy=P_NOVIRTUAL) + assert res == 0 Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py Fri Feb 29 11:19:22 2008 @@ -141,117 +141,3 @@ return calls class TestPortal(PortalTest): - def test_simple_recursive_portal_call(self): - - def main(code, x): - return evaluate(code, x) - - def evaluate(y, x): - hint(y, concrete=True) - if y <= 0: - return x - z = 1 + evaluate(y - 1, x) - return z - - res = self.timeshift_from_portal(main, evaluate, [3, 2]) - assert res == 5 - - res = self.timeshift_from_portal(main, evaluate, [3, 5]) - assert res == 8 - - res = self.timeshift_from_portal(main, evaluate, [4, 7]) - assert res == 11 - - - def test_simple_recursive_portal_call2(self): - - def main(code, x): - return evaluate(code, x) - - def evaluate(y, x): - hint(y, concrete=True) - if x <= 0: - return y - z = evaluate(y, x - 1) + 1 - return z - - res = self.timeshift_from_portal(main, evaluate, [3, 2]) - assert res == 5 - - res = self.timeshift_from_portal(main, evaluate, [3, 5]) - assert res == 8 - - res = self.timeshift_from_portal(main, evaluate, [4, 7]) - assert res == 11 - - def test_simple_recursive_portal_call_with_exc(self): - - def main(code, x): - return evaluate(code, x) - - class Bottom(Exception): - pass - - def evaluate(y, x): - hint(y, concrete=True) - if y <= 0: - raise Bottom - try: - z = 1 + evaluate(y - 1, x) - except Bottom: - z = 1 + x - return z - - res = self.timeshift_from_portal(main, evaluate, [3, 2]) - assert res == 5 - - res = self.timeshift_from_portal(main, evaluate, [3, 5]) - assert res == 8 - - res = self.timeshift_from_portal(main, evaluate, [4, 7]) - assert res == 11 - - - def test_portal_returns_none(self): - py.test.skip("portal returning None is not supported") - def g(x): - x = hint(x, promote=True) - if x == 42: - return None - def f(x): - return g(x) - - res = self.timeshift_from_portal(f, g, [42], policy=P_NOVIRTUAL) - - def test_portal_returns_none_with_origins(self): - py.test.skip("portal returning None is not supported") - def returnNone(): - pass - def returnNone2(): - pass - def g(x): - x = hint(x, promote=True) - if x == 42: - return returnNone() - return returnNone2() - def f(x): - return g(x) - - res = self.timeshift_from_portal(f, g, [42], policy=P_NOVIRTUAL) - - def test_recursive_portal_call(self): - def indirection(green, red): - newgreen = hint((green + red) % 100, promote=True) - return portal(newgreen, red + 1) - def portal(green, red): - hint(None, global_merge_point=True) - green = abs(green) - red = abs(red) - hint(green, concrete=True) - if green > 42: - return 0 - if red > 42: - return 1 - return indirection(green, red) - res = self.timeshift_from_portal(portal, portal, [41, 1], policy=P_NOVIRTUAL) - assert res == 0 From cfbolz at codespeak.net Fri Feb 29 11:20:49 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 29 Feb 2008 11:20:49 +0100 (CET) Subject: [pypy-svn] r51939 - pypy/branch/jit-refactoring/pypy/jit/timeshifter/test Message-ID: <20080229102049.B30021684FB@codespeak.net> Author: cfbolz Date: Fri Feb 29 11:20:49 2008 New Revision: 51939 Removed: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_portal.py Log: all those tests have been ported over to the rainbow interp From cfbolz at codespeak.net Fri Feb 29 11:38:54 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 29 Feb 2008 11:38:54 +0100 (CET) Subject: [pypy-svn] r51941 - in pypy/branch/jit-refactoring/pypy/jit: rainbow/test timeshifter/test Message-ID: <20080229103854.360FC1684F9@codespeak.net> Author: cfbolz Date: Fri Feb 29 11:38:53 2008 New Revision: 51941 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py Log: moving over the remaining interpretation test Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Fri Feb 29 11:38:53 2008 @@ -1113,7 +1113,6 @@ self.check_insns({'int_mul': 1, 'int_add': 1, 'int_sub': 1}) def test_green_red_mismatch_in_call(self): - #py.test.skip("WIP") def add(a,b, u): return a+b @@ -1126,5 +1125,605 @@ res = self.interpret(f, [4, 5, 0], [], policy=P_NOVIRTUAL) assert res == 20 + + def test_recursive_with_red_termination_condition(self): + py.test.skip('Does not terminate') + def ll_factorial(n): + if n <= 0: + return 1 + return n * ll_factorial(n - 1) + + res = self.interpret(ll_factorial, [5], []) + assert res == 120 + + def test_simple_indirect_call(self): + def g1(v): + return v * 2 + + def g2(v): + return v + 2 + + def f(flag, v): + if hint(flag, concrete=True): + g = g1 + else: + g = g2 + return g(v) + + res = self.interpret(f, [0, 40], [0]) + assert res == 42 + self.check_insns({'int_add': 1}) + + def test_normalize_indirect_call(self): + def g1(v): + return -17 + + def g2(v): + return v + 2 + + def f(flag, v): + if hint(flag, concrete=True): + g = g1 + else: + g = g2 + return g(v) + + res = self.interpret(f, [0, 40], [0]) + assert res == 42 + self.check_insns({'int_add': 1}) + + res = self.interpret(f, [1, 40], [0]) + assert res == -17 + self.check_insns({}) + + def test_normalize_indirect_call_more(self): + py.test.skip("not working yet") + def g1(v): + if v >= 0: + return -17 + else: + return -155 + + def g2(v): + return v + 2 + + def f(flag, v): + w = g1(v) + if hint(flag, concrete=True): + g = g1 + else: + g = g2 + return g(v) + w + + res = self.interpret(f, [0, 40], [0]) + assert res == 25 + self.check_insns({'int_add': 2, 'int_ge': 1}) + + res = self.interpret(f, [1, 40], [0]) + assert res == -34 + self.check_insns({'int_ge': 2, 'int_add': 1}) + + res = self.interpret(f, [0, -1000], [0]) + assert res == f(False, -1000) + self.check_insns({'int_add': 2, 'int_ge': 1}) + + res = self.interpret(f, [1, -1000], [0]) + assert res == f(True, -1000) + self.check_insns({'int_ge': 2, 'int_add': 1}) + + def test_green_char_at_merge(self): + def f(c, x): + c = chr(c) + c = hint(c, concrete=True) + if x: + x = 3 + else: + x = 1 + c = hint(c, variable=True) + return len(c*x) + + res = self.interpret(f, [ord('a'), 1], [], policy=P_NOVIRTUAL) + assert res == 3 + + res = self.interpret(f, [ord('b'), 0], [], policy=P_NOVIRTUAL) + assert res == 1 + + def test_self_referential_structures(self): + py.test.skip("not working yet") + S = lltype.GcForwardReference() + S.become(lltype.GcStruct('s', + ('ps', lltype.Ptr(S)))) + + def f(x): + s = lltype.malloc(S) + if x: + s.ps = lltype.malloc(S) + return s + def count_depth(s): + x = 0 + while s: + x += 1 + s = s.ps + return str(x) + + f.convert_result = count_depth + + res = self.interpret(f, [3], [], policy=P_NOVIRTUAL) + assert res == '2' + + def test_known_nonzero(self): + S = lltype.GcStruct('s', ('x', lltype.Signed)) + global_s = lltype.malloc(S, immortal=True) + global_s.x = 100 + + def h(): + s = lltype.malloc(S) + s.x = 50 + return s + def g(s, y): + if s: + return s.x * 5 + else: + return -12 + y + def f(x, y): + x = hint(x, concrete=True) + if x == 1: + return g(lltype.nullptr(S), y) + elif x == 2: + return g(global_s, y) + elif x == 3: + s = lltype.malloc(S) + s.x = y + return g(s, y) + elif x == 4: + s = h() + return g(s, y) + else: + s = h() + if s: + return g(s, y) + else: + return 0 + + P = StopAtXPolicy(h) + + res = self.interpret(f, [1, 10], [0], policy=P) + assert res == -2 + self.check_insns(int_mul=0, int_add=1) + + res = self.interpret(f, [2, 10], [0], policy=P) + assert res == 500 + self.check_insns(int_mul=1, int_add=0) + + res = self.interpret(f, [3, 10], [0], policy=P) + assert res == 50 + self.check_insns(int_mul=1, int_add=0) + + res = self.interpret(f, [4, 10], [0], policy=P) + assert res == 250 + self.check_insns(int_mul=1, int_add=1) + + res = self.interpret(f, [5, 10], [0], policy=P) + assert res == 250 + self.check_insns(int_mul=1, int_add=0) + + def test_debug_assert_ptr_nonzero(self): + py.test.skip("not working yet") + S = lltype.GcStruct('s', ('x', lltype.Signed)) + def h(): + s = lltype.malloc(S) + s.x = 42 + return s + def g(s): + # assumes that s is not null here + ll_assert(bool(s), "please don't give me a null") + return 5 + def f(m): + s = h() + n = g(s) + if not s: + n *= m + return n + + P = StopAtXPolicy(h) + + res = self.interpret(f, [17], [], policy=P) + assert res == 5 + self.check_insns(int_mul=0) + + def test_indirect_red_call(self): + def h1(n): + return n*2 + def h2(n): + return n*4 + l = [h1, h2] + def f(n, x): + h = l[n&1] + return h(n) + x + + P = StopAtXPolicy() + res = self.interpret(f, [7, 3], policy=P) + assert res == f(7,3) + self.check_insns(indirect_call=1, direct_call=1) + + def test_indirect_red_call_with_exc(self): + def h1(n): + if n < 0: + raise ValueError + return n*2 + def h2(n): + if n < 0: + raise ValueError + return n*4 + l = [h1, h2] + def g(n, x): + h = l[n&1] + return h(n) + x + + def f(n, x): + try: + return g(n, x) + except ValueError: + return -1111 + + P = StopAtXPolicy() + res = self.interpret(f, [7, 3], policy=P) + assert res == f(7,3) + self.check_insns(indirect_call=1) + + res = self.interpret(f, [-7, 3], policy=P) + assert res == -1111 + self.check_insns(indirect_call=1) + + def test_indirect_gray_call(self): + py.test.skip("not working yet") + def h1(w, n): + w[0] = n*2 + def h2(w, n): + w[0] = n*4 + l = [h1, h2] + def f(n, x): + w = [0] + h = l[n&1] + h(w, n) + return w[0] + x + + P = StopAtXPolicy() + res = self.interpret(f, [7, 3], policy=P) + assert res == f(7,3) + + def test_indirect_residual_red_call(self): + py.test.skip("not working yet") + def h1(n): + return n*2 + def h2(n): + return n*4 + l = [h1, h2] + def f(n, x): + h = l[n&1] + return h(n) + x + + P = StopAtXPolicy(h1, h2) + res = self.interpret(f, [7, 3], policy=P) + assert res == f(7,3) + self.check_insns(indirect_call=1) + + def test_constant_indirect_red_call(self): + def h1(x): + return x-2 + def h2(x): + return x*4 + l = [h1, h2] + def f(n, x): + frozenl = hint(l, deepfreeze=True) + h = frozenl[n&1] + return h(x) + + P = StopAtXPolicy() + res = self.interpret(f, [7, 3], [0], policy=P) + assert res == f(7,3) + self.check_insns({'int_mul': 1}) + res = self.interpret(f, [4, 113], [0], policy=P) + assert res == f(4,113) + self.check_insns({'int_sub': 1}) + + def test_indirect_sometimes_residual_pure_red_call(self): + def h1(x): + return x-2 + def h2(x): + return x*4 + l = [h1, h2] + def f(n, x): + hint(None, global_merge_point=True) + hint(n, concrete=True) + frozenl = hint(l, deepfreeze=True) + h = frozenl[n&1] + return h(x) + + P = StopAtXPolicy(h1) + P.oopspec = True + res = self.interpret(f, [7, 3], [], policy=P) + assert res == f(7,3) + self.check_insns({'int_mul': 1}) + res = self.interpret(f, [4, 113], [], policy=P) + assert res == f(4,113) + self.check_insns({'direct_call': 1}) + + def test_indirect_sometimes_residual_pure_but_fixed_red_call(self): + py.test.skip("not working yet") + def h1(x): + return x-2 + def h2(x): + return x*4 + l = [h1, h2] + def f(n, x): + hint(None, global_merge_point=True) + frozenl = hint(l, deepfreeze=True) + h = frozenl[n&1] + z = h(x) + hint(z, concrete=True) + return z + + P = StopAtXPolicy(h1) + P.oopspec = True + res = self.interpret(f, [7, 3], [], policy=P) + assert res == f(7,3) + self.check_insns({}) + res = self.interpret(f, [4, 113], [], policy=P) + assert res == f(4,113) + self.check_insns({}) + + def test_manual_marking_of_pure_functions(self): + py.test.skip("not working yet") + class A(object): + pass + class B(object): + pass + a1 = A() + a2 = A() + d = {} + def h1(b, s): + try: + return d[s] + except KeyError: + d[s] = r = A() + return r + h1._pure_function_ = True + b = B() + def f(n): + hint(None, global_merge_point=True) + hint(n, concrete=True) + if n == 0: + s = "abc" + else: + s = "123" + a = h1(b, s) + return hint(n, variable=True) + + P = StopAtXPolicy(h1) + P.oopspec = True + res = self.interpret(f, [0], [], policy=P) + assert res == 0 + self.check_insns({}) + res = self.interpret(f, [4], [], policy=P) + assert res == 4 + self.check_insns({}) + + + def test_red_int_add_ovf(self): + py.test.skip("not working yet") + def f(n, m): + try: + return ovfcheck(n + m) + except OverflowError: + return -42 + + res = self.interpret(f, [100, 20]) + assert res == 120 + self.check_insns(int_add_ovf=1) + res = self.interpret(f, [sys.maxint, 1]) + assert res == -42 + self.check_insns(int_add_ovf=1) + + def test_green_int_add_ovf(self): + py.test.skip("not working yet") + def f(n, m): + try: + res = ovfcheck(n + m) + except OverflowError: + res = -42 + hint(res, concrete=True) + return res + + res = self.interpret(f, [100, 20]) + assert res == 120 + self.check_insns({}) + res = self.interpret(f, [sys.maxint, 1]) + assert res == -42 + self.check_insns({}) + + def test_nonzeroness_assert_while_compiling(self): + class X: + pass + class Y: + pass + + def g(x, y): + if y.flag: + return x.value + else: + return -7 + + def h(n): + if n: + x = X() + x.value = n + return x + else: + return None + + y = Y() + + def f(n): + y.flag = True + g(h(n), y) + y.flag = False + return g(h(0), y) + + res = self.interpret(f, [42], policy=P_NOVIRTUAL) + assert res == -7 + + def test_segfault_while_compiling(self): + class X: + pass + class Y: + pass + + def g(x, y): + x = hint(x, deepfreeze=True) + if y.flag: + return x.value + else: + return -7 + + def h(n): + if n: + x = X() + x.value = n + return x + else: + return None + + y = Y() + + def f(n): + y.flag = True + g(h(n), y) + y.flag = False + return g(h(0), y) + + res = self.interpret(f, [42], policy=P_NOVIRTUAL) + assert res == -7 + + def test_switch(self): + py.test.skip("not working yet") + def g(n): + if n == 0: + return 12 + elif n == 1: + return 34 + elif n == 3: + return 56 + elif n == 7: + return 78 + else: + return 90 + def f(n, m): + x = g(n) # gives a red switch + y = g(hint(m, concrete=True)) # gives a green switch + return x - y + + res = self.interpret(f, [7, 2], backendoptimize=True) + assert res == 78 - 90 + res = self.interpret(f, [8, 1], backendoptimize=True) + assert res == 90 - 34 + + def test_switch_char(self): + py.test.skip("not working yet") + def g(n): + n = chr(n) + if n == '\x00': + return 12 + elif n == '\x01': + return 34 + elif n == '\x02': + return 56 + elif n == '\x03': + return 78 + else: + return 90 + def f(n, m): + x = g(n) # gives a red switch + y = g(hint(m, concrete=True)) # gives a green switch + return x - y + + res = self.interpret(f, [3, 0], backendoptimize=True) + assert res == 78 - 12 + res = self.interpret(f, [2, 4], backendoptimize=True) + assert res == 56 - 90 + + def test_substitute_graph(self): + py.test.skip("not working yet") + + class MetaG: + __metaclass__ = cachedtype + + def __init__(self, hrtyper): + pass + + def _freeze_(self): + return True + + def metafunc(self, jitstate, space, mbox): + from pypy.jit.timeshifter.rvalue import IntRedBox + builder = jitstate.curbuilder + gv_result = builder.genop1("int_neg", mbox.getgenvar(jitstate)) + return IntRedBox(mbox.kind, gv_result) + + class Fz(object): + x = 10 + + def _freeze_(self): + return True + + def g(fz, m): + return m * fz.x + + fz = Fz() + + def f(n, m): + x = g(fz, n) + y = g(fz, m) + hint(y, concrete=True) + return x + g(fz, y) + + class MyPolicy(HintAnnotatorPolicy): + novirtualcontainer = True + + def look_inside_graph(self, graph): + if graph.func is g: + return MetaG # replaces g with a meta-call to metafunc() + else: + return True + + res = self.interpret(f, [3, 6], policy=MyPolicy()) + assert res == -3 + 600 + self.check_insns({'int_neg': 1, 'int_add': 1}) + + def test_hash_of_green_string_is_green(self): + py.test.skip("unfortunately doesn't work") + def f(n): + if n == 0: + s = "abc" + elif n == 1: + s = "cde" + else: + s = "fgh" + return hash(s) + + res = self.interpret(f, [0]) + self.check_insns({'int_eq': 2}) + assert res == f(0) + + def test_misplaced_global_merge_point(self): + py.test.skip("not working yet") + def g(n): + hint(None, global_merge_point=True) + return n+1 + def f(n): + hint(None, global_merge_point=True) + return g(n) + py.test.raises(AssertionError, self.interpret, f, [7], []) + class TestLLType(SimpleTests): type_system = "lltype" Modified: pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/timeshifter/test/test_timeshift.py Fri Feb 29 11:38:53 2008 @@ -381,594 +381,7 @@ class BaseTestTimeshift(TimeshiftingTests): - - - - def test_recursive_with_red_termination_condition(self): - py.test.skip('Does not terminate') - def ll_factorial(n): - if n <= 0: - return 1 - return n * ll_factorial(n - 1) - - res = self.timeshift(ll_factorial, [5], []) - assert res == 120 - - def test_simple_indirect_call(self): - def g1(v): - return v * 2 - - def g2(v): - return v + 2 - - def f(flag, v): - if hint(flag, concrete=True): - g = g1 - else: - g = g2 - return g(v) - - res = self.timeshift(f, [0, 40], [0]) - assert res == 42 - self.check_insns({'int_add': 1}) - - def test_normalize_indirect_call(self): - def g1(v): - return -17 - - def g2(v): - return v + 2 - - def f(flag, v): - if hint(flag, concrete=True): - g = g1 - else: - g = g2 - return g(v) - - res = self.timeshift(f, [0, 40], [0]) - assert res == 42 - self.check_insns({'int_add': 1}) - - res = self.timeshift(f, [1, 40], [0]) - assert res == -17 - self.check_insns({}) - - def test_normalize_indirect_call_more(self): - def g1(v): - if v >= 0: - return -17 - else: - return -155 - - def g2(v): - return v + 2 - - def f(flag, v): - w = g1(v) - if hint(flag, concrete=True): - g = g1 - else: - g = g2 - return g(v) + w - - res = self.timeshift(f, [0, 40], [0]) - assert res == 25 - self.check_insns({'int_add': 2, 'int_ge': 1}) - - res = self.timeshift(f, [1, 40], [0]) - assert res == -34 - self.check_insns({'int_ge': 2, 'int_add': 1}) - - res = self.timeshift(f, [0, -1000], [0]) - assert res == f(False, -1000) - self.check_insns({'int_add': 2, 'int_ge': 1}) - - res = self.timeshift(f, [1, -1000], [0]) - assert res == f(True, -1000) - self.check_insns({'int_ge': 2, 'int_add': 1}) - - def test_green_char_at_merge(self): - def f(c, x): - c = chr(c) - c = hint(c, concrete=True) - if x: - x = 3 - else: - x = 1 - c = hint(c, variable=True) - return len(c*x) - - res = self.timeshift(f, [ord('a'), 1], [], policy=P_NOVIRTUAL) - assert res == 3 - - res = self.timeshift(f, [ord('b'), 0], [], policy=P_NOVIRTUAL) - assert res == 1 - - def test_self_referential_structures(self): - S = lltype.GcForwardReference() - S.become(lltype.GcStruct('s', - ('ps', lltype.Ptr(S)))) - - def f(x): - s = lltype.malloc(S) - if x: - s.ps = lltype.malloc(S) - return s - def count_depth(s): - x = 0 - while s: - x += 1 - s = s.ps - return str(x) - - f.convert_result = count_depth - - res = self.timeshift(f, [3], [], policy=P_NOVIRTUAL) - assert res == '2' - - def test_known_nonzero(self): - S = lltype.GcStruct('s', ('x', lltype.Signed)) - global_s = lltype.malloc(S, immortal=True) - global_s.x = 100 - - def h(): - s = lltype.malloc(S) - s.x = 50 - return s - def g(s, y): - if s: - return s.x * 5 - else: - return -12 + y - def f(x, y): - x = hint(x, concrete=True) - if x == 1: - return g(lltype.nullptr(S), y) - elif x == 2: - return g(global_s, y) - elif x == 3: - s = lltype.malloc(S) - s.x = y - return g(s, y) - elif x == 4: - s = h() - return g(s, y) - else: - s = h() - if s: - return g(s, y) - else: - return 0 - - P = StopAtXPolicy(h) - - res = self.timeshift(f, [1, 10], [0], policy=P) - assert res == -2 - self.check_insns(int_mul=0, int_add=1) - - res = self.timeshift(f, [2, 10], [0], policy=P) - assert res == 500 - self.check_insns(int_mul=1, int_add=0) - - res = self.timeshift(f, [3, 10], [0], policy=P) - assert res == 50 - self.check_insns(int_mul=1, int_add=0) - - res = self.timeshift(f, [4, 10], [0], policy=P) - assert res == 250 - self.check_insns(int_mul=1, int_add=1) - - res = self.timeshift(f, [5, 10], [0], policy=P) - assert res == 250 - self.check_insns(int_mul=1, int_add=0) - - def test_debug_assert_ptr_nonzero(self): - S = lltype.GcStruct('s', ('x', lltype.Signed)) - def h(): - s = lltype.malloc(S) - s.x = 42 - return s - def g(s): - # assumes that s is not null here - ll_assert(bool(s), "please don't give me a null") - return 5 - def f(m): - s = h() - n = g(s) - if not s: - n *= m - return n - - P = StopAtXPolicy(h) - - res = self.timeshift(f, [17], [], policy=P) - assert res == 5 - self.check_insns(int_mul=0) - - def test_indirect_red_call(self): - def h1(n): - return n*2 - def h2(n): - return n*4 - l = [h1, h2] - def f(n, x): - h = l[n&1] - return h(n) + x - - P = StopAtXPolicy() - res = self.timeshift(f, [7, 3], policy=P) - assert res == f(7,3) - self.check_insns(indirect_call=1, direct_call=1) - - def test_indirect_red_call_with_exc(self): - def h1(n): - if n < 0: - raise ValueError - return n*2 - def h2(n): - if n < 0: - raise ValueError - return n*4 - l = [h1, h2] - def g(n, x): - h = l[n&1] - return h(n) + x - - def f(n, x): - try: - return g(n, x) - except ValueError: - return -1111 - - P = StopAtXPolicy() - res = self.timeshift(f, [7, 3], policy=P) - assert res == f(7,3) - self.check_insns(indirect_call=1) - - res = self.timeshift(f, [-7, 3], policy=P) - assert res == -1111 - self.check_insns(indirect_call=1) - - def test_indirect_gray_call(self): - def h1(w, n): - w[0] = n*2 - def h2(w, n): - w[0] = n*4 - l = [h1, h2] - def f(n, x): - w = [0] - h = l[n&1] - h(w, n) - return w[0] + x - - P = StopAtXPolicy() - res = self.timeshift(f, [7, 3], policy=P) - assert res == f(7,3) - - def test_indirect_residual_red_call(self): - def h1(n): - return n*2 - def h2(n): - return n*4 - l = [h1, h2] - def f(n, x): - h = l[n&1] - return h(n) + x - - P = StopAtXPolicy(h1, h2) - res = self.timeshift(f, [7, 3], policy=P) - assert res == f(7,3) - self.check_insns(indirect_call=1) - - def test_constant_indirect_red_call(self): - def h1(x): - return x-2 - def h2(x): - return x*4 - l = [h1, h2] - def f(n, x): - frozenl = hint(l, deepfreeze=True) - h = frozenl[n&1] - return h(x) - - P = StopAtXPolicy() - res = self.timeshift(f, [7, 3], [0], policy=P) - assert res == f(7,3) - self.check_insns({'int_mul': 1}) - res = self.timeshift(f, [4, 113], [0], policy=P) - assert res == f(4,113) - self.check_insns({'int_sub': 1}) - - def test_indirect_sometimes_residual_pure_red_call(self): - def h1(x): - return x-2 - def h2(x): - return x*4 - l = [h1, h2] - def f(n, x): - hint(None, global_merge_point=True) - hint(n, concrete=True) - frozenl = hint(l, deepfreeze=True) - h = frozenl[n&1] - return h(x) - - P = StopAtXPolicy(h1) - P.oopspec = True - res = self.timeshift(f, [7, 3], [], policy=P) - assert res == f(7,3) - self.check_insns({'int_mul': 1}) - res = self.timeshift(f, [4, 113], [], policy=P) - assert res == f(4,113) - self.check_insns({'direct_call': 1}) - - def test_indirect_sometimes_residual_pure_but_fixed_red_call(self): - def h1(x): - return x-2 - def h2(x): - return x*4 - l = [h1, h2] - def f(n, x): - hint(None, global_merge_point=True) - frozenl = hint(l, deepfreeze=True) - h = frozenl[n&1] - z = h(x) - hint(z, concrete=True) - return z - - P = StopAtXPolicy(h1) - P.oopspec = True - res = self.timeshift(f, [7, 3], [], policy=P) - assert res == f(7,3) - self.check_insns({}) - res = self.timeshift(f, [4, 113], [], policy=P) - assert res == f(4,113) - self.check_insns({}) - - def test_manual_marking_of_pure_functions(self): - class A(object): - pass - class B(object): - pass - a1 = A() - a2 = A() - d = {} - def h1(b, s): - try: - return d[s] - except KeyError: - d[s] = r = A() - return r - h1._pure_function_ = True - b = B() - def f(n): - hint(None, global_merge_point=True) - hint(n, concrete=True) - if n == 0: - s = "abc" - else: - s = "123" - a = h1(b, s) - return hint(n, variable=True) - - P = StopAtXPolicy(h1) - P.oopspec = True - res = self.timeshift(f, [0], [], policy=P) - assert res == 0 - self.check_insns({}) - res = self.timeshift(f, [4], [], policy=P) - assert res == 4 - self.check_insns({}) - - - def test_red_int_add_ovf(self): - def f(n, m): - try: - return ovfcheck(n + m) - except OverflowError: - return -42 - - res = self.timeshift(f, [100, 20]) - assert res == 120 - self.check_insns(int_add_ovf=1) - res = self.timeshift(f, [sys.maxint, 1]) - assert res == -42 - self.check_insns(int_add_ovf=1) - - def test_green_int_add_ovf(self): - def f(n, m): - try: - res = ovfcheck(n + m) - except OverflowError: - res = -42 - hint(res, concrete=True) - return res - - res = self.timeshift(f, [100, 20]) - assert res == 120 - self.check_insns({}) - res = self.timeshift(f, [sys.maxint, 1]) - assert res == -42 - self.check_insns({}) - - def test_nonzeroness_assert_while_compiling(self): - class X: - pass - class Y: - pass - - def g(x, y): - if y.flag: - return x.value - else: - return -7 - - def h(n): - if n: - x = X() - x.value = n - return x - else: - return None - - y = Y() - - def f(n): - y.flag = True - g(h(n), y) - y.flag = False - return g(h(0), y) - - res = self.timeshift(f, [42], policy=P_NOVIRTUAL) - assert res == -7 - - def test_segfault_while_compiling(self): - class X: - pass - class Y: - pass - - def g(x, y): - x = hint(x, deepfreeze=True) - if y.flag: - return x.value - else: - return -7 - - def h(n): - if n: - x = X() - x.value = n - return x - else: - return None - - y = Y() - - def f(n): - y.flag = True - g(h(n), y) - y.flag = False - return g(h(0), y) - - res = self.timeshift(f, [42], policy=P_NOVIRTUAL) - assert res == -7 - - def test_switch(self): - def g(n): - if n == 0: - return 12 - elif n == 1: - return 34 - elif n == 3: - return 56 - elif n == 7: - return 78 - else: - return 90 - def f(n, m): - x = g(n) # gives a red switch - y = g(hint(m, concrete=True)) # gives a green switch - return x - y - - res = self.timeshift(f, [7, 2], backendoptimize=True) - assert res == 78 - 90 - res = self.timeshift(f, [8, 1], backendoptimize=True) - assert res == 90 - 34 - - def test_switch_char(self): - def g(n): - n = chr(n) - if n == '\x00': - return 12 - elif n == '\x01': - return 34 - elif n == '\x02': - return 56 - elif n == '\x03': - return 78 - else: - return 90 - def f(n, m): - x = g(n) # gives a red switch - y = g(hint(m, concrete=True)) # gives a green switch - return x - y - - res = self.timeshift(f, [3, 0], backendoptimize=True) - assert res == 78 - 12 - res = self.timeshift(f, [2, 4], backendoptimize=True) - assert res == 56 - 90 - - def test_substitute_graph(self): - - class MetaG: - __metaclass__ = cachedtype - - def __init__(self, hrtyper): - pass - - def _freeze_(self): - return True - - def metafunc(self, jitstate, space, mbox): - from pypy.jit.timeshifter.rvalue import IntRedBox - builder = jitstate.curbuilder - gv_result = builder.genop1("int_neg", mbox.getgenvar(jitstate)) - return IntRedBox(mbox.kind, gv_result) - - class Fz(object): - x = 10 - - def _freeze_(self): - return True - - def g(fz, m): - return m * fz.x - - fz = Fz() - - def f(n, m): - x = g(fz, n) - y = g(fz, m) - hint(y, concrete=True) - return x + g(fz, y) - - class MyPolicy(HintAnnotatorPolicy): - novirtualcontainer = True - - def look_inside_graph(self, graph): - if graph.func is g: - return MetaG # replaces g with a meta-call to metafunc() - else: - return True - - res = self.timeshift(f, [3, 6], policy=MyPolicy()) - assert res == -3 + 600 - self.check_insns({'int_neg': 1, 'int_add': 1}) - - def test_hash_of_green_string_is_green(self): - py.test.skip("unfortunately doesn't work") - def f(n): - if n == 0: - s = "abc" - elif n == 1: - s = "cde" - else: - s = "fgh" - return hash(s) - - res = self.timeshift(f, [0]) - self.check_insns({'int_eq': 2}) - assert res == f(0) - - def test_misplaced_global_merge_point(self): - def g(n): - hint(None, global_merge_point=True) - return n+1 - def f(n): - hint(None, global_merge_point=True) - return g(n) - py.test.raises(AssertionError, self.timeshift, f, [7], []) + pass class TestLLType(BaseTestTimeshift): type_system = 'lltype' From antocuni at codespeak.net Fri Feb 29 11:42:42 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 29 Feb 2008 11:42:42 +0100 (CET) Subject: [pypy-svn] r51942 - in pypy/dist/pypy/translator/cli: . test Message-ID: <20080229104242.6D0B91684F9@codespeak.net> Author: antocuni Date: Fri Feb 29 11:42:40 2008 New Revision: 51942 Modified: pypy/dist/pypy/translator/cli/constant.py pypy/dist/pypy/translator/cli/database.py pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: make sure we can mix the result of classof(cliClass) and classof(FUNCTYPE) Modified: pypy/dist/pypy/translator/cli/constant.py ============================================================================== --- pypy/dist/pypy/translator/cli/constant.py (original) +++ pypy/dist/pypy/translator/cli/constant.py Fri Feb 29 11:42:40 2008 @@ -340,8 +340,13 @@ def push_inline(self, gen, EXPECTED_TYPE): if not self.is_null(): - INSTANCE = self.value._INSTANCE - gen.ilasm.opcode('ldtoken', self.db.class_or_delegate_name(INSTANCE)) + if hasattr(self.value, '_FUNC'): + FUNC = self.value._FUNC + classname = self.db.record_delegate(FUNC) + else: + INSTANCE = self.value._INSTANCE + classname = self.db.class_name(INSTANCE) + gen.ilasm.opcode('ldtoken', classname) gen.ilasm.call('class [mscorlib]System.Type class [mscorlib]System.Type::GetTypeFromHandle(valuetype [mscorlib]System.RuntimeTypeHandle)') return super(CLIClassConst, self).push_inline(gen, EXPECTED_TYPE) Modified: pypy/dist/pypy/translator/cli/database.py ============================================================================== --- pypy/dist/pypy/translator/cli/database.py (original) +++ pypy/dist/pypy/translator/cli/database.py Fri Feb 29 11:42:40 2008 @@ -150,9 +150,3 @@ self.delegates[TYPE] = name self.pending_node(Delegate(self, TYPE, name)) return name - - def class_or_delegate_name(self, TYPE): - if isinstance(TYPE, ootype.StaticMethod): - return self.record_delegate(TYPE) - else: - return self.class_name(TYPE) Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Fri Feb 29 11:42:40 2008 @@ -569,7 +569,8 @@ FUNC = cliClass_or_type return known_delegates_class[FUNC] except KeyError: - cls = ootype._class(FUNC) + cls = ootype._class(ootype.ROOT) + cls._FUNC = FUNC known_delegates_class[FUNC] = cls return cls else: Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Fri Feb 29 11:42:40 2008 @@ -629,6 +629,18 @@ res = self.interpret(fn, []) assert res.startswith('StaticMethod__') + def test_mix_classof(self): + a = classof(System.Int32) + b = classof(FUNCTYPE) + def fn(flag): + if flag: + x = a + else: + x = b + return clidowncast(box(x), System.Type).get_Name() + res = self.interpret(fn, [True]) + assert res == 'Int32' + class TestPythonnet(TestDotnetRtyping): # don't interpreter functions but execute them directly through pythonnet From arigo at codespeak.net Fri Feb 29 11:44:09 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 29 Feb 2008 11:44:09 +0100 (CET) Subject: [pypy-svn] r51943 - pypy/dist/pypy/rpython Message-ID: <20080229104409.E058E1684F9@codespeak.net> Author: arigo Date: Fri Feb 29 11:44:09 2008 New Revision: 51943 Modified: pypy/dist/pypy/rpython/llinterp.py Log: Obscure hack to fix llinterp support for 'get_frame_base'. Modified: pypy/dist/pypy/rpython/llinterp.py ============================================================================== --- pypy/dist/pypy/rpython/llinterp.py (original) +++ pypy/dist/pypy/rpython/llinterp.py Fri Feb 29 11:44:09 2008 @@ -1012,6 +1012,7 @@ # read frame var support def op_get_frame_base(self): + self._obj0 = self # hack return llmemory.fakeaddress(self) def op_frame_info(self, *vars): From antocuni at codespeak.net Fri Feb 29 11:51:47 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 29 Feb 2008 11:51:47 +0100 (CET) Subject: [pypy-svn] r51946 - in pypy/dist/pypy/translator/cli: . src test Message-ID: <20080229105147.60F4616850B@codespeak.net> Author: antocuni Date: Fri Feb 29 11:51:46 2008 New Revision: 51946 Modified: pypy/dist/pypy/translator/cli/dotnet.py pypy/dist/pypy/translator/cli/src/pypylib.cs pypy/dist/pypy/translator/cli/test/test_dotnet.py Log: fix two failing tests Modified: pypy/dist/pypy/translator/cli/dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/dotnet.py (original) +++ pypy/dist/pypy/translator/cli/dotnet.py Fri Feb 29 11:51:46 2008 @@ -361,13 +361,15 @@ else: return CLR.System.String(x) elif isinstance(x, ootype._class): - TYPE = x._INSTANCE - if isinstance(TYPE, ootype.StaticMethod): + if hasattr(x, '_FUNC'): + TYPE = x._FUNC + assert isinstance(TYPE, ootype.StaticMethod) return typeof(TYPE) - name = '%s.%s' % (x._INSTANCE._namespace, x._INSTANCE._classname) - t = CLR.System.Type.GetType(name) - assert t is not None - return t + else: + name = '%s.%s' % (x._INSTANCE._namespace, x._INSTANCE._classname) + t = CLR.System.Type.GetType(name) + assert t is not None + return t elif isinstance(x, PythonNet.System.Object): return x elif x is None: Modified: pypy/dist/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/dist/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/dist/pypy/translator/cli/src/pypylib.cs Fri Feb 29 11:51:46 2008 @@ -57,6 +57,10 @@ public delegate int DelegateType_int__int_27(int a0, int a1, int a2, int a3, int a4, int a5, int a6, int a7, int a8, int a9, int a10, int a11, int a12, int a13, int a14, int a15, int a16, int a17, int a18, int a19, int a20, int a21, int a22, int a23, int a24, int a25, int a26); public delegate int DelegateType_int__int_100(int a0, int a1, int a2, int a3, int a4, int a5, int a6, int a7, int a8, int a9, int a10, int a11, int a12, int a13, int a14, int a15, int a16, int a17, int a18, int a19, int a20, int a21, int a22, int a23, int a24, int a25, int a26, int a27, int a28, int a29, int a30, int a31, int a32, int a33, int a34, int a35, int a36, int a37, int a38, int a39, int a40, int a41, int a42, int a43, int a44, int a45, int a46, int a47, int a48, int a49, int a50, int a51, int a52, int a53, int a54, int a55, int a56, int a57, int a58, int a59, int a60, int a61, int a62, int a63, int a64, int a65, int a66, int a67, int a68, int a69, int a70, int a71, int a72, int a73, int a74, int a75, int a76, int a77, int a78, int a79, int a80, int a81, int a82, int a83, int a84, int a85, int a86, int a87, int a88, int a89, int a90, int a91, int a92, int a93, int a94, int a95, int a96, int a97, int a98, int a99); + public class DummyClass { + public static object myfield; + } + } namespace pypy.runtime Modified: pypy/dist/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/dist/pypy/translator/cli/test/test_dotnet.py Fri Feb 29 11:51:46 2008 @@ -510,11 +510,11 @@ assert res == 42 def test_static_fields(self): - Constants = CLR.pypy.runtime.Constants + DummyClass = CLR.pypy.test.DummyClass def fn(): obj = System.Object() - Constants.const1 = obj - return Constants.const1 is obj + DummyClass.myfield = obj + return DummyClass.myfield is obj res = self.interpret(fn, []) assert res From arigo at codespeak.net Fri Feb 29 11:52:35 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 29 Feb 2008 11:52:35 +0100 (CET) Subject: [pypy-svn] r51947 - pypy/dist/pypy/module/bz2 Message-ID: <20080229105235.05B56168504@codespeak.net> Author: arigo Date: Fri Feb 29 11:52:35 2008 New Revision: 51947 Modified: pypy/dist/pypy/module/bz2/interp_bz2.py Log: Fix the bz2 module for 64-bit offsets support. Modified: pypy/dist/pypy/module/bz2/interp_bz2.py ============================================================================== --- pypy/dist/pypy/module/bz2/interp_bz2.py (original) +++ pypy/dist/pypy/module/bz2/interp_bz2.py Fri Feb 29 11:52:35 2008 @@ -8,7 +8,7 @@ from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped, interp2app, Arguments from pypy.rlib.streamio import Stream from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.rlib.rarithmetic import intmask +from pypy.rlib.rarithmetic import intmask, r_longlong import sys class CConfig: @@ -272,7 +272,7 @@ self.space = space self.stream = stream self.decompressor = W_BZ2Decompressor(space) - self.readlength = 0 + self.readlength = r_longlong(0) self.buffer = "" self.finished = False if buffering < 1024: @@ -286,29 +286,40 @@ return self.readlength def seek(self, offset, whence): + READMAX = 2**18 # 256KB if whence == 1: if offset >= 0: - read = 0 + read = r_longlong(0) while read < offset: - read += len(self.read(offset - read)) + count = offset - read + if count < READMAX: + count = intmask(count) + else: + count = READMAX + read += len(self.read(count)) else: pos = self.readlength + offset self.seek(pos, 0) elif whence == 0: self.stream.seek(0, 0) self.decompressor = W_BZ2Decompressor(self.space) - self.readlength = 0 + self.readlength = r_longlong(0) self.buffer = "" self.finished = False read = 0 while read < offset: - length = len(self.read(offset - read)) + count = offset - read + if count < READMAX: + count = intmask(count) + else: + count = READMAX + length = len(self.read(count)) read += length if not length: break else: # first measure the length by reading everything left - while len(self.read(65536)) > 0: + while len(self.read(READMAX)) > 0: pass pos = self.readlength + offset self.seek(pos, 0) From arigo at codespeak.net Fri Feb 29 11:55:08 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 29 Feb 2008 11:55:08 +0100 (CET) Subject: [pypy-svn] r51948 - pypy/dist/pypy/translator/jvm Message-ID: <20080229105508.B3271168511@codespeak.net> Author: arigo Date: Fri Feb 29 11:55:06 2008 New Revision: 51948 Modified: pypy/dist/pypy/translator/jvm/node.py Log: Python 2.4 compatibility. Modified: pypy/dist/pypy/translator/jvm/node.py ============================================================================== --- pypy/dist/pypy/translator/jvm/node.py (original) +++ pypy/dist/pypy/translator/jvm/node.py Fri Feb 29 11:55:06 2008 @@ -34,6 +34,7 @@ from pypy.translator.oosupport.constant import \ push_constant +import py import pypy.translator.jvm.generator as jvmgen import pypy.translator.jvm.typesystem as jvmtype from pypy.translator.jvm.log import log @@ -336,7 +337,7 @@ for pyexccls, jexcty in translation_table: for llexitcase in llexitcases: - assert issubclass(llexitcase, BaseException) + assert issubclass(llexitcase, py.builtin.BaseException) if issubclass(llexitcase, pyexccls): # Generate some converter code like: # try { ... } From arigo at codespeak.net Fri Feb 29 11:59:26 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 29 Feb 2008 11:59:26 +0100 (CET) Subject: [pypy-svn] r51949 - pypy/dist/pypy/translator/llvm/test Message-ID: <20080229105926.5AB30168514@codespeak.net> Author: arigo Date: Fri Feb 29 11:59:25 2008 New Revision: 51949 Modified: pypy/dist/pypy/translator/llvm/test/test_newgc.py Log: Fix test (GC header was reduced) Modified: pypy/dist/pypy/translator/llvm/test/test_newgc.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_newgc.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_newgc.py Fri Feb 29 11:59:25 2008 @@ -27,7 +27,7 @@ i5 = (res // 1) % 100 assert i1 % 4 == 0 assert 12 <= i1 <= 24 - assert 8 <= i2 <= i1 - 4 + assert 4 <= i2 <= i1 - 8 assert 4 <= i3 <= 12 assert i4 == i5 assert i3 + 4 <= i5 From antocuni at codespeak.net Fri Feb 29 12:02:43 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 29 Feb 2008 12:02:43 +0100 (CET) Subject: [pypy-svn] r51950 - pypy/dist/pypy/jit/codegen/cli Message-ID: <20080229110243.2CC0B168514@codespeak.net> Author: antocuni Date: Fri Feb 29 12:02:42 2008 New Revision: 51950 Modified: pypy/dist/pypy/jit/codegen/cli/operation.py pypy/dist/pypy/jit/codegen/cli/rgenop.py Log: use classof() to generate the kindtokens. This approach fixes the previous hack of storing the repr of the TYPE. Modified: pypy/dist/pypy/jit/codegen/cli/operation.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/operation.py (original) +++ pypy/dist/pypy/jit/codegen/cli/operation.py Fri Feb 29 12:02:42 2008 @@ -133,19 +133,19 @@ class Call(Operation): def __init__(self, builder, sigtoken, gv_fnptr, args_gv): - from pypy.jit.codegen.cli.rgenop import token2clitype + from pypy.jit.codegen.cli.rgenop import class2type self.builder = builder self.sigtoken = sigtoken self.gv_fnptr = gv_fnptr self.args_gv = args_gv - self._restype = token2clitype(sigtoken[1]) + self._restype = class2type(sigtoken.res) def restype(self): return self._restype def emit(self): - from pypy.jit.codegen.cli.rgenop import sigtoken2clitype - delegate_type = sigtoken2clitype(self.sigtoken) + from pypy.jit.codegen.cli.rgenop import class2type + delegate_type = class2type(self.sigtoken.funcclass) meth_invoke = delegate_type.GetMethod('Invoke') self.gv_fnptr.load(self.builder) self.builder.il.Emit(OpCodes.Castclass, delegate_type) Modified: pypy/dist/pypy/jit/codegen/cli/rgenop.py ============================================================================== --- pypy/dist/pypy/jit/codegen/cli/rgenop.py (original) +++ pypy/dist/pypy/jit/codegen/cli/rgenop.py Fri Feb 29 12:02:42 2008 @@ -5,7 +5,7 @@ from pypy.jit.codegen.model import GenVarOrConst, GenVar, GenConst, CodeGenSwitch from pypy.jit.codegen.cli import operation as ops from pypy.jit.codegen.cli.dumpgenerator import DumpGenerator -from pypy.translator.cli.dotnet import CLR, typeof, new_array, box, unbox +from pypy.translator.cli.dotnet import CLR, typeof, new_array, box, unbox, clidowncast, classof System = CLR.System Utils = CLR.pypy.runtime.Utils DelegateHolder = CLR.pypy.runtime.DelegateHolder @@ -14,34 +14,19 @@ DUMP_IL = False DEBUG = False -SM_INT__INT_1 = ootype.StaticMethod([ootype.Signed], ootype.Signed) -SM_INT__INT_2 = ootype.StaticMethod([ootype.Signed] * 2, ootype.Signed) -SM_INT__INT_3 = ootype.StaticMethod([ootype.Signed] * 3, ootype.Signed) -SM_INT__INT_5 = ootype.StaticMethod([ootype.Signed] * 5, ootype.Signed) -SM_INT__INT_27 = ootype.StaticMethod([ootype.Signed] * 27, ootype.Signed) -SM_INT__INT_100 = ootype.StaticMethod([ootype.Signed] * 100, ootype.Signed) - -def token2clitype(tok): - if tok == '': - return typeof(System.Int32) - else: - assert False - -def sigtoken2clitype(tok): - if tok == ([''], ''): - return typeof(SM_INT__INT_1) - elif tok == (['', ''], ''): - return typeof(SM_INT__INT_2) - elif tok == ([''] * 3, ''): - return typeof(SM_INT__INT_3) - elif tok == ([''] * 5, ''): - return typeof(SM_INT__INT_5) - elif tok == ([''] * 27, ''): - return typeof(SM_INT__INT_27) - elif tok == ([''] * 100, ''): - return typeof(SM_INT__INT_100) - else: - assert False +cInt32 = classof(System.Int32) +cBoolean = classof(System.Boolean) +cObject = classof(System.Object) + +class SigToken: + def __init__(self, args, res, funcclass): + self.args = args + self.res = res + self.funcclass = funcclass + +def class2type(cls): + 'Cast a PBC of type ootype.Class into a System.Type instance' + return clidowncast(box(cls), System.Type) class __extend__(GenVarOrConst): __metaclass__ = extendabletype @@ -207,7 +192,7 @@ @staticmethod def genzeroconst(kind): - if kind == '': + if kind is cInt32: return IntConst(0) else: return zero_const # ??? @@ -220,20 +205,28 @@ # represent typeof(t) as a pbc args = [RCliGenOp.kindToken(T) for T in FUNCTYPE.ARGS] res = RCliGenOp.kindToken(FUNCTYPE.RESULT) - return args, res + funcclass = classof(FUNCTYPE) + return SigToken(args, res, funcclass) @staticmethod @specialize.memo() def kindToken(T): - return repr(T) + if T is ootype.Signed: + return cInt32 + elif T is ootype.Bool: + return cBoolean + elif isinstance(T, ootype.Instance): + return cObject # XXX? + else: + assert False def newgraph(self, sigtoken, name): - argtoks, restok = sigtoken - args = new_array(System.Type, len(argtoks)+1) + argsclass = sigtoken.args + args = new_array(System.Type, len(argsclass)+1) args[0] = System.Type.GetType("System.Object[]") - for i in range(len(argtoks)): - args[i+1] = token2clitype(argtoks[i]) - res = token2clitype(restok) + for i in range(len(argsclass)): + args[i+1] = class2type(argsclass[i]) + res = class2type(sigtoken.res) builder = Builder(self, name, res, args, sigtoken) return builder, builder.gv_entrypoint, builder.inputargs_gv[:] @@ -250,7 +243,7 @@ # we start from 1 because the 1st arg is an Object[] containing the genconsts for i in range(1, len(args)): self.inputargs_gv.append(GenArgVar(i, args[i])) - self.delegatetype = sigtoken2clitype(sigtoken) + self.delegatetype = class2type(sigtoken.funcclass) self.gv_entrypoint = FunctionConst(self.delegatetype) self.isOpen = False self.operations = [] From cfbolz at codespeak.net Fri Feb 29 12:42:44 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 29 Feb 2008 12:42:44 +0100 (CET) Subject: [pypy-svn] r51951 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080229114244.18271168513@codespeak.net> Author: cfbolz Date: Fri Feb 29 12:42:43 2008 New Revision: 51951 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: convert_result not supported Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Fri Feb 29 12:42:43 2008 @@ -709,11 +709,6 @@ def test_degenerated_at_return(self): S = lltype.GcStruct('S', ('n', lltype.Signed)) T = lltype.GcStruct('T', ('s', S), ('n', lltype.Float)) - class Result: - def convert(self, s): - self.s = s - return str(s.n) - glob_result = Result() def ll_function(flag): t = lltype.malloc(T) @@ -724,7 +719,6 @@ if flag: s = t.s return s - ll_function.convert_result = glob_result.convert res = self.interpret(ll_function, [0], []) assert res.n == 4 @@ -758,7 +752,6 @@ s = lltype.malloc(S) s.x = 123 return s - ll_function.convert_result = lambda s: str(s.x) res = self.interpret(ll_function, [], []) assert res.x == 123 @@ -1229,7 +1222,6 @@ assert res == 1 def test_self_referential_structures(self): - py.test.skip("not working yet") S = lltype.GcForwardReference() S.become(lltype.GcStruct('s', ('ps', lltype.Ptr(S)))) @@ -1244,12 +1236,10 @@ while s: x += 1 s = s.ps - return str(x) + return x - f.convert_result = count_depth - res = self.interpret(f, [3], [], policy=P_NOVIRTUAL) - assert res == '2' + assert count_depth(res) == 2 def test_known_nonzero(self): S = lltype.GcStruct('s', ('x', lltype.Signed)) From cfbolz at codespeak.net Fri Feb 29 12:45:31 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 29 Feb 2008 12:45:31 +0100 (CET) Subject: [pypy-svn] r51952 - pypy/branch/gameboy-emulator Message-ID: <20080229114531.D6E1216851C@codespeak.net> Author: cfbolz Date: Fri Feb 29 12:45:31 2008 New Revision: 51952 Added: pypy/branch/gameboy-emulator/ - copied from r51951, pypy/dist/ Log: make a branch for the work on the gameboy emulator From cfbolz at codespeak.net Fri Feb 29 12:52:02 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 29 Feb 2008 12:52:02 +0100 (CET) Subject: [pypy-svn] r51953 - pypy/branch/jit-refactoring/pypy/jit/rainbow/test Message-ID: <20080229115202.79CCF16851C@codespeak.net> Author: cfbolz Date: Fri Feb 29 12:52:02 2008 New Revision: 51953 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: this needs an indirection because otherwise the portal reentry will take care of things and the test would not test the real problem. Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Fri Feb 29 12:52:02 2008 @@ -1121,12 +1121,14 @@ def test_recursive_with_red_termination_condition(self): py.test.skip('Does not terminate') + def indirection(n): + return ll_factorial def ll_factorial(n): if n <= 0: return 1 return n * ll_factorial(n - 1) - res = self.interpret(ll_factorial, [5], []) + res = self.interpret(indirection, [5], []) assert res == 120 def test_simple_indirect_call(self): From arigo at codespeak.net Fri Feb 29 12:53:53 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 29 Feb 2008 12:53:53 +0100 (CET) Subject: [pypy-svn] r51954 - pypy/dist/pypy/translator/llvm Message-ID: <20080229115353.C17CC16851C@codespeak.net> Author: arigo Date: Fri Feb 29 12:53:50 2008 New Revision: 51954 Modified: pypy/dist/pypy/translator/llvm/modwrapper.py Log: Must clear the RPython-level exception between two invocations via ctypes, otherwise Extreme Confusion occurs. Modified: pypy/dist/pypy/translator/llvm/modwrapper.py ============================================================================== --- pypy/dist/pypy/translator/llvm/modwrapper.py (original) +++ pypy/dist/pypy/translator/llvm/modwrapper.py Fri Feb 29 12:53:50 2008 @@ -18,6 +18,10 @@ rpyexc_occured.argtypes = [] rpyexc_occured.restype = ctypes.c_byte +rpyexc_clear = _c.pypy_rpyexc_clear +rpyexc_clear.argtypes = [] +rpyexc_clear.restype = None + rpyexc_fetch_type = _c.pypy_rpyexc_fetch_type rpyexc_fetch_type.argtypes = [] rpyexc_fetch_type.restype = ctypes.c_void_p @@ -39,6 +43,8 @@ if not startup_code(): raise Exception("Failed to startup") _setup = True + else: + rpyexc_clear() args = [f(a) for a, f in zip(args, to_llargs)] result = __entrypoint__(*args) if rpyexc_occured(): From cfbolz at codespeak.net Fri Feb 29 12:54:33 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 29 Feb 2008 12:54:33 +0100 (CET) Subject: [pypy-svn] r51955 - in pypy/branch/gameboy-emulator/pypy/lang/gameboy: . test Message-ID: <20080229115433.0FF1D16851C@codespeak.net> Author: cfbolz Date: Fri Feb 29 12:54:32 2008 New Revision: 51955 Added: pypy/branch/gameboy-emulator/pypy/lang/gameboy/ pypy/branch/gameboy-emulator/pypy/lang/gameboy/__init__.py (contents, props changed) pypy/branch/gameboy-emulator/pypy/lang/gameboy/test/ pypy/branch/gameboy-emulator/pypy/lang/gameboy/test/__init__.py (contents, props changed) Log: add a directory for the gameboy emulator Added: pypy/branch/gameboy-emulator/pypy/lang/gameboy/__init__.py ============================================================================== Added: pypy/branch/gameboy-emulator/pypy/lang/gameboy/test/__init__.py ============================================================================== From arigo at codespeak.net Fri Feb 29 13:09:02 2008 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 29 Feb 2008 13:09:02 +0100 (CET) Subject: [pypy-svn] r51958 - pypy/dist/pypy/lib/ctypes Message-ID: <20080229120902.ADA6B1684EC@codespeak.net> Author: arigo Date: Fri Feb 29 13:09:02 2008 New Revision: 51958 Modified: pypy/dist/pypy/lib/ctypes/__init__.py Log: Don't provide ctypes.pythonapi at all when running on top of PyPy. Modified: pypy/dist/pypy/lib/ctypes/__init__.py ============================================================================== --- pypy/dist/pypy/lib/ctypes/__init__.py (original) +++ pypy/dist/pypy/lib/ctypes/__init__.py Fri Feb 29 13:09:02 2008 @@ -425,7 +425,9 @@ cdll = LibraryLoader(CDLL) pydll = LibraryLoader(PyDLL) -if _os.name in ("nt", "ce"): +if '__pypy__' in _sys.builtin_module_names: + pass +elif _os.name in ("nt", "ce"): pythonapi = PyDLL("python dll", None, _sys.dllhandle) elif _sys.platform == "cygwin": pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2]) From niko at codespeak.net Fri Feb 29 13:35:38 2008 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 29 Feb 2008 13:35:38 +0100 (CET) Subject: [pypy-svn] r51960 - pypy/dist/pypy/translator/jvm Message-ID: <20080229123538.C44F71684F6@codespeak.net> Author: niko Date: Fri Feb 29 13:35:38 2008 New Revision: 51960 Modified: pypy/dist/pypy/translator/jvm/node.py Log: remove assertion that was py2.5 not py2.4 Modified: pypy/dist/pypy/translator/jvm/node.py ============================================================================== --- pypy/dist/pypy/translator/jvm/node.py (original) +++ pypy/dist/pypy/translator/jvm/node.py Fri Feb 29 13:35:38 2008 @@ -337,7 +337,7 @@ for pyexccls, jexcty in translation_table: for llexitcase in llexitcases: - assert issubclass(llexitcase, py.builtin.BaseException) + # llexitcase is an exception class, like ZeroDivisionError if issubclass(llexitcase, pyexccls): # Generate some converter code like: # try { ... } From antocuni at codespeak.net Fri Feb 29 17:52:48 2008 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 29 Feb 2008 17:52:48 +0100 (CET) Subject: [pypy-svn] r51961 - in pypy/dist/pypy: jit/codegen/llgraph jit/timeshifter jit/timeshifter/test rpython Message-ID: <20080229165248.7978416852C@codespeak.net> Author: antocuni Date: Fri Feb 29 17:52:46 2008 New Revision: 51961 Modified: pypy/dist/pypy/jit/codegen/llgraph/llimpl.py pypy/dist/pypy/jit/timeshifter/hrtyper.py pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py pypy/dist/pypy/rpython/typesystem.py Log: move a bit more the rtyper towards ootype by using staticmethods instead of function pointers Modified: pypy/dist/pypy/jit/codegen/llgraph/llimpl.py ============================================================================== --- pypy/dist/pypy/jit/codegen/llgraph/llimpl.py (original) +++ pypy/dist/pypy/jit/codegen/llgraph/llimpl.py Fri Feb 29 17:52:46 2008 @@ -14,6 +14,7 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.rpython.rclass import fishllattr from pypy.rpython.lltypesystem.lloperation import llop +from pypy.translator.simplify import get_funcobj def _from_opaque(opq): return opq._obj.externalobj @@ -43,6 +44,13 @@ else: return LLSupport.from_rstr(s) +def functionptr_general(TYPE, name, **attrs): + if isinstance(TYPE, lltype.FuncType): + return functionptr(TYPE, name, **attrs) + else: + assert isinstance(TYPE, ootype.StaticMethod) + return ootype.static_meth(TYPE, name, **attrs) + def newgraph(gv_FUNCTYPE, name): FUNCTYPE = _from_opaque(gv_FUNCTYPE).value # 'name' is just a way to track things @@ -74,12 +82,12 @@ casting_link(graph.prereturnblock, [v1], graph.returnblock) substartblock = flowmodel.Block(erasedinputargs) casting_link(graph.startblock, inputargs, substartblock) - fptr = lltype.functionptr(FUNCTYPE, name, - graph=graph) + fptr = functionptr_general(FUNCTYPE, name, + graph=graph) return genconst(fptr) def _getgraph(gv_func): - graph = _from_opaque(gv_func).value._obj.graph + graph = get_funcobj(_from_opaque(gv_func).value).graph return graph def end(gv_func): Modified: pypy/dist/pypy/jit/timeshifter/hrtyper.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/hrtyper.py (original) +++ pypy/dist/pypy/jit/timeshifter/hrtyper.py Fri Feb 29 17:52:46 2008 @@ -469,7 +469,7 @@ ARGS = [] for r_arg in args_r: ARGS += r_arg.residual_argtypes() - return lltype.FuncType(ARGS, RESTYPE) + return self.rtyper.type_system.getcallabletype(ARGS, RESTYPE) def make_new_lloplist(self, block): return HintLowLevelOpList(self) Modified: pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py Fri Feb 29 17:52:46 2008 @@ -22,6 +22,7 @@ from pypy import conftest from pypy.jit.conftest import Benchmark from pypy.jit.codegen.llgraph.rgenop import RGenOp as LLRGenOp +from pypy.translator.simplify import get_funcobj P_NOVIRTUAL = HintAnnotatorPolicy(novirtualcontainer=True) @@ -180,6 +181,7 @@ + timeshifted_entrypoint_args_s, hrtyper.s_JITState) FUNC = hrtyper.get_residual_functype(ha.translator.graphs[0]) + PFUNC = self.Ptr(FUNC) argcolors = unrolling_iterable(argcolors) self.argcolors = argcolors @@ -220,12 +222,11 @@ finish_jitstate(top_jitstate, exceptiondesc, sigtoken) builder.end() - generated = gv_generated.revealconst(lltype.Ptr(FUNC)) + generated = gv_generated.revealconst(PFUNC) return generated ml_generate_code.args_s = ["XXX rgenop"] + generate_code_args_s - ml_generate_code.s_result = annmodel.lltype_to_annotation( - lltype.Ptr(FUNC)) + ml_generate_code.s_result = annmodel.lltype_to_annotation(PFUNC) ## def ml_extract_residual_args(*args): ## result = () @@ -305,7 +306,7 @@ ll_generated = llinterp.eval_graph(self.maingraph, mainargs) # now try to run the residual graph generated by the builder - residual_graph = ll_generated._obj.graph + residual_graph = get_funcobj(ll_generated).graph self.ll_generated = ll_generated self.residual_graph = residual_graph if conftest.option.view: @@ -1795,6 +1796,9 @@ class TestLLType(BaseTestTimeshift): type_system = 'lltype' + def Ptr(self, T): + return lltype.Ptr(T) + passing_ootype_tests = set([ 'test_very_simple', 'test_convert_const_to_redbox', @@ -1813,6 +1817,9 @@ class TestOOType(BaseTestTimeshift): type_system = 'ootype' + def Ptr(self, T): + return T + def __getattribute__(self, name): if name.startswith('test_') and name not in passing_ootype_tests: def fn(): Modified: pypy/dist/pypy/rpython/typesystem.py ============================================================================== --- pypy/dist/pypy/rpython/typesystem.py (original) +++ pypy/dist/pypy/rpython/typesystem.py Fri Feb 29 17:52:46 2008 @@ -45,6 +45,10 @@ def null_callable(self, T): """null callable object of type T""" raise NotImplementedError() + + def getcallabletype(self, ARGS, RESTYPE): + cls = self.callable_trait[0] + return cls(ARGS, RESTYPE) def getcallable(self, graph, getconcretetype=None): """Return callable given a Python function.""" From cfbolz at codespeak.net Fri Feb 29 17:53:25 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 29 Feb 2008 17:53:25 +0100 (CET) Subject: [pypy-svn] r51962 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080229165325.CE4CD16852C@codespeak.net> Author: cfbolz Date: Fri Feb 29 17:53:25 2008 New Revision: 51962 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: fix another test Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Fri Feb 29 17:53:25 2008 @@ -283,7 +283,7 @@ assert 0, "unknown graph calling color %s" % (color, ) elif len(block.exits) == 1: link, = block.exits - self.insert_renaming(link) + self.emit(*self.insert_renaming(link)) self.make_bytecode_block(link.target, insert_goto=True) elif len(block.exits) == 2: linkfalse, linktrue = block.exits @@ -300,6 +300,9 @@ reverse = False elif srcopname == 'ptr_iszero': reverse = True + + falserenaming = self.insert_renaming(linkfalse) + truerenaming = self.insert_renaming(linktrue) if reverse is not None: ptrindex = self.serialize_oparg("red", srcargs[0]) self.emit("red_goto_ifptrnonzero") @@ -309,10 +312,10 @@ self.emit("%s_goto_iftrue" % color) self.emit(index) self.emit(tlabel(linktrue)) - self.insert_renaming(linkfalse) + self.emit(*falserenaming) self.make_bytecode_block(linkfalse.target, insert_goto=True) self.emit(label(linktrue)) - self.insert_renaming(linktrue) + self.emit(*truerenaming) self.make_bytecode_block(linktrue.target, insert_goto=True) else: XXX @@ -350,13 +353,12 @@ def insert_renaming(self, link): reds, greens = self.sort_by_color(link.args, link.target.inputargs) + result = [] for color, args in [("red", reds), ("green", greens)]: - result = [] + result += ["make_new_%svars" % (color, ), len(args)] for v in args: result.append(self.serialize_oparg(color, v)) - self.emit("make_new_%svars" % (color, )) - self.emit(len(args)) - self.emit(*result) + return result def serialize_op(self, op): specialcase = getattr(self, "serialize_op_%s" % (op.opname, ), None) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Fri Feb 29 17:53:25 2008 @@ -1172,7 +1172,6 @@ self.check_insns({}) def test_normalize_indirect_call_more(self): - py.test.skip("not working yet") def g1(v): if v >= 0: return -17 From cfbolz at codespeak.net Fri Feb 29 18:04:32 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 29 Feb 2008 18:04:32 +0100 (CET) Subject: [pypy-svn] r51963 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080229170432.2A70B16853A@codespeak.net> Author: cfbolz Date: Fri Feb 29 18:04:31 2008 New Revision: 51963 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: support debug_assert Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Fri Feb 29 18:04:31 2008 @@ -669,7 +669,11 @@ return result def serialize_op_debug_assert(self, op): - pass + v = op.args[0] + srcopname, srcargs = self.trace_back_bool_var(self.current_block, v) + if srcopname in ('ptr_iszero', 'ptr_nonzero'): + arg = self.serialize_oparg("red", srcargs[0]) + self.emit("learn_nonzeroness", arg, srcopname == "ptr_nonzero") def serialize_op_direct_call(self, op): kind, withexc = self.guess_call_kind(op) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Fri Feb 29 18:04:31 2008 @@ -403,6 +403,10 @@ return rtimeshift.genptreq(self.jitstate, ptrbox1, ptrbox2, True) + @arguments("red", "bool") + def opimpl_learn_nonzeroness(self, ptrbox, nonzero): + rtimeshift.learn_nonzeroness(self.jitstate, ptrbox, nonzero) + @arguments() def opimpl_red_return(self): rtimeshift.save_return(self.jitstate) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Fri Feb 29 18:04:31 2008 @@ -14,6 +14,7 @@ from pypy.rpython.module.support import LLSupport from pypy.annotation import model as annmodel from pypy.objspace.flow.model import summary, Variable +from pypy.rlib.debug import ll_assert from pypy.rlib.jit import hint from pypy.rlib.objectmodel import keepalive_until_here from pypy import conftest @@ -1299,7 +1300,6 @@ self.check_insns(int_mul=1, int_add=0) def test_debug_assert_ptr_nonzero(self): - py.test.skip("not working yet") S = lltype.GcStruct('s', ('x', lltype.Signed)) def h(): s = lltype.malloc(S) From cfbolz at codespeak.net Fri Feb 29 19:17:54 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 29 Feb 2008 19:17:54 +0100 (CET) Subject: [pypy-svn] r51964 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080229181754.D777116851A@codespeak.net> Author: cfbolz Date: Fri Feb 29 19:17:52 2008 New Revision: 51964 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: fix indirect gray calls Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Fri Feb 29 19:17:52 2008 @@ -691,7 +691,7 @@ emitted_args.append(self.serialize_oparg("red", v)) self.emit("red_residual_call") calldescindex = self.calldesc_position(op.args[0].concretetype) - self.emit(fnptrindex, calldescindex, withexc) + self.emit(fnptrindex, calldescindex, withexc, kind != "gray") self.emit(len(emitted_args), *emitted_args) self.emit(self.promotiondesc_position(lltype.Signed)) self.emit("goto", tlabel(("after indirect call", op))) @@ -706,14 +706,17 @@ if kind == "red": self.emit("red_after_direct_call") + self.register_redvar(op.result) elif kind == "yellow": self.emit("yellow_after_direct_call") self.emit("yellow_retrieve_result_as_red") self.emit(self.type_position(op.result.concretetype)) + self.register_redvar(op.result) + elif kind == "gray": + self.emit("red_after_direct_call") else: XXX - self.register_redvar(op.result) self.emit(label(("after indirect call", op))) def handle_oopspec_call(self, op, withexc): @@ -783,12 +786,14 @@ def handle_residual_call(self, op, withexc): fnptr = op.args[0] pos = self.calldesc_position(lltype.typeOf(fnptr.value)) + has_result = self.varcolor(op.result) != "gray" func = self.serialize_oparg("red", fnptr) emitted_args = [] for v in op.args[1:]: emitted_args.append(self.serialize_oparg("red", v)) self.emit("red_residual_call") - self.emit(func, pos, withexc, len(emitted_args), *emitted_args) + self.emit(func, pos, withexc, has_result, len(emitted_args)) + self.emit(*emitted_args) self.emit(self.promotiondesc_position(lltype.Signed)) self.register_redvar(op.result) Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Fri Feb 29 19:17:52 2008 @@ -228,7 +228,8 @@ bytecode = self.load_2byte() assert bytecode >= 0 result = self.opcode_implementations[bytecode](self) - #assert self.frame.local_boxes[-1] is not None + #assert (self.frame is None or not self.frame.local_boxes or + # self.frame.local_boxes[-1] is not None) if result is STOP: return else: @@ -581,12 +582,14 @@ rtimeshift.residual_fetch(self.jitstate, self.exceptiondesc, check_forced, flagbox) - @arguments("red", "calldesc", "bool", "red_varargs", "promotiondesc") - def opimpl_red_residual_call(self, funcbox, calldesc, withexc, - redargs, promotiondesc): + @arguments("red", "calldesc", "bool", "bool", "red_varargs", + "promotiondesc") + def opimpl_red_residual_call(self, funcbox, calldesc, withexc, has_result, + redargs, promotiondesc): result = rtimeshift.gen_residual_call(self.jitstate, calldesc, funcbox, redargs) - self.red_result(result) + if has_result: + self.red_result(result) if withexc: exceptiondesc = self.exceptiondesc else: Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Fri Feb 29 19:17:52 2008 @@ -1367,7 +1367,6 @@ self.check_insns(indirect_call=1) def test_indirect_gray_call(self): - py.test.skip("not working yet") def h1(w, n): w[0] = n*2 def h2(w, n): From cfbolz at codespeak.net Fri Feb 29 19:29:51 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 29 Feb 2008 19:29:51 +0100 (CET) Subject: [pypy-svn] r51965 - pypy/branch/jit-refactoring/pypy/jit/rainbow Message-ID: <20080229182951.48CB816851A@codespeak.net> Author: cfbolz Date: Fri Feb 29 19:29:50 2008 New Revision: 51965 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Log: enable an assert and fix a bug that it found Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Fri Feb 29 19:29:50 2008 @@ -786,7 +786,8 @@ def handle_residual_call(self, op, withexc): fnptr = op.args[0] pos = self.calldesc_position(lltype.typeOf(fnptr.value)) - has_result = self.varcolor(op.result) != "gray" + has_result = (self.varcolor(op.result) != "gray" and + op.result.concretetype != lltype.Void) func = self.serialize_oparg("red", fnptr) emitted_args = [] for v in op.args[1:]: @@ -795,7 +796,8 @@ self.emit(func, pos, withexc, has_result, len(emitted_args)) self.emit(*emitted_args) self.emit(self.promotiondesc_position(lltype.Signed)) - self.register_redvar(op.result) + if has_result: + self.register_redvar(op.result) def handle_rpyexc_raise_call(self, op, withexc): emitted_args = [] Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/interpreter.py Fri Feb 29 19:29:50 2008 @@ -228,8 +228,8 @@ bytecode = self.load_2byte() assert bytecode >= 0 result = self.opcode_implementations[bytecode](self) - #assert (self.frame is None or not self.frame.local_boxes or - # self.frame.local_boxes[-1] is not None) + assert (self.frame is None or not self.frame.local_boxes or + self.frame.local_boxes[-1] is not None) if result is STOP: return else: From cfbolz at codespeak.net Fri Feb 29 20:19:46 2008 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 29 Feb 2008 20:19:46 +0100 (CET) Subject: [pypy-svn] r51966 - in pypy/branch/jit-refactoring/pypy/jit/rainbow: . test Message-ID: <20080229191946.4A63C168530@codespeak.net> Author: cfbolz Date: Fri Feb 29 20:19:45 2008 New Revision: 51966 Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Log: fix two more tests Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/codewriter.py Fri Feb 29 20:19:45 2008 @@ -685,39 +685,46 @@ kind, withexc = self.guess_call_kind(op) targets = dict(self.graphs_from(op)) fnptrindex = self.serialize_oparg("red", op.args[0]) - self.emit("goto_if_constant", fnptrindex, tlabel(("direct call", op))) + has_result = (self.varcolor(op.result) != "gray" and + op.result.concretetype != lltype.Void) + if targets: + self.emit("goto_if_constant", fnptrindex, tlabel(("direct call", op))) + emitted_args = [] for v in op.args[1:-1]: emitted_args.append(self.serialize_oparg("red", v)) self.emit("red_residual_call") calldescindex = self.calldesc_position(op.args[0].concretetype) - self.emit(fnptrindex, calldescindex, withexc, kind != "gray") + self.emit(fnptrindex, calldescindex, withexc, has_result) self.emit(len(emitted_args), *emitted_args) self.emit(self.promotiondesc_position(lltype.Signed)) - self.emit("goto", tlabel(("after indirect call", op))) - - self.emit(label(("direct call", op))) - args = targets.values()[0].getargs() - emitted_args = self.args_of_call(op.args[1:-1], args) - self.emit("indirect_call_const") - self.emit(*emitted_args) - setdescindex = self.indirectcalldesc_position(targets) - self.emit(fnptrindex, setdescindex) - if kind == "red": - self.emit("red_after_direct_call") - self.register_redvar(op.result) - elif kind == "yellow": - self.emit("yellow_after_direct_call") - self.emit("yellow_retrieve_result_as_red") - self.emit(self.type_position(op.result.concretetype)) + if has_result: self.register_redvar(op.result) - elif kind == "gray": - self.emit("red_after_direct_call") - else: - XXX - self.emit(label(("after indirect call", op))) + if targets: + self.emit("goto", tlabel(("after indirect call", op))) + + self.emit(label(("direct call", op))) + args = targets.values()[0].getargs() + emitted_args = self.args_of_call(op.args[1:-1], args) + self.emit("indirect_call_const") + self.emit(*emitted_args) + setdescindex = self.indirectcalldesc_position(targets) + self.emit(fnptrindex, setdescindex) + + if kind == "red": + self.emit("red_after_direct_call") + elif kind == "yellow": + self.emit("yellow_after_direct_call") + self.emit("yellow_retrieve_result_as_red") + self.emit(self.type_position(op.result.concretetype)) + elif kind == "gray": + self.emit("red_after_direct_call") + else: + XXX + + self.emit(label(("after indirect call", op))) def handle_oopspec_call(self, op, withexc): from pypy.jit.timeshifter.oop import Index Modified: pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py ============================================================================== --- pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py (original) +++ pypy/branch/jit-refactoring/pypy/jit/rainbow/test/test_interpreter.py Fri Feb 29 20:19:45 2008 @@ -17,8 +17,11 @@ from pypy.rlib.debug import ll_assert from pypy.rlib.jit import hint from pypy.rlib.objectmodel import keepalive_until_here +from pypy.rlib.rarithmetic import ovfcheck from pypy import conftest +import sys + P_NOVIRTUAL = HintAnnotatorPolicy(novirtualcontainer=True) P_OOPSPEC = HintAnnotatorPolicy(novirtualcontainer=True, oopspec=True) P_OOPSPEC_NOVIRTUAL = HintAnnotatorPolicy(oopspec=True, @@ -1383,7 +1386,6 @@ assert res == f(7,3) def test_indirect_residual_red_call(self): - py.test.skip("not working yet") def h1(n): return n*2 def h2(n): @@ -1464,39 +1466,31 @@ self.check_insns({}) def test_manual_marking_of_pure_functions(self): - py.test.skip("not working yet") - class A(object): - pass - class B(object): - pass - a1 = A() - a2 = A() d = {} - def h1(b, s): + def h1(s): try: return d[s] except KeyError: - d[s] = r = A() + d[s] = r = s * 15 return r h1._pure_function_ = True - b = B() def f(n): hint(None, global_merge_point=True) hint(n, concrete=True) if n == 0: - s = "abc" + s = 123 else: - s = "123" - a = h1(b, s) - return hint(n, variable=True) + s = 567 + a = h1(s) + return hint(a, variable=True) P = StopAtXPolicy(h1) P.oopspec = True res = self.interpret(f, [0], [], policy=P) - assert res == 0 + assert res == 123 * 15 self.check_insns({}) res = self.interpret(f, [4], [], policy=P) - assert res == 4 + assert res == 567 * 15 self.check_insns({})