From fijal at codespeak.net Thu Oct 1 00:24:56 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 00:24:56 +0200 (CEST) Subject: [pypy-svn] r68075 - in pypy/branch/floats-via-sse2/pypy/jit/backend/x86: . test Message-ID: <20090930222456.1C63E168011@codespeak.net> Author: fijal Date: Thu Oct 1 00:24:55 2009 New Revision: 68075 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/ri386setup.py pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Log: I think this is correct encoding, not sure... Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/ri386setup.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/ri386setup.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/ri386setup.py Thu Oct 1 00:24:55 2009 @@ -489,10 +489,10 @@ FUCOMPP = Instruction() FUCOMPP.mode0(['\xDA\xE9']) -FSTPL = Instruction() -FSTPL.mode1(MODRM64, ['\xDD', orbyte(3<<3), modrm(1)]) -FSTL = Instruction() -FSTL.mode1(MODRM64, ['\xDD', orbyte(2<<3), modrm(1)]) +FSTP = Instruction() +FSTP.mode1(MODRM64, ['\xDD', orbyte(3<<3), modrm(1)]) +FST = Instruction() +FST.mode1(MODRM64, ['\xDD', orbyte(2<<3), modrm(1)]) FISTP = Instruction() FISTP.mode1(MODRM, ['\xDB', orbyte(3<<3), modrm(1)]) Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Thu Oct 1 00:24:55 2009 @@ -141,8 +141,10 @@ all = instr.as_all_suffixes for m, extra in args: if m in (i386.MODRM, i386.MODRM8) or all: - if not instrname == 'FNSTCW': + if instrname != 'FNSTCW': suffix = suffixes[sizes[m]] + suffix + if m is i386.MODRM64 and instrname in ['FST', 'FSTP']: + suffix = 'l' following = "" if instr.indirect: suffix = "" From fijal at codespeak.net Thu Oct 1 00:27:34 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 00:27:34 +0200 (CEST) Subject: [pypy-svn] r68076 - in pypy/branch/floats-via-sse2/pypy/jit/backend: llsupport llsupport/test x86 Message-ID: <20090930222734.6AFBF168011@codespeak.net> Author: fijal Date: Thu Oct 1 00:27:33 2009 New Revision: 68076 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/descr.py pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/llmodel.py pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/test/test_descr.py pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py pypy/branch/floats-via-sse2/pypy/jit/backend/x86/runner.py Log: Enough to pass the first, very basic call Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/descr.py Thu Oct 1 00:27:33 2009 @@ -1,6 +1,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.jit.backend.llsupport import symbolic -from pypy.jit.metainterp.history import AbstractDescr, getkind, BoxInt, BoxPtr +from pypy.jit.metainterp.history import AbstractDescr, getkind, BoxInt, BoxPtr,\ + BoxFloat from pypy.jit.metainterp.history import BasicFailDescr from pypy.jit.metainterp.resoperation import ResOperation, rop @@ -175,6 +176,7 @@ result = [] for c in self.arg_classes: if c == 'i': box = BoxInt() + elif c == 'f': box = BoxFloat() else: box = BoxPtr() result.append(box) return result @@ -189,12 +191,15 @@ if self.executable_token is not None: return self.executable_token args = [BoxInt()] + self.instantiate_arg_classes() - if self.get_result_size(cpu.translate_support_code) == 0: + res_size = self.get_result_size(cpu.translate_support_code) + if res_size == 0: result = None result_list = [] else: if self.returns_a_pointer(): result = BoxPtr() + elif res_size == 8: + result = BoxFloat() else: result = BoxInt() result_list = [result] @@ -240,6 +245,7 @@ kind = getkind(ARG) if kind == 'int': arg_classes.append('i') elif kind == 'ref': arg_classes.append('r') + elif kind == 'float': arg_classes.append('f') else: raise NotImplementedError('ARG = %r' % (ARG,)) arg_classes = ''.join(arg_classes) Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/llmodel.py Thu Oct 1 00:27:33 2009 @@ -4,7 +4,8 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.rpython.annlowlevel import llhelper from pypy.rlib.objectmodel import we_are_translated, specialize -from pypy.jit.metainterp.history import BoxInt, BoxPtr, set_future_values +from pypy.jit.metainterp.history import BoxInt, BoxPtr, set_future_values,\ + BoxFloat from pypy.jit.backend.model import AbstractCPU from pypy.jit.backend.llsupport import symbolic from pypy.jit.backend.llsupport.symbolic import WORD, unroll_basic_sizes @@ -414,9 +415,12 @@ self.execute_token(executable_token) # Note: if an exception is set, the rest of the code does a bit of # nonsense but nothing wrong (the return value should be ignored) + res = calldescr.get_result_size(self.translate_support_code) if calldescr.returns_a_pointer(): return BoxPtr(self.get_latest_value_ref(0)) - elif calldescr.get_result_size(self.translate_support_code) != 0: + elif res == self.FLOATSIZE: + return BoxFloat(self.get_latest_value_float(0)) + elif res > 0: return BoxInt(self.get_latest_value_int(0)) else: return None Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/test/test_descr.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/test/test_descr.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/test/test_descr.py Thu Oct 1 00:27:33 2009 @@ -130,7 +130,10 @@ assert isinstance(descr3.get_result_size(True), Symbolic) assert descr3.returns_a_pointer() assert descr3.arg_classes == "r" - + # + c1 = GcCache(True) + descr4 = get_call_descr(c1, [lltype.Float, lltype.Float], lltype.Float) + assert descr4.get_result_size(False) == rffi.sizeof(lltype.Float) def test_repr_of_descr(): c0 = GcCache(False) Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py Thu Oct 1 00:27:33 2009 @@ -255,6 +255,9 @@ else: self.mc.MOV(to_loc, from_loc) + def regalloc_fstp(self, loc): + self.mc.FSTP(loc) + def regalloc_push(self, loc): self.mc.PUSH(loc) @@ -780,15 +783,26 @@ tmp = ecx else: tmp = eax + p = 0 for i in range(2, nargs + 2): loc = arglocs[i] if isinstance(loc, REG): - self.mc.MOV(mem(esp, WORD * (i - 2)), loc) + if isinstance(loc, XMMREG): + self.mc.MOVSD(mem64(esp, p), loc) + p += 2*WORD + else: + self.mc.MOV(mem(esp, p), loc) + p += WORD + p = 0 for i in range(2, nargs + 2): loc = arglocs[i] if not isinstance(loc, REG): - self.mc.MOV(tmp, loc) - self.mc.MOV(mem(esp, WORD * (i - 2)), tmp) + if isinstance(loc, MODRM64): + xxx + else: + self.mc.MOV(tmp, loc) + self.mc.MOV(mem(esp, p), tmp) + p += WORD self.mc.CALL(x) self.mark_gc_roots() self.mc.ADD(esp, imm(WORD * extra_on_stack)) Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py Thu Oct 1 00:27:33 2009 @@ -93,12 +93,11 @@ return heap64(rffi.cast(lltype.Signed, arr) + num_pos * WORD * 2) def after_call(self, v): - xxx # test # the result is stored in st0, but we don't have this around, # so we move it to some stack location if v is not None: - loc = self.sm.stack_loc(v, 2) - self.assembler.regalloc_mov(st0, loc) + loc = self.stack_manager.loc(v, 2) + self.assembler.regalloc_fstp(loc) class X86StackManager(StackManager): @@ -569,8 +568,12 @@ def _call(self, op, arglocs, force_store=[]): self.rm.before_call(force_store) + self.xrm.before_call(force_store) self.Perform(op, arglocs, eax) - self.rm.after_call(op.result) + if op.result.type == FLOAT: + self.xrm.after_call(op.result) + else: + self.rm.after_call(op.result) def consider_call(self, op, ignored): calldescr = op.descr Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/runner.py Thu Oct 1 00:27:33 2009 @@ -12,6 +12,7 @@ class CPU386(AbstractLLCPU): debug = True supports_floats = True + FLOATSIZE = 8 BOOTSTRAP_TP = lltype.FuncType([], lltype.Signed) dont_keepalive_stuff = False # for tests From fijal at codespeak.net Thu Oct 1 00:38:55 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 00:38:55 +0200 (CEST) Subject: [pypy-svn] r68077 - in pypy/branch/floats-via-sse2/pypy/jit/backend: test x86 Message-ID: <20090930223855.3B94F168011@codespeak.net> Author: fijal Date: Thu Oct 1 00:38:54 2009 New Revision: 68077 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py Log: Yay. I think I'm done. Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Thu Oct 1 00:38:54 2009 @@ -281,6 +281,24 @@ 'int', descr=calldescr) assert res.value == 2 * num + if cpu.supports_floats: + def func(f0, f1, f2, f3, f4, f5, f6, i0, i1, f7, f8, f9): + return f0 + f1 + f2 + f3 + f4 + f5 + f6 + float(i0 + i1) + f7 + f8 + f9 + F = lltype.Float + I = lltype.Signed + FUNC = self.FuncType([F] * 7 + [I] * 2 + [F] * 3, F) + FPTR = self.Ptr(FUNC) + func_ptr = llhelper(FPTR, func) + calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + funcbox = self.get_funcbox(cpu, func_ptr) + args = ([BoxFloat(.1) for i in range(7)] + + [BoxInt(1), BoxInt(2), BoxFloat(.2), BoxFloat(.3), + BoxFloat(.4)]) + res = self.execute_operation(rop.CALL, + [funcbox] + args, + 'float', descr=calldescr) + assert res.value - 4.6 < 0.0001 + def test_executor(self): cpu = self.cpu x = execute(cpu, rop.INT_ADD, None, BoxInt(100), ConstInt(42)) Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py Thu Oct 1 00:38:54 2009 @@ -789,20 +789,20 @@ if isinstance(loc, REG): if isinstance(loc, XMMREG): self.mc.MOVSD(mem64(esp, p), loc) - p += 2*WORD else: self.mc.MOV(mem(esp, p), loc) - p += WORD + p += loc.width p = 0 for i in range(2, nargs + 2): loc = arglocs[i] if not isinstance(loc, REG): if isinstance(loc, MODRM64): - xxx + self.mc.MOVSD(xmm0, loc) + self.mc.MOVSD(mem64(esp, p), xmm0) else: self.mc.MOV(tmp, loc) self.mc.MOV(mem(esp, p), tmp) - p += WORD + p += loc.width self.mc.CALL(x) self.mark_gc_roots() self.mc.ADD(esp, imm(WORD * extra_on_stack)) Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py Thu Oct 1 00:38:54 2009 @@ -570,10 +570,11 @@ self.rm.before_call(force_store) self.xrm.before_call(force_store) self.Perform(op, arglocs, eax) - if op.result.type == FLOAT: - self.xrm.after_call(op.result) - else: - self.rm.after_call(op.result) + if op.result is not None: + if op.result.type == FLOAT: + self.xrm.after_call(op.result) + else: + self.rm.after_call(op.result) def consider_call(self, op, ignored): calldescr = op.descr From fijal at codespeak.net Thu Oct 1 00:41:21 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 00:41:21 +0200 (CEST) Subject: [pypy-svn] r68078 - pypy/branch/floats-via-sse2/pypy/jit/backend/x86 Message-ID: <20090930224121.7D00A168011@codespeak.net> Author: fijal Date: Thu Oct 1 00:41:20 2009 New Revision: 68078 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py Log: We use WORD for smaller stuff than WORD Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py Thu Oct 1 00:41:20 2009 @@ -791,7 +791,7 @@ self.mc.MOVSD(mem64(esp, p), loc) else: self.mc.MOV(mem(esp, p), loc) - p += loc.width + p += round_up_to_4(loc.width) p = 0 for i in range(2, nargs + 2): loc = arglocs[i] @@ -802,7 +802,7 @@ else: self.mc.MOV(tmp, loc) self.mc.MOV(mem(esp, p), tmp) - p += loc.width + p += round_up_to_4(loc.width) self.mc.CALL(x) self.mark_gc_roots() self.mc.ADD(esp, imm(WORD * extra_on_stack)) @@ -920,3 +920,8 @@ return heap(reg_or_imm1.value + offset) else: return mem(reg_or_imm1, offset) + +def round_up_to_4(size): + if size < 4: + return 4 + return size From fijal at codespeak.net Thu Oct 1 00:47:53 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 00:47:53 +0200 (CEST) Subject: [pypy-svn] r68079 - pypy/branch/floats-via-sse2/pypy/jit/backend/x86 Message-ID: <20090930224753.4E795168011@codespeak.net> Author: fijal Date: Thu Oct 1 00:47:52 2009 New Revision: 68079 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py Log: correctly adjust stack depth Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py Thu Oct 1 00:47:52 2009 @@ -773,7 +773,10 @@ assert isinstance(sizeloc, IMM32) size = sizeloc.value nargs = len(op.args)-1 - extra_on_stack = self.align_stack_for_call(nargs) + extra_on_stack = 0 + for arg in range(2, nargs + 2): + extra_on_stack += round_up_to_4(arglocs[arg].width) + extra_on_stack = self.align_stack_for_call(extra_on_stack) self.mc.SUB(esp, imm(WORD * extra_on_stack)) if isinstance(op.args[0], Const): x = rel32(op.args[0].getint()) From fijal at codespeak.net Thu Oct 1 01:31:45 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 01:31:45 +0200 (CEST) Subject: [pypy-svn] r68080 - pypy/branch/floats-via-sse2/pypy/module/pypyjit Message-ID: <20090930233145.E7273168011@codespeak.net> Author: fijal Date: Thu Oct 1 01:31:45 2009 New Revision: 68080 Modified: pypy/branch/floats-via-sse2/pypy/module/pypyjit/policy.py Log: We do support floats, but force_cast is a bit too much for us so far, ignore, keep in mind Modified: pypy/branch/floats-via-sse2/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/floats-via-sse2/pypy/module/pypyjit/policy.py Thu Oct 1 01:31:45 2009 @@ -35,10 +35,12 @@ # gc_id operation if func.__name__ == 'id__ANY': return False - # floats if mod == 'pypy.rlib.rbigint': #if func.__name__ == '_bigint_true_divide': return False + if mod == 'pypy.rpython.lltypesystem.ll_math': + # XXX temporary, contains force_cast + return False if '_geninterp_' in func.func_globals: # skip all geninterped stuff return False if mod.startswith('pypy.interpreter.astcompiler.'): From fijal at codespeak.net Thu Oct 1 02:09:16 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 02:09:16 +0200 (CEST) Subject: [pypy-svn] r68081 - pypy/branch/floats-via-sse2/pypy/jit/backend/x86 Message-ID: <20091001000916.D8702168020@codespeak.net> Author: fijal Date: Thu Oct 1 02:09:16 2009 New Revision: 68081 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py Log: Fix rpython Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py Thu Oct 1 02:09:16 2009 @@ -71,8 +71,9 @@ return lltype.malloc(rffi.CArray(lltype.Float), BASE_CONSTANT_SIZE, flavor='raw') - def __init__(self, *args, **kwds): - RegisterManager.__init__(self, *args, **kwds) + def __init__(self, longevity, stack_manager=None, assembler=None): + RegisterManager.__init__(self, longevity, stack_manager=stack_manager, + assembler=assembler) self.constant_arrays = [self.new_const_array()] self.constant_arrays[-1][0] = NEG_ZERO self.constant_arrays[-1][1] = NAN From fijal at codespeak.net Thu Oct 1 09:31:02 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 09:31:02 +0200 (CEST) Subject: [pypy-svn] r68082 - pypy/branch/floats-via-sse2/pypy/jit/backend/x86 Message-ID: <20091001073102.3D51D168015@codespeak.net> Author: fijal Date: Thu Oct 1 09:31:00 2009 New Revision: 68082 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py Log: RPython does not seem to be too happy with kwds, not do it Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py Thu Oct 1 09:31:00 2009 @@ -193,17 +193,26 @@ for var in vars: self.possibly_free_var(var) - def make_sure_var_in_reg(self, var, forbidden_vars=[], **kwds): + def make_sure_var_in_reg(self, var, forbidden_vars=[], + selected_reg=None, imm_fine=True, + need_lower_byte=False): if var.type == FLOAT: - return self.xrm.make_sure_var_in_reg(var, forbidden_vars, **kwds) - else: - return self.rm.make_sure_var_in_reg(var, forbidden_vars, **kwds) + return self.xrm.make_sure_var_in_reg(var, forbidden_vars, + selected_reg, imm_fine, + need_lower_byte) + else: + return self.rm.make_sure_var_in_reg(var, forbidden_vars, + selected_reg, imm_fine, + need_lower_byte) - def force_allocate_reg(self, var, forbidden_vars=[], **kwds): + def force_allocate_reg(self, var, forbidden_vars=[], selected_reg=None, + need_lower_byte=False): if var.type == FLOAT: - return self.xrm.force_allocate_reg(var, forbidden_vars, **kwds) + return self.xrm.force_allocate_reg(var, forbidden_vars, + selected_reg, need_lower_byte) else: - return self.rm.force_allocate_reg(var, forbidden_vars, **kwds) + return self.rm.force_allocate_reg(var, forbidden_vars, + selected_reg, need_lower_byte) def _compute_loop_consts(self, inputargs, jump): if jump.opnum != rop.JUMP or jump.jump_target is not None: From afa at codespeak.net Thu Oct 1 09:53:24 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 1 Oct 2009 09:53:24 +0200 (CEST) Subject: [pypy-svn] r68083 - pypy/trunk/pypy/jit/backend/llsupport Message-ID: <20091001075324.632DC168012@codespeak.net> Author: afa Date: Thu Oct 1 09:53:24 2009 New Revision: 68083 Modified: pypy/trunk/pypy/jit/backend/llsupport/gc.py Log: Boehm version 7.x doesn't define GC_local_malloc: call GC_malloc instead. This fixes translation with "-Ojit --gc=boehm" on Windows. Modified: pypy/trunk/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/gc.py Thu Oct 1 09:53:24 2009 @@ -37,8 +37,23 @@ def __init__(self, gcdescr, translator): GcLLDescription.__init__(self, gcdescr, translator) # grab a pointer to the Boehm 'malloc' function - compilation_info = ExternalCompilationInfo(libraries=['gc']) - malloc_fn_ptr = rffi.llexternal("GC_local_malloc", + from pypy.rpython.tool import rffi_platform + compilation_info = rffi_platform.check_boehm() + + # Versions 6.x of libgc needs to use GC_local_malloc(). + # Versions 7.x of libgc removed this function; GC_malloc() has + # the same behavior if libgc was compiled with + # THREAD_LOCAL_ALLOC. + class CConfig: + _compilation_info_ = compilation_info + HAS_LOCAL_MALLOC = rffi_platform.Has("GC_local_malloc") + config = rffi_platform.configure(CConfig) + if config['HAS_LOCAL_MALLOC']: + GC_MALLOC = "GC_local_malloc" + else: + GC_MALLOC = "GC_malloc" + + malloc_fn_ptr = rffi.llexternal(GC_MALLOC, [lltype.Signed], # size_t, but good enough llmemory.GCREF, compilation_info=compilation_info, @@ -48,6 +63,7 @@ # on some platform GC_init is required before any other # GC_* functions, call it here for the benefit of tests + # XXX move this to tests init_fn_ptr = rffi.llexternal("GC_init", [], lltype.Void, compilation_info=compilation_info, From fijal at codespeak.net Thu Oct 1 09:55:24 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 09:55:24 +0200 (CEST) Subject: [pypy-svn] r68084 - pypy/branch/floats-via-sse2/pypy/module/pypyjit Message-ID: <20091001075524.E5F4F168012@codespeak.net> Author: fijal Date: Thu Oct 1 09:55:24 2009 New Revision: 68084 Modified: pypy/branch/floats-via-sse2/pypy/module/pypyjit/policy.py Log: Oops, typo Modified: pypy/branch/floats-via-sse2/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/floats-via-sse2/pypy/module/pypyjit/policy.py Thu Oct 1 09:55:24 2009 @@ -38,7 +38,7 @@ if mod == 'pypy.rlib.rbigint': #if func.__name__ == '_bigint_true_divide': return False - if mod == 'pypy.rpython.lltypesystem.ll_math': + if mod == 'pypy.rpython.lltypesystem.module.ll_math': # XXX temporary, contains force_cast return False if '_geninterp_' in func.func_globals: # skip all geninterped stuff From fijal at codespeak.net Thu Oct 1 10:31:17 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 10:31:17 +0200 (CEST) Subject: [pypy-svn] r68085 - pypy/branch/floats-via-sse2/pypy/module/pypyjit Message-ID: <20091001083117.A3011168010@codespeak.net> Author: fijal Date: Thu Oct 1 10:31:16 2009 New Revision: 68085 Modified: pypy/branch/floats-via-sse2/pypy/module/pypyjit/policy.py Log: Reinstantiate float support Modified: pypy/branch/floats-via-sse2/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/floats-via-sse2/pypy/module/pypyjit/policy.py Thu Oct 1 10:31:16 2009 @@ -10,12 +10,6 @@ if (func.__name__.startswith('_mm_') or func.__name__.startswith('__mm_')): # multimethods - name = func.__name__.lstrip('_') - if (name.startswith('mm_truediv') or - name.startswith('mm_inplace_truediv') or - name.startswith('mm_float')): - # floats - return False return True if '_mth_mm_' in func.__name__: # e.g. str_mth_mm_join_xxx return True @@ -27,11 +21,6 @@ return False if mod.startswith('pypy.objspace.'): - # we don't support floats - if 'float' in mod or 'complex' in mod: - return False - if func.__name__ == 'format_float': - return False # gc_id operation if func.__name__ == 'id__ANY': return False From fijal at codespeak.net Thu Oct 1 11:23:58 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 11:23:58 +0200 (CEST) Subject: [pypy-svn] r68086 - pypy/branch/floats-via-sse2/pypy/jit/backend/test Message-ID: <20091001092358.036FA16801B@codespeak.net> Author: fijal Date: Thu Oct 1 11:23:58 2009 New Revision: 68086 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Log: missing abs Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Thu Oct 1 11:23:58 2009 @@ -297,7 +297,7 @@ res = self.execute_operation(rop.CALL, [funcbox] + args, 'float', descr=calldescr) - assert res.value - 4.6 < 0.0001 + assert abs(res.value - 4.6) < 0.0001 def test_executor(self): cpu = self.cpu From fijal at codespeak.net Thu Oct 1 11:46:01 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 11:46:01 +0200 (CEST) Subject: [pypy-svn] r68087 - pypy/branch/floats-via-sse2/pypy/jit/backend/test Message-ID: <20091001094601.778CA168012@codespeak.net> Author: fijal Date: Thu Oct 1 11:46:01 2009 New Revision: 68087 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Log: get/set arrayitem tests for floats Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Thu Oct 1 11:46:01 2009 @@ -961,6 +961,19 @@ descr_B) assert isinstance(x, BoxPtr) assert x.getref(lltype.Ptr(A)) == a + C = lltype.GcArray(lltype.Float) + c = lltype.malloc(C, 6) + c[3] = 3.5 + descr_C = cpu.arraydescrof(C) + x = cpu.do_getarrayitem_gc( + BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, c)), BoxInt(3), + descr_C) + assert isinstance(x, BoxFloat) + assert x.getfloat() == 3.5 + cpu.do_setarrayitem_gc( + BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, c)), BoxInt(4), + BoxFloat(4.5), descr_C) + assert c[4] == 4.5 # s = rstr.mallocstr(6) x = cpu.do_strlen( From fijal at codespeak.net Thu Oct 1 11:47:28 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 11:47:28 +0200 (CEST) Subject: [pypy-svn] r68088 - pypy/branch/floats-via-sse2/pypy/jit/backend/test Message-ID: <20091001094728.CF3FA168012@codespeak.net> Author: fijal Date: Thu Oct 1 11:47:28 2009 New Revision: 68088 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Log: Actually run those only if cpu supports floats Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Thu Oct 1 11:47:28 2009 @@ -961,19 +961,20 @@ descr_B) assert isinstance(x, BoxPtr) assert x.getref(lltype.Ptr(A)) == a - C = lltype.GcArray(lltype.Float) - c = lltype.malloc(C, 6) - c[3] = 3.5 - descr_C = cpu.arraydescrof(C) - x = cpu.do_getarrayitem_gc( - BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, c)), BoxInt(3), - descr_C) - assert isinstance(x, BoxFloat) - assert x.getfloat() == 3.5 - cpu.do_setarrayitem_gc( - BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, c)), BoxInt(4), - BoxFloat(4.5), descr_C) - assert c[4] == 4.5 + if self.cpu.supports_floats: + C = lltype.GcArray(lltype.Float) + c = lltype.malloc(C, 6) + c[3] = 3.5 + descr_C = cpu.arraydescrof(C) + x = cpu.do_getarrayitem_gc( + BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, c)), BoxInt(3), + descr_C) + assert isinstance(x, BoxFloat) + assert x.getfloat() == 3.5 + cpu.do_setarrayitem_gc( + BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, c)), BoxInt(4), + BoxFloat(4.5), descr_C) + assert c[4] == 4.5 # s = rstr.mallocstr(6) x = cpu.do_strlen( @@ -985,7 +986,8 @@ BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s)), BoxInt(3)) assert x.value == ord('X') # - S = lltype.GcStruct('S', ('x', lltype.Char), ('y', lltype.Ptr(A))) + S = lltype.GcStruct('S', ('x', lltype.Char), ('y', lltype.Ptr(A)), + ('z', lltype.Float)) descrfld_x = cpu.fielddescrof(S, 'x') s = lltype.malloc(S) s.x = 'Z' From fijal at codespeak.net Thu Oct 1 11:52:21 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 11:52:21 +0200 (CEST) Subject: [pypy-svn] r68089 - pypy/branch/floats-via-sse2/pypy/jit/backend/test Message-ID: <20091001095221.C80A2168014@codespeak.net> Author: fijal Date: Thu Oct 1 11:52:20 2009 New Revision: 68089 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Log: get/set field Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Thu Oct 1 11:52:20 2009 @@ -1031,6 +1031,19 @@ descrfld_rx) assert rs.x == '!' # + + if self.cpu.supports_floats: + descrfld_z = cpu.fielddescrof(S, 'z') + cpu.do_setfield_gc( + BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s)), + BoxFloat(3.5), + descrfld_z) + assert s.z == 3.5 + s.z = 3.2 + x = cpu.do_getfield_gc( + BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s)), + descrfld_z) + assert x.getfloat() == 3.2 ### we don't support in the JIT for now GC pointers ### stored inside non-GC structs. #descrfld_ry = cpu.fielddescrof(RS, 'y') From pedronis at codespeak.net Thu Oct 1 11:56:14 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 1 Oct 2009 11:56:14 +0200 (CEST) Subject: [pypy-svn] r68090 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20091001095614.D5719168014@codespeak.net> Author: pedronis Date: Thu Oct 1 11:56:14 2009 New Revision: 68090 Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: give the test its new proper name Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Thu Oct 1 11:56:14 2009 @@ -23,7 +23,7 @@ self.pc = pc self.exception_target = exc_target -def test_clone_guard_simple(): +def test_store_final_boxes_in_guard(): from pypy.jit.metainterp.compile import ResumeGuardDescr b0 = BoxInt() b1 = BoxInt() From arigo at codespeak.net Thu Oct 1 11:58:32 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 1 Oct 2009 11:58:32 +0200 (CEST) Subject: [pypy-svn] r68091 - in pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport: . test Message-ID: <20091001095832.50AD5168014@codespeak.net> Author: arigo Date: Thu Oct 1 11:58:31 2009 New Revision: 68091 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/descr.py pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/test/test_descr.py Log: Implement floats in descr.py properly. Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/descr.py Thu Oct 1 11:58:31 2009 @@ -1,8 +1,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.jit.backend.llsupport import symbolic -from pypy.jit.metainterp.history import AbstractDescr, getkind, BoxInt, BoxPtr,\ - BoxFloat -from pypy.jit.metainterp.history import BasicFailDescr +from pypy.jit.metainterp.history import AbstractDescr, getkind, BoxInt, BoxPtr +from pypy.jit.metainterp.history import BasicFailDescr, BoxFloat from pypy.jit.metainterp.resoperation import ResOperation, rop # The point of the class organization in this file is to make instances @@ -71,6 +70,9 @@ def is_pointer_field(self): return False # unless overridden by GcPtrFieldDescr + def is_float_field(self): + return False # unless overridden by FloatFieldDescr + def repr_of_descr(self): return '<%s %s>' % (self._clsname, self.offset) @@ -87,7 +89,8 @@ def getFieldDescrClass(TYPE): return getDescrClass(TYPE, BaseFieldDescr, GcPtrFieldDescr, - NonGcPtrFieldDescr, 'Field', 'get_field_size') + NonGcPtrFieldDescr, 'Field', 'get_field_size', + 'is_float_field') def get_field_descr(gccache, STRUCT, fieldname): cache = gccache._cache_field @@ -126,6 +129,9 @@ def is_array_of_pointers(self): return False # unless overridden by GcPtrArrayDescr + def is_array_of_floats(self): + return False # unless overridden by FloatArrayDescr + def repr_of_descr(self): return '<%s>' % self._clsname @@ -142,7 +148,8 @@ def getArrayDescrClass(ARRAY): return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr, - NonGcPtrArrayDescr, 'Array', 'get_item_size') + NonGcPtrArrayDescr, 'Array', 'get_item_size', + 'is_array_of_floats') def get_array_descr(gccache, ARRAY): cache = gccache._cache_array @@ -175,15 +182,18 @@ def instantiate_arg_classes(self): result = [] for c in self.arg_classes: - if c == 'i': box = BoxInt() + if c == 'i': box = BoxInt() elif c == 'f': box = BoxFloat() - else: box = BoxPtr() + else: box = BoxPtr() result.append(box) return result def returns_a_pointer(self): return False # unless overridden by GcPtrCallDescr + def returns_a_float(self): + return False # unless overridden by FloatCallDescr + def get_result_size(self, translate_support_code): raise NotImplementedError @@ -191,14 +201,13 @@ if self.executable_token is not None: return self.executable_token args = [BoxInt()] + self.instantiate_arg_classes() - res_size = self.get_result_size(cpu.translate_support_code) - if res_size == 0: + if self.get_result_size(cpu.translate_support_code) == 0: result = None result_list = [] else: if self.returns_a_pointer(): result = BoxPtr() - elif res_size == 8: + elif self.returns_a_float(): result = BoxFloat() else: result = BoxInt() @@ -237,7 +246,8 @@ if RESULT is lltype.Void: return VoidCallDescr return getDescrClass(RESULT, BaseCallDescr, GcPtrCallDescr, - NonGcPtrCallDescr, 'Call', 'get_result_size') + NonGcPtrCallDescr, 'Call', 'get_result_size', + 'returns_a_float') def get_call_descr(gccache, ARGS, RESULT): arg_classes = [] @@ -263,7 +273,7 @@ # ____________________________________________________________ def getDescrClass(TYPE, BaseDescr, GcPtrDescr, NonGcPtrDescr, - nameprefix, methodname, _cache={}): + nameprefix, methodname, floatcheckname, _cache={}): if isinstance(TYPE, lltype.Ptr): if TYPE.TO._gckind == 'gc': return GcPtrDescr @@ -281,5 +291,10 @@ return symbolic.get_size(TYPE, translate_support_code) setattr(Descr, methodname, method) # + if TYPE is lltype.Float: + def is_float(self): + return True + setattr(Descr, floatcheckname, is_float) + # _cache[nameprefix, TYPE] = Descr return Descr Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/test/test_descr.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/test/test_descr.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/test/test_descr.py Thu Oct 1 11:58:31 2009 @@ -26,12 +26,16 @@ T = lltype.GcStruct('T') S = lltype.GcStruct('S', ('x', lltype.Char), ('y', lltype.Ptr(T)), - ('z', lltype.Ptr(U))) + ('z', lltype.Ptr(U)), + ('f', lltype.Float)) assert getFieldDescrClass(lltype.Ptr(T)) is GcPtrFieldDescr assert getFieldDescrClass(lltype.Ptr(U)) is NonGcPtrFieldDescr cls = getFieldDescrClass(lltype.Char) assert cls != getFieldDescrClass(lltype.Signed) assert cls == getFieldDescrClass(lltype.Char) + clsf = getFieldDescrClass(lltype.Float) + assert clsf != cls + assert clsf == getFieldDescrClass(lltype.Float) # c0 = GcCache(False) c1 = GcCache(True) @@ -42,25 +46,35 @@ descr_x = get_field_descr(c2, S, 'x') descr_y = get_field_descr(c2, S, 'y') descr_z = get_field_descr(c2, S, 'z') + descr_f = get_field_descr(c2, S, 'f') assert descr_x.__class__ is cls assert descr_y.__class__ is GcPtrFieldDescr assert descr_z.__class__ is NonGcPtrFieldDescr + assert descr_f.__class__ is clsf if not tsc: assert descr_x.offset < descr_y.offset < descr_z.offset assert descr_x.sort_key() < descr_y.sort_key() < descr_z.sort_key() assert descr_x.get_field_size(False) == rffi.sizeof(lltype.Char) assert descr_y.get_field_size(False) == rffi.sizeof(lltype.Ptr(T)) assert descr_z.get_field_size(False) == rffi.sizeof(lltype.Ptr(U)) + assert descr_f.get_field_size(False) == rffi.sizeof(lltype.Float) else: assert isinstance(descr_x.offset, Symbolic) assert isinstance(descr_y.offset, Symbolic) assert isinstance(descr_z.offset, Symbolic) + assert isinstance(descr_f.offset, Symbolic) assert isinstance(descr_x.get_field_size(True), Symbolic) assert isinstance(descr_y.get_field_size(True), Symbolic) assert isinstance(descr_z.get_field_size(True), Symbolic) + assert isinstance(descr_f.get_field_size(True), Symbolic) assert not descr_x.is_pointer_field() assert descr_y.is_pointer_field() assert not descr_z.is_pointer_field() + assert not descr_f.is_pointer_field() + assert not descr_x.is_float_field() + assert not descr_y.is_float_field() + assert not descr_z.is_float_field() + assert descr_f.is_float_field() def test_get_array_descr(): @@ -69,71 +83,102 @@ A1 = lltype.GcArray(lltype.Char) A2 = lltype.GcArray(lltype.Ptr(T)) A3 = lltype.GcArray(lltype.Ptr(U)) + A4 = lltype.GcArray(lltype.Float) assert getArrayDescrClass(A2) is GcPtrArrayDescr assert getArrayDescrClass(A3) is NonGcPtrArrayDescr cls = getArrayDescrClass(A1) assert cls != getArrayDescrClass(lltype.GcArray(lltype.Signed)) assert cls == getArrayDescrClass(lltype.GcArray(lltype.Char)) + clsf = getArrayDescrClass(A4) + assert clsf != cls + assert clsf == getArrayDescrClass(lltype.GcArray(lltype.Float)) # c0 = GcCache(False) descr1 = get_array_descr(c0, A1) descr2 = get_array_descr(c0, A2) descr3 = get_array_descr(c0, A3) + descr4 = get_array_descr(c0, A4) assert descr1.__class__ is cls assert descr2.__class__ is GcPtrArrayDescr assert descr3.__class__ is NonGcPtrArrayDescr + assert descr4.__class__ is clsf assert descr1 == get_array_descr(c0, lltype.GcArray(lltype.Char)) assert not descr1.is_array_of_pointers() assert descr2.is_array_of_pointers() assert not descr3.is_array_of_pointers() + assert not descr4.is_array_of_pointers() + assert not descr1.is_array_of_floats() + assert not descr2.is_array_of_floats() + assert not descr3.is_array_of_floats() + assert descr4.is_array_of_floats() # WORD = rffi.sizeof(lltype.Signed) assert descr1.get_base_size(False) == WORD assert descr2.get_base_size(False) == WORD assert descr3.get_base_size(False) == WORD + assert descr4.get_base_size(False) == WORD assert descr1.get_ofs_length(False) == 0 assert descr2.get_ofs_length(False) == 0 assert descr3.get_ofs_length(False) == 0 + assert descr4.get_ofs_length(False) == 0 assert descr1.get_item_size(False) == rffi.sizeof(lltype.Char) assert descr2.get_item_size(False) == rffi.sizeof(lltype.Ptr(T)) assert descr3.get_item_size(False) == rffi.sizeof(lltype.Ptr(U)) + assert descr4.get_item_size(False) == rffi.sizeof(lltype.Float) # assert isinstance(descr1.get_base_size(True), Symbolic) assert isinstance(descr2.get_base_size(True), Symbolic) assert isinstance(descr3.get_base_size(True), Symbolic) + assert isinstance(descr4.get_base_size(True), Symbolic) assert isinstance(descr1.get_ofs_length(True), Symbolic) assert isinstance(descr2.get_ofs_length(True), Symbolic) assert isinstance(descr3.get_ofs_length(True), Symbolic) + assert isinstance(descr4.get_ofs_length(True), Symbolic) assert isinstance(descr1.get_item_size(True), Symbolic) assert isinstance(descr2.get_item_size(True), Symbolic) assert isinstance(descr3.get_item_size(True), Symbolic) + assert isinstance(descr4.get_item_size(True), Symbolic) -def test_get_call_descr(): +def test_get_call_descr_not_translated(): c0 = GcCache(False) descr1 = get_call_descr(c0, [lltype.Char, lltype.Signed], lltype.Char) assert descr1.get_result_size(False) == rffi.sizeof(lltype.Char) assert not descr1.returns_a_pointer() + assert not descr1.returns_a_float() assert descr1.arg_classes == "ii" # T = lltype.GcStruct('T') descr2 = get_call_descr(c0, [lltype.Ptr(T)], lltype.Ptr(T)) assert descr2.get_result_size(False) == rffi.sizeof(lltype.Ptr(T)) assert descr2.returns_a_pointer() + assert not descr2.returns_a_float() assert descr2.arg_classes == "r" # U = lltype.GcStruct('U', ('x', lltype.Signed)) assert descr2 == get_call_descr(c0, [lltype.Ptr(U)], lltype.Ptr(U)) # + descr4 = get_call_descr(c0, [lltype.Float, lltype.Float], lltype.Float) + assert descr4.get_result_size(False) == rffi.sizeof(lltype.Float) + assert not descr4.returns_a_pointer() + assert descr4.returns_a_float() + assert descr4.arg_classes == "ff" + +def test_get_call_descr_translated(): c1 = GcCache(True) + T = lltype.GcStruct('T') + U = lltype.GcStruct('U', ('x', lltype.Signed)) descr3 = get_call_descr(c1, [lltype.Ptr(T)], lltype.Ptr(U)) assert isinstance(descr3.get_result_size(True), Symbolic) assert descr3.returns_a_pointer() + assert not descr3.returns_a_float() assert descr3.arg_classes == "r" # - c1 = GcCache(True) descr4 = get_call_descr(c1, [lltype.Float, lltype.Float], lltype.Float) - assert descr4.get_result_size(False) == rffi.sizeof(lltype.Float) + assert isinstance(descr4.get_result_size(True), Symbolic) + assert not descr4.returns_a_pointer() + assert descr4.returns_a_float() + assert descr4.arg_classes == "ff" def test_repr_of_descr(): c0 = GcCache(False) @@ -165,3 +210,6 @@ # descr4i = get_call_descr(c0, [lltype.Char, lltype.Ptr(S)], lltype.Char) assert 'CharCallDescr' in descr4i.repr_of_descr() + # + descr4f = get_call_descr(c0, [lltype.Char, lltype.Ptr(S)], lltype.Float) + assert 'FloatCallDescr' in descr4f.repr_of_descr() From arigo at codespeak.net Thu Oct 1 12:07:16 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 1 Oct 2009 12:07:16 +0200 (CEST) Subject: [pypy-svn] r68092 - in pypy/trunk/pypy/jit: backend/llgraph metainterp metainterp/test Message-ID: <20091001100716.62FC9168014@codespeak.net> Author: arigo Date: Thu Oct 1 12:07:15 2009 New Revision: 68092 Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py pypy/trunk/pypy/jit/metainterp/codewriter.py pypy/trunk/pypy/jit/metainterp/policy.py pypy/trunk/pypy/jit/metainterp/test/test_basic.py pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py pypy/trunk/pypy/jit/metainterp/test/test_del.py pypy/trunk/pypy/jit/metainterp/warmspot.py Log: Get rid of the issue that instantiate() turns into an indirect_call with no graphs to hint what is called. Done by detecting exactly this situation and guessing that what is called is any 'instantiate' field of any vtable. If this guess is ever wrong it's not bad. Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/llimpl.py Thu Oct 1 12:07:15 2009 @@ -110,6 +110,7 @@ 'ooisnot' : (('ref', 'ref'), 'bool'), 'instanceof' : (('ref',), 'bool'), 'subclassof' : (('ref', 'ref'), 'bool'), + 'runtimenew' : (('ref',), 'ref'), 'setfield_gc' : (('ref', 'intorptr'), None), 'getfield_gc' : (('ref',), 'intorptr'), 'getfield_gc_pure': (('ref',), 'intorptr'), @@ -844,6 +845,11 @@ if ootype.classof(value) is not expected_class: raise GuardFailed + def op_runtimenew(self, _, cls): + cls = ootype.cast_from_object(ootype.Class, cls) + res = ootype.runtimenew(cls) + return ootype.cast_to_object(res) + def op_instanceof(self, typedescr, obj): inst = ootype.cast_from_object(ootype.ROOT, obj) return ootype.instanceof(inst, typedescr.TYPE) Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/codewriter.py Thu Oct 1 12:07:15 2009 @@ -148,7 +148,8 @@ return (cfnptr, calldescr) def register_indirect_call_targets(self, op): - targets = self.policy.graphs_from(op, self.cpu.supports_floats) + targets = self.policy.graphs_from(op, self.rtyper, + self.cpu.supports_floats) assert targets is not None for graph in targets: if graph in self.all_indirect_call_targets: @@ -1000,22 +1001,26 @@ def serialize_op_direct_call(self, op): kind = self.codewriter.policy.guess_call_kind(op, + self.codewriter.rtyper, self.codewriter.cpu.supports_floats) return getattr(self, 'handle_%s_call' % kind)(op) def serialize_op_indirect_call(self, op): kind = self.codewriter.policy.guess_call_kind(op, + self.codewriter.rtyper, self.codewriter.cpu.supports_floats) return getattr(self, 'handle_%s_indirect_call' % kind)(op) def serialize_op_oosend(self, op): kind = self.codewriter.policy.guess_call_kind(op, + self.codewriter.rtyper, self.codewriter.cpu.supports_floats) return getattr(self, 'handle_%s_oosend' % kind)(op) def handle_regular_call(self, op, oosend_methdescr=None): self.minimize_variables() [targetgraph] = self.codewriter.policy.graphs_from(op, + self.codewriter.rtyper, self.codewriter.cpu.supports_floats) jitbox = self.codewriter.get_jitcode(targetgraph, self.graph, oosend_methdescr=oosend_methdescr) Modified: pypy/trunk/pypy/jit/metainterp/policy.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/policy.py (original) +++ pypy/trunk/pypy/jit/metainterp/policy.py Thu Oct 1 12:07:15 2009 @@ -1,6 +1,6 @@ from pypy.translator.simplify import get_funcobj from pypy.jit.metainterp import support, history -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, rclass class JitPolicy(object): @@ -29,7 +29,7 @@ not contains_unsupported_variable_type(graph, supports_floats)) - def graphs_from(self, op, supports_floats): + def graphs_from(self, op, rtyper, supports_floats): if op.opname == 'direct_call': funcobj = get_funcobj(op.args[0].value) graph = funcobj.graph @@ -50,10 +50,24 @@ if result: return result # common case: look inside these graphs, # and ignore the others if there are any + else: + # special case: handle the indirect call that goes to + # the 'instantiate' methods. This check is a bit imprecise + # but it's not too bad if we mistake a random indirect call + # for the one to 'instantiate'. + CALLTYPE = op.args[0].concretetype + if (op.opname == 'indirect_call' and len(op.args) == 2 and + CALLTYPE == rclass.OBJECT_VTABLE.instantiate): + return list(self._graphs_of_all_instantiate(rtyper)) # residual call case: we don't need to look into any graph return None - def guess_call_kind(self, op, supports_floats): + def _graphs_of_all_instantiate(self, rtyper): + for vtable in rtyper.lltype_to_vtable_mapping().itervalues(): + if vtable.instantiate: + yield vtable.instantiate._obj.graph + + def guess_call_kind(self, op, rtyper, supports_floats): if op.opname == 'direct_call': funcptr = op.args[0].value funcobj = get_funcobj(funcptr) @@ -69,7 +83,7 @@ SELFTYPE, methname, opargs = support.decompose_oosend(op) if SELFTYPE.oopspec_name is not None: return 'builtin' - if self.graphs_from(op, supports_floats) is None: + if self.graphs_from(op, rtyper, supports_floats) is None: return 'residual' return 'regular' Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Thu Oct 1 12:07:15 2009 @@ -644,7 +644,6 @@ res = self.interp_operations(f, [13]) assert res == 72 - @py.test.mark.xfail def test_instantiate_does_not_call(self): mydriver = JitDriver(reds = ['n', 'x'], greens = []) class Base: pass @@ -664,7 +663,7 @@ x += inst.foo n -= 1 return x - res = self.meta_interp(f, [20]) + res = self.meta_interp(f, [20], optimizer=OPTIMIZER_SIMPLE) assert res == f(20) self.check_loops(call=0) Modified: pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py Thu Oct 1 12:07:15 2009 @@ -101,7 +101,6 @@ assert dict.fromkeys(names) == {'g': None} def test_instantiate(self): - py.test.skip("in-progress") class A1: id = 651 class A2(A1): id = 652 class B1: id = 661 @@ -119,7 +118,17 @@ graph2 = self.graphof(ll_instantiate) jitcode = cw.make_one_bytecode((graph2, None), False) assert 'residual_call' not in jitcode._source - xxx + names = [jitcode.name for (fnaddress, jitcode) + in self.metainterp_sd.indirectcalls] + names = dict.fromkeys(names) + assert len(names) >= 4 + for expected in ['A1', 'A2', 'B1', 'B2']: + for name in names: + if name.startswith('instantiate_') and name.endswith(expected): + break + else: + assert 0, "missing instantiate_*_%s in:\n%r" % (expected, + names) class ImmutableFieldsTests: Modified: pypy/trunk/pypy/jit/metainterp/test/test_del.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_del.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_del.py Thu Oct 1 12:07:15 2009 @@ -1,5 +1,5 @@ import py -from pypy.rlib.jit import JitDriver +from pypy.rlib.jit import JitDriver, OPTIMIZER_SIMPLE from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin @@ -54,6 +54,32 @@ res = self.meta_interp(f, [20]) assert res == 42 + def test_instantiate_with_or_without_del(self): + import gc + mydriver = JitDriver(reds = ['n', 'x'], greens = []) + class Base: pass + class A(Base): foo = 72 + class B(Base): + foo = 8 + def __del__(self): + pass + def f(n): + x = 0 + while n > 0: + mydriver.can_enter_jit(n=n, x=x) + mydriver.jit_merge_point(n=n, x=x) + if n % 2 == 0: + cls = A + else: + cls = B + inst = cls() + x += inst.foo + n -= 1 + return 1 + res = self.meta_interp(f, [20], optimizer=OPTIMIZER_SIMPLE) + assert res == 1 + self.check_loops(call=1) # for the case B(), but not for the case A() + class TestLLtype(DelTests, LLJitMixin): def test_signal_action(self): Modified: pypy/trunk/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmspot.py Thu Oct 1 12:07:15 2009 @@ -561,6 +561,7 @@ def find_all_graphs(portal, policy, translator, supports_floats): from pypy.translator.simplify import get_graph + rtyper = translator.rtyper all_graphs = [portal] seen = set([portal]) todo = [portal] @@ -569,10 +570,10 @@ for _, op in top_graph.iterblockops(): if op.opname not in ("direct_call", "indirect_call", "oosend"): continue - kind = policy.guess_call_kind(op, supports_floats) + kind = policy.guess_call_kind(op, rtyper, supports_floats) if kind != "regular": continue - for graph in policy.graphs_from(op, supports_floats): + for graph in policy.graphs_from(op, rtyper, supports_floats): if graph in seen: continue if policy.look_inside_graph(graph, supports_floats): From pedronis at codespeak.net Thu Oct 1 12:44:35 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 1 Oct 2009 12:44:35 +0200 (CEST) Subject: [pypy-svn] r68093 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20091001104435.A008C168012@codespeak.net> Author: pedronis Date: Thu Oct 1 12:44:35 2009 New Revision: 68093 Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: (cfbolz, pedronis) first step in reworking store_final_boxes_in_guard, make the optimizeopt test don't care about the exact order of fail_args which will be harder to keep unchanging, but still test enough Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Thu Oct 1 12:44:35 2009 @@ -47,7 +47,7 @@ assert fdescr.rd_frame_infos == fi # ____________________________________________________________ -def equaloplists(oplist1, oplist2, remap={}): +def equaloplists(oplist1, oplist2, strict_fail_args=True, remap={}): print '-'*20, 'Comparing lists', '-'*20 for op1, op2 in zip(oplist1, oplist2): txt1 = str(op1) @@ -68,8 +68,13 @@ assert op1.descr == op2.descr if op1.fail_args or op2.fail_args: assert len(op1.fail_args) == len(op2.fail_args) - for x, y in zip(op1.fail_args, op2.fail_args): - assert x == remap.get(y, y) + if strict_fail_args: + for x, y in zip(op1.fail_args, op2.fail_args): + assert x == remap.get(y, y) + else: + fail_args1 = set(op1.fail_args) + fail_args2 = set([remap.get(y, y) for y in op2.fail_args]) + assert fail_args1 == fail_args2 assert len(oplist1) == len(oplist2) print '-'*57 return True @@ -101,10 +106,16 @@ """ namespace = {} loop1 = pure_parse(ops, namespace=namespace) - loop2 = pure_parse(ops.replace("[i2, i1]", "[i2, i0]"), + loop2 = pure_parse(ops.replace("[i2, i1]", "[i1, i2]"), namespace=namespace) py.test.raises(AssertionError, "equaloplists(loop1.operations, loop2.operations)") + assert equaloplists(loop1.operations, loop2.operations, + strict_fail_args=False) + loop3 = pure_parse(ops.replace("[i2, i1]", "[i2, i0]"), + namespace=namespace) + py.test.raises(AssertionError, + "equaloplists(loop1.operations, loop3.operations)") # ____________________________________________________________ @@ -135,8 +146,7 @@ assert box1.__class__ == box2.__class__ remap[box2] = box1 assert equaloplists(optimized.operations, - expected.operations, - remap) + expected.operations, False, remap) def optimize_loop(self, ops, spectext, optops, checkspecnodes=True): loop = self.parse(ops) @@ -153,6 +163,7 @@ # combinations different from the one computed by optimizefindnode loop.token.specnodes = self.unpack_specnodes(spectext) # + self.loop = loop optimize_loop_1(self.cpu, loop) # expected = self.parse(optops) @@ -1253,18 +1264,17 @@ def make_fail_descr(self): class FailDescr(compile.ResumeGuardDescr): - args_seen = [] + oparse = None def _oparser_uses_descr_of_guard(self, oparse, fail_args): # typically called twice, before and after optimization - if len(self.args_seen) == 0: + if self.oparse is None: fdescr.rd_frame_info_list = resume.FrameInfo(None, FakeFrame()) fdescr.rd_snapshot = resume.Snapshot(None, fail_args) fdescr.virtuals = None - self.args_seen.append((fail_args, oparse)) + self.oparse = oparse # fdescr = instantiate(FailDescr) - self.fdescr = fdescr self.namespace['fdescr'] = fdescr def teardown_method(self, meth): @@ -1348,11 +1358,13 @@ index += 1 def check_expanded_fail_descr(self, expectedtext): - fdescr = self.fdescr - args, oparse = fdescr.args_seen[-1] - reader = resume.ResumeDataReader(fdescr, args, MyMetaInterp(self.cpu)) + guard_op, = [op for op in self.loop.operations if op.is_guard()] + fail_args = guard_op.fail_args + fdescr = guard_op.descr + reader = resume.ResumeDataReader(fdescr, fail_args, + MyMetaInterp(self.cpu)) boxes = reader.consume_boxes() - self._verify_fail_args(boxes, oparse, expectedtext) + self._verify_fail_args(boxes, fdescr.oparse, expectedtext) def test_expand_fail_1(self): self.make_fail_descr() From arigo at codespeak.net Thu Oct 1 12:46:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 1 Oct 2009 12:46:42 +0200 (CEST) Subject: [pypy-svn] r68094 - in pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport: . test Message-ID: <20091001104642.058EF168012@codespeak.net> Author: arigo Date: Thu Oct 1 12:46:42 2009 New Revision: 68094 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/llmodel.py pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/symbolic.py pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/test/test_runner.py Log: Reimplement get/set of fields and array items to also support floats. Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/llmodel.py Thu Oct 1 12:46:42 2009 @@ -206,7 +206,8 @@ ofs = fielddescr.offset size = fielddescr.get_field_size(self.translate_support_code) ptr = fielddescr.is_pointer_field() - return ofs, size, ptr + float = fielddescr.is_float_field() + return ofs, size, ptr, float unpack_fielddescr._always_inline_ = True def arraydescrof(self, A): @@ -217,7 +218,8 @@ ofs = arraydescr.get_base_size(self.translate_support_code) size = arraydescr.get_item_size(self.translate_support_code) ptr = arraydescr.is_array_of_pointers() - return ofs, size, ptr + float = arraydescr.is_array_of_floats() + return ofs, size, ptr, float unpack_arraydescr._always_inline_ = True def calldescrof(self, FUNC, ARGS, RESULT): @@ -247,40 +249,66 @@ def do_getarrayitem_gc(self, arraybox, indexbox, arraydescr): itemindex = indexbox.getint() gcref = arraybox.getref_base() - ofs, size, ptr = self.unpack_arraydescr(arraydescr) + ofs, size, ptr, float = self.unpack_arraydescr(arraydescr) + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + # + if ptr: + items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items) + pval = self._cast_int_to_gcref(items[itemindex]) + # --- end of GC unsafe code --- + return BoxPtr(pval) + # + if float: + items = rffi.cast(rffi.CArrayPtr(lltype.Float), items) + fval = items[itemindex] + # --- end of GC unsafe code --- + return BoxFloat(fval) # for TYPE, itemsize in unroll_basic_sizes: if size == itemsize: - val = (rffi.cast(rffi.CArrayPtr(TYPE), gcref) - [ofs/itemsize + itemindex]) - val = rffi.cast(lltype.Signed, val) - break + items = rffi.cast(rffi.CArrayPtr(TYPE), items) + val = items[itemindex] + # --- end of GC unsafe code --- + return BoxInt(rffi.cast(lltype.Signed, val)) else: raise NotImplementedError("size = %d" % size) - if ptr: - return BoxPtr(self._cast_int_to_gcref(val)) - else: - return BoxInt(val) def do_setarrayitem_gc(self, arraybox, indexbox, vbox, arraydescr): itemindex = indexbox.getint() gcref = arraybox.getref_base() - ofs, size, ptr = self.unpack_arraydescr(arraydescr) + ofs, size, ptr, float = self.unpack_arraydescr(arraydescr) # if ptr: vboxptr = vbox.getref_base() self.gc_ll_descr.do_write_barrier(gcref, vboxptr) - a = rffi.cast(rffi.CArrayPtr(lltype.Signed), gcref) - a[ofs/WORD + itemindex] = self.cast_gcref_to_int(vboxptr) + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items) + items[itemindex] = self.cast_gcref_to_int(vboxptr) + # --- end of GC unsafe code --- + return + # + if float: + fval = vbox.getfloat() + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + items = rffi.cast(rffi.CArrayPtr(lltype.Float), items) + items[itemindex] = fval + # --- end of GC unsafe code --- + return + # + val = vbox.getint() + for TYPE, itemsize in unroll_basic_sizes: + if size == itemsize: + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + items = rffi.cast(rffi.CArrayPtr(TYPE), items) + items[itemindex] = rffi.cast(TYPE, val) + # --- end of GC unsafe code --- + return else: - v = vbox.getint() - for TYPE, itemsize in unroll_basic_sizes: - if size == itemsize: - a = rffi.cast(rffi.CArrayPtr(TYPE), gcref) - a[ofs/itemsize + itemindex] = rffi.cast(TYPE, v) - break - else: - raise NotImplementedError("size = %d" % size) + raise NotImplementedError("size = %d" % size) def _new_do_len(TP): def do_strlen(self, stringbox): @@ -313,18 +341,29 @@ @specialize.argtype(1) def _base_do_getfield(self, gcref, fielddescr): - ofs, size, ptr = self.unpack_fielddescr(fielddescr) + ofs, size, ptr, float = self.unpack_fielddescr(fielddescr) + # --- start of GC unsafe code (no GC operation!) --- + field = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + # + if ptr: + pval = rffi.cast(rffi.CArrayPtr(lltype.Signed), field)[0] + pval = self._cast_int_to_gcref(pval) + # --- end of GC unsafe code --- + return BoxPtr(pval) + # + if float: + fval = rffi.cast(rffi.CArrayPtr(lltype.Float), field)[0] + # --- end of GC unsafe code --- + return BoxFloat(fval) + # for TYPE, itemsize in unroll_basic_sizes: if size == itemsize: - val = rffi.cast(rffi.CArrayPtr(TYPE), gcref)[ofs/itemsize] + val = rffi.cast(rffi.CArrayPtr(TYPE), field)[0] + # --- end of GC unsafe code --- val = rffi.cast(lltype.Signed, val) - break + return BoxInt(val) else: raise NotImplementedError("size = %d" % size) - if ptr: - return BoxPtr(self._cast_int_to_gcref(val)) - else: - return BoxInt(val) def do_getfield_gc(self, structbox, fielddescr): gcref = structbox.getref_base() @@ -335,23 +374,40 @@ @specialize.argtype(1) def _base_do_setfield(self, gcref, vbox, fielddescr): - ofs, size, ptr = self.unpack_fielddescr(fielddescr) + ofs, size, ptr, float = self.unpack_fielddescr(fielddescr) + # if ptr: assert lltype.typeOf(gcref) is not lltype.Signed, ( "can't handle write barriers for setfield_raw") ptr = vbox.getref_base() self.gc_ll_descr.do_write_barrier(gcref, ptr) - a = rffi.cast(rffi.CArrayPtr(lltype.Signed), gcref) - a[ofs/WORD] = self.cast_gcref_to_int(ptr) + # --- start of GC unsafe code (no GC operation!) --- + field = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + field = rffi.cast(rffi.CArrayPtr(lltype.Signed), field) + field[0] = self.cast_gcref_to_int(ptr) + # --- end of GC unsafe code --- + return + # + if float: + fval = vbox.getfloat() + # --- start of GC unsafe code (no GC operation!) --- + field = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + field = rffi.cast(rffi.CArrayPtr(lltype.Float), field) + field[0] = fval + # --- end of GC unsafe code --- + return + # + val = vbox.getint() + for TYPE, itemsize in unroll_basic_sizes: + if size == itemsize: + # --- start of GC unsafe code (no GC operation!) --- + field = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + field = rffi.cast(rffi.CArrayPtr(TYPE), field) + field[0] = rffi.cast(TYPE, val) + # --- end of GC unsafe code --- + return else: - v = vbox.getint() - for TYPE, itemsize in unroll_basic_sizes: - if size == itemsize: - v = rffi.cast(TYPE, v) - rffi.cast(rffi.CArrayPtr(TYPE), gcref)[ofs/itemsize] = v - break - else: - raise NotImplementedError("size = %d" % size) + raise NotImplementedError("size = %d" % size) def do_setfield_gc(self, structbox, vbox, fielddescr): gcref = structbox.getref_base() @@ -415,12 +471,11 @@ self.execute_token(executable_token) # Note: if an exception is set, the rest of the code does a bit of # nonsense but nothing wrong (the return value should be ignored) - res = calldescr.get_result_size(self.translate_support_code) if calldescr.returns_a_pointer(): return BoxPtr(self.get_latest_value_ref(0)) - elif res == self.FLOATSIZE: + elif calldescr.returns_a_float(): return BoxFloat(self.get_latest_value_float(0)) - elif res > 0: + elif calldescr.get_result_size(self.translate_support_code) > 0: return BoxInt(self.get_latest_value_int(0)) else: return None Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/symbolic.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/symbolic.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/symbolic.py Thu Oct 1 12:46:42 2009 @@ -61,8 +61,10 @@ SIZEOF_CHAR = get_size(lltype.Char, False) SIZEOF_SHORT = get_size(rffi.SHORT, False) SIZEOF_INT = get_size(rffi.INT, False) +SIZEOF_FLOAT = get_size(lltype.Float, False) unroll_basic_sizes = unrolling_iterable([(lltype.Signed, WORD), (lltype.Char, SIZEOF_CHAR), (rffi.SHORT, SIZEOF_SHORT), (rffi.INT, SIZEOF_INT)]) +# does not contain Float ^^^ which must be special-cased Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/test/test_runner.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/test/test_runner.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/test/test_runner.py Thu Oct 1 12:46:42 2009 @@ -7,6 +7,7 @@ pass class MyLLCPU(AbstractLLCPU): + supports_floats = True def compile_loop(self, inputargs, operations): py.test.skip("llsupport test: cannot compile operations") From fijal at codespeak.net Thu Oct 1 13:02:14 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 13:02:14 +0200 (CEST) Subject: [pypy-svn] r68095 - pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test Message-ID: <20091001110214.D73AD168012@codespeak.net> Author: fijal Date: Thu Oct 1 13:02:14 2009 New Revision: 68095 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py Log: Another missing abs Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py Thu Oct 1 13:02:14 2009 @@ -524,7 +524,7 @@ ops.append('fail(f%d)' % (BASE_CONSTANT_SIZE * 2)) ops = "\n".join(ops) self.interpret(ops, [0.1]) - assert self.getfloat(0) - (1 + BASE_CONSTANT_SIZE * 2) * 3.5 + 0.1 < 0.00001 + assert abs(self.getfloat(0) - (1 + BASE_CONSTANT_SIZE * 2) * 3.5 + 0.1) < 0.00001 def test_lt_const(self): ops = ''' From fijal at codespeak.net Thu Oct 1 13:10:47 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 13:10:47 +0200 (CEST) Subject: [pypy-svn] r68096 - pypy/branch/floats-via-sse2/pypy/jit/backend/test Message-ID: <20091001111047.8BB66168012@codespeak.net> Author: fijal Date: Thu Oct 1 13:10:46 2009 New Revision: 68096 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Log: GUARD_VALUE with floats Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Thu Oct 1 13:10:46 2009 @@ -527,11 +527,12 @@ assert res.value == 3.4 def test_passing_guards(self): - for (opname, args) in [(rop.GUARD_TRUE, [BoxInt(1)]), - (rop.GUARD_FALSE, [BoxInt(0)]), - (rop.GUARD_VALUE, [BoxInt(42), BoxInt(42)]), - #(rop.GUARD_VALUE_INVERSE, [BoxInt(42), BoxInt(41)]), - ]: + all = [(rop.GUARD_TRUE, [BoxInt(1)]), + (rop.GUARD_FALSE, [BoxInt(0)]), + (rop.GUARD_VALUE, [BoxInt(42), BoxInt(42)])] + if self.cpu.supports_floats: + all.append((rop.GUARD_VALUE, [BoxFloat(3.5), BoxFloat(3.5)])) + for (opname, args) in all: assert self.execute_operation(opname, args, 'void') == None assert not self.guard_failed @@ -545,16 +546,18 @@ # 'void') def test_failing_guards(self): - for opname, args in [(rop.GUARD_TRUE, [BoxInt(0)]), - (rop.GUARD_FALSE, [BoxInt(1)]), - (rop.GUARD_VALUE, [BoxInt(42), BoxInt(41)]), - ]: + all = [(rop.GUARD_TRUE, [BoxInt(0)]), + (rop.GUARD_FALSE, [BoxInt(1)]), + (rop.GUARD_VALUE, [BoxInt(42), BoxInt(41)])] + if self.cpu.supports_floats: + all.append((rop.GUARD_VALUE, [BoxFloat(-1.0), BoxFloat(1.0)])) + for opname, args in all: assert self.execute_operation(opname, args, 'void') == None assert self.guard_failed def test_failing_guard_class(self): t_box, T_box = self.alloc_instance(self.T) - u_box, U_box = self.alloc_instance(self.U) + u_box, U_box = self.alloc_instance(self.U) #null_box = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.nullptr(T))) for opname, args in [(rop.GUARD_CLASS, [t_box, U_box]), (rop.GUARD_CLASS, [u_box, T_box]), From pedronis at codespeak.net Thu Oct 1 14:36:28 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 1 Oct 2009 14:36:28 +0200 (CEST) Subject: [pypy-svn] r68097 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091001123628.DAADB168014@codespeak.net> Author: pedronis Date: Thu Oct 1 14:36:28 2009 New Revision: 68097 Modified: pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_resume.py Log: (cfbolz, pedronis) small steps, liveboxes_order is not needed if we don't mind getting reshuffling, adjust test to care less but still test enough Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Thu Oct 1 14:36:28 2009 @@ -110,81 +110,6 @@ storage.rd_virtuals = None return liveboxes - -## class ResumeDataBuilder(object): - -## def __init__(self, _other=None): -## if _other is None: -## self.memo = {} -## self.liveboxes = [] -## self.consts = [] -## self.nums = [] -## self.frame_infos = [] -## else: -## self.memo = _other.memo.copy() -## self.liveboxes = _other.liveboxes[:] -## self.consts = _other.consts[:] -## self.nums = _other.nums[:] -## self.frame_infos = _other.frame_infos[:] - -## def clone(self): -## return ResumeDataBuilder(self) - -## def generate_boxes(self, boxes): -## for box in boxes: -## assert box is not None -## if isinstance(box, Box): -## try: -## num = self.memo[box] -## except KeyError: -## num = len(self.liveboxes) -## self.liveboxes.append(box) -## self.memo[box] = num -## else: -## num = -2 - len(self.consts) -## self.consts.append(box) -## self.nums.append(num) -## self.nums.append(-1) - -## def generate_frame_info(self, *frame_info): -## self.frame_infos.append(frame_info) - -## def _add_level(self, frame): -## self.generate_frame_info(frame.jitcode, frame.pc, -## frame.exception_target) -## self.generate_boxes(frame.env) - -## @staticmethod -## def _get_fresh_parent_resumedata(framestack, n): -## target = framestack[n] -## if target.parent_resumedata is not None: -## return target.parent_resumedata.clone() -## if n == 0: -## parent_resumedata = ResumeDataBuilder() -## else: -## parent_resumedata = ResumeDataBuilder._get_fresh_parent_resumedata(framestack, n-1) -## parent_resumedata._add_level(framestack[n-1]) -## target.parent_resumedata = parent_resumedata -## return parent_resumedata.clone() - -## @staticmethod -## def make(framestack): -## n = len(framestack)-1 -## top = framestack[-1] -## builder = ResumeDataBuilder._get_fresh_parent_resumedata(framestack, n) -## builder._add_level(top) -## return builder - -## def finish(self, storage): -## storage.rd_frame_infos = self.frame_infos[:] -## storage.rd_nums = self.nums[:] -## storage.rd_consts = self.consts[:] -## storage.rd_virtuals = None -## if debug: -## dump_storage(storage) -## return self.liveboxes - - VIRTUAL_FLAG = int((sys.maxint+1) // 2) assert not (VIRTUAL_FLAG & (VIRTUAL_FLAG-1)) # a power of two @@ -196,7 +121,6 @@ assert storage.rd_virtuals is None self.original_liveboxes = liveboxes self.liveboxes = {} - self.liveboxes_order = [] self._register_boxes(liveboxes) self.virtuals = [] self.vfieldboxes = [] @@ -229,7 +153,6 @@ for box in boxes: if isinstance(box, Box) and box not in self.liveboxes: self.liveboxes[box] = 0 - self.liveboxes_order.append(box) def is_virtual(self, virtualbox): return self.liveboxes[virtualbox] >= VIRTUAL_FLAG @@ -237,7 +160,7 @@ def finish(self): storage = self.storage liveboxes = [] - for box in self.liveboxes_order: + for box in self.liveboxes.iterkeys(): if self.liveboxes[box] == 0: self.liveboxes[box] = len(liveboxes) liveboxes.append(box) Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Thu Oct 1 14:36:28 2009 @@ -40,8 +40,11 @@ fdescr.rd_snapshot = resume.Snapshot(snapshot0, [b1]) # opt.store_final_boxes_in_guard(op) - assert op.fail_args == [b0, b1] - assert fdescr.rd_nums == [0, -1, 1, -1] + if op.fail_args == [b0, b1]: + assert fdescr.rd_nums == [0, -1, 1, -1] + else: + assert op.fail_args == [b1, b0] + assert fdescr.rd_nums == [1, -1, 0, -1] assert fdescr.rd_virtuals is None assert fdescr.rd_consts == [] assert fdescr.rd_frame_infos == fi Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Thu Oct 1 14:36:28 2009 @@ -213,6 +213,13 @@ demo55 = lltype.malloc(LLtypeMixin.NODE) demo55o = lltype.cast_opaque_ptr(llmemory.GCREF, demo55) +def _resume_remap(liveboxes, expected, *newvalues): + newboxes = [] + for box in liveboxes: + i = expected.index(box) + newboxes.append(newvalues[i]) + assert len(newboxes) == len(expected) + return newboxes def test_virtual_adder_no_op(): storage = make_demo_storage() @@ -223,11 +230,10 @@ assert not modifier.is_virtual(b3s) # done liveboxes = modifier.finish() - assert liveboxes == [b1s, b2s, b3s] - # b1t, b2t, b3t = [BoxInt(11), BoxPtr(demo55o), BoxInt(33)] + newboxes = _resume_remap(liveboxes, [b1s, b2s, b3s], b1t, b2t, b3t) metainterp = MyMetaInterp(LLtypeMixin.cpu) - reader = ResumeDataReader(storage, [b1t, b2t, b3t], metainterp) + reader = ResumeDataReader(storage, newboxes, metainterp) lst = reader.consume_boxes() assert lst == [b1t, ConstInt(1), b1t, b2t] lst = reader.consume_boxes() @@ -264,17 +270,17 @@ assert not modifier.is_virtual(b5s) # done liveboxes = modifier.finish() - assert liveboxes == [b1s, - #b2s -- virtual - b3s, - #b4s -- virtual - #b2s -- again, shared - #b3s -- again, shared - b5s] - # b1t, b3t, b5t = [BoxInt(11), BoxInt(33), BoxPtr(demo55o)] + newboxes = _resume_remap(liveboxes, [b1s, + #b2s -- virtual + b3s, + #b4s -- virtual + #b2s -- again, shared + #b3s -- again, shared + b5s], b1t, b3t, b5t) + # metainterp = MyMetaInterp(LLtypeMixin.cpu) - reader = ResumeDataReader(storage, [b1t, b3t, b5t], metainterp) + reader = ResumeDataReader(storage, newboxes, metainterp) lst = reader.consume_boxes() b2t = lst[-1] assert lst == [b1t, ConstInt(1), b1t, b2t] @@ -325,11 +331,10 @@ assert not modifier.is_virtual(b3s) # done liveboxes = modifier.finish() - assert liveboxes == [b2s, b3s] - # b2t, b3t = [BoxPtr(demo55o), BoxInt(33)] + newboxes = _resume_remap(liveboxes, [b2s, b3s], b2t, b3t) metainterp = MyMetaInterp(LLtypeMixin.cpu) - reader = ResumeDataReader(storage, [b2t, b3t], metainterp) + reader = ResumeDataReader(storage, newboxes, metainterp) lst = reader.consume_boxes() c1t = ConstInt(111) assert lst == [c1t, ConstInt(1), c1t, b2t] @@ -357,14 +362,15 @@ assert not modifier.is_virtual(b4s) # done liveboxes = modifier.finish() - assert liveboxes == [b1s, - #b2s -- virtual - b3s, - b4s] - # b1t, b3t, b4t = [BoxInt(11), BoxInt(33), BoxInt(44)] + newboxes = _resume_remap(liveboxes, [b1s, + #b2s -- virtual + b3s, + b4s], + b1t, b3t, b4t) + # metainterp = MyMetaInterp(LLtypeMixin.cpu) - reader = ResumeDataReader(storage, [b1t, b3t, b4t], metainterp) + reader = ResumeDataReader(storage, newboxes, metainterp) lst = reader.consume_boxes() b2t = lst[-1] assert lst == [b1t, ConstInt(1), b1t, b2t] @@ -406,15 +412,16 @@ assert not modifier.is_virtual(b4s) # done liveboxes = modifier.finish() - assert liveboxes == [b1s, - #b2s -- virtual - b3s, - b4s] + b1t, b3t, b4t = [BoxInt(11), BoxInt(33), BoxPtr()] + newboxes = _resume_remap(liveboxes, [b1s, + #b2s -- virtual + b3s, + b4s], + b1t, b3t, b4t) # NULL = ConstPtr.value - b1t, b3t, b4t = [BoxInt(11), BoxInt(33), BoxPtr()] metainterp = MyMetaInterp(LLtypeMixin.cpu) - reader = ResumeDataReader(storage, [b1t, b3t, b4t], metainterp) + reader = ResumeDataReader(storage, newboxes, metainterp) lst = reader.consume_boxes() b2t = lst[-1] assert lst == [b1t, ConstInt(1), b1t, b2t] From arigo at codespeak.net Thu Oct 1 15:07:55 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 1 Oct 2009 15:07:55 +0200 (CEST) Subject: [pypy-svn] r68098 - pypy/branch/floats-via-sse2/pypy/jit/backend/x86 Message-ID: <20091001130755.AD0D9168010@codespeak.net> Author: arigo Date: Thu Oct 1 15:07:55 2009 New Revision: 68098 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/runner.py Log: Remove FLOATSIZE. Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/runner.py Thu Oct 1 15:07:55 2009 @@ -12,7 +12,6 @@ class CPU386(AbstractLLCPU): debug = True supports_floats = True - FLOATSIZE = 8 BOOTSTRAP_TP = lltype.FuncType([], lltype.Signed) dont_keepalive_stuff = False # for tests From arigo at codespeak.net Thu Oct 1 15:09:06 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 1 Oct 2009 15:09:06 +0200 (CEST) Subject: [pypy-svn] r68099 - pypy/branch/floats-via-sse2/pypy/jit/backend/test Message-ID: <20091001130906.C7590168010@codespeak.net> Author: arigo Date: Thu Oct 1 15:09:01 2009 New Revision: 68099 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Log: A (biggish) test about jumps. Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Thu Oct 1 15:09:01 2009 @@ -766,6 +766,108 @@ u_box = self.alloc_unicode(u"hello\u1234") r = self.execute_operation(rop.SAME_AS, [u_box.constbox()], 'ref') assert r.value == u_box.value + if self.cpu.supports_floats: + r = self.execute_operation(rop.SAME_AS, [ConstFloat(5.5)], 'float') + assert r.value == 5.5 + + def test_jump(self): + # this test generate small loops where the JUMP passes many + # arguments of various types, shuffling them around. + if self.cpu.supports_floats: + numkinds = 3 + else: + numkinds = 2 + for _ in range(50): + # + inputargs = [] + for k in range(random.randrange(1, 16)): + kind = random.randrange(0, numkinds) + if kind == 0: + inputargs.append(BoxInt()) + elif kind == 1: + inputargs.append(BoxPtr()) + else: + inputargs.append(BoxFloat()) + jumpargs = [] + remixing = [] + for srcbox in inputargs: + n = random.randrange(0, len(inputargs)) + otherbox = inputargs[n] + if otherbox.type == srcbox.type: + remixing.append((srcbox, otherbox)) + else: + otherbox = srcbox + jumpargs.append(otherbox) + # + index_counter = random.randrange(0, len(inputargs)+1) + i0 = BoxInt() + i1 = BoxInt() + i2 = BoxInt() + inputargs.insert(index_counter, i0) + jumpargs.insert(index_counter, i1) + # + faildescr = BasicFailDescr() + operations = [ + ResOperation(rop.INT_SUB, [i0, ConstInt(1)], i1), + ResOperation(rop.INT_GE, [i1, ConstInt(0)], i2), + ResOperation(rop.GUARD_TRUE, [i2], None), + ResOperation(rop.JUMP, jumpargs, None), + ] + operations[2].suboperations = [ + ResOperation(rop.FAIL, inputargs[:], None, descr=faildescr) + ] + operations[-1].jump_target = None + # + executable_token = self.cpu.compile_loop(inputargs, operations) + # + values = [] + S = lltype.GcStruct('S') + for box in inputargs: + if isinstance(box, BoxInt): + values.append(random.randrange(-10000, 10000)) + elif isinstance(box, BoxPtr): + p = lltype.malloc(S) + values.append(lltype.cast_opaque_ptr(llmemory.GCREF, p)) + elif isinstance(box, BoxFloat): + values.append(random.random()) + else: + assert 0 + values[index_counter] = 11 + # + for i, (box, val) in enumerate(zip(inputargs, values)): + if isinstance(box, BoxInt): + self.cpu.set_future_value_int(i, val) + elif isinstance(box, BoxPtr): + self.cpu.set_future_value_ref(i, val) + elif isinstance(box, BoxFloat): + self.cpu.set_future_value_float(i, val) + else: + assert 0 + # + fail = self.cpu.execute_token(executable_token) + assert fail is faildescr + # + dstvalues = values[:] + for _ in range(11): + expected = dstvalues[:] + for tgtbox, srcbox in remixing: + v = dstvalues[inputargs.index(srcbox)] + expected[inputargs.index(tgtbox)] = v + dstvalues = expected + # + assert dstvalues[index_counter] == 11 + dstvalues[index_counter] = 0 + for i, (box, val) in enumerate(zip(inputargs, dstvalues)): + if isinstance(box, BoxInt): + got = self.cpu.get_latest_value_int(i) + elif isinstance(box, BoxPtr): + got = self.cpu.get_latest_value_ref(i) + elif isinstance(box, BoxFloat): + got = self.cpu.get_latest_value_float(i) + else: + assert 0 + assert type(got) == type(val) + assert got == val class LLtypeBackendTest(BaseBackendTest): From arigo at codespeak.net Thu Oct 1 15:24:43 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 1 Oct 2009 15:24:43 +0200 (CEST) Subject: [pypy-svn] r68100 - pypy/branch/floats-via-sse2/pypy/jit/backend/test Message-ID: <20091001132443.E9BBD168009@codespeak.net> Author: arigo Date: Thu Oct 1 15:24:43 2009 New Revision: 68100 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Log: A test for operations whose result is not used. Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Thu Oct 1 15:24:43 2009 @@ -8,6 +8,7 @@ BoxObj, Const, ConstObj, BoxFloat, ConstFloat) from pypy.jit.metainterp.resoperation import ResOperation, rop +from pypy.jit.metainterp import resoperation from pypy.jit.metainterp.typesystem import deref from pypy.rpython.lltypesystem import lltype, llmemory, rstr, rffi, rclass from pypy.rpython.ootypesystem import ootype @@ -869,6 +870,47 @@ assert type(got) == type(val) assert got == val + def test_unused_result_int(self): + # check operations with no side effect and whose result + # is not used. The backend is free to do nothing from them. + # This is a "did not crash" kind of test. + from pypy.jit.metainterp.test import test_executor + # XXX added on trunk only so far + assert not hasattr(test_executor, 'get_int_tests') # yet + + def test_unused_result_float(self): + # same as test_unused_result_int, for float operations + from pypy.jit.metainterp.test.test_executor import get_float_tests + float_tests = list(get_float_tests(self.cpu)) + inputargs = [] + operations = [] + for opnum, boxargs, rettype, retvalue in float_tests: + inputargs += boxargs + if rettype == 'int': + boxres = BoxInt() + elif rettype == 'float': + boxres = BoxFloat() + else: + assert 0 + operations.append(ResOperation(opnum, boxargs, boxres)) + faildescr = BasicFailDescr() + operations.append(ResOperation(rop.FINISH, [], None, + descr=faildescr)) + operations[-1].jump_target = None + # + executable_token = self.cpu.compile_loop(inputargs, operations) + # + for i, box in enumerate(inputargs): + if isinstance(box, BoxInt): + self.cpu.set_future_value_int(i, box.getint()) + elif isinstance(box, BoxFloat): + self.cpu.set_future_value_float(i, box.getfloat()) + else: + assert 0 + # + fail = self.cpu.execute_token(executable_token) + assert fail is faildescr + class LLtypeBackendTest(BaseBackendTest): From arigo at codespeak.net Thu Oct 1 15:40:08 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 1 Oct 2009 15:40:08 +0200 (CEST) Subject: [pypy-svn] r68101 - pypy/branch/floats-via-sse2/pypy/jit/backend/x86 Message-ID: <20091001134008.F1E6E168009@codespeak.net> Author: arigo Date: Thu Oct 1 15:40:08 2009 New Revision: 68101 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py Log: Add an XXX. Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py Thu Oct 1 15:40:08 2009 @@ -58,6 +58,9 @@ import struct NEG_ZERO, = struct.unpack('d', struct.pack('ll', 0, -2147483648)) NAN, = struct.unpack('d', struct.pack('ll', -1, 2147483647)) +# XXX These are actually masks for float_neg and float_abs. +# They should not be converted to 'double' and given +# names that reflect their float value. class X86XMMRegisterManager(RegisterManager): From arigo at codespeak.net Thu Oct 1 15:44:24 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 1 Oct 2009 15:44:24 +0200 (CEST) Subject: [pypy-svn] r68102 - pypy/branch/floats-via-sse2/pypy/jit/backend/test Message-ID: <20091001134424.8CD53168009@codespeak.net> Author: arigo Date: Thu Oct 1 15:44:08 2009 New Revision: 68102 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Log: Improve this test: actually generate longer and longer jumps, with up to 50 arguments, to be sure to exhaust all the registers. Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Thu Oct 1 15:44:08 2009 @@ -772,16 +772,17 @@ assert r.value == 5.5 def test_jump(self): - # this test generate small loops where the JUMP passes many + # this test generates small loops where the JUMP passes many # arguments of various types, shuffling them around. if self.cpu.supports_floats: numkinds = 3 else: numkinds = 2 - for _ in range(50): + for nb_args in range(50): + print 'Passing %d arguments around...' % nb_args # inputargs = [] - for k in range(random.randrange(1, 16)): + for k in range(nb_args): kind = random.randrange(0, numkinds) if kind == 0: inputargs.append(BoxInt()) From fijal at codespeak.net Thu Oct 1 15:48:58 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 15:48:58 +0200 (CEST) Subject: [pypy-svn] r68103 - in pypy/branch/floats-via-sse2/pypy/jit/backend/x86: . test Message-ID: <20091001134858.34670168009@codespeak.net> Author: fijal Date: Thu Oct 1 15:48:41 2009 New Revision: 68103 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/jump.py pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test/test_jump.py Log: Some tests and a fix for remap_stack_layout Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/jump.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/jump.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/jump.py Thu Oct 1 15:48:41 2009 @@ -19,7 +19,7 @@ return self.position -def remap_stack_layout(assembler, src_locations, dst_locations, tmpreg): +def remap_stack_layout(assembler, src_locations, dst_locations, tmpreg, xmmtmp): pending_dests = len(dst_locations) srccount = {} # maps dst_locations to how many times the same # location appears in src_locations @@ -50,7 +50,7 @@ key = src._getregkey() if key in srccount: srccount[key] -= 1 - _move(assembler, src, dst, tmpreg) + _move(assembler, src, dst, tmpreg, xmmtmp) progress = True if not progress: # we are left with only pure disjoint cycles @@ -74,16 +74,20 @@ src = sources[key] if src._getregkey() == originalkey: break - _move(assembler, src, dst, tmpreg) + _move(assembler, src, dst, tmpreg, xmmtmp) dst = src assembler.regalloc_pop(dst) assert pending_dests == 0 -def _move(assembler, src, dst, tmpreg): +def _move(assembler, src, dst, tmpreg, xmmtmp): if isinstance(dst, MODRM): if isinstance(src, MODRM): - assembler.regalloc_mov(src, tmpreg) - src = tmpreg + if src.width == 8: + tmp = xmmtmp + else: + tmp = tmpreg + assembler.regalloc_mov(src, tmp) + src = tmp assembler.regalloc_mov(src, dst) else: assembler.regalloc_mov(src, dst) Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test/test_jump.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test/test_jump.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test/test_jump.py Thu Oct 1 15:48:41 2009 @@ -36,22 +36,22 @@ def test_trivial(): assembler = MockAssembler() - remap_stack_layout(assembler, [], [], '?') + remap_stack_layout(assembler, [], [], '?', '?') assert assembler.ops == [] remap_stack_layout(assembler, [eax, ebx, ecx, edx, esi, edi], - [eax, ebx, ecx, edx, esi, edi], '?') + [eax, ebx, ecx, edx, esi, edi], '?', '?') assert assembler.ops == [] s8 = stack_pos(1, 1) s12 = stack_pos(31, 1) s20 = stack_pos(6, 1) remap_stack_layout(assembler, [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], - '?') + '?', '?') assert assembler.ops == [] def test_simple_registers(): assembler = MockAssembler() - remap_stack_layout(assembler, [eax, ebx, ecx], [edx, esi, edi], '?') + remap_stack_layout(assembler, [eax, ebx, ecx], [edx, esi, edi], '?', '?') assert assembler.ops == [('mov', eax, edx), ('mov', ebx, esi), ('mov', ecx, edi)] @@ -62,7 +62,7 @@ s12 = stack_pos(13, 1) s20 = stack_pos(20, 1) s24 = stack_pos(221, 1) - remap_stack_layout(assembler, [s8, eax, s12], [s20, s24, edi], edx) + remap_stack_layout(assembler, [s8, eax, s12], [s20, s24, edi], edx, '?') assert assembler.ops == [('mov', s8, edx), ('mov', edx, s20), ('mov', eax, s24), @@ -75,7 +75,7 @@ s20 = stack_pos(19, 1) s24 = stack_pos(1, 1) remap_stack_layout(assembler, [eax, s8, s20, ebx], - [s8, ebx, eax, edi], '?') + [s8, ebx, eax, edi], '?', '?') assert assembler.got([('mov', ebx, edi), ('mov', s8, ebx), ('mov', eax, s8), @@ -88,7 +88,7 @@ s20 = stack_pos(19, 1) s24 = stack_pos(1, 1) remap_stack_layout(assembler, [eax, s8, s20, ebx], - [s8, ebx, eax, s20], '?') + [s8, ebx, eax, s20], '?', '?') assert assembler.got([('push', s8), ('mov', eax, s8), ('mov', s20, eax), @@ -106,7 +106,7 @@ remap_stack_layout(assembler, [eax, s8, edi, s20, eax, s20, s24, esi, s2, s3], [s8, s20, edi, eax, edx, s24, ebx, s12, s3, s2], - ecx) + ecx, '?') assert assembler.got([('mov', eax, edx), ('mov', s24, ebx), ('mov', esi, s12), @@ -124,11 +124,11 @@ def test_constants(): assembler = MockAssembler() c3 = imm(3) - remap_stack_layout(assembler, [c3], [eax], '?') + remap_stack_layout(assembler, [c3], [eax], '?', '?') assert assembler.ops == [('mov', c3, eax)] assembler = MockAssembler() s12 = stack_pos(12, 1) - remap_stack_layout(assembler, [c3], [s12], '?') + remap_stack_layout(assembler, [c3], [s12], '?', '?') assert assembler.ops == [('mov', c3, s12)] def test_constants_and_cycle(): @@ -136,8 +136,28 @@ c3 = imm(3) s12 = stack_pos(13, 1) remap_stack_layout(assembler, [ebx, c3, s12], - [s12, eax, ebx], edi) + [s12, eax, ebx], edi, '?') assert assembler.ops == [('mov', c3, eax), ('push', s12), ('mov', ebx, s12), ('pop', ebx)] + +def test_floats(): + assembler = MockAssembler() + s3 = stack_pos(3, 2) + remap_stack_layout(assembler, [xmm1, xmm2, s3], + [s3, xmm3, xmm1], edi, xmm0) + assert assembler.ops == [('mov', xmm2, xmm3), + ('push', s3), + ('mov', xmm1, s3), + ('pop', xmm1)] + +def test_floats_need_tmp(): + assembler = MockAssembler() + s3 = stack_pos(3, 2) + s5 = stack_pos(5, 2) + remap_stack_layout(assembler, [s3, s5], [s5, s3], edi, xmm0) + assert assembler.ops == [('push', s5), + ('mov', s3, xmm0), + ('mov', xmm0, s5), + ('pop', s3)] From fijal at codespeak.net Thu Oct 1 15:51:12 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 15:51:12 +0200 (CEST) Subject: [pypy-svn] r68104 - pypy/branch/floats-via-sse2/pypy/jit/backend/x86 Message-ID: <20091001135112.02C05168016@codespeak.net> Author: fijal Date: Thu Oct 1 15:51:02 2009 New Revision: 68104 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py Log: *IN-PROGRESS* Start supporting floats in various places, like jumps and guard_value. Does not pass test for obscure reasons Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py Thu Oct 1 15:51:02 2009 @@ -249,12 +249,14 @@ # ------------------------------------------------------------ - def regalloc_mov(self, from_loc, to_loc): + def mov(self, from_loc, to_loc): if isinstance(from_loc, XMMREG) or isinstance(to_loc, XMMREG): self.mc.MOVSD(to_loc, from_loc) else: self.mc.MOV(to_loc, from_loc) + regalloc_mov = mov # legacy interface + def regalloc_fstp(self, loc): self.mc.FSTP(loc) @@ -286,7 +288,7 @@ else: dispatch_opnum = op.opnum adr_jump_offset = genop_guard_list[dispatch_opnum](self, op, - guard_opnum, + guard_op, failaddr, arglocs, resloc) faildescr._x86_adr_jump_offset = adr_jump_offset @@ -329,7 +331,8 @@ return genop_cmp def _cmpop_guard(cond, rev_cond, false_cond, false_rev_cond): - def genop_cmp_guard(self, op, guard_opnum, addr, arglocs, result_loc): + def genop_cmp_guard(self, op, guard_op, addr, arglocs, result_loc): + guard_opnum = guard_op.opnum if isinstance(op.args[0], Const): self.mc.CMP(arglocs[1], arglocs[0]) if guard_opnum == rop.GUARD_FALSE: @@ -459,7 +462,8 @@ loc2 = cl self.mc.SHR(loc, loc2) - def genop_guard_oononnull(self, op, guard_opnum, addr, arglocs, resloc): + def genop_guard_oononnull(self, op, guard_op, addr, arglocs, resloc): + guard_opnum = guard_op.opnum loc = arglocs[0] self.mc.TEST(loc, loc) if guard_opnum == rop.GUARD_TRUE: @@ -467,7 +471,8 @@ else: return self.implement_guard(addr, self.mc.JNZ) - def genop_guard_ooisnull(self, op, guard_opnum, addr, arglocs, resloc): + def genop_guard_ooisnull(self, op, guard_op, addr, arglocs, resloc): + guard_opnum == guard_op.opnum loc = arglocs[0] self.mc.TEST(loc, loc) if guard_opnum == rop.GUARD_TRUE: @@ -490,7 +495,7 @@ self.mc.SETE(lower_byte(resloc)) def genop_same_as(self, op, arglocs, resloc): - self.mc.MOV(resloc, arglocs[0]) + self.mov(arglocs[0], resloc) genop_cast_ptr_to_int = genop_same_as def genop_int_mod(self, op, arglocs, resloc): @@ -654,17 +659,17 @@ else: assert 0, itemsize - def genop_guard_guard_true(self, ign_1, guard_opnum, addr, locs, ign_2): + def genop_guard_guard_true(self, ign_1, guard_op, addr, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) return self.implement_guard(addr, self.mc.JZ) - def genop_guard_guard_no_exception(self, ign_1, guard_opnum, addr, + def genop_guard_guard_no_exception(self, ign_1, guard_op, addr, locs, ign_2): self.mc.CMP(heap(self.cpu.pos_exception()), imm(0)) return self.implement_guard(addr, self.mc.JNZ) - def genop_guard_guard_exception(self, ign_1, guard_opnum, addr, + def genop_guard_guard_exception(self, ign_1, guard_op, addr, locs, resloc): loc = locs[0] loc1 = locs[1] @@ -677,24 +682,28 @@ self.mc.MOV(heap(self.cpu.pos_exc_value()), imm(0)) return addr - def genop_guard_guard_no_overflow(self, ign_1, guard_opnum, addr, + def genop_guard_guard_no_overflow(self, ign_1, guard_op, addr, locs, resloc): return self.implement_guard(addr, self.mc.JO) - def genop_guard_guard_overflow(self, ign_1, guard_opnum, addr, + def genop_guard_guard_overflow(self, ign_1, guard_op, addr, locs, resloc): return self.implement_guard(addr, self.mc.JNO) - def genop_guard_guard_false(self, ign_1, guard_opnum, addr, locs, ign_2): + def genop_guard_guard_false(self, ign_1, guard_op, addr, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) return self.implement_guard(addr, self.mc.JNZ) - def genop_guard_guard_value(self, ign_1, guard_opnum, addr, locs, ign_2): - self.mc.CMP(locs[0], locs[1]) + def genop_guard_guard_value(self, ign_1, guard_op, addr, locs, ign_2): + if guard_op.args[0].type == FLOAT: + assert guard_op.args[1].type == FLOAT + self.mc.UCOMISD(locs[0], locs[1]) + else: + self.mc.CMP(locs[0], locs[1]) return self.implement_guard(addr, self.mc.JNE) - def genop_guard_guard_class(self, ign_1, guard_opnum, addr, locs, ign_2): + def genop_guard_guard_class(self, ign_1, guard_op, addr, locs, ign_2): offset = self.cpu.vtable_offset self.mc.CMP(mem(locs[0], offset), locs[1]) return self.implement_guard(addr, self.mc.JNE) Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py Thu Oct 1 15:51:02 2009 @@ -161,8 +161,10 @@ # Don't use all_regs[0] for passing arguments around a loop. # Must be kept in sync with consider_jump(). # XXX this should probably go to llsupport/regalloc.py + xmmtmp = self.xrm.free_regs.pop(0) tmpreg = self.rm.free_regs.pop(0) assert tmpreg == X86RegisterManager.all_regs[0] + assert xmmtmp == X86XMMRegisterManager.all_regs[0] for i in range(len(inputargs)): arg = inputargs[i] assert not isinstance(arg, Const) @@ -181,9 +183,10 @@ locs[i] = loc # otherwise we have it saved on stack, so no worry self.rm.free_regs.insert(0, tmpreg) + self.xrm.free_regs.insert(0, xmmtmp) assert tmpreg not in locs - for arg in inputargs: - self.possibly_free_var(arg) + assert xmmtmp not in locs + self.possibly_free_vars(inputargs) return locs def possibly_free_var(self, var): @@ -271,7 +274,7 @@ arglocs, result_loc, self.sm.stack_depth) self.rm.possibly_free_var(op.result) - self.rm.possibly_free_vars(guard_op.suboperations[0].args) + self.possibly_free_vars(guard_op.suboperations[0].args) def perform_guard(self, guard_op, arglocs, result_loc): faillocs = self.locs_for_fail(guard_op) @@ -284,7 +287,7 @@ self.assembler.regalloc_perform_guard(guard_op, faillocs, arglocs, result_loc, self.sm.stack_depth) - self.rm.possibly_free_vars(guard_op.suboperations[0].args) + self.possibly_free_vars(guard_op.suboperations[0].args) def PerformDiscard(self, op, arglocs): if not we_are_translated(): @@ -401,10 +404,10 @@ consider_guard_overflow = consider_guard_no_exception def consider_guard_value(self, op, ignored): - x = self.rm.make_sure_var_in_reg(op.args[0]) + x = self.make_sure_var_in_reg(op.args[0]) y = self.loc(op.args[1]) self.perform_guard(op, [x, y], None) - self.rm.possibly_free_vars(op.args) + self.possibly_free_vars(op.args) def consider_guard_class(self, op, ignored): assert isinstance(op.args[0], Box) @@ -785,8 +788,8 @@ def consider_same_as(self, op, ignored): argloc = self.loc(op.args[0]) - self.rm.possibly_free_var(op.args[0]) - resloc = self.rm.force_allocate_reg(op.result) + self.possibly_free_var(op.args[0]) + resloc = self.force_allocate_reg(op.result) self.Perform(op, [argloc], resloc) consider_cast_ptr_to_int = consider_same_as @@ -823,14 +826,19 @@ arglocs = assembler.target_arglocs(self.jump_target) # compute 'tmploc' to be all_regs[0] by spilling what is there box = TempBox() + box1 = TempBox() tmpreg = X86RegisterManager.all_regs[0] - tmploc = self.rm.force_allocate_reg(box, [], selected_reg=tmpreg) - src_locations = [self.rm.loc(arg) for arg in op.args] + tmploc = self.rm.force_allocate_reg(box, selected_reg=tmpreg) + xmmtmp = X86XMMRegisterManager.all_regs[0] + xmmtmploc = self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp) + src_locations = [self.loc(arg) for arg in op.args] dst_locations = arglocs assert tmploc not in dst_locations - remap_stack_layout(assembler, src_locations, dst_locations, tmploc) + remap_stack_layout(assembler, src_locations, dst_locations, tmploc, + xmmtmp) self.rm.possibly_free_var(box) - self.rm.possibly_free_vars(op.args) + self.xrm.possibly_free_var(box1) + self.possibly_free_vars(op.args) assembler.closing_jump(self.jump_target) def consider_debug_merge_point(self, op, ignored): From arigo at codespeak.net Thu Oct 1 15:55:35 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 1 Oct 2009 15:55:35 +0200 (CEST) Subject: [pypy-svn] r68105 - pypy/branch/floats-via-sse2/pypy/jit/backend/test Message-ID: <20091001135535.4B892168013@codespeak.net> Author: arigo Date: Thu Oct 1 15:55:34 2009 New Revision: 68105 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Log: Print the random seed, and let a seed be reused (by editing the test). Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Thu Oct 1 15:55:34 2009 @@ -778,12 +778,16 @@ numkinds = 3 else: numkinds = 2 + seed = random.randrange(0, 10000) + print 'Seed is', seed # or choose it by changing the previous line + r = random.Random() + r.seed(seed) for nb_args in range(50): print 'Passing %d arguments around...' % nb_args # inputargs = [] for k in range(nb_args): - kind = random.randrange(0, numkinds) + kind = r.randrange(0, numkinds) if kind == 0: inputargs.append(BoxInt()) elif kind == 1: @@ -793,7 +797,7 @@ jumpargs = [] remixing = [] for srcbox in inputargs: - n = random.randrange(0, len(inputargs)) + n = r.randrange(0, len(inputargs)) otherbox = inputargs[n] if otherbox.type == srcbox.type: remixing.append((srcbox, otherbox)) @@ -801,7 +805,7 @@ otherbox = srcbox jumpargs.append(otherbox) # - index_counter = random.randrange(0, len(inputargs)+1) + index_counter = r.randrange(0, len(inputargs)+1) i0 = BoxInt() i1 = BoxInt() i2 = BoxInt() @@ -826,12 +830,12 @@ S = lltype.GcStruct('S') for box in inputargs: if isinstance(box, BoxInt): - values.append(random.randrange(-10000, 10000)) + values.append(r.randrange(-10000, 10000)) elif isinstance(box, BoxPtr): p = lltype.malloc(S) values.append(lltype.cast_opaque_ptr(llmemory.GCREF, p)) elif isinstance(box, BoxFloat): - values.append(random.random()) + values.append(r.random()) else: assert 0 values[index_counter] = 11 From fijal at codespeak.net Thu Oct 1 16:00:45 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 16:00:45 +0200 (CEST) Subject: [pypy-svn] r68106 - pypy/branch/floats-via-sse2/pypy/jit/metainterp Message-ID: <20091001140045.CBE3A168015@codespeak.net> Author: fijal Date: Thu Oct 1 16:00:44 2009 New Revision: 68106 Modified: pypy/branch/floats-via-sse2/pypy/jit/metainterp/history.py Log: improve printing Modified: pypy/branch/floats-via-sse2/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/metainterp/history.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/metainterp/history.py Thu Oct 1 16:00:44 2009 @@ -466,6 +466,8 @@ try: if self.type == INT: t = 'i' + elif self.type == FLOAT: + t = 'f' else: t = 'p' except AttributeError: From antocuni at codespeak.net Thu Oct 1 16:30:45 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 1 Oct 2009 16:30:45 +0200 (CEST) Subject: [pypy-svn] r68107 - in pypy/trunk/pypy/jit/backend/cli: . test Message-ID: <20091001143045.47392168015@codespeak.net> Author: antocuni Date: Thu Oct 1 16:30:44 2009 New Revision: 68107 Modified: pypy/trunk/pypy/jit/backend/cli/method.py pypy/trunk/pypy/jit/backend/cli/runner.py pypy/trunk/pypy/jit/backend/cli/test/test_basic.py pypy/trunk/pypy/jit/backend/cli/test/test_runner.py Log: refactor the cli jit backend after the merging of the remove-plfbid, remove-fail and kill-jumptarget branches. The main idea is that so far the cli backend recompiles the whole loop each time we add a path for a failing guard, and since the frontend no longer keeps the tree in memory, we have to keep it in the backend itself Modified: pypy/trunk/pypy/jit/backend/cli/method.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/method.py (original) +++ pypy/trunk/pypy/jit/backend/cli/method.py Thu Oct 1 16:30:44 2009 @@ -7,7 +7,7 @@ from pypy.translator.cli import opcodes from pypy.jit.metainterp import history from pypy.jit.metainterp.history import (AbstractValue, Const, ConstInt, - ConstObj, BoxInt) + ConstObj, BoxInt, LoopToken) from pypy.jit.metainterp.resoperation import rop, opname from pypy.jit.metainterp.typesystem import oohelper from pypy.jit.backend.cli import runner @@ -23,7 +23,6 @@ cVoid = ootype.nullruntimeclass - class __extend__(AbstractValue): __metaclass__ = extendabletype @@ -127,11 +126,11 @@ tailcall = True nocast = True - def __init__(self, cpu, name, loop): + def __init__(self, cpu, cliloop): self.setoptions() self.cpu = cpu - self.name = name - self.loop = loop + self.name = cliloop.get_fresh_cli_name() + self.cliloop = cliloop self.boxes = {} # box --> local var self.branches = [] self.branchlabels = [] @@ -149,18 +148,19 @@ else: self.av_OverflowError = None self.av_ZeroDivisionError = None + self.box2type = {} + def compile(self): # ---- - self.box2type = {} if self.nocast: self.compute_types() self.emit_load_inputargs() self.emit_preamble() - self.emit_operations(loop.operations) + self.emit_operations(self.cliloop.operations) self.emit_branches() self.emit_end() # ---- - self.finish_code() + return self.finish_code() def _parseopt(self, text): text = text.lower() @@ -194,12 +194,13 @@ descr = op.descr assert isinstance(descr, runner.FieldDescr) box2classes.setdefault(box, []).append(descr.selfclass) - if op.suboperations: - self._collect_types(op.suboperations, box2classes) + if op in self.cliloop.guard2ops: + _, suboperations = self.cliloop.guard2ops[op] + self._collect_types(suboperations, box2classes) def compute_types(self): box2classes = {} # box --> [ootype.Class] - self._collect_types(self.loop.operations, box2classes) + self._collect_types(self.cliloop.operations, box2classes) for box, classes in box2classes.iteritems(): cls = classes[0] for cls2 in classes[1:]: @@ -217,8 +218,7 @@ consts[i] = av_const.get_cliobj() # build the delegate func = self.meth_wrapper.create_delegate(delegatetype, consts) - func = dotnet.clidowncast(func, LoopDelegate) - self.loop._cli_funcbox.holder.SetFunc(func) + return dotnet.clidowncast(func, LoopDelegate) def _get_meth_wrapper(self): restype = dotnet.class2type(cVoid) @@ -240,6 +240,12 @@ self.boxes[box] = v return v + def match_var_fox_boxes(self, failargs, inputargs): + assert len(failargs) == len(inputargs) + for i in range(len(failargs)): + v = self.boxes[failargs[i]] + self.boxes[inputargs[i]] = v + def get_index_for_failing_op(self, op): try: return self.cpu.failing_ops.index(op) @@ -292,7 +298,7 @@ def emit_load_inputargs(self): self.emit_debug("executing: " + self.name) i = 0 - for box in self.loop.inputargs: + for box in self.cliloop.inputargs: self.load_inputarg(i, box.type, box.getCliType(self)) box.store(self) i+=1 @@ -324,7 +330,16 @@ op = branches[i] il_label = branchlabels[i] self.il.MarkLabel(il_label) - self.emit_operations(op.suboperations) + self.emit_guard_subops(op) + + def emit_guard_subops(self, op): + assert op.is_guard() + if op in self.cliloop.guard2ops: + inputargs, suboperations = self.cliloop.guard2ops[op] + self.match_var_fox_boxes(op.fail_args, inputargs) + self.emit_operations(suboperations) + else: + self.emit_return_failed_op(op, op.fail_args) def emit_end(self): assert self.branches == [] @@ -396,7 +411,6 @@ self.il.Emit(OpCodes.Leave, lbl) self.il.BeginCatchBlock(dotnet.typeof(System.OverflowException)) # emit the guard - assert opguard.suboperations assert len(opguard.args) == 0 il_label = self.newbranch(opguard) self.il.Emit(OpCodes.Leave, il_label) @@ -408,25 +422,27 @@ # -------------------------------- - def emit_op_fail(self, op): + def emit_return_failed_op(self, op, args): # store the index of the failed op index_op = self.get_index_for_failing_op(op) self.av_inputargs.load(self) self.il.Emit(OpCodes.Ldc_I4, index_op) field = dotnet.typeof(InputArgs).GetField('failed_op') self.il.Emit(OpCodes.Stfld, field) - self.emit_store_opargs(op) + self.emit_store_opargs(args) self.il.Emit(OpCodes.Ret) - def emit_store_opargs(self, op): + def emit_op_finish(self, op): + self.emit_return_failed_op(op, op.args) + + def emit_store_opargs(self, args): # store the latest values i = 0 - for box in op.args: + for box in args: self.store_inputarg(i, box.type, box.getCliType(self), box) i+=1 def emit_guard_bool(self, op, opcode): - assert op.suboperations assert len(op.args) == 1 il_label = self.newbranch(op) op.args[0].load(self) @@ -439,14 +455,12 @@ self.emit_guard_bool(op, OpCodes.Brtrue) def emit_op_guard_value(self, op): - assert op.suboperations assert len(op.args) == 2 il_label = self.newbranch(op) self.push_all_args(op) self.il.Emit(OpCodes.Bne_Un, il_label) def emit_op_guard_class(self, op): - assert op.suboperations assert len(op.args) == 2 il_label = self.newbranch(op) self.push_arg(op, 0) @@ -456,14 +470,12 @@ self.il.Emit(OpCodes.Bne_Un, il_label) def emit_op_guard_no_exception(self, op): - assert op.suboperations il_label = self.newbranch(op) self.av_inputargs.load(self) self.il.Emit(OpCodes.Ldfld, self.exc_value_field) self.il.Emit(OpCodes.Brtrue, il_label) def emit_op_guard_exception(self, op): - assert op.suboperations il_label = self.newbranch(op) classbox = op.args[0] assert isinstance(classbox, ConstObj) @@ -479,7 +491,6 @@ self.store_result(op) def emit_guard_overflow_impl(self, op, opcode): - assert op.suboperations assert len(op.args) == 0 il_label = self.newbranch(op) self.av_ovf_flag.load(self) @@ -492,19 +503,22 @@ self.emit_guard_overflow_impl(op, OpCodes.Brfalse) def emit_op_jump(self, op): - target = op.jump_target - assert len(op.args) == len(target.inputargs) - if target is self.loop: + target_token = op.descr + assert isinstance(target_token, LoopToken) + if target_token.cliloop is self.cliloop: + # jump to the beginning of the loop i = 0 for i in range(len(op.args)): op.args[i].load(self) - target.inputargs[i].store(self) + self.cliloop.inputargs[i].store(self) self.il.Emit(OpCodes.Br, self.il_loop_start) else: # it's a real bridge - self.emit_debug('jumping to ' + target.name) - self.emit_store_opargs(op) - target._cli_funcbox.load(self) + cliloop = target_token.cliloop + assert len(op.args) == len(cliloop.inputargs) + self.emit_debug('jumping to ' + cliloop.name) + self.emit_store_opargs(op.args) + cliloop.funcbox.load(self) self.av_inputargs.load(self) methinfo = dotnet.typeof(LoopDelegate).GetMethod('Invoke') if self.tailcall: Modified: pypy/trunk/pypy/jit/backend/cli/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/runner.py (original) +++ pypy/trunk/pypy/jit/backend/cli/runner.py Thu Oct 1 16:30:44 2009 @@ -3,8 +3,8 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.jit.metainterp import history from pypy.jit.metainterp.history import AbstractDescr, AbstractMethDescr +from pypy.jit.metainterp.history import AbstractFailDescr, LoopToken from pypy.jit.metainterp.history import Box, BoxInt, BoxObj, ConstObj, Const -from pypy.jit.metainterp.history import TreeLoop from pypy.jit.metainterp import executor from pypy.jit.metainterp.resoperation import rop, opname from pypy.jit.backend import model @@ -18,15 +18,24 @@ InputArgs = CLR.pypy.runtime.InputArgs cpypyString = dotnet.classof(CLR.pypy.runtime.String) -class __extend__(TreeLoop): - __metaclass__ = extendabletype +LoopToken.cliloop = None +AbstractFailDescr._loop_token = None +AbstractFailDescr._guard_op = None - _cli_funcbox = None - _cli_meth = None - _cli_count = 0 - - def _get_cli_name(self): - return '%s(r%d)' % (self.name, self._cli_count) +class CliLoop(object): + + def __init__(self, name, inputargs, operations): + self.name = name + self.inputargs = inputargs + self.operations = operations + self.guard2ops = {} # guard_op --> (inputargs, operations) + self.funcbox = None + self.methcount = 0 + + def get_fresh_cli_name(self): + name = '%s(r%d)' % (self.name, self.methcount) + self.methcount += 1 + return name class CliCPU(model.AbstractCPU): @@ -38,6 +47,7 @@ self.rtyper = rtyper if rtyper: assert rtyper.type_system.name == "ootypesystem" + self.loopcount = 0 self.stats = stats self.translate_support_code = translate_support_code self.inputargs = None @@ -90,21 +100,42 @@ # ---------------------- - def compile_operations(self, loop, bridge=None): - from pypy.jit.backend.cli.method import Method, ConstFunction - if loop._cli_funcbox is None: - loop._cli_funcbox = ConstFunction(loop.name) - else: - # discard previously compiled loop - loop._cli_funcbox.holder.SetFunc(None) - loop._cli_meth = Method(self, loop._get_cli_name(), loop) - loop._cli_count += 1 + def _attach_token_to_faildescrs(self, token, operations): + for op in operations: + if op.is_guard(): + descr = op.descr + assert isinstance(descr, AbstractFailDescr) + descr._loop_token = token + descr._guard_op = op - def execute_operations(self, loop): - func = loop._cli_funcbox.holder.GetFunc() + def compile_loop(self, inputargs, operations, looptoken): + from pypy.jit.backend.cli.method import Method, ConstFunction + name = 'Loop%d' % self.loopcount + self.loopcount += 1 + cliloop = CliLoop(name, inputargs, operations) + looptoken.cliloop = cliloop + cliloop.funcbox = ConstFunction(cliloop.name) + self._attach_token_to_faildescrs(cliloop, operations) + meth = Method(self, cliloop) + cliloop.funcbox.holder.SetFunc(meth.compile()) + + def compile_bridge(self, faildescr, inputargs, operations): + from pypy.jit.backend.cli.method import Method + op = faildescr._guard_op + token = faildescr._loop_token + token.guard2ops[op] = (inputargs, operations) + self._attach_token_to_faildescrs(token, operations) + meth = Method(self, token) + token.funcbox.holder.SetFunc(meth.compile()) + return token + + def execute_token(self, looptoken): + cliloop = looptoken.cliloop + func = cliloop.funcbox.holder.GetFunc() func(self.get_inputargs()) - return self.failing_ops[self.inputargs.get_failed_op()] - + op = self.failing_ops[self.inputargs.get_failed_op()] + return op.descr + def set_future_value_int(self, index, intvalue): self.get_inputargs().set_int(index, intvalue) @@ -180,10 +211,12 @@ def do_getfield_gc(self, instancebox, fielddescr): assert isinstance(fielddescr, FieldDescr) + assert fielddescr.getfield is not None return fielddescr.getfield(instancebox) def do_setfield_gc(self, instancebox, newvaluebox, fielddescr): assert isinstance(fielddescr, FieldDescr) + assert fielddescr.setfield is not None return fielddescr.setfield(instancebox, newvaluebox) def do_call(self, args, calldescr): @@ -205,6 +238,7 @@ def do_oosend(self, args, descr): assert isinstance(descr, MethDescr) + assert descr.callmeth is not None selfbox = args[0] argboxes = args[1:] return descr.callmeth(selfbox, argboxes) @@ -428,6 +462,7 @@ setfield = None selfclass = ootype.nullruntimeclass fieldname = '' + _is_pointer_field = False def __init__(self, TYPE, fieldname): DescrWithKey.__init__(self, (TYPE, fieldname)) Modified: pypy/trunk/pypy/jit/backend/cli/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/backend/cli/test/test_basic.py Thu Oct 1 16:30:44 2009 @@ -43,6 +43,10 @@ test_subclassof = skip test_assert_isinstance = skip test_dont_look_inside = skip + test_setfield_bool = skip + test_instantiate_does_not_call = skip + test_listcomp = skip + test_tuple_immutable = skip def test_fielddescr_ootype(): Modified: pypy/trunk/pypy/jit/backend/cli/test/test_runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/test/test_runner.py (original) +++ pypy/trunk/pypy/jit/backend/cli/test/test_runner.py Thu Oct 1 16:30:44 2009 @@ -13,7 +13,7 @@ CPUClass = CliCPU # for the individual tests see - # ====> ../../test/runner.py + # ====> ../../test/runner_test.py def setup_class(cls): cls.cpu = cls.CPUClass(rtyper=None, stats=FakeStats()) @@ -33,9 +33,15 @@ def test_ovf_operations(self, reversed=False): self.skip() + def test_do_unicode_basic(self): + py.test.skip('fixme!') + def test_unicode_basic(self): py.test.skip('fixme!') + def test_backends_dont_keep_loops_alive(self): + pass # the cli backend DOES keep loops alive + def test_pypycliopt(): import os from pypy.jit.backend.cli.method import Method From antocuni at codespeak.net Thu Oct 1 17:01:01 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 1 Oct 2009 17:01:01 +0200 (CEST) Subject: [pypy-svn] r68108 - pypy/trunk/pypy/jit/backend/cli Message-ID: <20091001150101.BBC4516800B@codespeak.net> Author: antocuni Date: Thu Oct 1 17:01:00 2009 New Revision: 68108 Modified: pypy/trunk/pypy/jit/backend/cli/method.py Log: implement op_runtimenew: this fixes test_instantiate_does_not_call Modified: pypy/trunk/pypy/jit/backend/cli/method.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/method.py (original) +++ pypy/trunk/pypy/jit/backend/cli/method.py Thu Oct 1 17:01:00 2009 @@ -538,7 +538,11 @@ self.store_result(op) def emit_op_runtimenew(self, op): - raise NotImplementedError + clitype_utils = dotnet.typeof(Utils) + methinfo = clitype_utils.GetMethod('RuntimeNew') + op.args[0].load(self) + self.il.Emit(OpCodes.Call, methinfo) + self.store_result(op) def emit_op_instanceof(self, op): descr = op.descr From fijal at codespeak.net Thu Oct 1 17:04:02 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 17:04:02 +0200 (CEST) Subject: [pypy-svn] r68109 - in pypy/branch/floats-via-sse2/pypy/jit/backend/x86: . test Message-ID: <20091001150402.94FD716800B@codespeak.net> Author: fijal Date: Thu Oct 1 17:04:02 2009 New Revision: 68109 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/jump.py pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test/test_jump.py Log: Revert 68103 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/jump.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/jump.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/jump.py Thu Oct 1 17:04:02 2009 @@ -19,7 +19,7 @@ return self.position -def remap_stack_layout(assembler, src_locations, dst_locations, tmpreg, xmmtmp): +def remap_stack_layout(assembler, src_locations, dst_locations, tmpreg): pending_dests = len(dst_locations) srccount = {} # maps dst_locations to how many times the same # location appears in src_locations @@ -50,7 +50,7 @@ key = src._getregkey() if key in srccount: srccount[key] -= 1 - _move(assembler, src, dst, tmpreg, xmmtmp) + _move(assembler, src, dst, tmpreg) progress = True if not progress: # we are left with only pure disjoint cycles @@ -74,20 +74,16 @@ src = sources[key] if src._getregkey() == originalkey: break - _move(assembler, src, dst, tmpreg, xmmtmp) + _move(assembler, src, dst, tmpreg) dst = src assembler.regalloc_pop(dst) assert pending_dests == 0 -def _move(assembler, src, dst, tmpreg, xmmtmp): +def _move(assembler, src, dst, tmpreg): if isinstance(dst, MODRM): if isinstance(src, MODRM): - if src.width == 8: - tmp = xmmtmp - else: - tmp = tmpreg - assembler.regalloc_mov(src, tmp) - src = tmp + assembler.regalloc_mov(src, tmpreg) + src = tmpreg assembler.regalloc_mov(src, dst) else: assembler.regalloc_mov(src, dst) Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test/test_jump.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test/test_jump.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test/test_jump.py Thu Oct 1 17:04:02 2009 @@ -36,22 +36,22 @@ def test_trivial(): assembler = MockAssembler() - remap_stack_layout(assembler, [], [], '?', '?') + remap_stack_layout(assembler, [], [], '?') assert assembler.ops == [] remap_stack_layout(assembler, [eax, ebx, ecx, edx, esi, edi], - [eax, ebx, ecx, edx, esi, edi], '?', '?') + [eax, ebx, ecx, edx, esi, edi], '?') assert assembler.ops == [] s8 = stack_pos(1, 1) s12 = stack_pos(31, 1) s20 = stack_pos(6, 1) remap_stack_layout(assembler, [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], - '?', '?') + '?') assert assembler.ops == [] def test_simple_registers(): assembler = MockAssembler() - remap_stack_layout(assembler, [eax, ebx, ecx], [edx, esi, edi], '?', '?') + remap_stack_layout(assembler, [eax, ebx, ecx], [edx, esi, edi], '?') assert assembler.ops == [('mov', eax, edx), ('mov', ebx, esi), ('mov', ecx, edi)] @@ -62,7 +62,7 @@ s12 = stack_pos(13, 1) s20 = stack_pos(20, 1) s24 = stack_pos(221, 1) - remap_stack_layout(assembler, [s8, eax, s12], [s20, s24, edi], edx, '?') + remap_stack_layout(assembler, [s8, eax, s12], [s20, s24, edi], edx) assert assembler.ops == [('mov', s8, edx), ('mov', edx, s20), ('mov', eax, s24), @@ -75,7 +75,7 @@ s20 = stack_pos(19, 1) s24 = stack_pos(1, 1) remap_stack_layout(assembler, [eax, s8, s20, ebx], - [s8, ebx, eax, edi], '?', '?') + [s8, ebx, eax, edi], '?') assert assembler.got([('mov', ebx, edi), ('mov', s8, ebx), ('mov', eax, s8), @@ -88,7 +88,7 @@ s20 = stack_pos(19, 1) s24 = stack_pos(1, 1) remap_stack_layout(assembler, [eax, s8, s20, ebx], - [s8, ebx, eax, s20], '?', '?') + [s8, ebx, eax, s20], '?') assert assembler.got([('push', s8), ('mov', eax, s8), ('mov', s20, eax), @@ -106,7 +106,7 @@ remap_stack_layout(assembler, [eax, s8, edi, s20, eax, s20, s24, esi, s2, s3], [s8, s20, edi, eax, edx, s24, ebx, s12, s3, s2], - ecx, '?') + ecx) assert assembler.got([('mov', eax, edx), ('mov', s24, ebx), ('mov', esi, s12), @@ -124,11 +124,11 @@ def test_constants(): assembler = MockAssembler() c3 = imm(3) - remap_stack_layout(assembler, [c3], [eax], '?', '?') + remap_stack_layout(assembler, [c3], [eax], '?') assert assembler.ops == [('mov', c3, eax)] assembler = MockAssembler() s12 = stack_pos(12, 1) - remap_stack_layout(assembler, [c3], [s12], '?', '?') + remap_stack_layout(assembler, [c3], [s12], '?') assert assembler.ops == [('mov', c3, s12)] def test_constants_and_cycle(): @@ -136,28 +136,8 @@ c3 = imm(3) s12 = stack_pos(13, 1) remap_stack_layout(assembler, [ebx, c3, s12], - [s12, eax, ebx], edi, '?') + [s12, eax, ebx], edi) assert assembler.ops == [('mov', c3, eax), ('push', s12), ('mov', ebx, s12), ('pop', ebx)] - -def test_floats(): - assembler = MockAssembler() - s3 = stack_pos(3, 2) - remap_stack_layout(assembler, [xmm1, xmm2, s3], - [s3, xmm3, xmm1], edi, xmm0) - assert assembler.ops == [('mov', xmm2, xmm3), - ('push', s3), - ('mov', xmm1, s3), - ('pop', xmm1)] - -def test_floats_need_tmp(): - assembler = MockAssembler() - s3 = stack_pos(3, 2) - s5 = stack_pos(5, 2) - remap_stack_layout(assembler, [s3, s5], [s5, s3], edi, xmm0) - assert assembler.ops == [('push', s5), - ('mov', s3, xmm0), - ('mov', xmm0, s5), - ('pop', s3)] From fijal at codespeak.net Thu Oct 1 17:53:19 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 17:53:19 +0200 (CEST) Subject: [pypy-svn] r68110 - pypy/branch/floats-via-sse2/pypy/jit/backend/x86 Message-ID: <20091001155319.C9DF8168014@codespeak.net> Author: fijal Date: Thu Oct 1 17:53:19 2009 New Revision: 68110 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py Log: Implement jumps, without touching jump.py. There is inneficient trick about pushing MODRM64 on top of stack Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py Thu Oct 1 17:53:19 2009 @@ -7,7 +7,8 @@ from pypy.rpython.lltypesystem.rclass import OBJECT from pypy.rpython.lltypesystem.lloperation import llop from pypy.tool.uid import fixid -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, lower_byte +from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, lower_byte,\ + X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.jit.backend.x86 import codebuf from pypy.jit.backend.x86.ri386 import * @@ -192,6 +193,7 @@ mc.done() def _assemble_bootstrap_code(self, inputargs, arglocs): + nonfloatlocs, floatlocs = arglocs self.mc.PUSH(ebp) self.mc.MOV(ebp, esp) self.mc.PUSH(ebx) @@ -200,41 +202,37 @@ # NB. exactly 4 pushes above; if this changes, fix stack_pos(). # You must also keep _get_callshape() in sync. adr_stackadjust = self._patchable_stackadjust() - for i in range(len(arglocs)): - loc = arglocs[i] - if not isinstance(loc, REG): - if inputargs[i].type == FLOAT: - self.mc.MOVSD(xmm0, - addr64_add(imm(self.fail_box_float_addr), - imm(i*WORD*2))) - self.mc.MOVSD(loc, xmm0) - else: - if inputargs[i].type == REF: - # This uses XCHG to put zeroes in fail_boxes_ptr after - # reading them - self.mc.XOR(ecx, ecx) - self.mc.XCHG(ecx, addr_add(imm(self.fail_box_ptr_addr), - imm(i*WORD))) - else: - self.mc.MOV(ecx, addr_add(imm(self.fail_box_int_addr), - imm(i*WORD))) - self.mc.MOV(loc, ecx) - for i in range(len(arglocs)): - loc = arglocs[i] + tmp = X86RegisterManager.all_regs[0] + xmmtmp = X86XMMRegisterManager.all_regs[0] + for i in range(len(nonfloatlocs)): + loc = nonfloatlocs[i] + if loc is None: + continue if isinstance(loc, REG): - if inputargs[i].type == FLOAT: - self.mc.MOVSD(loc, - addr64_add(imm(self.fail_box_float_addr), - imm(i*WORD*2))) - elif inputargs[i].type == REF: - # This uses XCHG to put zeroes in fail_boxes_ptr after - # reading them - self.mc.XOR(loc, loc) - self.mc.XCHG(loc, addr_add(imm(self.fail_box_ptr_addr), - imm(i*WORD))) - else: - self.mc.MOV(loc, addr_add(imm(self.fail_box_int_addr), + target = loc + else: + target = tmp + if inputargs[i].type == REF: + # This uses XCHG to put zeroes in fail_boxes_ptr after + # reading them + self.mc.XOR(target, target) + self.mc.XCHG(target, addr_add(imm(self.fail_box_ptr_addr), imm(i*WORD))) + else: + self.mc.MOV(target, addr_add(imm(self.fail_box_int_addr), + imm(i*WORD))) + self.mc.MOV(loc, target) + for i in range(len(floatlocs)): + loc = floatlocs[i] + if loc is None: + continue + if isinstance(loc, REG): + self.mc.MOVSD(loc, addr64_add(imm(self.fail_box_float_addr), + imm(i*WORD*2))) + else: + self.mc.MOVSD(xmmtmp, addr64_add(imm(self.fail_box_float_addr), + imm(i*WORD*2))) + self.mc.MOVSD(loc, xmmtmp) return adr_stackadjust def dump(self, text): @@ -261,10 +259,26 @@ self.mc.FSTP(loc) def regalloc_push(self, loc): - self.mc.PUSH(loc) + if isinstance(loc, XMMREG): + self.mc.SUB(esp, imm(2*WORD)) + self.mc.MOVSD(mem64(esp, 0), loc) + elif isinstance(loc, MODRM64): + # XXX evil trick + self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position))) + self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position + 1))) + else: + self.mc.PUSH(loc) def regalloc_pop(self, loc): - self.mc.POP(loc) + if isinstance(loc, XMMREG): + self.mc.MOVSD(loc, mem64(esp, 0)) + self.mc.ADD(esp, imm(2*WORD)) + elif isinstance(loc, MODRM64): + # XXX evil trick + self.mc.POP(mem(ebp, get_ebp_ofs(loc.position + 1))) + self.mc.POP(mem(ebp, get_ebp_ofs(loc.position))) + else: + self.mc.POP(loc) def regalloc_perform(self, op, arglocs, resloc): genop_list[op.opnum](self, op, arglocs, resloc) @@ -472,7 +486,7 @@ return self.implement_guard(addr, self.mc.JNZ) def genop_guard_ooisnull(self, op, guard_op, addr, arglocs, resloc): - guard_opnum == guard_op.opnum + guard_opnum = guard_op.opnum loc = arglocs[0] self.mc.TEST(loc, loc) if guard_opnum == rop.GUARD_TRUE: @@ -786,7 +800,7 @@ for arg in range(2, nargs + 2): extra_on_stack += round_up_to_4(arglocs[arg].width) extra_on_stack = self.align_stack_for_call(extra_on_stack) - self.mc.SUB(esp, imm(WORD * extra_on_stack)) + self.mc.SUB(esp, imm(extra_on_stack)) if isinstance(op.args[0], Const): x = rel32(op.args[0].getint()) else: @@ -817,7 +831,7 @@ p += round_up_to_4(loc.width) self.mc.CALL(x) self.mark_gc_roots() - self.mc.ADD(esp, imm(WORD * extra_on_stack)) + self.mc.ADD(esp, imm(extra_on_stack)) if size == 1: self.mc.AND(eax, imm(0xff)) elif size == 2: Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py Thu Oct 1 17:53:19 2009 @@ -110,7 +110,7 @@ if size == 1: res = mem(ebp, get_ebp_ofs(i)) elif size == 2: - res = mem64(ebp, get_ebp_ofs(i)) + res = mem64(ebp, get_ebp_ofs(i + 1)) else: print "Unimplemented size %d" % i raise NotImplementedError("unimplemented size %d" % i) @@ -157,7 +157,8 @@ def _process_inputargs(self, inputargs): # XXX we can sort out here by longevity if we need something # more optimal - locs = [None] * len(inputargs) + floatlocs = [None] * len(inputargs) + nonfloatlocs = [None] * len(inputargs) # Don't use all_regs[0] for passing arguments around a loop. # Must be kept in sync with consider_jump(). # XXX this should probably go to llsupport/regalloc.py @@ -177,17 +178,20 @@ else: reg = self.rm.try_allocate_reg(arg) if reg: - locs[i] = reg + loc = reg else: loc = self.sm.loc(arg, width_of_type[arg.type]) - locs[i] = loc + if arg.type == FLOAT: + floatlocs[i] = loc + else: + nonfloatlocs[i] = loc # otherwise we have it saved on stack, so no worry self.rm.free_regs.insert(0, tmpreg) self.xrm.free_regs.insert(0, xmmtmp) - assert tmpreg not in locs - assert xmmtmp not in locs + assert tmpreg not in nonfloatlocs + assert xmmtmp not in floatlocs self.possibly_free_vars(inputargs) - return locs + return nonfloatlocs, floatlocs def possibly_free_var(self, var): if var.type == FLOAT: @@ -317,7 +321,7 @@ self.xrm.position = i if op.has_no_side_effect() and op.result not in self.longevity: i += 1 - self.rm.possibly_free_vars(op.args) + self.possibly_free_vars(op.args) continue if self.can_optimize_cmp_op(op, i, operations): oplist[op.opnum](self, op, operations[i + 1]) @@ -823,7 +827,7 @@ assembler = self.assembler assert self.jump_target is None self.jump_target = op.jump_target - arglocs = assembler.target_arglocs(self.jump_target) + nonfloatlocs, floatlocs = assembler.target_arglocs(self.jump_target) # compute 'tmploc' to be all_regs[0] by spilling what is there box = TempBox() box1 = TempBox() @@ -831,11 +835,15 @@ tmploc = self.rm.force_allocate_reg(box, selected_reg=tmpreg) xmmtmp = X86XMMRegisterManager.all_regs[0] xmmtmploc = self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp) - src_locations = [self.loc(arg) for arg in op.args] - dst_locations = arglocs - assert tmploc not in dst_locations - remap_stack_layout(assembler, src_locations, dst_locations, tmploc, - xmmtmp) + # Part about non-floats + src_locations = [self.loc(arg) for arg in op.args if arg.type != FLOAT] + assert tmploc not in nonfloatlocs + dst_locations = [loc for loc in nonfloatlocs if loc is not None] + remap_stack_layout(assembler, src_locations, dst_locations, tmploc) + # Part about floats + src_locations = [self.loc(arg) for arg in op.args if arg.type == FLOAT] + dst_locations = [loc for loc in floatlocs if loc is not None] + remap_stack_layout(assembler, src_locations, dst_locations, xmmtmp) self.rm.possibly_free_var(box) self.xrm.possibly_free_var(box1) self.possibly_free_vars(op.args) From arigo at codespeak.net Thu Oct 1 18:32:27 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 1 Oct 2009 18:32:27 +0200 (CEST) Subject: [pypy-svn] r68111 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091001163227.303BF168025@codespeak.net> Author: arigo Date: Thu Oct 1 18:32:25 2009 New Revision: 68111 Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/test/test_basic.py pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py Log: Fix for ootype, sorry. Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/codewriter.py Thu Oct 1 18:32:25 2009 @@ -183,8 +183,12 @@ _, meth = T._lookup(methname) if not getattr(meth, 'abstract', False): assert meth.graph - jitcode = self.get_jitcode(meth.graph, - oosend_methdescr=methdescr) + if self.policy.look_inside_graph(meth.graph, + self.cpu.supports_floats): + jitcode = self.get_jitcode(meth.graph, + oosend_methdescr=methdescr) + else: + jitcode = None oocls = ootype.runtimeClass(T) jitcodes[oocls] = jitcode methdescr.setup(jitcodes) Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Thu Oct 1 18:32:25 2009 @@ -677,7 +677,13 @@ self.generate_guard(pc, rop.GUARD_CLASS, objbox, [clsbox]) oocls = clsbox.getref(ootype.Class) jitcode = methdescr.get_jitcode_for_class(oocls) - return self.perform_call(jitcode, varargs) + if jitcode is not None: + # we should follow calls to this graph + return self.perform_call(jitcode, varargs) + else: + # but we should not follow calls to that graph + return self.execute_varargs(rop.OOSEND, varargs, + descr=methdescr, exc=True) @arguments("box") def opimpl_strlen(self, str): Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Thu Oct 1 18:32:25 2009 @@ -906,7 +906,28 @@ res = self.interp_operations(f, [3, 5]) assert res == 5 self.check_history_(setfield_gc=2, getfield_gc_pure=1) - + + def test_oosend_look_inside_only_one(self): + class A: + pass + class B(A): + def g(self): + return 123 + class C(A): + @dont_look_inside + def g(self): + return 456 + def f(n): + if n > 3: + x = B() + else: + x = C() + return x.g() + x.g() + res = self.interp_operations(f, [10]) + assert res == 123 * 2 + res = self.interp_operations(f, [-10]) + assert res == 456 * 2 + class TestOOtype(BasicTests, OOJitMixin): def test_oohash(self): Modified: pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py Thu Oct 1 18:32:25 2009 @@ -14,9 +14,8 @@ class TestCodeWriter: - type_system = 'lltype' - def setup_method(self, _): + def make_graph(self, func, values, type_system='lltype'): class FakeMetaInterpSd: virtualizable_info = None def find_opcode(self, name): @@ -25,13 +24,31 @@ def _register_indirect_call_target(self, fnaddress, jitcode): self.indirectcalls.append((fnaddress, jitcode)) + class FakeMethDescr: + def __init__(self1, CLASS, methname): + self.methdescrs.append(self1) + self1.CLASS = CLASS + self1.methname = methname + self1.jitcodes = None + def setup(self1, jitcodes): + self1.jitcodes = jitcodes + self.methdescrs = [] + class FakeCPU: - ts = typesystem.llhelper supports_floats = False def fielddescrof(self, STRUCT, fieldname): return ('fielddescr', STRUCT, fieldname) def calldescrof(self, FUNC, NON_VOID_ARGS, RESULT): return ('calldescr', FUNC, NON_VOID_ARGS, RESULT) + def typedescrof(self, CLASS): + return ('typedescr', CLASS) + def methdescrof(self, CLASS, methname): + return FakeMethDescr(CLASS, methname) + + if type_system == 'lltype': + FakeCPU.ts = typesystem.llhelper + else: + FakeCPU.ts = typesystem.oohelper self.metainterp_sd = FakeMetaInterpSd() self.metainterp_sd.opcode_implementations = None @@ -39,9 +56,7 @@ self.metainterp_sd.indirectcalls = [] self.metainterp_sd.cpu = FakeCPU() - def make_graph(self, func, values): - rtyper = support.annotate(func, values, - type_system=self.type_system) + rtyper = support.annotate(func, values, type_system=type_system) self.metainterp_sd.cpu.rtyper = rtyper return rtyper.annotator.translator.graphs[0] @@ -100,6 +115,36 @@ in self.metainterp_sd.indirectcalls] assert dict.fromkeys(names) == {'g': None} + def test_oosend_look_inside_only_one(self): + class A: + pass + class B(A): + def g(self): + return 123 + class C(A): + @jit.dont_look_inside + def g(self): + return 456 + def f(n): + if n > 3: + x = B() + else: + x = C() + return x.g() + x.g() + graph = self.make_graph(f, [5], type_system='ootype') + cw = CodeWriter(self.metainterp_sd, JitPolicy()) + jitcode = cw.make_one_bytecode((graph, None), False) + assert len(self.methdescrs) == 1 + assert self.methdescrs[0].CLASS._name.endswith('.A') + assert self.methdescrs[0].methname == 'og' + assert len(self.methdescrs[0].jitcodes.keys()) == 2 + values = self.methdescrs[0].jitcodes.values() + values.sort() + assert values[0] is None + assert values[1].name == 'B.g' + for graph, _ in cw.all_graphs.keys(): + assert graph.name in ['f', 'B.g'] + def test_instantiate(self): class A1: id = 651 class A2(A1): id = 652 From arigo at codespeak.net Thu Oct 1 18:45:29 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 1 Oct 2009 18:45:29 +0200 (CEST) Subject: [pypy-svn] r68112 - pypy/branch/floats-via-sse2/pypy/jit/backend/x86 Message-ID: <20091001164529.AAA28168024@codespeak.net> Author: arigo Date: Thu Oct 1 18:45:28 2009 New Revision: 68112 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/jump.py Log: Simplify this, now that regalloc_load() and regalloc_store() are one and the same. Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/jump.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/jump.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/jump.py Thu Oct 1 18:45:28 2009 @@ -80,10 +80,7 @@ assert pending_dests == 0 def _move(assembler, src, dst, tmpreg): - if isinstance(dst, MODRM): - if isinstance(src, MODRM): - assembler.regalloc_mov(src, tmpreg) - src = tmpreg - assembler.regalloc_mov(src, dst) - else: - assembler.regalloc_mov(src, dst) + if isinstance(dst, MODRM) and isinstance(src, MODRM): + assembler.regalloc_mov(src, tmpreg) + src = tmpreg + assembler.regalloc_mov(src, dst) From arigo at codespeak.net Thu Oct 1 18:45:49 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 1 Oct 2009 18:45:49 +0200 (CEST) Subject: [pypy-svn] r68113 - pypy/branch/floats-via-sse2/pypy/jit/backend/test Message-ID: <20091001164549.629D1168024@codespeak.net> Author: arigo Date: Thu Oct 1 18:45:48 2009 New Revision: 68113 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Log: A test to pass float arguments across bridges. Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Thu Oct 1 18:45:48 2009 @@ -875,6 +875,45 @@ assert type(got) == type(val) assert got == val + def test_compile_bridge_float(self): + if not self.cpu.supports_floats: + py.test.skip("requires floats") + fboxes = [BoxFloat() for i in range(12)] + i2 = BoxInt() + faildescr1 = BasicFailDescr() + faildescr2 = BasicFailDescr() + operations = [ + ResOperation(rop.FLOAT_LE, [fboxes[0], ConstFloat(9.2)], i2), + ResOperation(rop.GUARD_TRUE, [i2], None), + ResOperation(rop.FINISH, fboxes, None, descr=faildescr2), + ] + operations[-2].suboperations = [ + ResOperation(rop.FAIL, fboxes, None, descr=faildescr1) + ] + operations[-1].jump_target = None + executable_token = self.cpu.compile_loop(fboxes, operations) + loop_token = LoopToken() + loop_token.executable_token = executable_token + + fboxes2 = [BoxFloat() for i in range(12)] + f3 = BoxFloat() + bridge = [ + ResOperation(rop.FLOAT_SUB, [fboxes2[0], ConstFloat(1.0)], f3), + ResOperation(rop.JUMP, [f3] + fboxes2[1:], None), + ] + bridge[-1].jump_target = loop_token + + self.cpu.compile_bridge(faildescr1, fboxes2, bridge) + + for i in range(len(fboxes)): + self.cpu.set_future_value_int(i, 13.5 + 6.73 * i) + fail = self.cpu.execute_token(executable_token) + assert fail is faildescr2 + res = self.cpu.get_latest_value_float(0) + assert res == 8.5 + for i in range(1, len(fboxes)): + assert self.cpu.get_latest_value_float(i) == fboxes[i].value + def test_unused_result_int(self): # check operations with no side effect and whose result # is not used. The backend is free to do nothing from them. From arigo at codespeak.net Thu Oct 1 18:53:08 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 1 Oct 2009 18:53:08 +0200 (CEST) Subject: [pypy-svn] r68114 - pypy/trunk/pypy/translator/c/gcc Message-ID: <20091001165308.674A7168024@codespeak.net> Author: arigo Date: Thu Oct 1 18:53:08 2009 New Revision: 68114 Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Log: Some people also see the JC operation sometimes, apparently. Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Thu Oct 1 18:53:08 2009 @@ -773,6 +773,8 @@ visit_jns = conditional_jump visit_jo = conditional_jump visit_jno = conditional_jump + visit_jc = conditional_jump + visit_jnc = conditional_jump def visit_xchgl(self, line): # only support the format used in VALGRIND_DISCARD_TRANSLATIONS From pedronis at codespeak.net Thu Oct 1 19:04:24 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 1 Oct 2009 19:04:24 +0200 (CEST) Subject: [pypy-svn] r68115 - in pypy/trunk/pypy/annotation: . test Message-ID: <20091001170424.53050168012@codespeak.net> Author: pedronis Date: Thu Oct 1 19:04:23 2009 New Revision: 68115 Modified: pypy/trunk/pypy/annotation/test/test_annrpython.py pypy/trunk/pypy/annotation/unaryop.py Log: (cfbolz, pedronis) op_contains of a dict should annotate SomeBool(const=False) as long as it is considered empty Modified: pypy/trunk/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/trunk/pypy/annotation/test/test_annrpython.py (original) +++ pypy/trunk/pypy/annotation/test/test_annrpython.py Thu Oct 1 19:04:23 2009 @@ -3196,6 +3196,33 @@ s = a.build_types(f, []) assert s.knowntype == int + def test_contains_of_empty_dict(self): + class A(object): + def meth(self): + return 1 + + def g(x, y): + d1 = {} + for i in range(y): + if x in d1: + return d1[x].meth() + d1[i+1] = A() + return 0 + + a = self.RPythonAnnotator() + s = a.build_types(g, [int, int]) + assert s.knowntype is int + + def f(x): + d0 = {} + if x in d0: + d0[x].meth() + return x+1 + + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + assert s.knowntype is int + def g(n): return [0,1,2,n] Modified: pypy/trunk/pypy/annotation/unaryop.py ============================================================================== --- pypy/trunk/pypy/annotation/unaryop.py (original) +++ pypy/trunk/pypy/annotation/unaryop.py Thu Oct 1 19:04:23 2009 @@ -375,10 +375,14 @@ class __extend__(SomeDict): - def len(dct): + def _is_empty(dct): s_key = dct.dictdef.read_key() s_value = dct.dictdef.read_value() - if isinstance(s_key, SomeImpossibleValue) or isinstance(s_value, SomeImpossibleValue): + return (isinstance(s_key, SomeImpossibleValue) or + isinstance(s_value, SomeImpossibleValue)) + + def len(dct): + if dct._is_empty(): return immutablevalue(0) return SomeObject.len(dct) @@ -443,6 +447,10 @@ def op_contains(dct, s_element): dct.dictdef.generalize_key(s_element) + if dct._is_empty(): + s_bool = SomeBool() + s_bool.const = False + return s_bool return s_Bool op_contains.can_only_throw = _can_only_throw From pedronis at codespeak.net Thu Oct 1 19:06:03 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 1 Oct 2009 19:06:03 +0200 (CEST) Subject: [pypy-svn] r68116 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091001170603.BE9E7168013@codespeak.net> Author: pedronis Date: Thu Oct 1 19:06:03 2009 New Revision: 68116 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/jit/metainterp/simple_optimize.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_resume.py Log: (cfbolz, pedronis) avoid flattening of the resume data first, tried to reduce loops and list construction, we compute a numbering only once at the end now Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Thu Oct 1 19:06:03 2009 @@ -56,6 +56,11 @@ def get_key_box(self): return self.box + def register_value(self, modifier): + box = self.get_key_box() # may be a Const, too + if modifier.register_box(box): + self.get_args_for_fail(modifier) + def get_args_for_fail(self, modifier): pass @@ -499,18 +504,9 @@ def store_final_boxes_in_guard(self, op): descr = op.descr assert isinstance(descr, compile.ResumeGuardDescr) - oldboxes = [] - args = resume.flatten_resumedata(descr) # xxx expensive - for box in args: - if box in self.values: - box = self.values[box].get_key_box() # may be a Const, too - oldboxes.append(box) - modifier = resume.ResumeDataVirtualAdder(descr, oldboxes) - for box in args: - if box in self.values: - value = self.values[box] - value.get_args_for_fail(modifier) - newboxes = modifier.finish() + modifier = resume.ResumeDataVirtualAdder(descr) + modifier.walk_snapshots(self.values) + newboxes = modifier.finish(self.values) descr.store_final_boxes(op, newboxes) def clean_fields_of_values(self, descr=None): Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Thu Oct 1 19:06:03 2009 @@ -58,73 +58,39 @@ snapshot = Snapshot(snapshot, virtualizable_boxes[:]) # xxx for now storage.rd_snapshot = snapshot -_placeholder = (None, 0, 0) - -def flatten_resumedata(storage): - frame_info_list = storage.rd_frame_info_list - storage.rd_frame_info_list = None - j = frame_info_list.level - frame_infos = [_placeholder]*j - j -= 1 - while True: # at least one - frame_infos[j] = (frame_info_list.jitcode, frame_info_list.pc, - frame_info_list.exception_target) - frame_info_list = frame_info_list.prev - if frame_info_list is None: - break - j -= 1 - storage.rd_frame_infos = frame_infos - memo = {} - consts = [] - liveboxes = [] - nums = [] - snapshots = [] - snapshot = storage.rd_snapshot - storage.rd_snapshot = None - while True: - snapshots.append(snapshot) - snapshot = snapshot.prev - if snapshot is None: - break - n = len(snapshots)-1 - while n >= 0: - boxes = snapshots[n].boxes - n -= 1 - for box in boxes: - assert box is not None - if isinstance(box, Box): - try: - num = memo[box] - except KeyError: - num = len(liveboxes) - liveboxes.append(box) - memo[box] = num - else: - num = -2 - len(consts) - consts.append(box) - nums.append(num) - nums.append(-1) - nums = nums[:] - storage.rd_nums = nums - storage.rd_consts = consts[:] - storage.rd_virtuals = None - return liveboxes VIRTUAL_FLAG = int((sys.maxint+1) // 2) assert not (VIRTUAL_FLAG & (VIRTUAL_FLAG-1)) # a power of two +_frame_info_placeholder = (None, 0, 0) + class ResumeDataVirtualAdder(object): - def __init__(self, storage, liveboxes): + def __init__(self, storage): self.storage = storage - self.consts = storage.rd_consts[:] - assert storage.rd_virtuals is None - self.original_liveboxes = liveboxes self.liveboxes = {} - self._register_boxes(liveboxes) + self.consts = [] self.virtuals = [] self.vfieldboxes = [] + def walk_snapshots(self, values): + nnums = 0 + snapshot = self.storage.rd_snapshot + assert snapshot + while True: # at least one + boxes = snapshot.boxes + nnums += len(boxes)+1 + for box in boxes: + if box in values: + value = values[box] + value.register_value(self) + else: + self.register_box(box) + snapshot = snapshot.prev + if snapshot is None: + break + self.nnums = nnums + def make_constant(self, box, const): # this part of the interface is not used so far by optimizeopt.py if self.liveboxes[box] == 0: @@ -149,26 +115,60 @@ self.vfieldboxes.append(fieldboxes) self._register_boxes(fieldboxes) + def register_box(self, box): + if isinstance(box, Box) and box not in self.liveboxes: + self.liveboxes[box] = 0 + return True + return False + def _register_boxes(self, boxes): for box in boxes: - if isinstance(box, Box) and box not in self.liveboxes: - self.liveboxes[box] = 0 + self.register_box(box) def is_virtual(self, virtualbox): return self.liveboxes[virtualbox] >= VIRTUAL_FLAG - def finish(self): + def _flatten_frame_info(self): + storage = self.storage + frame_info_list = storage.rd_frame_info_list + storage.rd_frame_info_list = None + j = frame_info_list.level + frame_infos = [_frame_info_placeholder]*j + j -= 1 + while True: # at least one + frame_infos[j] = (frame_info_list.jitcode, frame_info_list.pc, + frame_info_list.exception_target) + frame_info_list = frame_info_list.prev + if frame_info_list is None: + break + j -= 1 + storage.rd_frame_infos = frame_infos + + def finish(self, values): + self._flatten_frame_info() storage = self.storage liveboxes = [] for box in self.liveboxes.iterkeys(): if self.liveboxes[box] == 0: self.liveboxes[box] = len(liveboxes) liveboxes.append(box) - for i in range(len(storage.rd_nums)): - num = storage.rd_nums[i] - if num >= 0: - box = self.original_liveboxes[num] - storage.rd_nums[i] = self._getboxindex(box) + nums = storage.rd_nums = [0]*self.nnums + i = self.nnums-1 + snapshot = self.storage.rd_snapshot + while True: # at least one + boxes = snapshot.boxes + nums[i] = -1 + i -= 1 + for j in range(len(boxes)-1, -1, -1): + box = boxes[j] + if box in values: + box = values[box].get_key_box() + nums[i] = self._getboxindex(box) + i -= 1 + snapshot = snapshot.prev + if snapshot is None: + break + storage.rd_virtuals = None if len(self.virtuals) > 0: storage.rd_virtuals = self.virtuals[:] for i in range(len(storage.rd_virtuals)): @@ -177,6 +177,7 @@ vinfo.fieldnums = [self._getboxindex(box) for box in fieldboxes] storage.rd_consts = self.consts[:] + storage.rd_snapshot = None if debug: dump_storage(storage) return liveboxes Modified: pypy/trunk/pypy/jit/metainterp/simple_optimize.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/simple_optimize.py (original) +++ pypy/trunk/pypy/jit/metainterp/simple_optimize.py Thu Oct 1 19:06:03 2009 @@ -5,6 +5,8 @@ from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp import resume, compile +EMPTY_VALUES = {} + def optimize_loop(options, old_loops, loop, cpu=None): if old_loops: assert len(old_loops) == 1 @@ -19,8 +21,10 @@ if op.is_guard(): descr = op.descr assert isinstance(descr, compile.ResumeGuardDescr) - args = resume.flatten_resumedata(descr) - descr.store_final_boxes(op, args) + modifier = resume.ResumeDataVirtualAdder(descr) + modifier.walk_snapshots(EMPTY_VALUES) + newboxes = modifier.finish(EMPTY_VALUES) + descr.store_final_boxes(op, newboxes) newoperations.append(op) loop.operations = newoperations return None Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Thu Oct 1 19:06:03 2009 @@ -32,7 +32,6 @@ op = ResOperation(rop.GUARD_TRUE, [], None, descr=fdescr) # setup rd data fi = [("code0", 1, 2), ("code1", 3, -1)] - fdescr.rd_virtuals = None fi0 = resume.FrameInfo(None, FakeFrame("code0", 1, 2)) fdescr.rd_frame_info_list = resume.FrameInfo(fi0, FakeFrame("code1", 3, -1)) @@ -139,7 +138,6 @@ descr = Storage() descr.rd_frame_info_list = resume.FrameInfo(None, FakeFrame()) descr.rd_snapshot = resume.Snapshot(None, fail_args) - descr.rd_virtuals = None return descr def assert_equal(self, optimized, expected): @@ -1274,7 +1272,6 @@ fdescr.rd_frame_info_list = resume.FrameInfo(None, FakeFrame()) fdescr.rd_snapshot = resume.Snapshot(None, fail_args) - fdescr.virtuals = None self.oparse = oparse # fdescr = instantiate(FailDescr) Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Thu Oct 1 19:06:03 2009 @@ -9,21 +9,17 @@ class Storage: pass -def make_demo_storage(): +def test_simple_read(): b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] storage = Storage() storage.rd_frame_infos = [] - storage.rd_virtuals = None storage.rd_consts = [c1, c2, c3] storage.rd_nums = [0, -2, 0, 1, -1, -3, -4, -1, 0, 1, 2, -1 ] - return storage - -def test_simple(): - storage = make_demo_storage() + storage.rd_virtuals = None b1s, b2s, b3s = [BoxInt(), BoxPtr(), BoxInt()] assert b1s != b3s reader = ResumeDataReader(storage, [b1s, b2s, b3s]) @@ -40,7 +36,7 @@ storage.rd_frame_infos = [(1, 2, 5), (3, 4, 7)] storage.rd_consts = [] storage.rd_nums = [] - storage.rd_virtuals = None + storage.rd_virtuals = None # reader = ResumeDataReader(storage, []) assert reader.has_more_frame_infos() @@ -95,40 +91,6 @@ assert fi1.exception_target == 4 assert fi1.level == 2 -def test_flatten_resumedata(): - # temporary "expensive" mean to go from the new to the old world - b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] - c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] - - env = [b1, c1, b2, b1, c2] - snap = Snapshot(None, env) - frame = FakeFrame("JITCODE", 1, 2) - fi = FrameInfo(None, frame) - env1 = [c3, b3, b1, c1] - snap1 = Snapshot(snap, env1) - frame1 = FakeFrame("JITCODE1", 3, 4) - fi1 = FrameInfo(fi, frame1) - - storage = Storage() - storage.rd_snapshot = snap1 - storage.rd_frame_info_list = fi1 - - liveboxes = flatten_resumedata(storage) - assert storage.rd_snapshot is None - assert storage.rd_frame_info_list is None - - assert storage.rd_frame_infos == [("JITCODE", 1, 2), - ("JITCODE1", 3, 4)] - assert storage.rd_virtuals is None - assert liveboxes == [b1, b2, b3] - assert storage.rd_consts == [c1, c2, c3, c1] - # check with reading - reader = ResumeDataReader(storage, liveboxes) - l = reader.consume_boxes() - assert l == env - l = reader.consume_boxes() - assert l == env1 - def test_capture_resumedata(): b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] @@ -198,6 +160,57 @@ # ____________________________________________________________ +def test_walk_snapshots(): + b1, b2, b3 = [BoxInt(), BoxInt(), BoxInt()] + c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] + + env = [b1, c1, b2, b1, c2] + snap = Snapshot(None, env) + env1 = [c3, b3, b1, c1] + snap1 = Snapshot(snap, env1) + + storage = Storage() + storage.rd_snapshot = snap1 + + modifier = ResumeDataVirtualAdder(storage) + modifier.walk_snapshots({}) + + assert modifier.liveboxes == {b1: 0, b2: 0, b3: 0} + assert modifier.nnums == len(env)+1+len(env1)+1 + + b1_2 = BoxInt() + class FakeValue(object): + + def register_value(self, modifier): + modifier.register_box(b1_2) + val = FakeValue() + + storage = Storage() + storage.rd_snapshot = snap1 + + modifier = ResumeDataVirtualAdder(storage) + modifier.walk_snapshots({b1: val, b2: val}) + + assert modifier.liveboxes == {b1_2: 0, b3: 0} + assert modifier.nnums == len(env)+1+len(env1)+1 + +def test_flatten_frame_info(): + frame = FakeFrame("JITCODE", 1, 2) + fi = FrameInfo(None, frame) + frame1 = FakeFrame("JITCODE1", 3, 4) + fi1 = FrameInfo(fi, frame1) + + storage = Storage() + storage.rd_frame_info_list = fi1 + + modifier = ResumeDataVirtualAdder(storage) + modifier._flatten_frame_info() + assert storage.rd_frame_info_list is None + + assert storage.rd_frame_infos == [("JITCODE", 1, 2), + ("JITCODE1", 3, 4)] + + class MyMetaInterp: def __init__(self, cpu): self.cpu = cpu @@ -221,15 +234,27 @@ assert len(newboxes) == len(expected) return newboxes +def make_storage(b1, b2, b3): + storage = Storage() + snapshot = Snapshot(None, [b1, ConstInt(1), b1, b2]) + snapshot = Snapshot(snapshot, [ConstInt(2), ConstInt(3)]) + snapshot = Snapshot(snapshot, [b1, b2, b3]) + storage.rd_snapshot = snapshot + # just to placate _flatten_frame_info + storage.rd_frame_info_list = FrameInfo(None, FakeFrame("", 0, -1)) + return storage + def test_virtual_adder_no_op(): - storage = make_demo_storage() b1s, b2s, b3s = [BoxInt(1), BoxPtr(), BoxInt(3)] - modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s]) + storage = make_storage(b1s, b2s, b3s) + modifier = ResumeDataVirtualAdder(storage) + modifier.walk_snapshots({}) assert not modifier.is_virtual(b1s) assert not modifier.is_virtual(b2s) assert not modifier.is_virtual(b3s) # done - liveboxes = modifier.finish() + liveboxes = modifier.finish({}) + assert storage.rd_snapshot is None b1t, b2t, b3t = [BoxInt(11), BoxPtr(demo55o), BoxInt(33)] newboxes = _resume_remap(liveboxes, [b1s, b2s, b3s], b1t, b2t, b3t) metainterp = MyMetaInterp(LLtypeMixin.cpu) @@ -242,16 +267,46 @@ assert lst == [b1t, b2t, b3t] assert metainterp.trace == [] +def test_virtual_adder_no_op_renaming(): + b1s, b2s, b3s = [BoxInt(1), BoxInt(2), BoxInt(3)] + storage = make_storage(b1s, b2s, b3s) + modifier = ResumeDataVirtualAdder(storage) + b1_2 = BoxInt() + class FakeValue(object): + + def register_value(self, modifier): + modifier.register_box(b1_2) + + def get_key_box(self): + return b1_2 + + val = FakeValue() + values = {b1s: val, b2s: val} + modifier.walk_snapshots(values) + assert not modifier.is_virtual(b1_2) + assert not modifier.is_virtual(b3s) + # done + liveboxes = modifier.finish(values) + assert storage.rd_snapshot is None + b1t, b3t = [BoxInt(11), BoxInt(33)] + newboxes = _resume_remap(liveboxes, [b1_2, b3s], b1t, b3t) + metainterp = MyMetaInterp(LLtypeMixin.cpu) + reader = ResumeDataReader(storage, newboxes, metainterp) + lst = reader.consume_boxes() + assert lst == [b1t, ConstInt(1), b1t, b1t] + lst = reader.consume_boxes() + assert lst == [ConstInt(2), ConstInt(3)] + lst = reader.consume_boxes() + assert lst == [b1t, b1t, b3t] + assert metainterp.trace == [] def test_virtual_adder_make_virtual(): - storage = make_demo_storage() b1s, b2s, b3s, b4s, b5s = [BoxInt(1), BoxPtr(), BoxInt(3), - BoxPtr(), BoxPtr()] + BoxPtr(), BoxPtr()] c1s = ConstInt(111) - modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s]) - assert not modifier.is_virtual(b1s) - assert not modifier.is_virtual(b2s) - assert not modifier.is_virtual(b3s) + storage = make_storage(b1s, b2s, b3s) + modifier = ResumeDataVirtualAdder(storage) + modifier.walk_snapshots({}) modifier.make_virtual(b2s, ConstAddr(LLtypeMixin.node_vtable_adr, LLtypeMixin.cpu), @@ -269,7 +324,7 @@ assert modifier.is_virtual(b4s) assert not modifier.is_virtual(b5s) # done - liveboxes = modifier.finish() + liveboxes = modifier.finish({}) b1t, b3t, b5t = [BoxInt(11), BoxInt(33), BoxPtr(demo55o)] newboxes = _resume_remap(liveboxes, [b1s, #b2s -- virtual @@ -315,22 +370,24 @@ def test_virtual_adder_make_constant(): for testnumber in [0, 1]: - storage = make_demo_storage() b1s, b2s, b3s = [BoxInt(1), BoxPtr(), BoxInt(3)] if testnumber == 0: - # I. making a constant with make_constant() - modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s]) + # I. making a constant by directly specifying a constant in + # the list of liveboxes + b1s = ConstInt(111) + storage = make_storage(b1s, b2s, b3s) + modifier = ResumeDataVirtualAdder(storage) + modifier.walk_snapshots({}) + + if testnumber == 1: + # II. making a constant with make_constant() modifier.make_constant(b1s, ConstInt(111)) assert not modifier.is_virtual(b1s) - else: - # II. making a constant by directly specifying a constant in - # the list of liveboxes - b1s = ConstInt(111) - modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s]) + assert not modifier.is_virtual(b2s) assert not modifier.is_virtual(b3s) # done - liveboxes = modifier.finish() + liveboxes = modifier.finish({}) b2t, b3t = [BoxPtr(demo55o), BoxInt(33)] newboxes = _resume_remap(liveboxes, [b2s, b3s], b2t, b3t) metainterp = MyMetaInterp(LLtypeMixin.cpu) @@ -346,13 +403,11 @@ def test_virtual_adder_make_varray(): - storage = make_demo_storage() b1s, b2s, b3s, b4s = [BoxInt(1), BoxPtr(), BoxInt(3), BoxInt(4)] c1s = ConstInt(111) - modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s]) - assert not modifier.is_virtual(b1s) - assert not modifier.is_virtual(b2s) - assert not modifier.is_virtual(b3s) + storage = make_storage(b1s, b2s, b3s) + modifier = ResumeDataVirtualAdder(storage) + modifier.walk_snapshots({}) modifier.make_varray(b2s, LLtypeMixin.arraydescr, [b4s, c1s]) # new fields @@ -361,7 +416,7 @@ assert not modifier.is_virtual(b3s) assert not modifier.is_virtual(b4s) # done - liveboxes = modifier.finish() + liveboxes = modifier.finish({}) b1t, b3t, b4t = [BoxInt(11), BoxInt(33), BoxInt(44)] newboxes = _resume_remap(liveboxes, [b1s, #b2s -- virtual @@ -395,13 +450,11 @@ def test_virtual_adder_make_vstruct(): - storage = make_demo_storage() b1s, b2s, b3s, b4s = [BoxInt(1), BoxPtr(), BoxInt(3), BoxPtr()] c1s = ConstInt(111) - modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s]) - assert not modifier.is_virtual(b1s) - assert not modifier.is_virtual(b2s) - assert not modifier.is_virtual(b3s) + storage = make_storage(b1s, b2s, b3s) + modifier = ResumeDataVirtualAdder(storage) + modifier.walk_snapshots({}) modifier.make_vstruct(b2s, LLtypeMixin.ssize, [LLtypeMixin.adescr, LLtypeMixin.bdescr], @@ -411,7 +464,7 @@ assert not modifier.is_virtual(b3s) assert not modifier.is_virtual(b4s) # done - liveboxes = modifier.finish() + liveboxes = modifier.finish({}) b1t, b3t, b4t = [BoxInt(11), BoxInt(33), BoxPtr()] newboxes = _resume_remap(liveboxes, [b1s, #b2s -- virtual From fijal at codespeak.net Thu Oct 1 19:16:45 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 19:16:45 +0200 (CEST) Subject: [pypy-svn] r68117 - pypy/branch/floats-via-sse2/pypy/jit/backend/test Message-ID: <20091001171645.9FFD0168013@codespeak.net> Author: fijal Date: Thu Oct 1 19:16:45 2009 New Revision: 68117 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Log: I suppose that was the idea... Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Thu Oct 1 19:16:45 2009 @@ -906,7 +906,7 @@ self.cpu.compile_bridge(faildescr1, fboxes2, bridge) for i in range(len(fboxes)): - self.cpu.set_future_value_int(i, 13.5 + 6.73 * i) + self.cpu.set_future_value_float(i, 13.5 + 6.73 * i) fail = self.cpu.execute_token(executable_token) assert fail is faildescr2 res = self.cpu.get_latest_value_float(0) From arigo at codespeak.net Thu Oct 1 19:23:16 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 1 Oct 2009 19:23:16 +0200 (CEST) Subject: [pypy-svn] r68118 - pypy/branch/floats-via-sse2/pypy/jit/backend/test Message-ID: <20091001172316.5DB81168013@codespeak.net> Author: arigo Date: Thu Oct 1 19:23:15 2009 New Revision: 68118 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Log: Fix the test. Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/test/runner_test.py Thu Oct 1 19:23:15 2009 @@ -912,7 +912,7 @@ res = self.cpu.get_latest_value_float(0) assert res == 8.5 for i in range(1, len(fboxes)): - assert self.cpu.get_latest_value_float(i) == fboxes[i].value + assert self.cpu.get_latest_value_float(i) == 13.5 + 6.73 * i def test_unused_result_int(self): # check operations with no side effect and whose result From fijal at codespeak.net Thu Oct 1 19:23:50 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 19:23:50 +0200 (CEST) Subject: [pypy-svn] r68119 - pypy/branch/floats-via-sse2/pypy/jit/backend/x86 Message-ID: <20091001172350.778E0168013@codespeak.net> Author: fijal Date: Thu Oct 1 19:23:50 2009 New Revision: 68119 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py Log: Good, now that test is fixed, it passes even on x86 backend Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py Thu Oct 1 19:23:50 2009 @@ -732,29 +732,36 @@ return addr def generate_failure(self, mc, faildescr, failargs, locs, exc): + nonfloatlocs, floatlocs = locs assert len(failargs) < MAX_FAIL_BOXES pos = mc.tell() - for i in range(len(locs)): - loc = locs[i] - if isinstance(loc, REG): - if failargs[i].type == FLOAT: + for i in range(len(failargs)): + arg = failargs[i] + if arg.type == FLOAT: + loc = floatlocs[i] + if isinstance(loc, REG): mc.MOVSD(addr64_add(imm(self.fail_box_float_addr), imm(i*WORD*2)), loc) - else: - if failargs[i].type == REF: + else: + loc = nonfloatlocs[i] + if isinstance(loc, REG): + if arg.type == REF: base = self.fail_box_ptr_addr else: base = self.fail_box_int_addr mc.MOV(addr_add(imm(base), imm(i*WORD)), loc) - for i in range(len(locs)): - loc = locs[i] - if not isinstance(loc, REG): - if failargs[i].type == FLOAT: + for i in range(len(failargs)): + arg = failargs[i] + if arg.type == FLOAT: + loc = floatlocs[i] + if not isinstance(loc, REG): mc.MOVSD(xmm0, loc) mc.MOVSD(addr64_add(imm(self.fail_box_float_addr), imm(i*WORD*2)), xmm0) - else: - if failargs[i].type == REF: + else: + loc = nonfloatlocs[i] + if not isinstance(loc, REG): + if arg.type == REF: base = self.fail_box_ptr_addr else: base = self.fail_box_int_addr @@ -763,7 +770,7 @@ if self.debug_markers: mc.MOV(eax, imm(pos)) mc.MOV(addr_add(imm(self.fail_box_int_addr), - imm(len(locs) * WORD)), + imm(len(nonfloatlocs) * WORD)), eax) # we call a provided function that will Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py Thu Oct 1 19:23:50 2009 @@ -236,26 +236,34 @@ def _update_bindings(self, locs, args): # XXX this should probably go to llsupport/regalloc.py - newlocs = [] - for loc in locs: - if not isinstance(loc, IMM8) and not isinstance(loc, IMM32): - newlocs.append(loc) - locs = newlocs - assert len(locs) == len(args) + nonfloatlocs, floatlocs = locs used = {} - for i in range(len(locs)): - v = args[i] - loc = locs[i] - if isinstance(loc, REG) and self.longevity[v][1] > -1: - # XXX xmm regs - self.rm.reg_bindings[v] = loc - used[loc] = None + for i in range(len(args)): + arg = args[i] + if arg.type == FLOAT: + loc = floatlocs[i] + if isinstance(loc, REG): + self.xrm.reg_bindings[arg] = loc + used[loc] = None + else: + self.sm.stack_bindings[arg] = loc else: - self.sm.stack_bindings[v] = loc + loc = nonfloatlocs[i] + if isinstance(loc, IMM8) or isinstance(loc, IMM32): + continue + if isinstance(loc, REG): + self.rm.bindings[arg] = loc + used[loc] = None + else: + self.sm.stack_bindings[arg] = loc self.rm.free_regs = [] for reg in X86RegisterManager.all_regs: if reg not in used: self.rm.free_regs.append(reg) + self.xrm.free_regs = [] + for reg in X86XMMRegisterManager.all_regs: + if reg not in used: + self.xrm.free_regs.append(reg) self.rm._check_invariants() self.xrm._check_invariants() @@ -268,7 +276,19 @@ assert len(guard_op.suboperations) == 1 fail_op = guard_op.suboperations[0] assert fail_op.opnum == rop.FAIL - return [self.loc(v) for v in fail_op.args] + return self.locs_for_op(fail_op) + + def locs_for_op(self, fail_op): + nonfloatlocs = [None] * len(fail_op.args) + floatlocs = [None] * len(fail_op.args) + for i in range(len(fail_op.args)): + v = fail_op.args[i] + loc = self.loc(v) + if v.type == FLOAT: + floatlocs[i] = loc + else: + nonfloatlocs[i] = loc + return nonfloatlocs, floatlocs def perform_with_guard(self, op, guard_op, arglocs, result_loc): faillocs = self.locs_for_fail(guard_op) @@ -381,7 +401,7 @@ consider_guard_false = _consider_guard def consider_fail(self, op, ignored): - locs = [self.loc(arg) for arg in op.args] + locs = self.locs_for_op(op) self.assembler.generate_failure(self.assembler.mc, op.descr, op.args, locs, self.exc) self.possibly_free_vars(op.args) From fijal at codespeak.net Thu Oct 1 19:24:42 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 19:24:42 +0200 (CEST) Subject: [pypy-svn] r68120 - pypy/branch/floats-via-sse2/pypy/jit/backend/x86 Message-ID: <20091001172442.8930D168013@codespeak.net> Author: fijal Date: Thu Oct 1 19:24:41 2009 New Revision: 68120 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py Log: typo Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py Thu Oct 1 19:24:41 2009 @@ -252,7 +252,7 @@ if isinstance(loc, IMM8) or isinstance(loc, IMM32): continue if isinstance(loc, REG): - self.rm.bindings[arg] = loc + self.rm.reg_bindings[arg] = loc used[loc] = None else: self.sm.stack_bindings[arg] = loc From arigo at codespeak.net Thu Oct 1 19:36:52 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 1 Oct 2009 19:36:52 +0200 (CEST) Subject: [pypy-svn] r68121 - pypy/branch/kill-jumptarget Message-ID: <20091001173652.B9DBB168020@codespeak.net> Author: arigo Date: Thu Oct 1 19:36:50 2009 New Revision: 68121 Removed: pypy/branch/kill-jumptarget/ Log: Branch was merged. From fijal at codespeak.net Thu Oct 1 19:40:35 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 19:40:35 +0200 (CEST) Subject: [pypy-svn] r68122 - in pypy/branch/floats-via-sse2/pypy/jit/backend/x86: . test Message-ID: <20091001174035.C8493168013@codespeak.net> Author: fijal Date: Thu Oct 1 19:40:35 2009 New Revision: 68122 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py Log: Fix it fix it fix it. Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py Thu Oct 1 19:40:35 2009 @@ -722,28 +722,36 @@ self.mc.CMP(mem(locs[0], offset), locs[1]) return self.implement_guard(addr, self.mc.JNE) + def _no_const_locs(self, args, locs): + """ returns those locs which correspond to non-const args + """ + newlocs = [] + for i in range(len(args)): + arg = args[i] + if isinstance(arg, Box): + newlocs.append(locs[i]) + return newlocs + def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): addr = self.mc2.tell() exc = (guard_opnum == rop.GUARD_EXCEPTION or guard_opnum == rop.GUARD_NO_EXCEPTION) - faildescr._x86_faillocs = fail_locs + faildescr._x86_faillocs = self._no_const_locs(failargs, fail_locs) self.generate_failure(self.mc2, faildescr, failargs, fail_locs, exc) return addr def generate_failure(self, mc, faildescr, failargs, locs, exc): - nonfloatlocs, floatlocs = locs assert len(failargs) < MAX_FAIL_BOXES pos = mc.tell() for i in range(len(failargs)): arg = failargs[i] + loc = locs[i] if arg.type == FLOAT: - loc = floatlocs[i] if isinstance(loc, REG): mc.MOVSD(addr64_add(imm(self.fail_box_float_addr), imm(i*WORD*2)), loc) else: - loc = nonfloatlocs[i] if isinstance(loc, REG): if arg.type == REF: base = self.fail_box_ptr_addr @@ -752,14 +760,13 @@ mc.MOV(addr_add(imm(base), imm(i*WORD)), loc) for i in range(len(failargs)): arg = failargs[i] + loc = locs[i] if arg.type == FLOAT: - loc = floatlocs[i] if not isinstance(loc, REG): mc.MOVSD(xmm0, loc) mc.MOVSD(addr64_add(imm(self.fail_box_float_addr), imm(i*WORD*2)), xmm0) else: - loc = nonfloatlocs[i] if not isinstance(loc, REG): if arg.type == REF: base = self.fail_box_ptr_addr @@ -770,7 +777,7 @@ if self.debug_markers: mc.MOV(eax, imm(pos)) mc.MOV(addr_add(imm(self.fail_box_int_addr), - imm(len(nonfloatlocs) * WORD)), + imm(len(locs) * WORD)), eax) # we call a provided function that will Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/regalloc.py Thu Oct 1 19:40:35 2009 @@ -236,21 +236,17 @@ def _update_bindings(self, locs, args): # XXX this should probably go to llsupport/regalloc.py - nonfloatlocs, floatlocs = locs used = {} for i in range(len(args)): arg = args[i] + loc = locs[i] if arg.type == FLOAT: - loc = floatlocs[i] if isinstance(loc, REG): self.xrm.reg_bindings[arg] = loc used[loc] = None else: self.sm.stack_bindings[arg] = loc else: - loc = nonfloatlocs[i] - if isinstance(loc, IMM8) or isinstance(loc, IMM32): - continue if isinstance(loc, REG): self.rm.reg_bindings[arg] = loc used[loc] = None @@ -264,6 +260,7 @@ for reg in X86XMMRegisterManager.all_regs: if reg not in used: self.xrm.free_regs.append(reg) + self.possibly_free_vars(args) self.rm._check_invariants() self.xrm._check_invariants() @@ -279,16 +276,7 @@ return self.locs_for_op(fail_op) def locs_for_op(self, fail_op): - nonfloatlocs = [None] * len(fail_op.args) - floatlocs = [None] * len(fail_op.args) - for i in range(len(fail_op.args)): - v = fail_op.args[i] - loc = self.loc(v) - if v.type == FLOAT: - floatlocs[i] = loc - else: - nonfloatlocs[i] = loc - return nonfloatlocs, floatlocs + return [self.loc(v) for v in fail_op.args] def perform_with_guard(self, op, guard_op, arglocs, result_loc): faillocs = self.locs_for_fail(guard_op) Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py Thu Oct 1 19:40:35 2009 @@ -524,7 +524,7 @@ ops.append('fail(f%d)' % (BASE_CONSTANT_SIZE * 2)) ops = "\n".join(ops) self.interpret(ops, [0.1]) - assert abs(self.getfloat(0) - (1 + BASE_CONSTANT_SIZE * 2) * 3.5 + 0.1) < 0.00001 + assert abs(self.getfloat(0) - (BASE_CONSTANT_SIZE * 2) * 3.5 - 0.1) < 0.00001 def test_lt_const(self): ops = ''' @@ -534,3 +534,24 @@ ''' self.interpret(ops, [0.1]) assert self.getint(0) == 0 + + def test_bug_wrong_stack_adj(self): + ops = ''' + [i0, i1, i2, i3, i4, i5, i6, i7, i8] + guard_true(i0) + fail(3.5, i0, i1, i2, i3, i4, i5, i6, i7, i8) + fail(4.5, i0, i1, i2, i3, i4, i5, i6, i7, i8) + ''' + loop = self.interpret(ops, [0, 1, 2, 3, 4, 5, 6, 7, 8]) + assert self.getint(0) == 0 + bridge_ops = ''' + [i0, i1, i2, i3, i4, i5, i6, i7, i8] + call(ConstClass(raising_fptr), 0, descr=raising_calldescr) + fail(i0, i1, i2, i3, i4, i5, i6, i7, i8) + ''' + self.attach_bridge(bridge_ops, loop, 0) + for i in range(9): + self.cpu.set_future_value_int(i, i) + self.run(loop) + assert self.getints(9) == range(9) + assert self.getfloat(0) == 3.5 From fijal at codespeak.net Thu Oct 1 20:05:56 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 1 Oct 2009 20:05:56 +0200 (CEST) Subject: [pypy-svn] r68123 - pypy/branch/floats-via-sse2/pypy/jit/backend/x86 Message-ID: <20091001180556.600C0168014@codespeak.net> Author: fijal Date: Thu Oct 1 20:05:55 2009 New Revision: 68123 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py Log: Fix translation Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/x86/assembler.py Thu Oct 1 20:05:55 2009 @@ -67,7 +67,7 @@ _x86_loop_code = 0 _x86_bootstrap_code = 0 _x86_stack_depth = 0 - _x86_arglocs = None + _x86_arglocs = (None, None) class Assembler386(object): mc = None From cfbolz at codespeak.net Thu Oct 1 21:06:04 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 1 Oct 2009 21:06:04 +0200 (CEST) Subject: [pypy-svn] r68124 - pypy/trunk/pypy/objspace/std Message-ID: <20091001190604.D2CCC16801E@codespeak.net> Author: cfbolz Date: Thu Oct 1 21:06:01 2009 New Revision: 68124 Modified: pypy/trunk/pypy/objspace/std/sharingdict.py Log: kill this assert: when jitting it produces sillyness Modified: pypy/trunk/pypy/objspace/std/sharingdict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/sharingdict.py (original) +++ pypy/trunk/pypy/objspace/std/sharingdict.py Thu Oct 1 21:06:01 2009 @@ -119,7 +119,6 @@ self.entries[new_structure.length - 1] = w_value assert self.structure.length + 1 == new_structure.length self.structure = new_structure - assert self.structure.keys[key] >= 0 return self def delitem(self, w_key): From pedronis at codespeak.net Thu Oct 1 22:55:52 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 1 Oct 2009 22:55:52 +0200 (CEST) Subject: [pypy-svn] r68125 - pypy/trunk/pypy/translator/platform/test Message-ID: <20091001205552.5A8B8168024@codespeak.net> Author: pedronis Date: Thu Oct 1 22:55:50 2009 New Revision: 68125 Modified: pypy/trunk/pypy/translator/platform/test/test_darwin.py Log: fix skip Modified: pypy/trunk/pypy/translator/platform/test/test_darwin.py ============================================================================== --- pypy/trunk/pypy/translator/platform/test/test_darwin.py (original) +++ pypy/trunk/pypy/translator/platform/test/test_darwin.py Thu Oct 1 22:55:50 2009 @@ -2,8 +2,8 @@ """ File containing darwin platform tests """ -import py, os -if os.name != 'darwin': +import py, sys +if sys.platform != 'darwin': py.test.skip("Darwin only") from pypy.tool.udir import udir From arigo at codespeak.net Fri Oct 2 13:24:16 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Oct 2009 13:24:16 +0200 (CEST) Subject: [pypy-svn] r68126 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091002112416.40FF2168043@codespeak.net> Author: arigo Date: Fri Oct 2 13:24:13 2009 New Revision: 68126 Modified: pypy/trunk/pypy/jit/metainterp/compile.py pypy/trunk/pypy/jit/metainterp/jitprof.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/test/test_basic.py pypy/trunk/pypy/jit/metainterp/test/test_compile.py pypy/trunk/pypy/jit/metainterp/test/test_jitprof.py pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py pypy/trunk/pypy/jit/metainterp/test/test_zrpy_basic.py pypy/trunk/pypy/jit/metainterp/warmspot.py Log: (pedronis, arigo) Make debug_level and profiler interaction sane. The profiler class is no longer selectable at run-time (see issue #448), but we can turn on or off the final printing. Modified: pypy/trunk/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/compile.py (original) +++ pypy/trunk/pypy/jit/metainterp/compile.py Fri Oct 2 13:24:13 2009 @@ -12,7 +12,6 @@ from pypy.jit.metainterp.specnode import NotSpecNode, more_general_specnodes from pypy.jit.metainterp.typesystem import llhelper, oohelper from pypy.jit.metainterp.optimizeutil import InvalidLoop -from pypy.rlib.debug import debug_print def show_loop(metainterp_sd, loop=None, error=None): # debugging @@ -64,8 +63,7 @@ except InvalidLoop: return None if old_loop_token is not None: - if metainterp.staticdata.state.debug > 1: - debug_print("reusing old loop") + metainterp.staticdata.log("reusing old loop") return old_loop_token send_loop_to_backend(metainterp_sd, loop, "loop") insert_loop_token(old_loop_tokens, loop_token) @@ -86,40 +84,32 @@ def send_loop_to_backend(metainterp_sd, loop, type): metainterp_sd.options.logger_ops.log_loop(loop.inputargs, loop.operations) - metainterp_sd.state.profiler.start_backend() + metainterp_sd.profiler.start_backend() if not we_are_translated(): show_loop(metainterp_sd, loop) loop.check_consistency() metainterp_sd.cpu.compile_loop(loop.inputargs, loop.operations, loop.token) - metainterp_sd.state.profiler.end_backend() + metainterp_sd.profiler.end_backend() metainterp_sd.stats.add_new_loop(loop) if not we_are_translated(): if type != "entry bridge": metainterp_sd.stats.compiled() else: loop._ignore_during_counting = True - if metainterp_sd.state.debug > 1: - log.info("compiled new " + type) - else: - if metainterp_sd.state.debug > 1: - debug_print("compiled new " + type) + metainterp_sd.log("compiled new " + type) def send_bridge_to_backend(metainterp_sd, faildescr, inputargs, operations): metainterp_sd.options.logger_ops.log_loop(inputargs, operations) - metainterp_sd.state.profiler.start_backend() + metainterp_sd.profiler.start_backend() if not we_are_translated(): show_loop(metainterp_sd) TreeLoop.check_consistency_of(inputargs, operations) pass metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations) - metainterp_sd.state.profiler.end_backend() + metainterp_sd.profiler.end_backend() if not we_are_translated(): - if metainterp_sd.state.debug > 1: - metainterp_sd.stats.compiled() - log.info("compiled new bridge") - else: - if metainterp_sd.state.debug > 1: - debug_print("compiled new bridge") + metainterp_sd.stats.compiled() + metainterp_sd.log("compiled new bridge") # ____________________________________________________________ Modified: pypy/trunk/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/jitprof.py (original) +++ pypy/trunk/pypy/jit/metainterp/jitprof.py Fri Oct 2 13:24:13 2009 @@ -36,6 +36,9 @@ def finish(self): pass + def set_printing(self, printing): + pass + def start_tracing(self): pass @@ -75,6 +78,7 @@ counters = None calls = None current = None + printing = True def start(self): self.starttime = self.timer() @@ -86,7 +90,11 @@ def finish(self): self.tk = self.timer() - self.print_stats() + if self.printing: + self.print_stats() + + def set_printing(self, printing): + self.printing = printing def _start(self, event): t0 = self.t1 Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Fri Oct 2 13:24:13 2009 @@ -10,7 +10,7 @@ from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp import codewriter, executor from pypy.jit.metainterp.logger import Logger -from pypy.jit.metainterp.jitprof import BLACKHOLED_OPS +from pypy.jit.metainterp.jitprof import BLACKHOLED_OPS, EmptyProfiler from pypy.jit.metainterp.jitprof import GUARDS, RECORDED_OPS from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import specialize @@ -796,8 +796,8 @@ def opimpl_jit_merge_point(self, pc): if not self.metainterp.is_blackholing(): self.generate_merge_point(pc, self.env) - if self.metainterp.staticdata.state.debug > DEBUG_PROFILE: - self.debug_merge_point() + # xxx we may disable the following line in some context later + self.debug_merge_point() if self.metainterp.seen_can_enter_jit: self.metainterp.seen_can_enter_jit = False try: @@ -872,7 +872,7 @@ self.pc = pc self.exception_target = exception_target self.env = env - if self.metainterp.staticdata.state.debug >= DEBUG_DETAILED: + if self.metainterp.staticdata.state.debug_level >= DEBUG_DETAILED: values = ' '.join([box.repr_rpython() for box in self.env]) log = self.metainterp.staticdata.log log('setup_resume_at_op %s:%d [%s] %d' % (self.jitcode.name, @@ -916,7 +916,7 @@ virtualizable_boxes = metainterp.virtualizable_boxes resume.capture_resumedata(metainterp.framestack, virtualizable_boxes, resumedescr) - self.metainterp.staticdata.state.profiler.count_ops(opnum, GUARDS) + self.metainterp.staticdata.profiler.count_ops(opnum, GUARDS) # count metainterp.attach_debug_info(guard_op) self.pc = saved_pc @@ -960,7 +960,7 @@ virtualizable_info = None def __init__(self, portal_graph, graphs, cpu, stats, options, - profile=None, warmrunnerdesc=None, + ProfilerClass=EmptyProfiler, warmrunnerdesc=None, leave_graph=None): self.portal_graph = portal_graph self.cpu = cpu @@ -976,6 +976,8 @@ self.opcode_names = [] self.opname_to_index = {} + self.profiler = ProfilerClass() + self.indirectcall_keys = [] self.indirectcall_values = [] @@ -1008,6 +1010,9 @@ self._setup_class_sizes() self.cpu.setup_once() self.log(self.jit_starting_line) + if not self.profiler.initialized: + self.profiler.start() + self.profiler.initialized = True self.globaldata.initialized = True self.options.logger_noopt.create_log('.noopt') self.options.logger_ops.create_log('.ops') @@ -1074,7 +1079,7 @@ # ---------------- logging ------------------------ def log(self, msg, event_kind='info'): - if self.state.debug > DEBUG_PROFILE: + if self.state.debug_level > DEBUG_PROFILE: if not we_are_translated(): getattr(history.log, event_kind)(msg) else: @@ -1224,7 +1229,7 @@ history.check_descr(descr) assert opnum != rop.CALL and opnum != rop.OOSEND # execute the operation - profiler = self.staticdata.state.profiler + profiler = self.staticdata.profiler profiler.count_ops(opnum) resbox = executor.execute(self.cpu, opnum, descr, *argboxes) if self.is_blackholing(): @@ -1246,7 +1251,7 @@ if require_attention and not self.is_blackholing(): self.before_residual_call() # execute the operation - profiler = self.staticdata.state.profiler + profiler = self.staticdata.profiler profiler.count_ops(opnum) resbox = executor.execute_varargs(self.cpu, opnum, argboxes, descr) if self.is_blackholing(): @@ -1285,7 +1290,7 @@ def _record_helper_nonpure_varargs(self, opnum, resbox, descr, argboxes): assert resbox is None or isinstance(resbox, Box) # record the operation - profiler = self.staticdata.state.profiler + profiler = self.staticdata.profiler profiler.count_ops(opnum, RECORDED_OPS) op = self.history.record(opnum, argboxes, resbox, descr) self.attach_debug_info(op) @@ -1301,8 +1306,8 @@ self.history = None # start blackholing self.staticdata.stats.aborted() self.staticdata.log('~~~ ABORTING TRACING', event_kind='event') - self.staticdata.state.profiler.end_tracing() - self.staticdata.state.profiler.start_blackhole() + self.staticdata.profiler.end_tracing() + self.staticdata.profiler.start_blackhole() def switch_to_blackhole_if_trace_too_long(self): if not self.is_blackholing(): @@ -1324,9 +1329,9 @@ self.check_recursion_invariant() finally: if self.is_blackholing(): - self.staticdata.state.profiler.end_blackhole() + self.staticdata.profiler.end_blackhole() else: - self.staticdata.state.profiler.end_tracing() + self.staticdata.profiler.end_tracing() self.staticdata.log('~~~ LEAVE' + self.blackholing_text(), event_kind='event') @@ -1545,7 +1550,7 @@ def initialize_state_from_start(self, *args): self.in_recursion = -1 # always one portal around self.staticdata._setup_once() - self.staticdata.state.profiler.start_tracing() + self.staticdata.profiler.start_tracing() self.create_empty_history() num_green_args = self.staticdata.num_green_args original_boxes = [] @@ -1567,9 +1572,9 @@ if must_compile: self.history = history.History(self.cpu) self.history.inputargs = inputargs - self.staticdata.state.profiler.start_tracing() + self.staticdata.profiler.start_tracing() else: - self.staticdata.state.profiler.start_blackhole() + self.staticdata.profiler.start_blackhole() self.history = None # this means that is_blackholing() is true self.rebuild_state_after_failure(resumedescr, inputargs) return resumedescr Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Fri Oct 2 13:24:13 2009 @@ -11,7 +11,6 @@ from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype -from pypy.jit.metainterp.jitprof import EmptyProfiler def get_metainterp(func, values, CPUClass, type_system, policy, listops=False, optimizer=OPTIMIZER_FULL): @@ -71,7 +70,7 @@ def __init__(self, cpu, *args): DoneWithThisFrame.__init__(self, *args) - class FakeWarmRunnerDesc: + class FakeWarmRunnerState: def attach_unoptimized_bridge_from_interp(self, greenkey, newloop): pass @@ -80,8 +79,7 @@ optimize_bridge = staticmethod(simple_optimize.optimize_bridge) trace_limit = sys.maxint - profiler = EmptyProfiler() - debug = 2 + debug_level = 2 if policy is None: policy = JitPolicy() @@ -96,7 +94,7 @@ cw.finish_making_bytecodes() metainterp.staticdata.portal_code = maingraph metainterp.staticdata._class_sizes = cw.class_sizes - metainterp.staticdata.state = FakeWarmRunnerDesc() + metainterp.staticdata.state = FakeWarmRunnerState() metainterp.staticdata.DoneWithThisFrameInt = DoneWithThisFrame metainterp.staticdata.DoneWithThisFrameRef = DoneWithThisFrameRef metainterp.staticdata.DoneWithThisFrameFloat = DoneWithThisFrame Modified: pypy/trunk/pypy/jit/metainterp/test/test_compile.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_compile.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_compile.py Fri Oct 2 13:24:13 2009 @@ -41,13 +41,15 @@ class FakeState: optimize_loop = staticmethod(optimize.optimize_loop) - profiler = jitprof.EmptyProfiler() - debug = 0 + debug_level = 0 class FakeMetaInterpStaticData: options = FakeOptions() state = FakeState() stats = Stats() + profiler = jitprof.EmptyProfiler() + def log(self, msg, event_kind=None): + pass class FakeMetaInterp: pass Modified: pypy/trunk/pypy/jit/metainterp/test/test_jitprof.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_jitprof.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_jitprof.py Fri Oct 2 13:24:13 2009 @@ -25,7 +25,7 @@ class ProfilerMixin(LLJitMixin): def meta_interp(self, *args, **kwds): kwds = kwds.copy() - kwds['profile'] = FakeProfiler + kwds['ProfilerClass'] = FakeProfiler return LLJitMixin.meta_interp(self, *args, **kwds) class TestProfile(ProfilerMixin): @@ -42,7 +42,7 @@ return res * 2 res = self.meta_interp(f, [6, 7]) assert res == 84 - profiler = pyjitpl._warmrunnerdesc.metainterp_sd.state.profiler + profiler = pyjitpl._warmrunnerdesc.metainterp_sd.profiler expected = [ TRACING, BACKEND, @@ -74,7 +74,7 @@ return res * 2 res = self.meta_interp(f, [6, 7]) assert res == 84 - profiler = pyjitpl._warmrunnerdesc.metainterp_sd.state.profiler + profiler = pyjitpl._warmrunnerdesc.metainterp_sd.profiler # calls = (executed, recorded, blackholed) x (inpure, pure) assert profiler.calls == [[1, 0], [1, 0], [0, 0]] @@ -96,6 +96,6 @@ return res * 2 res = self.meta_interp(f, [6, 7, 2]) assert res == 90 - profiler = pyjitpl._warmrunnerdesc.metainterp_sd.state.profiler + profiler = pyjitpl._warmrunnerdesc.metainterp_sd.profiler # calls = (executed, recorded, blackholed) x (inpure, pure) assert profiler.calls == [[0, 1], [0, 0], [0, 1]] Modified: pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py Fri Oct 2 13:24:13 2009 @@ -152,9 +152,55 @@ assert warmrunnerdescr.state.optimize_loop is optimize.optimize_loop assert warmrunnerdescr.state.optimize_bridge is optimize.optimize_bridge + def test_static_debug_level(self): + from pypy.rlib.jit import DEBUG_PROFILE, DEBUG_OFF, DEBUG_STEPS + from pypy.jit.metainterp.jitprof import EmptyProfiler, Profiler + + myjitdriver = JitDriver(greens = [], reds = ['n']) + def f(n): + while n > 0: + myjitdriver.can_enter_jit(n=n) + myjitdriver.jit_merge_point(n=n) + n -= 1 + return n + + outerr = py.io.StdCaptureFD() + self.meta_interp(f, [10], debug_level=DEBUG_OFF, + ProfilerClass=Profiler) + out, errf = outerr.done() + err = errf.read() + assert not 'ENTER' in err + assert not 'LEAVE' in err + assert not "Running asm" in err + outerr = py.io.StdCaptureFD() + self.meta_interp(f, [10], debug_level=DEBUG_PROFILE, + ProfilerClass=Profiler) + out, errf = outerr.done() + err = errf.read() + assert not 'ENTER' in err + assert not 'LEAVE' in err + assert not 'compiled new' in err + assert "Running asm" in err + outerr = py.io.StdCaptureFD() + self.meta_interp(f, [10], debug_level=DEBUG_STEPS, + ProfilerClass=Profiler) + out, errf = outerr.done() + err = errf.read() + assert 'ENTER' in err + assert 'LEAVE' in err + assert "Running asm" in err + outerr = py.io.StdCaptureFD() + self.meta_interp(f, [10], debug_level=DEBUG_STEPS, + ProfilerClass=EmptyProfiler) + out, errf = outerr.done() + err = errf.read() + assert 'ENTER' in err + assert 'LEAVE' in err + assert not "Running asm" in err + def test_set_param_debug(self): - from pypy.jit.metainterp.jitprof import Profiler, EmptyProfiler from pypy.rlib.jit import DEBUG_PROFILE, DEBUG_OFF, DEBUG_STEPS + from pypy.jit.metainterp.jitprof import EmptyProfiler, Profiler myjitdriver = JitDriver(greens = [], reds = ['n']) def f(n): @@ -165,18 +211,20 @@ return n def main(n, debug): - myjitdriver.set_param_debug(debug) + myjitdriver.set_param("debug", debug) print f(n) outerr = py.io.StdCaptureFD() - self.meta_interp(f, [10], debug=DEBUG_OFF) + self.meta_interp(main, [10, DEBUG_OFF], debug_level=DEBUG_STEPS, + ProfilerClass=Profiler) out, errf = outerr.done() err = errf.read() assert not 'ENTER' in err assert not 'LEAVE' in err - assert "Running asm" not in err + assert not "Running asm" in err outerr = py.io.StdCaptureFD() - self.meta_interp(f, [10], debug=DEBUG_PROFILE) + self.meta_interp(main, [10, DEBUG_PROFILE], debug_level=DEBUG_STEPS, + ProfilerClass=Profiler) out, errf = outerr.done() err = errf.read() assert not 'ENTER' in err @@ -184,7 +232,8 @@ assert not 'compiled new' in err assert "Running asm" in err outerr = py.io.StdCaptureFD() - self.meta_interp(f, [10], debug=DEBUG_STEPS) + self.meta_interp(main, [10, DEBUG_STEPS], debug_level=DEBUG_OFF, + ProfilerClass=Profiler) out, errf = outerr.done() err = errf.read() assert 'ENTER' in err Modified: pypy/trunk/pypy/jit/metainterp/test/test_zrpy_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_zrpy_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_zrpy_basic.py Fri Oct 2 13:24:13 2009 @@ -50,7 +50,7 @@ res = rpython_ll_meta_interp(f, [17], loops=2, CPUClass=self.CPUClass, type_system=self.type_system, optimizer=OPTIMIZER_FULL, - profile=Profiler) + ProfilerClass=Profiler) assert res == (17+14+11+8+7+6+5+4) * 10 Modified: pypy/trunk/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmspot.py Fri Oct 2 13:24:13 2009 @@ -33,10 +33,15 @@ if 'CPUClass' not in kwds: from pypy.jit.backend.detect_cpu import getcpuclass kwds['CPUClass'] = getcpuclass(backend_name) + if debug_level > DEBUG_OFF: + ProfilerClass = Profiler + else: + ProfilerClass = EmptyProfiler warmrunnerdesc = WarmRunnerDesc(translator, translate_support_code=True, listops=True, no_stats = True, + ProfilerClass = ProfilerClass, **kwds) warmrunnerdesc.state.set_param_inlining(inline) warmrunnerdesc.state.set_param_debug(debug_level) @@ -56,8 +61,9 @@ clear_tcache() return jittify_and_run(interp, graph, args, backendopt=backendopt, **kwds) -def jittify_and_run(interp, graph, args, repeat=1, hash_bits=None, backendopt=False, trace_limit=sys.maxint, debug=DEBUG_STEPS, profile=None, - inline=False, **kwds): +def jittify_and_run(interp, graph, args, repeat=1, hash_bits=None, + backendopt=False, trace_limit=sys.maxint, + debug_level=DEBUG_STEPS, inline=False, **kwds): translator = interp.typer.annotator.translator translator.config.translation.gc = "boehm" warmrunnerdesc = WarmRunnerDesc(translator, backendopt=backendopt, **kwds) @@ -65,18 +71,14 @@ warmrunnerdesc.state.set_param_trace_eagerness(2) # for tests warmrunnerdesc.state.set_param_trace_limit(trace_limit) warmrunnerdesc.state.set_param_inlining(inline) - warmrunnerdesc.state.set_param_debug(debug) - if not we_are_translated() and profile is not None: - # for tests - warmrunnerdesc.state.profiler = profile() - warmrunnerdesc.state.profiler.start() + warmrunnerdesc.state.set_param_debug(debug_level) warmrunnerdesc.state.create_tables_now() # for tests if hash_bits: warmrunnerdesc.state.set_param_hash_bits(hash_bits) warmrunnerdesc.finish() res = interp.eval_graph(graph, args) if not kwds.get('translate_support_code', False): - warmrunnerdesc.metainterp_sd.state.profiler.finish() + warmrunnerdesc.metainterp_sd.profiler.finish() print '~~~ return value:', res while repeat > 1: print '~' * 79 @@ -229,8 +231,8 @@ really_remove_asserts=True) def build_meta_interp(self, CPUClass, translate_support_code=False, - view="auto", profile=None, no_stats=False, - **kwds): + view="auto", no_stats=False, + ProfilerClass=EmptyProfiler, **kwds): assert CPUClass is not None opt = history.Options(**kwds) if no_stats: @@ -249,6 +251,7 @@ self.metainterp_sd = MetaInterpStaticData(self.portal_graph, self.translator.graphs, cpu, self.stats, opt, + ProfilerClass=ProfilerClass, warmrunnerdesc=self, leave_graph=self.leave_graph) @@ -530,8 +533,8 @@ def add_profiler_finish(self): def finish_profiler(): - if self.state.profiler.initialized: - self.state.profiler.finish() + if self.metainterp_sd.profiler.initialized: + self.metainterp_sd.profiler.finish() if self.cpu.translate_support_code: call_final_function(self.translator, finish_profiler, @@ -797,14 +800,8 @@ raise ValueError("unknown optimizer") def set_param_debug(self, value): - if value >= DEBUG_PROFILE: - self.profiler = Profiler() - else: - self.profiler = EmptyProfiler() - if not self.profiler.initialized: - self.profiler.start() - self.profiler.initialized = True - self.debug = value + self.debug_level = value + metainterp_sd.profiler.set_printing(value >= DEBUG_PROFILE) def create_tables_now(self): count = 1 << self.hashbits @@ -859,9 +856,9 @@ loop_token = cell.entry_loop_token # ---------- execute assembler ---------- while True: # until interrupted by an exception - metainterp_sd.state.profiler.start_running() + metainterp_sd.profiler.start_running() fail_descr = metainterp_sd.cpu.execute_token(loop_token) - metainterp_sd.state.profiler.end_running() + metainterp_sd.profiler.end_running() loop_token = fail_descr.handle_fail(metainterp_sd) maybe_compile_and_run._dont_inline_ = True From antocuni at codespeak.net Fri Oct 2 14:41:10 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 2 Oct 2009 14:41:10 +0200 (CEST) Subject: [pypy-svn] r68127 - in pypy/trunk/pypy: rpython/ootypesystem rpython/test translator/cli/src Message-ID: <20091002124110.52DFD16803A@codespeak.net> Author: antocuni Date: Fri Oct 2 14:41:08 2009 New Revision: 68127 Modified: pypy/trunk/pypy/rpython/ootypesystem/ootype.py pypy/trunk/pypy/rpython/test/test_rdict.py pypy/trunk/pypy/translator/cli/src/pypylib.cs Log: make sure that it's possible to change an ootype dictionary during iteration, as long as its size does not change. .NET does not support it natively, so we have to copy the keys of the dictionary when we create the iterator :-( Modified: pypy/trunk/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/trunk/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/trunk/pypy/rpython/ootypesystem/ootype.py Fri Oct 2 14:41:08 2009 @@ -1654,8 +1654,9 @@ # NOT_RPYTHON assert typeOf(key) == self._TYPE._KEYTYPE assert typeOf(value) == self._TYPE._VALUETYPE + if key not in self._dict: + self._stamp += 1 self._dict[key] = value - self._stamp += 1 def ll_remove(self, key): # NOT_RPYTHON Modified: pypy/trunk/pypy/rpython/test/test_rdict.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_rdict.py (original) +++ pypy/trunk/pypy/rpython/test/test_rdict.py Fri Oct 2 14:41:08 2009 @@ -533,6 +533,30 @@ res = self.interpret(f, [5]) assert res == 25019999 + def test_resize_during_iteration(self): + def func(): + d = {5: 1, 6: 2, 7: 3} + try: + for key, value in d.iteritems(): + d[key^16] = value*2 + except RuntimeError: + pass + total = 0 + for key in d: + total += key + return total + res = self.interpret(func, []) + assert 5 + 6 + 7 <= res <= 5 + 6 + 7 + (5^16) + (6^16) + (7^16) + + def test_change_during_iteration(self): + def func(): + d = {'a': 1, 'b': 2} + for key in d: + d[key] = 42 + return d['a'] + assert self.interpret(func, []) == 42 + + class TestLLtype(BaseTestRdict, LLRtypeMixin): def test_dict_but_not_with_char_keys(self): def func(i): @@ -735,21 +759,6 @@ assert lltype.typeOf(res.item1) == lltype.typeOf(res.item2) assert lltype.typeOf(res.item1) == lltype.typeOf(res.item3) - def test_resize_during_iteration(self): - def func(): - d = {5: 1, 6: 2, 7: 3} - try: - for key, value in d.iteritems(): - d[key^16] = value*2 - except RuntimeError: - pass - total = 0 - for key in d: - total += key - return total - res = self.interpret(func, []) - assert 5 + 6 + 7 <= res <= 5 + 6 + 7 + (5^16) + (6^16) + (7^16) - def test_prebuilt_list_of_addresses(self): from pypy.rpython.lltypesystem import llmemory @@ -805,17 +814,6 @@ res = self.interpret(func, [5]) assert res.ll_get(5) is res - def test_invalid_iterator(self): - def func(): - try: - d = {'a': 1, 'b': 2} - for key in d: - d[key] = 0 - return True - except RuntimeError: - return False - assert self.interpret(func, []) == False - # ____________________________________________________________ class TestStress: Modified: pypy/trunk/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/trunk/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/trunk/pypy/translator/cli/src/pypylib.cs Fri Oct 2 14:41:08 2009 @@ -847,7 +847,7 @@ public DictItemsIterator ll_get_items_iterator() { - return new DictItemsIterator(this.GetEnumerator()); + return new DictItemsIterator(this); } // XXX: this is CustomDict specific, maybe we should have a separate class for it @@ -881,7 +881,7 @@ public DictItemsIterator ll_get_items_iterator() { - return new DictItemsIterator(this.GetEnumerator()); + return new DictItemsIterator(this); } public DictOfVoid ll_copy() // XXX: why it should return a Dict? @@ -924,10 +924,10 @@ public DictItemsIterator ll_get_items_iterator() { - List> foo = new List>(); + Dictionary foo = new Dictionary(); if (length == 1) - foo.Add(new KeyValuePair(default(TKey), this.elem)); - return new DictItemsIterator(foo.GetEnumerator()); + foo[default(TKey)] = this.elem; + return new DictItemsIterator(foo); } } @@ -950,34 +950,46 @@ public DictItemsIterator ll_get_items_iterator() { - List> foo = new List>(); + Dictionary foo = new Dictionary(); if (length == 1) - foo.Add(new KeyValuePair(0, 0)); - return new DictItemsIterator(foo.GetEnumerator()); + foo[0] = 0; + return new DictItemsIterator(foo); } } public class DictItemsIterator { - IEnumerator> it; - - public DictItemsIterator(IEnumerator> it) - { - this.it = it; + Dictionary dict; + int size; + TKey[] keys; + int pos; + + public DictItemsIterator(Dictionary dict) + { + this.dict = dict; + this.size = dict.Count; + this.keys = new TKey[this.size]; + dict.Keys.CopyTo(this.keys, 0); + this.pos = -1; } public bool ll_go_next() { - try { - return it.MoveNext(); - } - catch(InvalidOperationException e) { + if (this.size != this.dict.Count) Helpers.raise_RuntimeError(); + if (this.pos >= this.size-1) return false; - } + pos++; + return true; + } + + public TKey ll_current_key() { + return this.keys[this.pos]; } - public TKey ll_current_key() { return it.Current.Key; } - public TValue ll_current_value() { return it.Current.Value; } + public TValue ll_current_value() { + TKey key = this.ll_current_key(); + return this.dict[key]; + } } public class WeakReference From pedronis at codespeak.net Fri Oct 2 15:33:42 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 2 Oct 2009 15:33:42 +0200 (CEST) Subject: [pypy-svn] r68128 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091002133342.B63EB168041@codespeak.net> Author: pedronis Date: Fri Oct 2 15:33:42 2009 New Revision: 68128 Modified: pypy/trunk/pypy/jit/metainterp/compile.py pypy/trunk/pypy/jit/metainterp/history.py pypy/trunk/pypy/jit/metainterp/optimize.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/test/test_compile.py Log: move the loggers to the static data Modified: pypy/trunk/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/compile.py (original) +++ pypy/trunk/pypy/jit/metainterp/compile.py Fri Oct 2 15:33:42 2009 @@ -59,7 +59,7 @@ metainterp_sd = metainterp.staticdata try: old_loop_token = metainterp_sd.state.optimize_loop( - metainterp_sd.options, old_loop_tokens, loop, metainterp.cpu) + metainterp_sd, old_loop_tokens, loop, metainterp.cpu) except InvalidLoop: return None if old_loop_token is not None: @@ -83,7 +83,7 @@ old_loop_tokens.append(loop_token) def send_loop_to_backend(metainterp_sd, loop, type): - metainterp_sd.options.logger_ops.log_loop(loop.inputargs, loop.operations) + metainterp_sd.logger_ops.log_loop(loop.inputargs, loop.operations) metainterp_sd.profiler.start_backend() if not we_are_translated(): show_loop(metainterp_sd, loop) @@ -99,7 +99,7 @@ metainterp_sd.log("compiled new " + type) def send_bridge_to_backend(metainterp_sd, faildescr, inputargs, operations): - metainterp_sd.options.logger_ops.log_loop(inputargs, operations) + metainterp_sd.logger_ops.log_loop(inputargs, operations) metainterp_sd.profiler.start_backend() if not we_are_translated(): show_loop(metainterp_sd) @@ -246,12 +246,11 @@ new_loop.inputargs = metainterp.history.inputargs new_loop.operations = metainterp.history.operations metainterp_sd = metainterp.staticdata - options = metainterp_sd.options try: - target_loop_token = metainterp_sd.state.optimize_bridge(options, - old_loop_tokens, - new_loop, - metainterp.cpu) + target_loop_token = metainterp_sd.state.optimize_bridge(metainterp_sd, + old_loop_tokens, + new_loop, + metainterp.cpu) except InvalidLoop: assert 0, "InvalidLoop in optimize_bridge?" return None Modified: pypy/trunk/pypy/jit/metainterp/history.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/history.py (original) +++ pypy/trunk/pypy/jit/metainterp/history.py Fri Oct 2 15:33:42 2009 @@ -899,7 +899,6 @@ # ---------------------------------------------------------------- class Options: - logger_noopt = None def __init__(self, listops=False): self.listops = listops def _freeze_(self): Modified: pypy/trunk/pypy/jit/metainterp/optimize.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimize.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimize.py Fri Oct 2 15:33:42 2009 @@ -4,8 +4,8 @@ from pypy.jit.metainterp.optimizeopt import optimize_loop_1 from pypy.jit.metainterp.specnode import equals_specnodes -def optimize_loop(options, old_loop_tokens, loop, cpu): - options.logger_noopt.log_loop(loop.inputargs, loop.operations) +def optimize_loop(metainterp_sd, old_loop_tokens, loop, cpu): + metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations) finder = PerfectSpecializationFinder(cpu) finder.find_nodes_loop(loop) for old_loop_token in old_loop_tokens: @@ -19,8 +19,8 @@ from pypy.jit.metainterp.optimizefindnode import BridgeSpecializationFinder from pypy.jit.metainterp.optimizeopt import optimize_bridge_1 -def optimize_bridge(options, old_loop_tokens, bridge, cpu): - options.logger_noopt.log_loop(bridge.inputargs, bridge.operations) +def optimize_bridge(metainterp_sd, old_loop_tokens, bridge, cpu): + metainterp_sd.logger_noopt.log_loop(bridge.inputargs, bridge.operations) finder = BridgeSpecializationFinder(cpu) finder.find_nodes_bridge(bridge) for old_loop_token in old_loop_tokens: Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Fri Oct 2 15:33:42 2009 @@ -958,6 +958,8 @@ class MetaInterpStaticData(object): virtualizable_info = None + logger_noopt = None + logger_ops = None def __init__(self, portal_graph, graphs, cpu, stats, options, ProfilerClass=EmptyProfiler, warmrunnerdesc=None, @@ -966,8 +968,8 @@ self.cpu = cpu self.stats = stats self.options = options - options.logger_noopt = Logger(cpu.ts) - options.logger_ops = Logger(cpu.ts) + self.logger_noopt = Logger(cpu.ts) + self.logger_ops = Logger(cpu.ts) RESULT = portal_graph.getreturnvar().concretetype self.result_type = history.getkind(RESULT) @@ -1014,8 +1016,8 @@ self.profiler.start() self.profiler.initialized = True self.globaldata.initialized = True - self.options.logger_noopt.create_log('.noopt') - self.options.logger_ops.create_log('.ops') + self.logger_noopt.create_log('.noopt') + self.logger_ops.create_log('.ops') def _setup_class_sizes(self): class_sizes = {} Modified: pypy/trunk/pypy/jit/metainterp/test/test_compile.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_compile.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_compile.py Fri Oct 2 15:33:42 2009 @@ -35,16 +35,14 @@ def log_loop(self, inputargs, operations): pass -class FakeOptions: - logger_noopt = FakeLogger() - logger_ops = FakeLogger() - class FakeState: optimize_loop = staticmethod(optimize.optimize_loop) debug_level = 0 class FakeMetaInterpStaticData: - options = FakeOptions() + logger_noopt = FakeLogger() + logger_ops = FakeLogger() + state = FakeState() stats = Stats() profiler = jitprof.EmptyProfiler() From arigo at codespeak.net Fri Oct 2 16:37:19 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 2 Oct 2009 16:37:19 +0200 (CEST) Subject: [pypy-svn] r68129 - in pypy/trunk/pypy/rpython: . test Message-ID: <20091002143719.CC9C816803A@codespeak.net> Author: arigo Date: Fri Oct 2 16:37:19 2009 New Revision: 68129 Modified: pypy/trunk/pypy/rpython/rrange.py pypy/trunk/pypy/rpython/test/test_rrange.py Log: Small performance improvement for len(range(1-or-2-arguments)). Don't call the general 3-arguments version of ll_rangelen() in that case. Modified: pypy/trunk/pypy/rpython/rrange.py ============================================================================== --- pypy/trunk/pypy/rpython/rrange.py (original) +++ pypy/trunk/pypy/rpython/rrange.py Fri Oct 2 16:37:19 2009 @@ -20,11 +20,13 @@ def rtype_len(self, hop): v_rng, = hop.inputargs(self) - if self.step != 0: - cstep = hop.inputconst(Signed, self.step) + if self.step == 1: + return hop.gendirectcall(ll_rangelen1, v_rng) + elif self.step != 0: + v_step = hop.inputconst(Signed, self.step) else: - cstep = self._getstep(v_rng, hop) - return hop.gendirectcall(ll_rangelen, v_rng, cstep) + v_step = self._getstep(v_rng, hop) + return hop.gendirectcall(ll_rangelen, v_rng, v_step) class __extend__(pairtype(AbstractRangeRepr, IntegerRepr)): @@ -62,6 +64,12 @@ def ll_rangelen(l, step): return _ll_rangelen(l.start, l.stop, step) +def ll_rangelen1(l): + result = l.stop - l.start + if result < 0: + result = 0 + return result + def ll_rangeitem_nonneg(func, l, index, step): if func is dum_checkidx and index >= _ll_rangelen(l.start, l.stop, step): raise IndexError Modified: pypy/trunk/pypy/rpython/test/test_rrange.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_rrange.py (original) +++ pypy/trunk/pypy/rpython/test/test_rrange.py Fri Oct 2 16:37:19 2009 @@ -67,13 +67,38 @@ res = self.interpret(dummyfn, [10]) assert res == 45 - def test_range_len(self): + def test_range_len_nostep(self): def dummyfn(start, stop): r = range(start, stop) return len(r) start, stop = 10, 17 res = self.interpret(dummyfn, [start, stop]) assert res == dummyfn(start, stop) + start, stop = 17, 10 + res = self.interpret(dummyfn, [start, stop]) + assert res == 0 + + def test_range_len_step_const(self): + def dummyfn(start, stop): + r = range(start, stop, -2) + return len(r) + start, stop = 10, 17 + res = self.interpret(dummyfn, [start, stop]) + assert res == 0 + start, stop = 17, 10 + res = self.interpret(dummyfn, [start, stop]) + assert res == dummyfn(start, stop) + + def test_range_len_step_nonconst(self): + def dummyfn(start, stop, step): + r = range(start, stop, step) + return len(r) + start, stop, step = 10, 17, -3 + res = self.interpret(dummyfn, [start, stop, step]) + assert res == 0 + start, stop, step = 17, 10, -3 + res = self.interpret(dummyfn, [start, stop, step]) + assert res == dummyfn(start, stop, step) def test_range2list(self): def dummyfn(start, stop): From pedronis at codespeak.net Fri Oct 2 17:47:57 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 2 Oct 2009 17:47:57 +0200 (CEST) Subject: [pypy-svn] r68130 - pypy/trunk/pypy/translator/c/test Message-ID: <20091002154757.8F6FF16803D@codespeak.net> Author: pedronis Date: Fri Oct 2 17:47:55 2009 New Revision: 68130 Modified: pypy/trunk/pypy/translator/c/test/test_genc.py Log: some cleanup of this test file just because Modified: pypy/trunk/pypy/translator/c/test/test_genc.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_genc.py (original) +++ pypy/trunk/pypy/translator/c/test/test_genc.py Fri Oct 2 17:47:55 2009 @@ -290,7 +290,7 @@ res = g2(0) assert isnan(res) -def test_x(): +def test_prebuilt_instance_with_dict(): class A: pass a = A() @@ -375,23 +375,13 @@ pass print f(C) -def test_oswrite(): +def test_print(): def f(): - import os - os.write(1,"o") - - t = TranslationContext() - s = t.buildannotator().build_types(f, []) - rtyper = t.buildrtyper(type_system="lltype") - rtyper.specialize() - -def test_x(): - py.test.skip("Failing test. Seems that print allocs one bit too much") - def f(): - print "xxx" + for i in range(10): + print "xxx" fn = compile(f, []) - fn() + fn(expected_extra_mallocs=1) def test_name(): def f(): From cfbolz at codespeak.net Fri Oct 2 23:20:26 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 2 Oct 2009 23:20:26 +0200 (CEST) Subject: [pypy-svn] r68131 - pypy/trunk/pypy/objspace/std Message-ID: <20091002212026.11991168026@codespeak.net> Author: cfbolz Date: Fri Oct 2 23:20:25 2009 New Revision: 68131 Modified: pypy/trunk/pypy/objspace/std/typeobject.py Log: (pedronis, cfbolz): the instancetypedef attribute is immutable Modified: pypy/trunk/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/typeobject.py (original) +++ pypy/trunk/pypy/objspace/std/typeobject.py Fri Oct 2 23:20:25 2009 @@ -54,7 +54,8 @@ 'needsdel', 'weakrefable', 'hasdict', - 'nslots'] + 'nslots', + 'instancetypedef'] # for config.objspace.std.getattributeshortcut # (False is a conservative default, fixed during real usage) From cfbolz at codespeak.net Fri Oct 2 23:58:12 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 2 Oct 2009 23:58:12 +0200 (CEST) Subject: [pypy-svn] r68132 - in pypy/trunk/pypy/jit: backend/llsupport metainterp metainterp/test Message-ID: <20091002215812.3C255168401@codespeak.net> Author: cfbolz Date: Fri Oct 2 23:58:10 2009 New Revision: 68132 Modified: pypy/trunk/pypy/jit/backend/llsupport/gc.py pypy/trunk/pypy/jit/metainterp/executor.py pypy/trunk/pypy/jit/metainterp/history.py pypy/trunk/pypy/jit/metainterp/optimizefindnode.py pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/jit/metainterp/specnode.py pypy/trunk/pypy/jit/metainterp/test/test_history.py pypy/trunk/pypy/jit/metainterp/typesystem.py pypy/trunk/pypy/jit/metainterp/warmspot.py Log: (pedronis awake a bit, cfbolz): prebuild const int values for fun and profit. Modified: pypy/trunk/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/gc.py Fri Oct 2 23:58:10 2009 @@ -4,7 +4,7 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.jit.metainterp.history import BoxInt, BoxPtr, ConstInt, ConstPtr +from pypy.jit.metainterp.history import BoxInt, BoxPtr, constint, ConstInt, ConstPtr from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.backend.llsupport import symbolic from pypy.jit.backend.llsupport.symbolic import WORD @@ -334,7 +334,7 @@ self.HDRPTR = lltype.Ptr(self.GCClass.HDR) self.gcheaderbuilder = GCHeaderBuilder(self.HDRPTR.TO) self.fielddescr_tid = get_field_descr(self, self.GCClass.HDR, 'tid') - self.c_jit_wb_if_flag = ConstInt(self.GCClass.JIT_WB_IF_FLAG) + self.c_jit_wb_if_flag = constint(self.GCClass.JIT_WB_IF_FLAG) self.calldescr_jit_wb = get_call_descr(self, [llmemory.GCREF, llmemory.GCREF], lltype.Void) @@ -488,7 +488,7 @@ addr = self.gcrefs.get_address_of_gcref(v.value) addr = cpu.cast_adr_to_int(addr) newops.append(ResOperation(rop.GETFIELD_RAW, - [ConstInt(addr)], box, + [constint(addr)], box, self.single_gcref_descr)) op.args[i] = box # ---------- write barrier for SETFIELD_GC ---------- @@ -519,6 +519,8 @@ llop1 = self.llop1 funcptr = llop1.get_write_barrier_failing_case(self.WB_FUNCPTR) funcaddr = llmemory.cast_ptr_to_adr(funcptr) + # this is not going to be a small integer, so don't use constint + # (bit of a micro-optimization) c_func = ConstInt(cpu.cast_adr_to_int(funcaddr)) args = [v_tid, self.c_jit_wb_if_flag, c_func, v_base, v_value] newops.append(ResOperation(rop.COND_CALL_GC_WB, args, None, Modified: pypy/trunk/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/executor.py (original) +++ pypy/trunk/pypy/jit/metainterp/executor.py Fri Oct 2 23:58:10 2009 @@ -6,7 +6,7 @@ from pypy.rpython.ootypesystem import ootype from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck, r_uint, intmask -from pypy.jit.metainterp.history import BoxInt, ConstInt, check_descr +from pypy.jit.metainterp.history import BoxInt, constint, check_descr from pypy.jit.metainterp.history import INT, REF, ConstFloat from pypy.jit.metainterp import resoperation from pypy.jit.metainterp.resoperation import rop @@ -19,86 +19,86 @@ # ____________________________________________________________ def do_int_add(cpu, box1, box2): - return ConstInt(intmask(box1.getint() + box2.getint())) + return constint(intmask(box1.getint() + box2.getint())) def do_int_sub(cpu, box1, box2): - return ConstInt(intmask(box1.getint() - box2.getint())) + return constint(intmask(box1.getint() - box2.getint())) def do_int_mul(cpu, box1, box2): - return ConstInt(intmask(box1.getint() * box2.getint())) + return constint(intmask(box1.getint() * box2.getint())) def do_int_floordiv(cpu, box1, box2): z = llop.int_floordiv(lltype.Signed, box1.getint(), box2.getint()) - return ConstInt(z) + return constint(z) def do_int_mod(cpu, box1, box2): z = llop.int_mod(lltype.Signed, box1.getint(), box2.getint()) - return ConstInt(z) + return constint(z) def do_int_and(cpu, box1, box2): - return ConstInt(box1.getint() & box2.getint()) + return constint(box1.getint() & box2.getint()) def do_int_or(cpu, box1, box2): - return ConstInt(box1.getint() | box2.getint()) + return constint(box1.getint() | box2.getint()) def do_int_xor(cpu, box1, box2): - return ConstInt(box1.getint() ^ box2.getint()) + return constint(box1.getint() ^ box2.getint()) def do_int_rshift(cpu, box1, box2): - return ConstInt(box1.getint() >> box2.getint()) + return constint(box1.getint() >> box2.getint()) def do_int_lshift(cpu, box1, box2): - return ConstInt(intmask(box1.getint() << box2.getint())) + return constint(intmask(box1.getint() << box2.getint())) def do_uint_rshift(cpu, box1, box2): v = r_uint(box1.getint()) >> r_uint(box2.getint()) - return ConstInt(intmask(v)) + return constint(intmask(v)) # ---------- def do_int_lt(cpu, box1, box2): - return ConstInt(box1.getint() < box2.getint()) + return constint(box1.getint() < box2.getint()) def do_int_le(cpu, box1, box2): - return ConstInt(box1.getint() <= box2.getint()) + return constint(box1.getint() <= box2.getint()) def do_int_eq(cpu, box1, box2): - return ConstInt(box1.getint() == box2.getint()) + return constint(box1.getint() == box2.getint()) def do_int_ne(cpu, box1, box2): - return ConstInt(box1.getint() != box2.getint()) + return constint(box1.getint() != box2.getint()) def do_int_gt(cpu, box1, box2): - return ConstInt(box1.getint() > box2.getint()) + return constint(box1.getint() > box2.getint()) def do_int_ge(cpu, box1, box2): - return ConstInt(box1.getint() >= box2.getint()) + return constint(box1.getint() >= box2.getint()) def do_uint_lt(cpu, box1, box2): - return ConstInt(r_uint(box1.getint()) < r_uint(box2.getint())) + return constint(r_uint(box1.getint()) < r_uint(box2.getint())) def do_uint_le(cpu, box1, box2): - return ConstInt(r_uint(box1.getint()) <= r_uint(box2.getint())) + return constint(r_uint(box1.getint()) <= r_uint(box2.getint())) def do_uint_gt(cpu, box1, box2): - return ConstInt(r_uint(box1.getint()) > r_uint(box2.getint())) + return constint(r_uint(box1.getint()) > r_uint(box2.getint())) def do_uint_ge(cpu, box1, box2): - return ConstInt(r_uint(box1.getint()) >= r_uint(box2.getint())) + return constint(r_uint(box1.getint()) >= r_uint(box2.getint())) # ---------- def do_int_is_true(cpu, box1): - return ConstInt(bool(box1.getint())) + return constint(bool(box1.getint())) def do_int_neg(cpu, box1): - return ConstInt(intmask(-box1.getint())) + return constint(intmask(-box1.getint())) def do_int_invert(cpu, box1): - return ConstInt(~box1.getint()) + return constint(~box1.getint()) def do_bool_not(cpu, box1): - return ConstInt(not box1.getint()) + return constint(not box1.getint()) def do_same_as(cpu, box1): return box1 @@ -111,7 +111,7 @@ x = bool(box1.getref_base()) else: assert False - return ConstInt(x) + return constint(x) def do_ooisnull(cpu, box1): tp = box1.type @@ -121,7 +121,7 @@ x = bool(box1.getref_base()) else: assert False - return ConstInt(not x) + return constint(not x) def do_oois(cpu, box1, box2): tp = box1.type @@ -132,7 +132,7 @@ x = box1.getref_base() == box2.getref_base() else: assert False - return ConstInt(x) + return constint(x) def do_ooisnot(cpu, box1, box2): tp = box1.type @@ -143,14 +143,14 @@ x = box1.getref_base() != box2.getref_base() else: assert False - return ConstInt(x) + return constint(x) def do_ooidentityhash(cpu, box1): obj = box1.getref_base() - return ConstInt(cpu.ts.ooidentityhash(obj)) + return constint(cpu.ts.ooidentityhash(obj)) def do_subclassof(cpu, box1, box2): - return ConstInt(cpu.ts.subclassOf(cpu, box1, box2)) + return constint(cpu.ts.subclassOf(cpu, box1, box2)) # ---------- @@ -202,7 +202,7 @@ return ConstFloat(abs(box1.getfloat())) def do_float_is_true(cpu, box1): - return ConstInt(bool(box1.getfloat())) + return constint(bool(box1.getfloat())) def do_float_add(cpu, box1, box2): return ConstFloat(box1.getfloat() + box2.getfloat()) @@ -217,25 +217,25 @@ return ConstFloat(box1.getfloat() / box2.getfloat()) def do_float_lt(cpu, box1, box2): - return ConstInt(box1.getfloat() < box2.getfloat()) + return constint(box1.getfloat() < box2.getfloat()) def do_float_le(cpu, box1, box2): - return ConstInt(box1.getfloat() <= box2.getfloat()) + return constint(box1.getfloat() <= box2.getfloat()) def do_float_eq(cpu, box1, box2): - return ConstInt(box1.getfloat() == box2.getfloat()) + return constint(box1.getfloat() == box2.getfloat()) def do_float_ne(cpu, box1, box2): - return ConstInt(box1.getfloat() != box2.getfloat()) + return constint(box1.getfloat() != box2.getfloat()) def do_float_gt(cpu, box1, box2): - return ConstInt(box1.getfloat() > box2.getfloat()) + return constint(box1.getfloat() > box2.getfloat()) def do_float_ge(cpu, box1, box2): - return ConstInt(box1.getfloat() >= box2.getfloat()) + return constint(box1.getfloat() >= box2.getfloat()) def do_cast_float_to_int(cpu, box1): - return ConstInt(int(box1.getfloat())) + return constint(int(box1.getfloat())) def do_cast_int_to_float(cpu, box1): return ConstFloat(float(box1.getint())) Modified: pypy/trunk/pypy/jit/metainterp/history.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/history.py (original) +++ pypy/trunk/pypy/jit/metainterp/history.py Fri Oct 2 23:58:10 2009 @@ -160,7 +160,7 @@ intval = cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x)) else: intval = lltype.cast_primitive(lltype.Signed, x) - return ConstInt(intval) + return constint(intval) elif kind == "ref": return cpu.ts.new_ConstRef(x) elif kind == "float": @@ -244,9 +244,27 @@ def repr_rpython(self): return repr_rpython(self, 'ci') +constint_lower = -5 +constint_upper = 200 +ConstInt.PREBUILT = [ConstInt(i) for i in range(constint_lower, constint_upper)] + +def constint(x): + from pypy.rlib.rarithmetic import r_uint + # use r_uint to perform a single comparison (this whole function + # is getting inlined into every caller so keeping the branching + # to a minimum is a good idea) + if not we_are_translated() and not isinstance(x, int): # e.g. TotalOrderSymbolic + return ConstInt(x) + index = r_uint(x - constint_lower) + if index >= r_uint(constint_upper - constint_lower): + const = ConstInt(x) + else: + const = ConstInt.PREBUILT[index] + return const -CONST_FALSE = ConstInt(0) -CONST_TRUE = ConstInt(1) + +CONST_0 = CONST_FALSE = constint(0) +CONST_1 = CONST_TRUE = constint(1) class ConstAddr(Const): # only for constants built before translation type = INT @@ -490,7 +508,7 @@ return BoxInt(self.value) def constbox(self): - return ConstInt(self.value) + return constint(self.value) def getint(self): return self.value Modified: pypy/trunk/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizefindnode.py Fri Oct 2 23:58:10 2009 @@ -4,7 +4,7 @@ from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.specnode import VirtualArraySpecNode from pypy.jit.metainterp.specnode import VirtualStructSpecNode -from pypy.jit.metainterp.history import AbstractValue, ConstInt, Const +from pypy.jit.metainterp.history import AbstractValue, constint, Const from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.executor import execute_nonspec from pypy.jit.metainterp.optimizeutil import av_newdict, _findall, sort_descrs @@ -199,7 +199,7 @@ def find_nodes_ARRAYLEN_GC(self, op): arraynode = self.getnode(op.args[0]) if arraynode.arraydescr is not None: - resbox = ConstInt(arraynode.arraysize) + resbox = constint(arraynode.arraysize) self.set_constant_node(op.result, resbox) def find_nodes_GUARD_CLASS(self, op): Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Fri Oct 2 23:58:10 2009 @@ -1,5 +1,6 @@ from pypy.jit.metainterp.history import Box, BoxInt, LoopToken -from pypy.jit.metainterp.history import Const, ConstInt, ConstPtr, ConstObj, REF +from pypy.jit.metainterp.history import Const, constint, ConstInt, ConstPtr, ConstObj, REF +from pypy.jit.metainterp.history import CONST_0, CONST_1 from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.metainterp.executor import execute_nonspec from pypy.jit.metainterp.specnode import SpecNode, NotSpecNode, ConstantSpecNode @@ -140,8 +141,6 @@ def __init__(self, box): self.box = box -CONST_0 = ConstInt(0) -CONST_1 = ConstInt(1) CVAL_ZERO = ConstantValue(CONST_0) llhelper.CONST_NULL = ConstPtr(ConstPtr.value) llhelper.CVAL_NULLREF = ConstantValue(llhelper.CONST_NULL) @@ -269,7 +268,7 @@ if subvalue is not None: subbox = subvalue.force_box() op = ResOperation(rop.SETARRAYITEM_GC, - [box, ConstInt(index), subbox], None, + [box, constint(index), subbox], None, descr=self.arraydescr) newoperations.append(op) return self.box @@ -410,7 +409,7 @@ self.make_equal_to(box, ConstantValue(constbox)) def make_constant_int(self, box, intvalue): - self.make_constant(box, ConstInt(intvalue)) + self.make_constant(box, constint(intvalue)) def make_virtual(self, known_class, box, source_op=None): vvalue = VirtualValue(self, known_class, box, source_op) Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Fri Oct 2 23:58:10 2009 @@ -6,7 +6,7 @@ from pypy.rlib.debug import debug_print from pypy.jit.metainterp import history, compile, resume -from pypy.jit.metainterp.history import Const, ConstInt, Box +from pypy.jit.metainterp.history import Const, constint, Box, CONST_0 from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp import codewriter, executor from pypy.jit.metainterp.logger import Logger @@ -351,7 +351,7 @@ @arguments("orgpc", "box", "descr", "box") def opimpl_check_neg_index(self, pc, arraybox, arraydesc, indexbox): negbox = self.metainterp.execute_and_record( - rop.INT_LT, None, indexbox, ConstInt(0)) + rop.INT_LT, None, indexbox, CONST_0) # xxx inefficient negbox = self.implement_guard_value(pc, negbox) if negbox.getint(): @@ -391,7 +391,7 @@ def opimpl_check_resizable_neg_index(self, pc, listbox, lengthdesc, indexbox): negbox = self.metainterp.execute_and_record( - rop.INT_LT, None, indexbox, ConstInt(0)) + rop.INT_LT, None, indexbox, CONST_0) # xxx inefficient negbox = self.implement_guard_value(pc, negbox) if negbox.getint(): @@ -405,7 +405,7 @@ @arguments("orgpc", "box") def opimpl_check_zerodivisionerror(self, pc, box): nonzerobox = self.metainterp.execute_and_record( - rop.INT_NE, None, box, ConstInt(0)) + rop.INT_NE, None, box, CONST_0) # xxx inefficient nonzerobox = self.implement_guard_value(pc, nonzerobox) if nonzerobox.getint(): @@ -419,11 +419,11 @@ # detect the combination "box1 = -sys.maxint-1, box2 = -1". import sys tmp1 = self.metainterp.execute_and_record( # combination to detect: - rop.INT_ADD, None, box1, ConstInt(sys.maxint)) # tmp1=-1, box2=-1 + rop.INT_ADD, None, box1, constint(sys.maxint)) # tmp1=-1, box2=-1 tmp2 = self.metainterp.execute_and_record( rop.INT_AND, None, tmp1, box2) # tmp2=-1 tmp3 = self.metainterp.execute_and_record( - rop.INT_EQ, None, tmp2, ConstInt(-1)) # tmp3? + rop.INT_EQ, None, tmp2, constint(-1)) # tmp3? # xxx inefficient tmp4 = self.implement_guard_value(pc, tmp3) # tmp4? if not tmp4.getint(): @@ -439,7 +439,7 @@ @arguments("orgpc", "box") def opimpl_int_abs(self, pc, box): nonneg = self.metainterp.execute_and_record( - rop.INT_GE, None, box, ConstInt(0)) + rop.INT_GE, None, box, CONST_0) # xxx inefficient nonneg = self.implement_guard_value(pc, nonneg) if nonneg.getint(): @@ -585,7 +585,7 @@ virtualizable_box = self.metainterp.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) result = vinfo.get_array_length(virtualizable, arrayindex) - self.make_result_box(ConstInt(result)) + self.make_result_box(constint(result)) def perform_call(self, jitcode, varargs): if (self.metainterp.is_blackholing() and @@ -1725,7 +1725,7 @@ descr = vinfo.array_descrs[k] for j in range(vinfo.get_array_length(virtualizable, k)): itembox = self.execute_and_record(rop.GETARRAYITEM_GC, - descr, abox, ConstInt(j)) + descr, abox, constint(j)) self.virtualizable_boxes[i] = itembox i += 1 assert i + 1 == len(self.virtualizable_boxes) @@ -1749,7 +1749,7 @@ itembox = self.virtualizable_boxes[i] i += 1 self.execute_and_record(rop.SETARRAYITEM_GC, descr, - abox, ConstInt(j), itembox) + abox, constint(j), itembox) assert i + 1 == len(self.virtualizable_boxes) def gen_store_back_in_virtualizable_no_perform(self): @@ -1770,7 +1770,7 @@ itembox = self.virtualizable_boxes[i] i += 1 self.history.record(rop.SETARRAYITEM_GC, - [abox, ConstInt(j), itembox], + [abox, constint(j), itembox], None, descr=vinfo.array_descrs[k]) assert i + 1 == len(self.virtualizable_boxes) Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Fri Oct 2 23:58:10 2009 @@ -1,5 +1,5 @@ import sys -from pypy.jit.metainterp.history import Box, Const, ConstInt +from pypy.jit.metainterp.history import Box, Const, constint from pypy.jit.metainterp.resoperation import rop # Logic to encode the chain of frames and the state of the boxes at a @@ -251,14 +251,14 @@ length = len(self.fieldnums) return metainterp.execute_and_record(rop.NEW_ARRAY, self.arraydescr, - ConstInt(length)) + constint(length)) def setfields(self, metainterp, box, fn_decode_box): for i in range(len(self.fieldnums)): itembox = fn_decode_box(self.fieldnums[i]) metainterp.execute_and_record(rop.SETARRAYITEM_GC, self.arraydescr, - box, ConstInt(i), itembox) + box, constint(i), itembox) def repr_rpython(self): return 'VArrayInfo("%s", %s)' % (self.arraydescr, Modified: pypy/trunk/pypy/jit/metainterp/specnode.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/specnode.py (original) +++ pypy/trunk/pypy/jit/metainterp/specnode.py Fri Oct 2 23:58:10 2009 @@ -97,7 +97,7 @@ for i in range(len(self.items)): itembox = executor.execute(cpu, resoperation.rop.GETARRAYITEM_GC, self.arraydescr, - valuebox, history.ConstInt(i)) + valuebox, history.constint(i)) subspecnode = self.items[i] subspecnode.extract_runtime_data(cpu, itembox, resultlist) Modified: pypy/trunk/pypy/jit/metainterp/test/test_history.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_history.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_history.py Fri Oct 2 23:58:10 2009 @@ -9,3 +9,13 @@ s = lltype.cast_pointer(lltype.Ptr(S), t) const = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s)) assert const._getrepr_() == "*T" + +def test_constint_sharing(): + for val in [-1, 0, 1, 5, 99]: + c1 = constint(val) + c2 = constint(val) + assert c1 is c2 + for val in [100000, -10000]: + c1 = constint(val) + c2 = constint(val) + assert c1 is not c2 Modified: pypy/trunk/pypy/jit/metainterp/typesystem.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/typesystem.py (original) +++ pypy/trunk/pypy/jit/metainterp/typesystem.py Fri Oct 2 23:58:10 2009 @@ -77,7 +77,7 @@ def cls_of_box(self, cpu, box): obj = box.getref(lltype.Ptr(rclass.OBJECT)) cls = llmemory.cast_ptr_to_adr(obj.typeptr) - return history.ConstInt(cpu.cast_adr_to_int(cls)) + return history.constint(cpu.cast_adr_to_int(cls)) def subclassOf(self, cpu, clsbox1, clsbox2): adr = clsbox2.getaddr(cpu) @@ -87,7 +87,7 @@ return rclass.ll_issubclass(real_class, bounding_class) def get_exception_box(self, etype): - return history.ConstInt(etype) + return history.constint(etype) def get_exc_value_box(self, evalue): return history.BoxPtr(evalue) Modified: pypy/trunk/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmspot.py Fri Oct 2 23:58:10 2009 @@ -637,7 +637,7 @@ else: value = intmask(value) if in_const_box: - return history.ConstInt(value) + return history.constint(value) else: return history.BoxInt(value) wrap._annspecialcase_ = 'specialize:ll' From cfbolz at codespeak.net Sat Oct 3 00:46:32 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 3 Oct 2009 00:46:32 +0200 (CEST) Subject: [pypy-svn] r68133 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20091002224632.82DA21683E8@codespeak.net> Author: cfbolz Date: Sat Oct 3 00:46:31 2009 New Revision: 68133 Modified: pypy/trunk/pypy/jit/metainterp/test/oparser.py Log: make the oparser use caching Modified: pypy/trunk/pypy/jit/metainterp/test/oparser.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/oparser.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/oparser.py Sat Oct 3 00:46:31 2009 @@ -3,7 +3,7 @@ in a nicer fashion """ -from pypy.jit.metainterp.history import TreeLoop, BoxInt, ConstInt,\ +from pypy.jit.metainterp.history import TreeLoop, BoxInt, constint,\ ConstAddr, ConstObj, ConstPtr, Box, BasicFailDescr, LoopToken from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.metainterp.typesystem import llhelper @@ -98,9 +98,9 @@ def getvar(self, arg): if not arg: - return ConstInt(0) + return constint(0) try: - return ConstInt(int(arg)) + return constint(int(arg)) except ValueError: if arg.startswith('"') or arg.startswith("'"): # XXX ootype From benjamin at codespeak.net Sat Oct 3 01:20:11 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 3 Oct 2009 01:20:11 +0200 (CEST) Subject: [pypy-svn] r68134 - in pypy/trunk/pypy/interpreter: . pyparser pyparser/test Message-ID: <20091002232011.EF2C01683E9@codespeak.net> Author: benjamin Date: Sat Oct 3 01:20:10 2009 New Revision: 68134 Modified: pypy/trunk/pypy/interpreter/pycompiler.py pypy/trunk/pypy/interpreter/pyparser/future.py pypy/trunk/pypy/interpreter/pyparser/test/test_futureautomaton.py Log: rename future.py to be consistent with the rest of the compiler Modified: pypy/trunk/pypy/interpreter/pycompiler.py ============================================================================== --- pypy/trunk/pypy/interpreter/pycompiler.py (original) +++ pypy/trunk/pypy/interpreter/pycompiler.py Sat Oct 3 01:20:10 2009 @@ -220,8 +220,8 @@ PyCodeCompiler.__init__(self, space) self.parser = PythonParser(space) self.additional_rules = {} - self.futureFlags = future.futureFlags_2_5 - self.compiler_flags = self.futureFlags.allowed_flags + self.future_flags = future.futureFlags_2_5 + self.compiler_flags = self.future_flags.allowed_flags def compile_ast(self, node, filename, mode, flags): from pypy.interpreter.pyparser.pyparse import CompileInfo @@ -248,14 +248,14 @@ return self._compile_to_ast(source, info) def _compile_to_ast(self, source, info): - from pypy.interpreter.pyparser.future import getFutures + from pypy.interpreter.pyparser.future import get_futures from pypy.interpreter.pyparser.error import (SyntaxError, IndentationError, TokenIndentationError) from pypy.interpreter.astcompiler.astbuilder import ast_from_node space = self.space try: - f_flags, future_info = getFutures(self.futureFlags, source) + f_flags, future_info = get_futures(self.future_flags, source) info.last_future_import = future_info info.flags |= f_flags parse_tree = self.parser.parse_source(source, info) Modified: pypy/trunk/pypy/interpreter/pyparser/future.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/future.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/future.py Sat Oct 3 01:20:10 2009 @@ -6,8 +6,8 @@ The resulting legal futures are avaliable in self.flags after the pass has ended. -Invocation is through getFutures(src), which returns a field of flags, -one per found correct future import. +Invocation is through get_futures(src), which returns a field of flags, one per +found correct future import. The flags can then be used to set up the parser. All error detection is left to the parser. @@ -27,8 +27,8 @@ from pypy.interpreter.astcompiler.consts import CO_GENERATOR_ALLOWED, \ CO_FUTURE_DIVISION, CO_FUTURE_WITH_STATEMENT, CO_FUTURE_ABSOLUTE_IMPORT -def getFutures(futureFlags, source): - futures = FutureAutomaton(futureFlags, source) +def get_futures(future_flags, source): + futures = FutureAutomaton(future_flags, source) try: futures.start() except DoneException, e: @@ -63,15 +63,15 @@ precede a future statement. """ - def __init__(self, futureFlags, string): - self.futureFlags = futureFlags + def __init__(self, future_flags, string): + self.future_flags = future_flags self.s = string self.pos = 0 self.current_lineno = 1 self.lineno = -1 self.line_start_pos = 0 self.col_offset = 0 - self.docstringConsumed = False + self.docstring_consumed = False self.flags = 0 self.got_features = 0 @@ -83,14 +83,14 @@ def start(self): c = self.getc() - if c in ["'", '"'] and not self.docstringConsumed: - self.consumeDocstring() + if c in ["'", '"'] and not self.docstring_consumed: + self.consume_docstring() elif c in whitespace_or_newline: - self.consumeEmptyLine() + self.consume_empty_line() elif c == '#': - self.consumeComment() + self.consume_comment() elif c == 'f': - self.consumeFrom() + self.consume_from() else: return @@ -98,8 +98,8 @@ self.current_lineno += 1 self.line_start_pos = self.pos - def consumeDocstring(self): - self.docstringConsumed = True + def consume_docstring(self): + self.docstring_consumed = True endchar = self.getc() if (self.getc() == self.getc(+1) and self.getc() == self.getc(+2)): @@ -122,7 +122,7 @@ if (self.getc() == endchar and self.getc(+1) == endchar): self.pos += 2 - self.consumeEmptyLine() + self.consume_empty_line() break else: # Deal with a single quoted docstring @@ -131,7 +131,7 @@ c = self.getc() self.pos += 1 if c == endchar: - self.consumeEmptyLine() + self.consume_empty_line() return elif c == '\\': # Deal with linefeeds @@ -145,7 +145,7 @@ # Syntax error return - def consumeEmptyLine(self): + def consume_empty_line(self): """ Called when the remainder of the line can only contain whitespace and comments. @@ -153,10 +153,10 @@ while self.getc() in whitespace: self.pos += 1 if self.getc() == '#': - self.consumeComment() + self.consume_comment() elif self.getc() == ';': self.pos += 1 - self.consumeWhitespace() + self.consume_whitespace() self.start() elif self.getc() in '\r\n': c = self.getc() @@ -169,53 +169,53 @@ self.atbol() self.start() - def consumeComment(self): + def consume_comment(self): self.pos += 1 while self.getc() not in '\r\n': self.pos += 1 - self.consumeEmptyLine() + self.consume_empty_line() - def consumeFrom(self): + def consume_from(self): col_offset = self.pos - self.line_start_pos line = self.current_lineno self.pos += 1 if self.getc() == 'r' and self.getc(+1) == 'o' and self.getc(+2) == 'm': - self.docstringConsumed = True + self.docstring_consumed = True self.pos += 3 - self.consumeMandatoryWhitespace() + self.consume_mandatory_whitespace() if self.s[self.pos:self.pos+10] != '__future__': raise DoneException self.pos += 10 - self.consumeMandatoryWhitespace() + self.consume_mandatory_whitespace() if self.s[self.pos:self.pos+6] != 'import': raise DoneException self.pos += 6 - self.consumeWhitespace() + self.consume_whitespace() old_got = self.got_features try: if self.getc() == '(': self.pos += 1 - self.consumeWhitespace() - self.setFlag(self.getName()) + self.consume_whitespace() + self.set_flag(self.get_name()) # Set flag corresponding to name - self.getMore(parenList=True) + self.get_more(paren_list=True) else: - self.setFlag(self.getName()) - self.getMore() + self.set_flag(self.get_name()) + self.get_more() finally: if self.got_features > old_got: self.col_offset = col_offset self.lineno = line - self.consumeEmptyLine() + self.consume_empty_line() else: return - def consumeMandatoryWhitespace(self): + def consume_mandatory_whitespace(self): if self.getc() not in whitespace + '\\': raise DoneException - self.consumeWhitespace() + self.consume_whitespace() - def consumeWhitespace(self): + def consume_whitespace(self): while 1: c = self.getc() if c in whitespace: @@ -238,7 +238,7 @@ else: return - def getName(self): + def get_name(self): if self.getc() not in letters: raise DoneException p = self.pos @@ -247,36 +247,36 @@ if self.getc() not in alphanumerics: break name = self.s[p:self.pos] - self.consumeWhitespace() + self.consume_whitespace() return name - def getMore(self, parenList=False): - if parenList and self.getc() == ')': + def get_more(self, paren_list=False): + if paren_list and self.getc() == ')': self.pos += 1 return if (self.getc() == 'a' and self.getc(+1) == 's' and self.getc(+2) in whitespace): - self.getName() - self.getName() - self.getMore(parenList=parenList) + self.get_name() + self.get_name() + self.get_more(paren_list=paren_list) return elif self.getc() != ',': return else: self.pos += 1 - self.consumeWhitespace() - if parenList and self.getc() == ')': + self.consume_whitespace() + if paren_list and self.getc() == ')': self.pos += 1 return # Handles trailing comma inside parenthesis - self.setFlag(self.getName()) - self.getMore(parenList=parenList) + self.set_flag(self.get_name()) + self.get_more(paren_list=paren_list) - def setFlag(self, feature): + def set_flag(self, feature): self.got_features += 1 try: - self.flags |= self.futureFlags.compiler_features[feature] + self.flags |= self.future_flags.compiler_features[feature] except KeyError: pass @@ -286,6 +286,7 @@ from pypy.tool import stdlib___future__ as future class FutureFlags(object): + def __init__(self, version): compiler_flags = 0 self.compiler_features = {} Modified: pypy/trunk/pypy/interpreter/pyparser/test/test_futureautomaton.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/test/test_futureautomaton.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/test/test_futureautomaton.py Sat Oct 3 01:20:10 2009 @@ -124,7 +124,7 @@ def test_full_chain(): s = '"abc" #def\n #ghi\nfrom __future__ import (division as b, generators,); from __future__ import with_statement\n' - flags, pos = future.getFutures(future.futureFlags_2_5, s) + flags, pos = future.get_futures(future.futureFlags_2_5, s) assert flags == (fut.CO_FUTURE_DIVISION | fut.CO_GENERATOR_ALLOWED | fut.CO_FUTURE_WITH_STATEMENT) @@ -132,7 +132,7 @@ def test_intervening_code(): s = 'from __future__ import (division as b, generators,)\nfrom sys import modules\nfrom __future__ import with_statement\n' - flags, pos = future.getFutures(future.futureFlags_2_5, s) + flags, pos = future.get_futures(future.futureFlags_2_5, s) assert flags & fut.CO_FUTURE_WITH_STATEMENT == 0 assert pos == (1, 0) From antocuni at codespeak.net Sat Oct 3 10:54:34 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 3 Oct 2009 10:54:34 +0200 (CEST) Subject: [pypy-svn] r68135 - pypy/trunk/pypy/translator/jvm/test Message-ID: <20091003085434.B077D1683E1@codespeak.net> Author: antocuni Date: Sat Oct 3 10:54:33 2009 New Revision: 68135 Modified: pypy/trunk/pypy/translator/jvm/test/test_dict.py Log: skip this test for now Modified: pypy/trunk/pypy/translator/jvm/test/test_dict.py ============================================================================== --- pypy/trunk/pypy/translator/jvm/test/test_dict.py (original) +++ pypy/trunk/pypy/translator/jvm/test/test_dict.py Sat Oct 3 10:54:33 2009 @@ -3,8 +3,8 @@ import pypy.translator.oosupport.test_template.dict as oodict class TestJvmDict(JvmTest, oodict.BaseTestDict): - def test_invalid_iterator(self): - py.test.skip("test_invalid_iterator() doesn't work yet") + def test_resize_during_iteration(self): + py.test.skip("test_resize_during_iteration() doesn't work yet") def test_recursive(self): py.test.skip("JVM doesn't support recursive dicts") From cfbolz at codespeak.net Sat Oct 3 12:02:18 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 3 Oct 2009 12:02:18 +0200 (CEST) Subject: [pypy-svn] r68136 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091003100218.DE57D1683DC@codespeak.net> Author: cfbolz Date: Sat Oct 3 12:02:17 2009 New Revision: 68136 Modified: pypy/trunk/pypy/jit/metainterp/history.py pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: (pedronis, cfbolz): do common expression elimination of pure ops. This will be reverted, it's a bit lost in the noise. Checking it in to not lose it. Modified: pypy/trunk/pypy/jit/metainterp/history.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/history.py (original) +++ pypy/trunk/pypy/jit/metainterp/history.py Sat Oct 3 12:02:17 2009 @@ -246,6 +246,7 @@ return repr_rpython(self, 'ci') constint_lower = -5 constint_upper = 200 +assert rop._LAST < constint_upper ConstInt.PREBUILT = [ConstInt(i) for i in range(constint_lower, constint_upper)] def constint(x): Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Sat Oct 3 12:02:17 2009 @@ -12,7 +12,7 @@ from pypy.jit.metainterp.optimizeutil import InvalidLoop from pypy.jit.metainterp import resume, compile from pypy.jit.metainterp.typesystem import llhelper, oohelper -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, r_dict from pypy.rpython.lltypesystem import lltype def optimize_loop_1(cpu, loop): @@ -348,6 +348,23 @@ subspecnode.teardown_virtual_node(optimizer, subvalue, newexitargs) +def hash_op(op): + from pypy.rlib.rarithmetic import intmask + mult = 1000003 + x = 0x345678 ^ op.opnum + z = len(op.args) + for box in op.args: + y = hash(box) + x = (x ^ y) * mult + z -= 1 + mult += 82520 + z + z + x += 97531 + return intmask(x) + +def eq_op(op1, op2): + return op1.opnum == op2.opnum and op1.args == op2.args + + class Optimizer(object): def __init__(self, cpu, loop): @@ -362,6 +379,7 @@ self.values_to_clean = {} self.interned_refs = {} + self.emitted_pure_ops = r_dict(eq_op, hash_op) def getinterned(self, box): constbox = self.get_constant_box(box) @@ -378,7 +396,7 @@ self.interned_refs[key] = box return box else: - return box + return constbox def getvalue(self, box): box = self.getinterned(box) @@ -494,7 +512,14 @@ op = op.clone() must_clone = False op.args[i] = box - if op.is_guard(): + if op.is_always_pure(): + oldresult = self.emitted_pure_ops.get(op, None) + if oldresult: + self.make_equal_to(op.result, self.getvalue(oldresult)) + return + else: + self.emitted_pure_ops[op] = op.result + elif op.is_guard(): self.store_final_boxes_in_guard(op) elif op.can_raise(): self.exception_might_have_happened = True @@ -534,6 +559,7 @@ resbox = execute_nonspec(self.cpu, op.opnum, argboxes, op.descr) self.make_constant(op.result, resbox.constbox()) return + elif not op.has_no_side_effect() and not op.is_ovf(): self.clean_fields_of_values() # otherwise, the operation remains Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Sat Oct 3 12:02:17 2009 @@ -47,6 +47,7 @@ assert fdescr.rd_virtuals is None assert fdescr.rd_consts == [] assert fdescr.rd_frame_infos == fi + # ____________________________________________________________ def equaloplists(oplist1, oplist2, strict_fail_args=True, remap={}): @@ -132,6 +133,17 @@ class BaseTestOptimizeOpt(BaseTest): + def test_getinterned(self): + opt = optimizeopt.Optimizer(self.cpu, None) + b1 = BoxInt() + assert opt.getinterned(b1) is b1 + opt.make_constant(b1, ConstInt(1)) + c1 = opt.getinterned(b1) + assert isinstance(c1, ConstInt) + c1 = self.cpu.ts.ConstRef(self.myptr) + c2 = self.cpu.ts.ConstRef(self.myptr) + assert opt.getinterned(c1) is opt.getinterned(c2) + def invent_fail_descr(self, fail_args): if fail_args is None: return None @@ -409,6 +421,72 @@ """ self.optimize_loop(ops, '', expected) + def test_common_expression_elimination(self): + ops = """ + [i0, i1, i2, i3] + ir1 = int_add(i0, 1) + ir2 = int_add(i0, 1) + ir3 = int_add(i1, i2) + ir4 = int_add(i1, i2) + jump(ir1, ir2, ir3, ir4) + """ + expected = """ + [i0, i1, i2, i3] + ir1 = int_add(i0, 1) + ir3 = int_add(i1, i2) + jump(ir1, ir1, ir3, ir3) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', expected) + + ops = """ + [p0, p1] + pr1 = getfield_gc_pure(p0, descr=valuedescr) + pr2 = getfield_gc_pure(p0, descr=valuedescr) + jump(pr1, pr2) + """ + expected = """ + [p0, p1] + pr1 = getfield_gc_pure(p0, descr=valuedescr) + jump(pr1, pr1) + """ + self.optimize_loop(ops, 'Not, Not', expected) + + ops = """ + [p0, p1] + i1 = oois(p0, p1) + guard_value(i1, 1) [] + i2 = oois(p0, p1) + guard_value(i2, 1) [] + jump(p0, p1) + """ + expected = """ + [p0, p1] + i1 = oois(p0, p1) + guard_value(i1, 1) [] + jump(p0, p1) + """ + self.optimize_loop(ops, 'Not, Not', expected) + + def test_common_expression_elimination_virtual(self): + ops = """ + [p0, p1] + i1 = oois(p0, p1) + guard_value(i1, 1) [] + p2 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p2, p0, descr=valuedescr) + p3 = getfield_gc(p2, descr=valuedescr) + i2 = oois(p3, p1) + guard_value(i2, 1) [] + jump(p0, p1) + """ + expected = """ + [p0, p1] + i1 = oois(p0, p1) + guard_value(i1, 1) [] + jump(p0, p1) + """ + self.optimize_loop(ops, 'Not, Not', expected) + def test_ooisnull_oononnull_via_virtual(self): ops = """ [p0] From cfbolz at codespeak.net Sat Oct 3 12:13:49 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 3 Oct 2009 12:13:49 +0200 (CEST) Subject: [pypy-svn] r68137 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091003101349.B0E8D1683E1@codespeak.net> Author: cfbolz Date: Sat Oct 3 12:13:48 2009 New Revision: 68137 Modified: pypy/trunk/pypy/jit/metainterp/executor.py pypy/trunk/pypy/jit/metainterp/history.py pypy/trunk/pypy/jit/metainterp/optimizefindnode.py pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/jit/metainterp/specnode.py pypy/trunk/pypy/jit/metainterp/test/oparser.py pypy/trunk/pypy/jit/metainterp/test/test_history.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/metainterp/typesystem.py pypy/trunk/pypy/jit/metainterp/warmspot.py Log: Revert the checkins 68136, 68133, 68132. Their time will come. Modified: pypy/trunk/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/executor.py (original) +++ pypy/trunk/pypy/jit/metainterp/executor.py Sat Oct 3 12:13:48 2009 @@ -6,7 +6,7 @@ from pypy.rpython.ootypesystem import ootype from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck, r_uint, intmask -from pypy.jit.metainterp.history import BoxInt, constint, check_descr +from pypy.jit.metainterp.history import BoxInt, ConstInt, check_descr from pypy.jit.metainterp.history import INT, REF, ConstFloat from pypy.jit.metainterp import resoperation from pypy.jit.metainterp.resoperation import rop @@ -19,86 +19,86 @@ # ____________________________________________________________ def do_int_add(cpu, box1, box2): - return constint(intmask(box1.getint() + box2.getint())) + return ConstInt(intmask(box1.getint() + box2.getint())) def do_int_sub(cpu, box1, box2): - return constint(intmask(box1.getint() - box2.getint())) + return ConstInt(intmask(box1.getint() - box2.getint())) def do_int_mul(cpu, box1, box2): - return constint(intmask(box1.getint() * box2.getint())) + return ConstInt(intmask(box1.getint() * box2.getint())) def do_int_floordiv(cpu, box1, box2): z = llop.int_floordiv(lltype.Signed, box1.getint(), box2.getint()) - return constint(z) + return ConstInt(z) def do_int_mod(cpu, box1, box2): z = llop.int_mod(lltype.Signed, box1.getint(), box2.getint()) - return constint(z) + return ConstInt(z) def do_int_and(cpu, box1, box2): - return constint(box1.getint() & box2.getint()) + return ConstInt(box1.getint() & box2.getint()) def do_int_or(cpu, box1, box2): - return constint(box1.getint() | box2.getint()) + return ConstInt(box1.getint() | box2.getint()) def do_int_xor(cpu, box1, box2): - return constint(box1.getint() ^ box2.getint()) + return ConstInt(box1.getint() ^ box2.getint()) def do_int_rshift(cpu, box1, box2): - return constint(box1.getint() >> box2.getint()) + return ConstInt(box1.getint() >> box2.getint()) def do_int_lshift(cpu, box1, box2): - return constint(intmask(box1.getint() << box2.getint())) + return ConstInt(intmask(box1.getint() << box2.getint())) def do_uint_rshift(cpu, box1, box2): v = r_uint(box1.getint()) >> r_uint(box2.getint()) - return constint(intmask(v)) + return ConstInt(intmask(v)) # ---------- def do_int_lt(cpu, box1, box2): - return constint(box1.getint() < box2.getint()) + return ConstInt(box1.getint() < box2.getint()) def do_int_le(cpu, box1, box2): - return constint(box1.getint() <= box2.getint()) + return ConstInt(box1.getint() <= box2.getint()) def do_int_eq(cpu, box1, box2): - return constint(box1.getint() == box2.getint()) + return ConstInt(box1.getint() == box2.getint()) def do_int_ne(cpu, box1, box2): - return constint(box1.getint() != box2.getint()) + return ConstInt(box1.getint() != box2.getint()) def do_int_gt(cpu, box1, box2): - return constint(box1.getint() > box2.getint()) + return ConstInt(box1.getint() > box2.getint()) def do_int_ge(cpu, box1, box2): - return constint(box1.getint() >= box2.getint()) + return ConstInt(box1.getint() >= box2.getint()) def do_uint_lt(cpu, box1, box2): - return constint(r_uint(box1.getint()) < r_uint(box2.getint())) + return ConstInt(r_uint(box1.getint()) < r_uint(box2.getint())) def do_uint_le(cpu, box1, box2): - return constint(r_uint(box1.getint()) <= r_uint(box2.getint())) + return ConstInt(r_uint(box1.getint()) <= r_uint(box2.getint())) def do_uint_gt(cpu, box1, box2): - return constint(r_uint(box1.getint()) > r_uint(box2.getint())) + return ConstInt(r_uint(box1.getint()) > r_uint(box2.getint())) def do_uint_ge(cpu, box1, box2): - return constint(r_uint(box1.getint()) >= r_uint(box2.getint())) + return ConstInt(r_uint(box1.getint()) >= r_uint(box2.getint())) # ---------- def do_int_is_true(cpu, box1): - return constint(bool(box1.getint())) + return ConstInt(bool(box1.getint())) def do_int_neg(cpu, box1): - return constint(intmask(-box1.getint())) + return ConstInt(intmask(-box1.getint())) def do_int_invert(cpu, box1): - return constint(~box1.getint()) + return ConstInt(~box1.getint()) def do_bool_not(cpu, box1): - return constint(not box1.getint()) + return ConstInt(not box1.getint()) def do_same_as(cpu, box1): return box1 @@ -111,7 +111,7 @@ x = bool(box1.getref_base()) else: assert False - return constint(x) + return ConstInt(x) def do_ooisnull(cpu, box1): tp = box1.type @@ -121,7 +121,7 @@ x = bool(box1.getref_base()) else: assert False - return constint(not x) + return ConstInt(not x) def do_oois(cpu, box1, box2): tp = box1.type @@ -132,7 +132,7 @@ x = box1.getref_base() == box2.getref_base() else: assert False - return constint(x) + return ConstInt(x) def do_ooisnot(cpu, box1, box2): tp = box1.type @@ -143,14 +143,14 @@ x = box1.getref_base() != box2.getref_base() else: assert False - return constint(x) + return ConstInt(x) def do_ooidentityhash(cpu, box1): obj = box1.getref_base() - return constint(cpu.ts.ooidentityhash(obj)) + return ConstInt(cpu.ts.ooidentityhash(obj)) def do_subclassof(cpu, box1, box2): - return constint(cpu.ts.subclassOf(cpu, box1, box2)) + return ConstInt(cpu.ts.subclassOf(cpu, box1, box2)) # ---------- @@ -202,7 +202,7 @@ return ConstFloat(abs(box1.getfloat())) def do_float_is_true(cpu, box1): - return constint(bool(box1.getfloat())) + return ConstInt(bool(box1.getfloat())) def do_float_add(cpu, box1, box2): return ConstFloat(box1.getfloat() + box2.getfloat()) @@ -217,25 +217,25 @@ return ConstFloat(box1.getfloat() / box2.getfloat()) def do_float_lt(cpu, box1, box2): - return constint(box1.getfloat() < box2.getfloat()) + return ConstInt(box1.getfloat() < box2.getfloat()) def do_float_le(cpu, box1, box2): - return constint(box1.getfloat() <= box2.getfloat()) + return ConstInt(box1.getfloat() <= box2.getfloat()) def do_float_eq(cpu, box1, box2): - return constint(box1.getfloat() == box2.getfloat()) + return ConstInt(box1.getfloat() == box2.getfloat()) def do_float_ne(cpu, box1, box2): - return constint(box1.getfloat() != box2.getfloat()) + return ConstInt(box1.getfloat() != box2.getfloat()) def do_float_gt(cpu, box1, box2): - return constint(box1.getfloat() > box2.getfloat()) + return ConstInt(box1.getfloat() > box2.getfloat()) def do_float_ge(cpu, box1, box2): - return constint(box1.getfloat() >= box2.getfloat()) + return ConstInt(box1.getfloat() >= box2.getfloat()) def do_cast_float_to_int(cpu, box1): - return constint(int(box1.getfloat())) + return ConstInt(int(box1.getfloat())) def do_cast_int_to_float(cpu, box1): return ConstFloat(float(box1.getint())) Modified: pypy/trunk/pypy/jit/metainterp/history.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/history.py (original) +++ pypy/trunk/pypy/jit/metainterp/history.py Sat Oct 3 12:13:48 2009 @@ -160,7 +160,7 @@ intval = cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x)) else: intval = lltype.cast_primitive(lltype.Signed, x) - return constint(intval) + return ConstInt(intval) elif kind == "ref": return cpu.ts.new_ConstRef(x) elif kind == "float": @@ -244,28 +244,9 @@ def repr_rpython(self): return repr_rpython(self, 'ci') -constint_lower = -5 -constint_upper = 200 -assert rop._LAST < constint_upper -ConstInt.PREBUILT = [ConstInt(i) for i in range(constint_lower, constint_upper)] - -def constint(x): - from pypy.rlib.rarithmetic import r_uint - # use r_uint to perform a single comparison (this whole function - # is getting inlined into every caller so keeping the branching - # to a minimum is a good idea) - if not we_are_translated() and not isinstance(x, int): # e.g. TotalOrderSymbolic - return ConstInt(x) - index = r_uint(x - constint_lower) - if index >= r_uint(constint_upper - constint_lower): - const = ConstInt(x) - else: - const = ConstInt.PREBUILT[index] - return const - -CONST_0 = CONST_FALSE = constint(0) -CONST_1 = CONST_TRUE = constint(1) +CONST_FALSE = ConstInt(0) +CONST_TRUE = ConstInt(1) class ConstAddr(Const): # only for constants built before translation type = INT @@ -509,7 +490,7 @@ return BoxInt(self.value) def constbox(self): - return constint(self.value) + return ConstInt(self.value) def getint(self): return self.value Modified: pypy/trunk/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizefindnode.py Sat Oct 3 12:13:48 2009 @@ -4,7 +4,7 @@ from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.specnode import VirtualArraySpecNode from pypy.jit.metainterp.specnode import VirtualStructSpecNode -from pypy.jit.metainterp.history import AbstractValue, constint, Const +from pypy.jit.metainterp.history import AbstractValue, ConstInt, Const from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.executor import execute_nonspec from pypy.jit.metainterp.optimizeutil import av_newdict, _findall, sort_descrs @@ -199,7 +199,7 @@ def find_nodes_ARRAYLEN_GC(self, op): arraynode = self.getnode(op.args[0]) if arraynode.arraydescr is not None: - resbox = constint(arraynode.arraysize) + resbox = ConstInt(arraynode.arraysize) self.set_constant_node(op.result, resbox) def find_nodes_GUARD_CLASS(self, op): Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Sat Oct 3 12:13:48 2009 @@ -1,6 +1,5 @@ from pypy.jit.metainterp.history import Box, BoxInt, LoopToken -from pypy.jit.metainterp.history import Const, constint, ConstInt, ConstPtr, ConstObj, REF -from pypy.jit.metainterp.history import CONST_0, CONST_1 +from pypy.jit.metainterp.history import Const, ConstInt, ConstPtr, ConstObj, REF from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.metainterp.executor import execute_nonspec from pypy.jit.metainterp.specnode import SpecNode, NotSpecNode, ConstantSpecNode @@ -12,7 +11,7 @@ from pypy.jit.metainterp.optimizeutil import InvalidLoop from pypy.jit.metainterp import resume, compile from pypy.jit.metainterp.typesystem import llhelper, oohelper -from pypy.rlib.objectmodel import we_are_translated, r_dict +from pypy.rlib.objectmodel import we_are_translated from pypy.rpython.lltypesystem import lltype def optimize_loop_1(cpu, loop): @@ -141,6 +140,8 @@ def __init__(self, box): self.box = box +CONST_0 = ConstInt(0) +CONST_1 = ConstInt(1) CVAL_ZERO = ConstantValue(CONST_0) llhelper.CONST_NULL = ConstPtr(ConstPtr.value) llhelper.CVAL_NULLREF = ConstantValue(llhelper.CONST_NULL) @@ -268,7 +269,7 @@ if subvalue is not None: subbox = subvalue.force_box() op = ResOperation(rop.SETARRAYITEM_GC, - [box, constint(index), subbox], None, + [box, ConstInt(index), subbox], None, descr=self.arraydescr) newoperations.append(op) return self.box @@ -348,23 +349,6 @@ subspecnode.teardown_virtual_node(optimizer, subvalue, newexitargs) -def hash_op(op): - from pypy.rlib.rarithmetic import intmask - mult = 1000003 - x = 0x345678 ^ op.opnum - z = len(op.args) - for box in op.args: - y = hash(box) - x = (x ^ y) * mult - z -= 1 - mult += 82520 + z + z - x += 97531 - return intmask(x) - -def eq_op(op1, op2): - return op1.opnum == op2.opnum and op1.args == op2.args - - class Optimizer(object): def __init__(self, cpu, loop): @@ -379,7 +363,6 @@ self.values_to_clean = {} self.interned_refs = {} - self.emitted_pure_ops = r_dict(eq_op, hash_op) def getinterned(self, box): constbox = self.get_constant_box(box) @@ -396,7 +379,7 @@ self.interned_refs[key] = box return box else: - return constbox + return box def getvalue(self, box): box = self.getinterned(box) @@ -427,7 +410,7 @@ self.make_equal_to(box, ConstantValue(constbox)) def make_constant_int(self, box, intvalue): - self.make_constant(box, constint(intvalue)) + self.make_constant(box, ConstInt(intvalue)) def make_virtual(self, known_class, box, source_op=None): vvalue = VirtualValue(self, known_class, box, source_op) @@ -512,14 +495,7 @@ op = op.clone() must_clone = False op.args[i] = box - if op.is_always_pure(): - oldresult = self.emitted_pure_ops.get(op, None) - if oldresult: - self.make_equal_to(op.result, self.getvalue(oldresult)) - return - else: - self.emitted_pure_ops[op] = op.result - elif op.is_guard(): + if op.is_guard(): self.store_final_boxes_in_guard(op) elif op.can_raise(): self.exception_might_have_happened = True @@ -559,7 +535,6 @@ resbox = execute_nonspec(self.cpu, op.opnum, argboxes, op.descr) self.make_constant(op.result, resbox.constbox()) return - elif not op.has_no_side_effect() and not op.is_ovf(): self.clean_fields_of_values() # otherwise, the operation remains Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Sat Oct 3 12:13:48 2009 @@ -6,7 +6,7 @@ from pypy.rlib.debug import debug_print from pypy.jit.metainterp import history, compile, resume -from pypy.jit.metainterp.history import Const, constint, Box, CONST_0 +from pypy.jit.metainterp.history import Const, ConstInt, Box from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp import codewriter, executor from pypy.jit.metainterp.logger import Logger @@ -351,7 +351,7 @@ @arguments("orgpc", "box", "descr", "box") def opimpl_check_neg_index(self, pc, arraybox, arraydesc, indexbox): negbox = self.metainterp.execute_and_record( - rop.INT_LT, None, indexbox, CONST_0) + rop.INT_LT, None, indexbox, ConstInt(0)) # xxx inefficient negbox = self.implement_guard_value(pc, negbox) if negbox.getint(): @@ -391,7 +391,7 @@ def opimpl_check_resizable_neg_index(self, pc, listbox, lengthdesc, indexbox): negbox = self.metainterp.execute_and_record( - rop.INT_LT, None, indexbox, CONST_0) + rop.INT_LT, None, indexbox, ConstInt(0)) # xxx inefficient negbox = self.implement_guard_value(pc, negbox) if negbox.getint(): @@ -405,7 +405,7 @@ @arguments("orgpc", "box") def opimpl_check_zerodivisionerror(self, pc, box): nonzerobox = self.metainterp.execute_and_record( - rop.INT_NE, None, box, CONST_0) + rop.INT_NE, None, box, ConstInt(0)) # xxx inefficient nonzerobox = self.implement_guard_value(pc, nonzerobox) if nonzerobox.getint(): @@ -419,11 +419,11 @@ # detect the combination "box1 = -sys.maxint-1, box2 = -1". import sys tmp1 = self.metainterp.execute_and_record( # combination to detect: - rop.INT_ADD, None, box1, constint(sys.maxint)) # tmp1=-1, box2=-1 + rop.INT_ADD, None, box1, ConstInt(sys.maxint)) # tmp1=-1, box2=-1 tmp2 = self.metainterp.execute_and_record( rop.INT_AND, None, tmp1, box2) # tmp2=-1 tmp3 = self.metainterp.execute_and_record( - rop.INT_EQ, None, tmp2, constint(-1)) # tmp3? + rop.INT_EQ, None, tmp2, ConstInt(-1)) # tmp3? # xxx inefficient tmp4 = self.implement_guard_value(pc, tmp3) # tmp4? if not tmp4.getint(): @@ -439,7 +439,7 @@ @arguments("orgpc", "box") def opimpl_int_abs(self, pc, box): nonneg = self.metainterp.execute_and_record( - rop.INT_GE, None, box, CONST_0) + rop.INT_GE, None, box, ConstInt(0)) # xxx inefficient nonneg = self.implement_guard_value(pc, nonneg) if nonneg.getint(): @@ -585,7 +585,7 @@ virtualizable_box = self.metainterp.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) result = vinfo.get_array_length(virtualizable, arrayindex) - self.make_result_box(constint(result)) + self.make_result_box(ConstInt(result)) def perform_call(self, jitcode, varargs): if (self.metainterp.is_blackholing() and @@ -1725,7 +1725,7 @@ descr = vinfo.array_descrs[k] for j in range(vinfo.get_array_length(virtualizable, k)): itembox = self.execute_and_record(rop.GETARRAYITEM_GC, - descr, abox, constint(j)) + descr, abox, ConstInt(j)) self.virtualizable_boxes[i] = itembox i += 1 assert i + 1 == len(self.virtualizable_boxes) @@ -1749,7 +1749,7 @@ itembox = self.virtualizable_boxes[i] i += 1 self.execute_and_record(rop.SETARRAYITEM_GC, descr, - abox, constint(j), itembox) + abox, ConstInt(j), itembox) assert i + 1 == len(self.virtualizable_boxes) def gen_store_back_in_virtualizable_no_perform(self): @@ -1770,7 +1770,7 @@ itembox = self.virtualizable_boxes[i] i += 1 self.history.record(rop.SETARRAYITEM_GC, - [abox, constint(j), itembox], + [abox, ConstInt(j), itembox], None, descr=vinfo.array_descrs[k]) assert i + 1 == len(self.virtualizable_boxes) Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Sat Oct 3 12:13:48 2009 @@ -1,5 +1,5 @@ import sys -from pypy.jit.metainterp.history import Box, Const, constint +from pypy.jit.metainterp.history import Box, Const, ConstInt from pypy.jit.metainterp.resoperation import rop # Logic to encode the chain of frames and the state of the boxes at a @@ -251,14 +251,14 @@ length = len(self.fieldnums) return metainterp.execute_and_record(rop.NEW_ARRAY, self.arraydescr, - constint(length)) + ConstInt(length)) def setfields(self, metainterp, box, fn_decode_box): for i in range(len(self.fieldnums)): itembox = fn_decode_box(self.fieldnums[i]) metainterp.execute_and_record(rop.SETARRAYITEM_GC, self.arraydescr, - box, constint(i), itembox) + box, ConstInt(i), itembox) def repr_rpython(self): return 'VArrayInfo("%s", %s)' % (self.arraydescr, Modified: pypy/trunk/pypy/jit/metainterp/specnode.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/specnode.py (original) +++ pypy/trunk/pypy/jit/metainterp/specnode.py Sat Oct 3 12:13:48 2009 @@ -97,7 +97,7 @@ for i in range(len(self.items)): itembox = executor.execute(cpu, resoperation.rop.GETARRAYITEM_GC, self.arraydescr, - valuebox, history.constint(i)) + valuebox, history.ConstInt(i)) subspecnode = self.items[i] subspecnode.extract_runtime_data(cpu, itembox, resultlist) Modified: pypy/trunk/pypy/jit/metainterp/test/oparser.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/oparser.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/oparser.py Sat Oct 3 12:13:48 2009 @@ -3,7 +3,7 @@ in a nicer fashion """ -from pypy.jit.metainterp.history import TreeLoop, BoxInt, constint,\ +from pypy.jit.metainterp.history import TreeLoop, BoxInt, ConstInt,\ ConstAddr, ConstObj, ConstPtr, Box, BasicFailDescr, LoopToken from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.metainterp.typesystem import llhelper @@ -98,9 +98,9 @@ def getvar(self, arg): if not arg: - return constint(0) + return ConstInt(0) try: - return constint(int(arg)) + return ConstInt(int(arg)) except ValueError: if arg.startswith('"') or arg.startswith("'"): # XXX ootype Modified: pypy/trunk/pypy/jit/metainterp/test/test_history.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_history.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_history.py Sat Oct 3 12:13:48 2009 @@ -9,13 +9,3 @@ s = lltype.cast_pointer(lltype.Ptr(S), t) const = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s)) assert const._getrepr_() == "*T" - -def test_constint_sharing(): - for val in [-1, 0, 1, 5, 99]: - c1 = constint(val) - c2 = constint(val) - assert c1 is c2 - for val in [100000, -10000]: - c1 = constint(val) - c2 = constint(val) - assert c1 is not c2 Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Sat Oct 3 12:13:48 2009 @@ -47,7 +47,6 @@ assert fdescr.rd_virtuals is None assert fdescr.rd_consts == [] assert fdescr.rd_frame_infos == fi - # ____________________________________________________________ def equaloplists(oplist1, oplist2, strict_fail_args=True, remap={}): @@ -133,17 +132,6 @@ class BaseTestOptimizeOpt(BaseTest): - def test_getinterned(self): - opt = optimizeopt.Optimizer(self.cpu, None) - b1 = BoxInt() - assert opt.getinterned(b1) is b1 - opt.make_constant(b1, ConstInt(1)) - c1 = opt.getinterned(b1) - assert isinstance(c1, ConstInt) - c1 = self.cpu.ts.ConstRef(self.myptr) - c2 = self.cpu.ts.ConstRef(self.myptr) - assert opt.getinterned(c1) is opt.getinterned(c2) - def invent_fail_descr(self, fail_args): if fail_args is None: return None @@ -421,72 +409,6 @@ """ self.optimize_loop(ops, '', expected) - def test_common_expression_elimination(self): - ops = """ - [i0, i1, i2, i3] - ir1 = int_add(i0, 1) - ir2 = int_add(i0, 1) - ir3 = int_add(i1, i2) - ir4 = int_add(i1, i2) - jump(ir1, ir2, ir3, ir4) - """ - expected = """ - [i0, i1, i2, i3] - ir1 = int_add(i0, 1) - ir3 = int_add(i1, i2) - jump(ir1, ir1, ir3, ir3) - """ - self.optimize_loop(ops, 'Not, Not, Not, Not', expected) - - ops = """ - [p0, p1] - pr1 = getfield_gc_pure(p0, descr=valuedescr) - pr2 = getfield_gc_pure(p0, descr=valuedescr) - jump(pr1, pr2) - """ - expected = """ - [p0, p1] - pr1 = getfield_gc_pure(p0, descr=valuedescr) - jump(pr1, pr1) - """ - self.optimize_loop(ops, 'Not, Not', expected) - - ops = """ - [p0, p1] - i1 = oois(p0, p1) - guard_value(i1, 1) [] - i2 = oois(p0, p1) - guard_value(i2, 1) [] - jump(p0, p1) - """ - expected = """ - [p0, p1] - i1 = oois(p0, p1) - guard_value(i1, 1) [] - jump(p0, p1) - """ - self.optimize_loop(ops, 'Not, Not', expected) - - def test_common_expression_elimination_virtual(self): - ops = """ - [p0, p1] - i1 = oois(p0, p1) - guard_value(i1, 1) [] - p2 = new_with_vtable(ConstClass(node_vtable)) - setfield_gc(p2, p0, descr=valuedescr) - p3 = getfield_gc(p2, descr=valuedescr) - i2 = oois(p3, p1) - guard_value(i2, 1) [] - jump(p0, p1) - """ - expected = """ - [p0, p1] - i1 = oois(p0, p1) - guard_value(i1, 1) [] - jump(p0, p1) - """ - self.optimize_loop(ops, 'Not, Not', expected) - def test_ooisnull_oononnull_via_virtual(self): ops = """ [p0] Modified: pypy/trunk/pypy/jit/metainterp/typesystem.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/typesystem.py (original) +++ pypy/trunk/pypy/jit/metainterp/typesystem.py Sat Oct 3 12:13:48 2009 @@ -77,7 +77,7 @@ def cls_of_box(self, cpu, box): obj = box.getref(lltype.Ptr(rclass.OBJECT)) cls = llmemory.cast_ptr_to_adr(obj.typeptr) - return history.constint(cpu.cast_adr_to_int(cls)) + return history.ConstInt(cpu.cast_adr_to_int(cls)) def subclassOf(self, cpu, clsbox1, clsbox2): adr = clsbox2.getaddr(cpu) @@ -87,7 +87,7 @@ return rclass.ll_issubclass(real_class, bounding_class) def get_exception_box(self, etype): - return history.constint(etype) + return history.ConstInt(etype) def get_exc_value_box(self, evalue): return history.BoxPtr(evalue) Modified: pypy/trunk/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmspot.py Sat Oct 3 12:13:48 2009 @@ -637,7 +637,7 @@ else: value = intmask(value) if in_const_box: - return history.constint(value) + return history.ConstInt(value) else: return history.BoxInt(value) wrap._annspecialcase_ = 'specialize:ll' From cfbolz at codespeak.net Sat Oct 3 13:11:27 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 3 Oct 2009 13:11:27 +0200 (CEST) Subject: [pypy-svn] r68138 - pypy/trunk/pypy/jit/backend/llsupport Message-ID: <20091003111127.07DE61683DC@codespeak.net> Author: cfbolz Date: Sat Oct 3 13:11:27 2009 New Revision: 68138 Modified: pypy/trunk/pypy/jit/backend/llsupport/gc.py Log: this belongs to 68137, sorry armin Modified: pypy/trunk/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/gc.py Sat Oct 3 13:11:27 2009 @@ -4,7 +4,7 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.jit.metainterp.history import BoxInt, BoxPtr, constint, ConstInt, ConstPtr +from pypy.jit.metainterp.history import BoxInt, BoxPtr, ConstInt, ConstPtr from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.backend.llsupport import symbolic from pypy.jit.backend.llsupport.symbolic import WORD @@ -334,7 +334,7 @@ self.HDRPTR = lltype.Ptr(self.GCClass.HDR) self.gcheaderbuilder = GCHeaderBuilder(self.HDRPTR.TO) self.fielddescr_tid = get_field_descr(self, self.GCClass.HDR, 'tid') - self.c_jit_wb_if_flag = constint(self.GCClass.JIT_WB_IF_FLAG) + self.c_jit_wb_if_flag = ConstInt(self.GCClass.JIT_WB_IF_FLAG) self.calldescr_jit_wb = get_call_descr(self, [llmemory.GCREF, llmemory.GCREF], lltype.Void) @@ -488,7 +488,7 @@ addr = self.gcrefs.get_address_of_gcref(v.value) addr = cpu.cast_adr_to_int(addr) newops.append(ResOperation(rop.GETFIELD_RAW, - [constint(addr)], box, + [ConstInt(addr)], box, self.single_gcref_descr)) op.args[i] = box # ---------- write barrier for SETFIELD_GC ---------- @@ -519,8 +519,6 @@ llop1 = self.llop1 funcptr = llop1.get_write_barrier_failing_case(self.WB_FUNCPTR) funcaddr = llmemory.cast_ptr_to_adr(funcptr) - # this is not going to be a small integer, so don't use constint - # (bit of a micro-optimization) c_func = ConstInt(cpu.cast_adr_to_int(funcaddr)) args = [v_tid, self.c_jit_wb_if_flag, c_func, v_base, v_value] newops.append(ResOperation(rop.COND_CALL_GC_WB, args, None, From arigo at codespeak.net Sat Oct 3 13:41:04 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 3 Oct 2009 13:41:04 +0200 (CEST) Subject: [pypy-svn] r68139 - in pypy/trunk/pypy: rpython translator Message-ID: <20091003114104.37CA11683DC@codespeak.net> Author: arigo Date: Sat Oct 3 13:41:03 2009 New Revision: 68139 Modified: pypy/trunk/pypy/rpython/exceptiondata.py pypy/trunk/pypy/translator/exceptiontransform.py Log: Also crash early on NotImplementedError in debug builds. Modified: pypy/trunk/pypy/rpython/exceptiondata.py ============================================================================== --- pypy/trunk/pypy/rpython/exceptiondata.py (original) +++ pypy/trunk/pypy/rpython/exceptiondata.py Sat Oct 3 13:41:03 2009 @@ -18,6 +18,7 @@ RuntimeError : True, UnicodeDecodeError: True, UnicodeEncodeError: True, + NotImplementedError: True, } Modified: pypy/trunk/pypy/translator/exceptiontransform.py ============================================================================== --- pypy/trunk/pypy/translator/exceptiontransform.py (original) +++ pypy/trunk/pypy/translator/exceptiontransform.py Sat Oct 3 13:41:03 2009 @@ -57,14 +57,12 @@ exc_data, null_type, null_value = self.setup_excdata() rclass = translator.rtyper.type_system.rclass - runtime_error_def = translator.annotator.bookkeeper.getuniqueclassdef(RuntimeError) - runtime_error_ll_exc = edata.get_standard_ll_exc_instance(translator.rtyper, runtime_error_def) - runtime_error_ll_exc_type = rclass.ll_inst_type(runtime_error_ll_exc) - bk = translator.annotator.bookkeeper - assertion_error_def = bk.getuniqueclassdef(AssertionError) - assertion_error_ll_exc = edata.get_standard_ll_exc_instance( - translator.rtyper, assertion_error_def) - assertion_error_ll_exc_type=rclass.ll_inst_type(assertion_error_ll_exc) + (runtime_error_ll_exc_type, + runtime_error_ll_exc) = self.get_builtin_exception(RuntimeError) + (assertion_error_ll_exc_type, + assertion_error_ll_exc) = self.get_builtin_exception(AssertionError) + (n_i_error_ll_exc_type, + n_i_error_ll_exc) = self.get_builtin_exception(NotImplementedError) def rpyexc_occured(): exc_type = exc_data.exc_type @@ -83,6 +81,7 @@ def rpyexc_raise(etype, evalue): # assert(!RPyExceptionOccurred()); ll_assert(etype != assertion_error_ll_exc_type, "AssertionError!") + ll_assert(etype != n_i_error_ll_exc_type, "NotImplementedError!") exc_data.exc_type = etype exc_data.exc_value = evalue @@ -151,6 +150,16 @@ return self.constant_func(name, inputtypes, rettype, graph, exception_policy="exc_helper", **kwds) + def get_builtin_exception(self, Class): + edata = self.translator.rtyper.getexceptiondata() + rclass = self.translator.rtyper.type_system.rclass + bk = self.translator.annotator.bookkeeper + error_def = bk.getuniqueclassdef(Class) + error_ll_exc = edata.get_standard_ll_exc_instance( + self.translator.rtyper, error_def) + error_ll_exc_type = rclass.ll_inst_type(error_ll_exc) + return error_ll_exc_type, error_ll_exc + def transform_completely(self): for graph in self.translator.graphs: self.create_exception_handling(graph) From fijal at codespeak.net Sat Oct 3 13:49:39 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 3 Oct 2009 13:49:39 +0200 (CEST) Subject: [pypy-svn] r68140 - in pypy/branch/floats-via-sse2/pypy/jit/metainterp: . test Message-ID: <20091003114939.BEC201683DC@codespeak.net> Author: fijal Date: Sat Oct 3 13:49:39 2009 New Revision: 68140 Modified: pypy/branch/floats-via-sse2/pypy/jit/metainterp/logger.py pypy/branch/floats-via-sse2/pypy/jit/metainterp/test/test_logger.py Log: Fix the logger of floats, additionally don't explode when we don't know Modified: pypy/branch/floats-via-sse2/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/metainterp/logger.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/metainterp/logger.py Sat Oct 3 13:49:39 2009 @@ -2,7 +2,7 @@ from pypy.rlib.objectmodel import compute_unique_id from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.history import Const, ConstInt, Box, \ - BoxInt, ConstAddr + BoxInt, ConstAddr, ConstFloat, BoxFloat from pypy.rlib.streamio import open_file_as_stream class Logger(object): @@ -45,10 +45,14 @@ return 'ConstPtr(ptr' + str(mv) + ')' elif isinstance(arg, self.ts.BoxRef): return 'p' + str(mv) + elif isinstance(arg, ConstFloat): + return str(arg.value) + elif isinstance(arg, BoxFloat): + return 'f' + str(mv) elif isinstance(arg, self.ts.ConstAddr): return 'ConstClass(cls' + str(mv) + ')' else: - raise NotImplementedError + return '?' def log_operations(self, inputargs, operations, memo, indent=0): if self.log_stream is None: Modified: pypy/branch/floats-via-sse2/pypy/jit/metainterp/test/test_logger.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/metainterp/test/test_logger.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/metainterp/test/test_logger.py Sat Oct 3 13:49:39 2009 @@ -78,3 +78,10 @@ loop, oloop = self.reparse(inp) assert oloop.operations[0].args[0]._get_str() == 'info' + def test_floats(self): + inp = ''' + [f0] + f1 = float_add(3.5, f0) + ''' + loop, oloop = self.reparse(inp) + equaloplists(loop.operations, oloop.operations) From arigo at codespeak.net Sun Oct 4 11:24:24 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 4 Oct 2009 11:24:24 +0200 (CEST) Subject: [pypy-svn] r68141 - pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport Message-ID: <20091004092424.9233D1683E7@codespeak.net> Author: arigo Date: Sun Oct 4 11:24:22 2009 New Revision: 68141 Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/descr.py Log: Use a class attribute instead of overriding the method from "return False" to "return True". Modified: pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/floats-via-sse2/pypy/jit/backend/llsupport/descr.py Sun Oct 4 11:24:22 2009 @@ -67,11 +67,14 @@ def get_field_size(self, translate_support_code): raise NotImplementedError + _is_pointer_field = False # unless overridden by GcPtrFieldDescr + _is_float_field = False # unless overridden by FloatFieldDescr + def is_pointer_field(self): - return False # unless overridden by GcPtrFieldDescr + return self._is_pointer_field def is_float_field(self): - return False # unless overridden by FloatFieldDescr + return self._is_float_field def repr_of_descr(self): return '<%s %s>' % (self._clsname, self.offset) @@ -84,13 +87,12 @@ class GcPtrFieldDescr(NonGcPtrFieldDescr): _clsname = 'GcPtrFieldDescr' - def is_pointer_field(self): - return True + _is_pointer_field = True def getFieldDescrClass(TYPE): return getDescrClass(TYPE, BaseFieldDescr, GcPtrFieldDescr, NonGcPtrFieldDescr, 'Field', 'get_field_size', - 'is_float_field') + '_is_float_field') def get_field_descr(gccache, STRUCT, fieldname): cache = gccache._cache_field @@ -126,11 +128,14 @@ def get_item_size(self, translate_support_code): raise NotImplementedError + _is_array_of_pointers = False # unless overridden by GcPtrArrayDescr + _is_array_of_floats = False # unless overridden by FloatArrayDescr + def is_array_of_pointers(self): - return False # unless overridden by GcPtrArrayDescr + return self._is_array_of_pointers def is_array_of_floats(self): - return False # unless overridden by FloatArrayDescr + return self._is_array_of_floats def repr_of_descr(self): return '<%s>' % self._clsname @@ -143,13 +148,12 @@ class GcPtrArrayDescr(NonGcPtrArrayDescr): _clsname = 'GcPtrArrayDescr' - def is_array_of_pointers(self): - return True + _is_array_of_pointers = True def getArrayDescrClass(ARRAY): return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr, NonGcPtrArrayDescr, 'Array', 'get_item_size', - 'is_array_of_floats') + '_is_array_of_floats') def get_array_descr(gccache, ARRAY): cache = gccache._cache_array @@ -188,11 +192,14 @@ result.append(box) return result + _returns_a_pointer = False # unless overridden by GcPtrCallDescr + _returns_a_float = False # unless overridden by FloatCallDescr + def returns_a_pointer(self): - return False # unless overridden by GcPtrCallDescr + return self._returns_a_pointer def returns_a_float(self): - return False # unless overridden by FloatCallDescr + return self._returns_a_float def get_result_size(self, translate_support_code): raise NotImplementedError @@ -234,8 +241,7 @@ class GcPtrCallDescr(NonGcPtrCallDescr): _clsname = 'GcPtrCallDescr' - def returns_a_pointer(self): - return True + _returns_a_pointer = True class VoidCallDescr(NonGcPtrCallDescr): _clsname = 'VoidCallDescr' @@ -247,7 +253,7 @@ return VoidCallDescr return getDescrClass(RESULT, BaseCallDescr, GcPtrCallDescr, NonGcPtrCallDescr, 'Call', 'get_result_size', - 'returns_a_float') + '_returns_a_float') def get_call_descr(gccache, ARGS, RESULT): arg_classes = [] @@ -273,7 +279,7 @@ # ____________________________________________________________ def getDescrClass(TYPE, BaseDescr, GcPtrDescr, NonGcPtrDescr, - nameprefix, methodname, floatcheckname, _cache={}): + nameprefix, methodname, floatattrname, _cache={}): if isinstance(TYPE, lltype.Ptr): if TYPE.TO._gckind == 'gc': return GcPtrDescr @@ -292,9 +298,7 @@ setattr(Descr, methodname, method) # if TYPE is lltype.Float: - def is_float(self): - return True - setattr(Descr, floatcheckname, is_float) + setattr(Descr, floatattrname, True) # _cache[nameprefix, TYPE] = Descr return Descr From arigo at codespeak.net Sun Oct 4 13:22:39 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 4 Oct 2009 13:22:39 +0200 (CEST) Subject: [pypy-svn] r68142 - pypy/trunk/pypy/translator/c/gcc Message-ID: <20091004112239.C3F641683E7@codespeak.net> Author: arigo Date: Sun Oct 4 13:22:38 2009 New Revision: 68142 Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Log: Raise the line as a message, to see it. Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Sun Oct 4 13:22:38 2009 @@ -753,6 +753,8 @@ def conditional_jump(self, line): match = r_jump.match(line) + if not match: + raise UnrecognizedOperation(line) label = match.group(1) self.register_jump_to(label) return [] From arigo at codespeak.net Sun Oct 4 16:55:01 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 4 Oct 2009 16:55:01 +0200 (CEST) Subject: [pypy-svn] r68143 - in pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86: . test Message-ID: <20091004145501.28C451683E1@codespeak.net> Author: arigo Date: Sun Oct 4 16:54:59 2009 New Revision: 68143 Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/assembler.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/codebuf.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/jump.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/ri386.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Log: Add some more instructions. Start converting assembler.py, but I should soon throw away these changes and start from a fresh copy from trunk. Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/assembler.py Sun Oct 4 16:54:59 2009 @@ -10,12 +10,12 @@ from pypy.jit.backend.x86.regalloc import (RegAlloc, WORD, REGS, TempBox, lower_byte, stack_pos) from pypy.rlib.objectmodel import we_are_translated, specialize -from pypy.jit.backend.x86 import codebuf -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86 import codebuf, ri386 +from pypy.jit.backend.x86.ri386 import WORD +from pypy.jit.backend.x86.ri386 import eax, ecx, edx, ebx, esp, ebp, esi, edi from pypy.jit.metainterp.resoperation import rop - # our calling convention - we pass first 6 args in registers # and the rest stays on the stack @@ -59,9 +59,8 @@ method.func_name = name return method -for name in dir(codebuf.MachineCodeBlock): - if name.upper() == name: - setattr(MachineCodeBlockWrapper, name, _new_method(name)) +for name in ri386.all_instructions: + setattr(MachineCodeBlockWrapper, name, _new_method(name)) class ExecutableToken386(object): _x86_loop_code = 0 @@ -177,28 +176,28 @@ def _patchable_stackadjust(self): # stack adjustment LEA - self.mc.LEA(esp, fixedsize_ebp_ofs(0)) + self.mc.LEA32_rs(esp, 0) return self.mc.tell() - 4 def _patch_stackadjust(self, adr_lea, stack_depth): # patch stack adjustment LEA # possibly align, e.g. for Mac OS X mc = codebuf.InMemoryCodeBuilder(adr_lea, adr_lea + 4) - mc.write(packimm32(-(stack_depth + RET_BP - 2) * WORD)) + mc.writeimm32(-(stack_depth + RET_BP - 2) * WORD) mc.done() def _assemble_bootstrap_code(self, inputargs, arglocs): - self.mc.PUSH(ebp) - self.mc.MOV(ebp, esp) - self.mc.PUSH(ebx) - self.mc.PUSH(esi) - self.mc.PUSH(edi) + self.mc.PUSH_r(ebp) + self.mc.MOV_rr(ebp, esp) + self.mc.PUSH_r(ebx) + self.mc.PUSH_r(esi) + self.mc.PUSH_r(edi) # NB. exactly 4 pushes above; if this changes, fix stack_pos(). # You must also keep _get_callshape() in sync. adr_stackadjust = self._patchable_stackadjust() for i in range(len(arglocs)): loc = arglocs[i] - if not isinstance(loc, REG): + if is_stack(loc): if inputargs[i].type == REF: # This uses XCHG to put zeroes in fail_boxes_ptr after # reading them @@ -825,3 +824,9 @@ return heap(reg_or_imm1.value + offset) else: return mem(reg_or_imm1, offset) + +def is_reg(loc): + return loc >= 0 + +def is_stack(loc): + return loc < 0 Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/codebuf.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/codebuf.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/codebuf.py Sun Oct 4 16:54:59 2009 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.jit.backend.x86.ri386 import I386CodeBuilder from pypy.rlib.rmmap import PTR, alloc, free +from pypy.rlib.debug import ll_assert class InMemoryCodeBuilder(I386CodeBuilder): @@ -18,15 +19,11 @@ self._size = map_size self._pos = 0 - def overwrite(self, pos, data): - assert pos + len(data) <= self._size - for c in data: - self._data[pos] = c - pos += 1 - return pos - - def write(self, data): - self._pos = self.overwrite(self._pos, data) + def writechar(self, char): + pos = self._pos + ll_assert(pos < self._size, "buffer overflow in codebuf.py") + self._data[pos] = char + self._pos = pos + 1 def get_relative_pos(self): return self._pos Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/jump.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/jump.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/jump.py Sun Oct 4 16:54:59 2009 @@ -2,15 +2,15 @@ from pypy.tool.pairtype import extendabletype from pypy.jit.backend.x86.ri386 import * -class __extend__(REG): - __metaclass__ = extendabletype - def _getregkey(self): - return ~self.op - -class __extend__(MODRM): - __metaclass__ = extendabletype - def _getregkey(self): - return self.position +##class __extend__(REG): +## __metaclass__ = extendabletype +## def _getregkey(self): +## return ~self.op + +##class __extend__(MODRM): +## __metaclass__ = extendabletype +## def _getregkey(self): +## return self.position def remap_stack_layout(assembler, src_locations, dst_locations, tmpreg): Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/ri386.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/ri386.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/ri386.py Sun Oct 4 16:54:59 2009 @@ -65,8 +65,8 @@ return -128 <= value < 128 @specialize.arg(2) -def encode_stack(mc, arg, allow_single_byte, orbyte): - if allow_single_byte and single_byte(arg): +def encode_stack(mc, arg, force_32bits, orbyte): + if not force_32bits and single_byte(arg): mc.writechar(chr(0x40 | ebp | orbyte)) mc.writeimm8(arg) else: @@ -74,8 +74,8 @@ mc.writeimm32(arg) return 0 -def stack(argnum, allow_single_byte=True): - return encode_stack, argnum, allow_single_byte +def stack(argnum, force_32bits=False): + return encode_stack, argnum, force_32bits # ____________________________________________________________ @@ -153,3 +153,14 @@ NOP = insn('\x90') RET = insn('\xC3') + + PUSH_r = insn(register(1), '\x50') + + LEA_rs = insn('\x8D', register(1,8), stack(2)) + LEA32_rs = insn('\x8D', register(1,8), stack(2, force_32bits=True)) + +# ____________________________________________________________ + +all_instructions = [name for name in I386CodeBuilder.__dict__ + if name.split('_')[0].isupper()] +all_instructions.sort() Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386.py Sun Oct 4 16:54:59 2009 @@ -6,7 +6,7 @@ self.buffer = [] def writechar(self, c): - self.buffer.append(c) # extend the list of characters + self.buffer.append(c) # append a character def getvalue(self): return ''.join(self.buffer) @@ -42,3 +42,13 @@ s.NOP() s.ADD_rr(eax, eax) assert s.getvalue() == '\x90\x01\xC0' + +def test_lea_rs(): + s = CodeBuilder() + s.LEA_rs(ecx, -36) + assert s.getvalue() == '\x8D\x4D\xDC' + +def test_lea32_rs(): + s = CodeBuilder() + s.LEA32_rs(ecx, -36) + assert s.getvalue() == '\x8D\x8D\xDC\xFF\xFF\xFF' Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Sun Oct 4 16:54:59 2009 @@ -3,8 +3,8 @@ from pypy.jit.backend.x86 import ri386 from pypy.tool.udir import udir -INPUTNAME = str(udir.join('checkfile.s')) -FILENAME = str(udir.join('checkfile.tmp')) +INPUTNAME = str(udir.join('checkfile_%s.s')) +FILENAME = str(udir.join('checkfile_%s.o')) BEGIN_TAG = '<<>>' END_TAG = '<<>>' @@ -77,10 +77,7 @@ return regnames[regnum] def assembler_operand_stack(position): - if position == 0: - return '(%ebp)' - else: - return '%d(%%ebp)' % position + return '%d(%%ebp)' % position def assembler_operand_imm(value): return '$%d' % value @@ -91,11 +88,11 @@ 'i': assembler_operand_imm, } -def run_test(instrname, argmodes, args_lists): +def run_test(methname, instrname, argmodes, args_lists): global labelcount labelcount = 0 oplist = [] - g = open(INPUTNAME, 'w') + g = open(INPUTNAME % methname, 'w') g.write('\x09.string "%s"\n' % BEGIN_TAG) for args in args_lists: suffix = "" @@ -132,9 +129,9 @@ oplist.append(op) g.write('\t.string "%s"\n' % END_TAG) g.close() - os.system('as "%s" -o "%s"' % (INPUTNAME, FILENAME)) + os.system('as "%s" -o "%s"' % (INPUTNAME % methname, FILENAME % methname)) try: - f = open(FILENAME, 'rb') + f = open(FILENAME % methname, 'rb') except IOError: raise Exception("Assembler error") data = f.read() @@ -227,7 +224,7 @@ instrname, argmodes = methname, '' print "Testing %s with argmodes=%r" % (instrname, argmodes) ilist = make_all_tests(methname, argmodes) - oplist, as_code = run_test(instrname, argmodes, ilist) + oplist, as_code = run_test(methname, instrname, argmodes, ilist) cc = CodeChecker(as_code) for op, args in zip(oplist, ilist): if op: @@ -246,12 +243,10 @@ def do_test(name): if name in ('CMOVPE', 'CMOVPO'): py.test.skip("why doesn't 'as' know about CMOVPE/CMOVPO?") + if name.split('_')[0][-1].isdigit(): + print "artificial instruction: %r" % (name,) + return complete_test(name) - for name in all_instructions: + for name in ri386.all_instructions: yield do_test, name - - -all_instructions = [name for name in ri386.I386CodeBuilder.__dict__ - if name.split('_')[0].isupper()] -all_instructions.sort() From cfbolz at codespeak.net Sun Oct 4 17:11:57 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 4 Oct 2009 17:11:57 +0200 (CEST) Subject: [pypy-svn] r68144 - pypy/trunk/pypy/jit/backend Message-ID: <20091004151157.A25B41683E3@codespeak.net> Author: cfbolz Date: Sun Oct 4 17:11:57 2009 New Revision: 68144 Modified: pypy/trunk/pypy/jit/backend/showstats.py Log: fix showstats Modified: pypy/trunk/pypy/jit/backend/showstats.py ============================================================================== --- pypy/trunk/pypy/jit/backend/showstats.py (original) +++ pypy/trunk/pypy/jit/backend/showstats.py Sun Oct 4 17:11:57 2009 @@ -11,7 +11,7 @@ alldict = AllDict() def main(argv): - lst = py.path.local(argv[0]).read().split("[") + lst = ("\n" + py.path.local(argv[0]).read()).split("\n[") lst = ['[' + i for i in lst if i] for oplist in lst: loop = parse(oplist, namespace=alldict) From arigo at codespeak.net Sun Oct 4 17:45:31 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 4 Oct 2009 17:45:31 +0200 (CEST) Subject: [pypy-svn] r68145 - in pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86: . test Message-ID: <20091004154531.378391683E3@codespeak.net> Author: arigo Date: Sun Oct 4 17:45:30 2009 New Revision: 68145 Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/ri386.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Log: The MOVs necessary to load and store fields and arrays. Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/ri386.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/ri386.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/ri386.py Sun Oct 4 17:45:30 2009 @@ -1,4 +1,4 @@ -from pypy.rlib.rarithmetic import intmask +from pypy.rlib.rarithmetic import intmask, r_uint, r_ulonglong from pypy.rlib.objectmodel import ComputedIntSymbolic, we_are_translated from pypy.rlib.objectmodel import specialize from pypy.rlib.unroll import unrolling_iterable @@ -25,9 +25,9 @@ # Encode a register number in the orbyte @specialize.arg(2) -def encode_register(mc, arg, factor, orbyte): - assert 0 <= arg < 8 - return orbyte | (arg * factor) +def encode_register(mc, reg, factor, orbyte): + assert 0 <= reg < 8 + return orbyte | (reg * factor) def register(argnum, factor=1): return encode_register, argnum, factor @@ -45,39 +45,92 @@ # Emit an immediate value @specialize.arg(2) -def encode_immediate(mc, arg, width, orbyte): +def encode_immediate(mc, immediate, width, orbyte): assert orbyte == 0 if width == 'b': - mc.writeimm8(arg) + mc.writeimm8(immediate) elif width == 'h': - mc.writeimm16(arg) + mc.writeimm16(immediate) else: - mc.writeimm32(arg) + mc.writeimm32(immediate) return 0 def immediate(argnum, width='i'): return encode_immediate, argnum, width # ____________________________________________________________ -# Emit a mod/rm referencing a stack location (%ebp+arg) +# Emit a mod/rm referencing a stack location [EBP+offset] def single_byte(value): return -128 <= value < 128 @specialize.arg(2) -def encode_stack(mc, arg, force_32bits, orbyte): - if not force_32bits and single_byte(arg): +def encode_stack(mc, offset, force_32bits, orbyte): + if not force_32bits and single_byte(offset): mc.writechar(chr(0x40 | ebp | orbyte)) - mc.writeimm8(arg) + mc.writeimm8(offset) else: mc.writechar(chr(0x80 | ebp | orbyte)) - mc.writeimm32(arg) + mc.writeimm32(offset) return 0 def stack(argnum, force_32bits=False): return encode_stack, argnum, force_32bits # ____________________________________________________________ +# Emit a mod/rm referencing a memory location [reg1+offset] + +def reg_offset(reg, offset): + assert 0 <= reg < 8 and reg != esp and reg != ebp + return (r_ulonglong(reg) << 32) | r_ulonglong(r_uint(offset)) + +def encode_mem_reg_plus_const(mc, reg1_offset, _, orbyte): + reg1 = intmask(reg1_offset >> 32) + offset = intmask(reg1_offset) + if offset == 0: + mc.writechar(chr(0x00 | reg1 | orbyte)) + elif single_byte(offset): + mc.writechar(chr(0x40 | reg1 | orbyte)) + mc.writeimm8(offset) + else: + mc.writechar(chr(0x80 | reg1 | orbyte)) + mc.writeimm32(offset) + return 0 + +def mem_reg_plus_const(argnum): + return encode_mem_reg_plus_const, argnum, None + +# ____________________________________________________________ +# Emit a mod/rm referencing an array memory location [reg1+reg2*scale+offset] + +def reg_reg_scaleshift_offset(reg1, reg2, scaleshift, offset): + assert 0 <= reg1 < 8 and reg1 != ebp + assert 0 <= reg2 < 8 and reg2 != esp + assert 0 <= scaleshift < 4 + SIB = (scaleshift<<6) | (reg2<<3) | reg1 + return (r_ulonglong(SIB) << 32) | r_ulonglong(r_uint(offset)) + +def encode_mem_reg_plus_scaled_reg_plus_const(mc, reg1_reg2_scaleshift_offset, + _, orbyte): + SIB = chr(intmask(reg1_reg2_scaleshift_offset >> 32)) + offset = intmask(reg1_reg2_scaleshift_offset) + if offset == 0: + mc.writechar(chr(0x04 | orbyte)) + mc.writechar(SIB) + elif single_byte(offset): + mc.writechar(chr(0x44 | orbyte)) + mc.writechar(SIB) + mc.writeimm8(offset) + else: + mc.writechar(chr(0x84 | orbyte)) + mc.writechar(SIB) + mc.writeimm32(offset) + return 0 + +def mem_reg_plus_scaled_reg_plus_const(argnum): + return encode_mem_reg_plus_scaled_reg_plus_const, argnum, None + +# ____________________________________________________________ def insn(*encoding): def encode(mc, *args): @@ -144,6 +197,14 @@ MOV_sr = insn('\x89', register(2,8), stack(1)) MOV_rs = insn('\x8B', register(1,8), stack(2)) + # "MOV reg1, [reg2+offset]" and the opposite direction + MOV_rm = insn('\x8B', register(1,8), mem_reg_plus_const(2)) + MOV_mr = insn('\x89', register(2,8), mem_reg_plus_const(1)) + + # "MOV reg1, [reg2+reg3*scale+offset]" and the opposite direction + MOV_ra = insn('\x8B', register(1,8), mem_reg_plus_scaled_reg_plus_const(2)) + MOV_ar = insn('\x89', register(2,8), mem_reg_plus_scaled_reg_plus_const(1)) + ADD_ri, ADD_rr, ADD_rs = common_modes(0) OR_ri, OR_rr, OR_rs = common_modes(1) AND_ri, AND_rr, AND_rs = common_modes(4) Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386.py Sun Oct 4 17:45:30 2009 @@ -37,6 +37,38 @@ s.MOV_rs(edx, -36) assert s.getvalue() == '\x8B\x55\xDC' +def test_mov_rm(): + s = CodeBuilder() + s.MOV_rm(edx, reg_offset(edi, 0)) + s.MOV_rm(edx, reg_offset(edi, -128)) + s.MOV_rm(edx, reg_offset(edi, 128)) + assert s.getvalue() == '\x8B\x17\x8B\x57\x80\x8B\x97\x80\x00\x00\x00' + +def test_mov_mr(): + s = CodeBuilder() + s.MOV_mr(reg_offset(edi, 0), edx) + s.MOV_mr(reg_offset(edi, -128), edx) + s.MOV_mr(reg_offset(edi, 128), edx) + assert s.getvalue() == '\x89\x17\x89\x57\x80\x89\x97\x80\x00\x00\x00' + +def test_mov_ra(): + s = CodeBuilder() + s.MOV_ra(edx, reg_reg_scaleshift_offset(esi, edi, 2, 0)) + s.MOV_ra(edx, reg_reg_scaleshift_offset(esi, edi, 2, -128)) + s.MOV_ra(edx, reg_reg_scaleshift_offset(esi, edi, 2, 128)) + assert s.getvalue() == ('\x8B\x14\xBE' + + '\x8B\x54\xBE\x80' + + '\x8B\x94\xBE\x80\x00\x00\x00') + +def test_mov_ar(): + s = CodeBuilder() + s.MOV_ar(reg_reg_scaleshift_offset(esi, edi, 2, 0), edx) + s.MOV_ar(reg_reg_scaleshift_offset(esi, edi, 2, -128), edx) + s.MOV_ar(reg_reg_scaleshift_offset(esi, edi, 2, 128), edx) + assert s.getvalue() == ('\x89\x14\xBE' + + '\x89\x54\xBE\x80' + + '\x89\x94\xBE\x80\x00\x00\x00') + def test_nop_add_rr(): s = CodeBuilder() s.NOP() Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Sun Oct 4 17:45:30 2009 @@ -1,6 +1,7 @@ import os, random, struct import py from pypy.jit.backend.x86 import ri386 +from pypy.rlib.rarithmetic import intmask from pypy.tool.udir import udir INPUTNAME = str(udir.join('checkfile_%s.s')) @@ -16,10 +17,27 @@ def reg_tests(): return range(8) -def stack_tests(): +def stack_tests(count=COUNT1): return ([0, 4, -4, 124, 128, -128, -132] + [random.randrange(-0x20000000, 0x20000000) * 4 - for i in range(COUNT1)]) + for i in range(count)]) + +def memory_tests(): + return [ri386.reg_offset(reg, ofs) + for reg in [ri386.eax, ri386.ecx, ri386.edx, ri386.ebx, + ri386.esi, ri386.edi] + for ofs in stack_tests(5) + ] + +def array_tests(): + return [ri386.reg_reg_scaleshift_offset(reg1, reg2, scaleshift, ofs) + for reg1 in [ri386.eax, ri386.ecx, ri386.edx, ri386.ebx, + ri386.esi, ri386.edi] #, ri386.esp + for reg2 in [ri386.eax, ri386.ecx, ri386.edx, ri386.ebx, + ri386.esi, ri386.edi] #, ri386.ebp + for scaleshift in [0, 1, 2, 3] + for ofs in stack_tests(1) + ] def imm8_tests(): v = [-128,-1,0,1,127] + [random.randrange(-127, 127) for i in range(COUNT1)] @@ -69,6 +87,8 @@ tests = { 'r': reg_tests, 's': stack_tests, + 'm': memory_tests, + 'a': array_tests, 'i': imm32_tests, } @@ -79,12 +99,30 @@ def assembler_operand_stack(position): return '%d(%%ebp)' % position +def assembler_operand_memory(reg1_offset): + reg1 = intmask(reg1_offset >> 32) + offset = intmask(reg1_offset) + if not offset: offset = '' + return '%s(%s)' % (offset, regnames[reg1]) + +def assembler_operand_array(reg1_reg2_scaleshift_offset): + SIB = intmask(reg1_reg2_scaleshift_offset >> 32) + offset = intmask(reg1_reg2_scaleshift_offset) + if not offset: offset = '' + reg1 = SIB & 7 + reg2 = (SIB >> 3) & 7 + scaleshift = SIB >> 6 + return '%s(%s,%s,%d)' % (offset, regnames[reg1], regnames[reg2], + 1< Author: arigo Date: Sun Oct 4 18:48:01 2009 New Revision: 68146 Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Log: Support strange labels reported by NickDaly by supporting the full label syntax of 'as', plus the @ sign which is apparently Mac OS/X-specific. Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Sun Oct 4 18:48:01 2009 @@ -25,7 +25,7 @@ OFFSET_LABELS = 2**30 # inside functions -LABEL = r'([.]?[\w$@]+)' +LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' r_label = re.compile(LABEL+"[:]\s*$") r_globl = re.compile(r"\t[.]globl\t(\w+)\s*$") r_globllabel = re.compile(LABEL+r"=[.][+]%d\s*$"%OFFSET_LABELS) From arigo at codespeak.net Sun Oct 4 21:20:45 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 4 Oct 2009 21:20:45 +0200 (CEST) Subject: [pypy-svn] r68147 - in pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86: . test Message-ID: <20091004192045.4E3371683E3@codespeak.net> Author: arigo Date: Sun Oct 4 21:20:43 2009 New Revision: 68147 Added: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py - copied, changed from r68145, pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/ri386.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86.py - copied, changed from r68145, pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py - copied, changed from r68145, pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py (contents, props changed) Removed: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/ri386.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/ri386setup.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Log: Rename ri386.py to rx86.py and write the equivalent encodings for the 64-bits mode. Copied: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py (from r68145, pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/ri386.py) ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/ri386.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py Sun Oct 4 21:20:43 2009 @@ -1,10 +1,10 @@ -from pypy.rlib.rarithmetic import intmask, r_uint, r_ulonglong +from pypy.rlib.rarithmetic import intmask, r_ulonglong from pypy.rlib.objectmodel import ComputedIntSymbolic, we_are_translated from pypy.rlib.objectmodel import specialize from pypy.rlib.unroll import unrolling_iterable +from pypy.rpython.lltypesystem import rffi -WORD = 4 - +# the following are synonyms for rax, rcx, etc. on 64 bits eax = 0 ecx = 1 edx = 2 @@ -14,6 +14,22 @@ esi = 6 edi = 7 +# the following are extra registers available only on 64 bits +r8 = 8 +r9 = 9 +r10 = 10 +r11 = 11 +r12 = 12 +r13 = 13 +r14 = 14 +r15 = 15 + +def single_byte(value): + return -128 <= value < 128 + +def fits_in_32bits(value): + return -2147483648 <= value <= 2147483647 + # ____________________________________________________________ # Emit a single char @@ -24,13 +40,31 @@ # ____________________________________________________________ # Encode a register number in the orbyte +def reg_number_3bits(mc, reg): + if mc.WORD == 4: + assert 0 <= reg < 8 + return reg + else: + assert 0 <= reg < 16 + return reg & 7 + @specialize.arg(2) def encode_register(mc, reg, factor, orbyte): - assert 0 <= reg < 8 - return orbyte | (reg * factor) + return orbyte | (reg_number_3bits(mc, reg) * factor) + + at specialize.arg(2) +def rex_register(mc, reg, factor): + if reg >= 8: + if factor == 1: + return REX_B + elif factor == 8: + return REX_R + else: + raise ValueError(factor) + return 0 def register(argnum, factor=1): - return encode_register, argnum, factor + return encode_register, argnum, factor, rex_register # ____________________________________________________________ # Encode a constant in the orbyte @@ -39,7 +73,7 @@ return orbyte | constant def orbyte(value): - return encode_orbyte, None, value + return encode_orbyte, None, value, None # ____________________________________________________________ # Emit an immediate value @@ -51,70 +85,115 @@ mc.writeimm8(immediate) elif width == 'h': mc.writeimm16(immediate) + elif width == 'q': + mc.writeimm64(immediate) else: mc.writeimm32(immediate) return 0 def immediate(argnum, width='i'): - return encode_immediate, argnum, width + return encode_immediate, argnum, width, None # ____________________________________________________________ # Emit a mod/rm referencing a stack location [EBP+offset] -def single_byte(value): - return -128 <= value < 128 - @specialize.arg(2) def encode_stack(mc, offset, force_32bits, orbyte): if not force_32bits and single_byte(offset): - mc.writechar(chr(0x40 | ebp | orbyte)) + mc.writechar(chr(0x40 | orbyte | ebp)) mc.writeimm8(offset) else: - mc.writechar(chr(0x80 | ebp | orbyte)) + mc.writechar(chr(0x80 | orbyte | ebp)) mc.writeimm32(offset) return 0 def stack(argnum, force_32bits=False): - return encode_stack, argnum, force_32bits + return encode_stack, argnum, force_32bits, None # ____________________________________________________________ # Emit a mod/rm referencing a memory location [reg1+offset] def reg_offset(reg, offset): - assert 0 <= reg < 8 and reg != esp and reg != ebp - return (r_ulonglong(reg) << 32) | r_ulonglong(r_uint(offset)) + # returns a 64-bit integer encoding "reg1+offset". + # * 'offset' is stored as bytes 1-4 of the result; + # * 'reg1' is stored as byte 5 of the result. + assert reg != esp and reg != ebp + return (r_ulonglong(reg) << 32) | r_ulonglong(rffi.r_uint(offset)) def encode_mem_reg_plus_const(mc, reg1_offset, _, orbyte): - reg1 = intmask(reg1_offset >> 32) + reg1 = reg_number_3bits(mc, intmask(reg1_offset >> 32)) offset = intmask(reg1_offset) - if offset == 0: - mc.writechar(chr(0x00 | reg1 | orbyte)) + no_offset = offset == 0 + SIB = -1 + # 64-bits special cases for reg1 == r12 or r13 + # (which look like esp or ebp after being truncated to 3 bits) + if mc.WORD == 8: + if reg1 == esp: + SIB = (esp<<3) | esp + elif reg1 == ebp: + no_offset = False + # end of 64-bits special cases + if no_offset: + mc.writechar(chr(0x00 | orbyte | reg1)) + if SIB >= 0: mc.writechar(chr(SIB)) elif single_byte(offset): - mc.writechar(chr(0x40 | reg1 | orbyte)) + mc.writechar(chr(0x40 | orbyte | reg1)) + if SIB >= 0: mc.writechar(chr(SIB)) mc.writeimm8(offset) else: - mc.writechar(chr(0x80 | reg1 | orbyte)) + mc.writechar(chr(0x80 | orbyte | reg1)) + if SIB >= 0: mc.writechar(chr(SIB)) mc.writeimm32(offset) return 0 +def rex_mem_reg_plus_const(mc, reg1_offset, _): + reg1 = intmask(reg1_offset >> 32) + if reg1 >= 8: + return REX_B + return 0 + def mem_reg_plus_const(argnum): - return encode_mem_reg_plus_const, argnum, None + return encode_mem_reg_plus_const, argnum, None, rex_mem_reg_plus_const # ____________________________________________________________ # Emit a mod/rm referencing an array memory location [reg1+reg2*scale+offset] def reg_reg_scaleshift_offset(reg1, reg2, scaleshift, offset): - assert 0 <= reg1 < 8 and reg1 != ebp - assert 0 <= reg2 < 8 and reg2 != esp + # returns a 64-bit integer encoding "reg1+reg2<= 8: + encoding |= REX_B << 8 + reg1 &= 7 + if reg2 >= 8: + encoding |= REX_X << 8 + reg2 &= 7 + encoding |= (scaleshift<<6) | (reg2<<3) | reg1 + return (r_ulonglong(encoding) << 32) | r_ulonglong(rffi.r_uint(offset)) def encode_mem_reg_plus_scaled_reg_plus_const(mc, reg1_reg2_scaleshift_offset, _, orbyte): - SIB = chr(intmask(reg1_reg2_scaleshift_offset >> 32)) + encoding = intmask(reg1_reg2_scaleshift_offset >> 32) + if mc.WORD == 4: + assert encoding <= 0xFF # else registers r8..r15 have been used + SIB = chr(encoding) + else: + SIB = chr(encoding & 0xFF) offset = intmask(reg1_reg2_scaleshift_offset) - if offset == 0: + no_offset = offset == 0 + # 64-bits special cases for reg1 == r13 + # (which look like ebp after being truncated to 3 bits) + if mc.WORD == 8: + if (encoding & 7) == ebp: + no_offset = False + # end of 64-bits special cases + if no_offset: mc.writechar(chr(0x04 | orbyte)) mc.writechar(SIB) elif single_byte(offset): @@ -127,15 +206,48 @@ mc.writeimm32(offset) return 0 +def rex_mem_reg_plus_scaled_reg_plus_const(mc, reg1_reg2_scaleshift_offset, _): + return intmask(reg1_reg2_scaleshift_offset >> (32+8)) + def mem_reg_plus_scaled_reg_plus_const(argnum): - return encode_mem_reg_plus_scaled_reg_plus_const, argnum, None + return (encode_mem_reg_plus_scaled_reg_plus_const, argnum, None, + rex_mem_reg_plus_scaled_reg_plus_const) + +# ____________________________________________________________ +# For 64-bit mode: the REX.W, REX.R, REX.X, REG.B prefixes + +REX_W = 8 +REX_R = 4 +REX_X = 2 +REX_B = 1 + + at specialize.arg(2) +def encode_rex(mc, _, basevalue, orbyte): + if mc.WORD == 8: + assert 0 <= orbyte < 8 + if basevalue != 0x40 or orbyte != 0: + mc.writechar(chr(basevalue | orbyte)) + else: + assert orbyte == 0 + return 0 + +rex_w = encode_rex, None, (0x40 | REX_W), None +rex_nw = encode_rex, None, 0x40, None # ____________________________________________________________ def insn(*encoding): def encode(mc, *args): orbyte = 0 - for encode_step, arg, extra in encoding_steps: + if mc.WORD == 8: + # compute the REX byte, if any + for encode_step, arg, extra, rex_step in encoding_steps: + if rex_step: + if arg is not None: + arg = args[arg-1] + orbyte |= rex_step(mc, arg, extra) + # emit the bytes of the instruction + for encode_step, arg, extra, rex_step in encoding_steps: if arg is not None: arg = args[arg-1] orbyte = encode_step(mc, arg, extra, orbyte) @@ -145,21 +257,21 @@ for step in encoding: if isinstance(step, str): for c in step: - encoding_steps.append((encode_char, None, ord(c))) + encoding_steps.append((encode_char, None, ord(c), None)) else: - assert type(step) is tuple and len(step) == 3 + assert type(step) is tuple and len(step) == 4 encoding_steps.append(step) encoding_steps = unrolling_iterable(encoding_steps) return encode def common_modes(group): base = group * 8 - INSN_ri8 = insn('\x83', orbyte(group<<3), register(1), '\xC0', + INSN_ri8 = insn(rex_w, '\x83', orbyte(group<<3), register(1), '\xC0', immediate(2,'b')) - INSN_ri32 = insn('\x81', orbyte(group<<3), register(1), '\xC0', + INSN_ri32 = insn(rex_w, '\x81', orbyte(group<<3), register(1), '\xC0', immediate(2)) - INSN_rr = insn(chr(base+1), register(2,8), register(1,1), '\xC0') - INSN_rs = insn(chr(base+3), register(1,8), stack(2)) + INSN_rr = insn(rex_w, chr(base+1), register(2,8), register(1,1), '\xC0') + INSN_rs = insn(rex_w, chr(base+3), register(1,8), stack(2)) def INSN_ri(mc, reg, immed): if single_byte(immed): @@ -172,7 +284,7 @@ # ____________________________________________________________ -class I386CodeBuilder(object): +class AbstractX86CodeBuilder(object): """Abstract base class.""" def writechar(self, char): @@ -186,24 +298,26 @@ self.writechar(chr((imm >> 8) & 0xFF)) def writeimm32(self, imm): + assert fits_in_32bits(imm) self.writechar(chr(imm & 0xFF)) self.writechar(chr((imm >> 8) & 0xFF)) self.writechar(chr((imm >> 16) & 0xFF)) self.writechar(chr((imm >> 24) & 0xFF)) - MOV_ri = insn(register(1), '\xB8', immediate(2)) - MOV_si = insn('\xC7', orbyte(0<<3), stack(1), immediate(2)) - MOV_rr = insn('\x89', register(2,8), register(1), '\xC0') - MOV_sr = insn('\x89', register(2,8), stack(1)) - MOV_rs = insn('\x8B', register(1,8), stack(2)) + #MOV_si = insn(rex_w, '\xC7', orbyte(0<<3), stack(1), immediate(2)) + MOV_rr = insn(rex_w, '\x89', register(2,8), register(1), '\xC0') + MOV_sr = insn(rex_w, '\x89', register(2,8), stack(1)) + MOV_rs = insn(rex_w, '\x8B', register(1,8), stack(2)) # "MOV reg1, [reg2+offset]" and the opposite direction - MOV_rm = insn('\x8B', register(1,8), mem_reg_plus_const(2)) - MOV_mr = insn('\x89', register(2,8), mem_reg_plus_const(1)) + MOV_rm = insn(rex_w, '\x8B', register(1,8), mem_reg_plus_const(2)) + MOV_mr = insn(rex_w, '\x89', register(2,8), mem_reg_plus_const(1)) # "MOV reg1, [reg2+reg3*scale+offset]" and the opposite direction - MOV_ra = insn('\x8B', register(1,8), mem_reg_plus_scaled_reg_plus_const(2)) - MOV_ar = insn('\x89', register(2,8), mem_reg_plus_scaled_reg_plus_const(1)) + MOV_ra = insn(rex_w, '\x8B', register(1,8), + mem_reg_plus_scaled_reg_plus_const(2)) + MOV_ar = insn(rex_w, '\x89', register(2,8), + mem_reg_plus_scaled_reg_plus_const(1)) ADD_ri, ADD_rr, ADD_rs = common_modes(0) OR_ri, OR_rr, OR_rs = common_modes(1) @@ -215,13 +329,35 @@ NOP = insn('\x90') RET = insn('\xC3') - PUSH_r = insn(register(1), '\x50') + PUSH_r = insn(rex_nw, register(1), '\x50') + + LEA_rs = insn(rex_w, '\x8D', register(1,8), stack(2)) + LEA32_rs = insn(rex_w, '\x8D', register(1,8), stack(2, force_32bits=True)) + + +class X86_32_CodeBuilder(AbstractX86CodeBuilder): + WORD = 4 + + MOV_ri = insn(register(1), '\xB8', immediate(2)) + + +class X86_64_CodeBuilder(AbstractX86CodeBuilder): + WORD = 8 - LEA_rs = insn('\x8D', register(1,8), stack(2)) - LEA32_rs = insn('\x8D', register(1,8), stack(2, force_32bits=True)) + _MOV_ri32 = insn(rex_w, '\xC7', register(1), '\xC0', immediate(2, 'i')) + _MOV_ri64 = insn(rex_w, register(1), '\xB8', immediate(2, 'q')) + + def MOV_ri(self, reg, immed): + if fits_in_32bits(immed): + self._MOV_ri32(reg, immed) + else: + self._MOV_ri64(reg, immed) # ____________________________________________________________ -all_instructions = [name for name in I386CodeBuilder.__dict__ +all_instructions = [name for name in AbstractX86CodeBuilder.__dict__ if name.split('_')[0].isupper()] +all_instructions += [name for name in X86_32_CodeBuilder.__dict__ + if name.split('_')[0].isupper()] +all_instructions.remove('WORD') all_instructions.sort() Copied: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86.py (from r68145, pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386.py) ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86.py Sun Oct 4 21:20:43 2009 @@ -1,5 +1,5 @@ import py -from pypy.jit.backend.x86.ri386 import * +from pypy.jit.backend.x86.rx86 import * class CodeBuilder(I386CodeBuilder): def __init__(self): Copied: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py (from r68145, pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py) ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py Sun Oct 4 21:20:43 2009 @@ -1,240 +1,15 @@ import os, random, struct import py -from pypy.jit.backend.x86 import ri386 +from pypy.jit.backend.x86 import rx86 from pypy.rlib.rarithmetic import intmask from pypy.tool.udir import udir -INPUTNAME = str(udir.join('checkfile_%s.s')) -FILENAME = str(udir.join('checkfile_%s.o')) -BEGIN_TAG = '<<>>' -END_TAG = '<<>>' +INPUTNAME = 'checkfile_%s.s' +FILENAME = 'checkfile_%s.o' +BEGIN_TAG = '<<>>' +END_TAG = '<<>>' -COUNT1 = 15 -COUNT2 = 25 - -suffixes = {0:'', 1:'b', 2:'w', 4:'l'} - -def reg_tests(): - return range(8) - -def stack_tests(count=COUNT1): - return ([0, 4, -4, 124, 128, -128, -132] + - [random.randrange(-0x20000000, 0x20000000) * 4 - for i in range(count)]) - -def memory_tests(): - return [ri386.reg_offset(reg, ofs) - for reg in [ri386.eax, ri386.ecx, ri386.edx, ri386.ebx, - ri386.esi, ri386.edi] - for ofs in stack_tests(5) - ] - -def array_tests(): - return [ri386.reg_reg_scaleshift_offset(reg1, reg2, scaleshift, ofs) - for reg1 in [ri386.eax, ri386.ecx, ri386.edx, ri386.ebx, - ri386.esi, ri386.edi] #, ri386.esp - for reg2 in [ri386.eax, ri386.ecx, ri386.edx, ri386.ebx, - ri386.esi, ri386.edi] #, ri386.ebp - for scaleshift in [0, 1, 2, 3] - for ofs in stack_tests(1) - ] - -def imm8_tests(): - v = [-128,-1,0,1,127] + [random.randrange(-127, 127) for i in range(COUNT1)] - return v - -def imm16_tests(): - v = [-32768,32767] + [random.randrange(-32767, -128) for i in range(COUNT1)] \ - + [random.randrange(128, 32767) for i in range(COUNT1)] - return v - -def imm32_tests(): - v = ([0x80000000, 0x7FFFFFFF, 128, 256, -129, -255] + - [random.randrange(0,65536)<<16 | - random.randrange(0,65536) for i in range(COUNT1)] + - [random.randrange(128, 256) for i in range(COUNT1)]) - return imm8_tests() + v - -def pick1(memSIB, cache=[]): - base = random.choice([None, None, None] + i386.registers) - index = random.choice([None, None, None] + i386.registers) - if index is i386.esp: index=None - scale = random.randrange(0,4) - if not cache: - cache[:] = [x.value for x in imm8_tests() + imm32_tests()] + [0,0] - random.shuffle(cache) - offset = cache.pop() - if base is None and scale==0: - base = index - index = None - return memSIB(base, index, scale, offset) - -def modrm_tests(): - return i386.registers + [pick1(i386.memSIB) for i in range(COUNT2)] - -def modrm_noreg_tests(): - return [pick1(i386.memSIB) for i in range(COUNT2)] - -def modrm64_tests(): - return [pick1(i386.memSIB64) for i in range(COUNT2)] - -def xmm_tests(): - return i386.xmm_registers - -def modrm8_tests(): - return i386.registers8 + [pick1(i386.memSIB8) for i in range(COUNT2)] - -tests = { - 'r': reg_tests, - 's': stack_tests, - 'm': memory_tests, - 'a': array_tests, - 'i': imm32_tests, - } - -regnames = ['%eax', '%ecx', '%edx', '%ebx', '%esp', '%ebp', '%esi', '%edi'] -def assembler_operand_reg(regnum): - return regnames[regnum] - -def assembler_operand_stack(position): - return '%d(%%ebp)' % position - -def assembler_operand_memory(reg1_offset): - reg1 = intmask(reg1_offset >> 32) - offset = intmask(reg1_offset) - if not offset: offset = '' - return '%s(%s)' % (offset, regnames[reg1]) - -def assembler_operand_array(reg1_reg2_scaleshift_offset): - SIB = intmask(reg1_reg2_scaleshift_offset >> 32) - offset = intmask(reg1_reg2_scaleshift_offset) - if not offset: offset = '' - reg1 = SIB & 7 - reg2 = (SIB >> 3) & 7 - scaleshift = SIB >> 6 - return '%s(%s,%s,%d)' % (offset, regnames[reg1], regnames[reg2], - 1<=0 - j = data.find(END_TAG, i) - assert j>=0 - as_code = data[i+len(BEGIN_TAG)+1:j] - return oplist, as_code - -def make_all_tests(methname, modes, args=[]): - if modes: - m = modes[0] - lst = tests[m]() - random.shuffle(lst) - result = [] - for v in lst: - result += make_all_tests(methname, modes[1:], args+[v]) - return result - else: - # special cases - if methname in ('ADD_ri', 'AND_ri', 'CMP_ri', 'OR_ri', - 'SUB_ri', 'XOR_ri'): - if args[0] == ri386.eax: - return [] # ADD EAX, constant: there is a special encoding -## if methname == "MOV_": -#### if args[0] == args[1]: -#### return [] # MOV reg, same reg -## if ((args[0][1] in (i386.eax, i386.al)) -## and args[1][1].assembler().lstrip('-').isdigit()): -## return [] # MOV accum, [constant-address] -## if ((args[1][1] in (i386.eax, i386.al)) -## and args[0][1].assembler().lstrip('-').isdigit()): -## return [] # MOV [constant-address], accum -## if instrname == "MOV16": -## return [] # skipped -## if instrname == "LEA": -## if (args[1][1].__class__ != i386.MODRM or -## args[1][1].is_register()): -## return [] -## if instrname == "INT": -## if args[0][1].value == 3: -## return [] -## if instrname in ('SHL', 'SHR', 'SAR'): -## if args[1][1].assembler() == '$1': -## return [] -## if instrname in ('MOVZX', 'MOVSX'): -## if args[1][1].width == 4: -## return [] -## if instrname == "TEST": -## if (args[0] != args[1] and -## isinstance(args[0][1], i386.REG) and -## isinstance(args[1][1], i386.REG)): -## return [] # TEST reg1, reg2 <=> TEST reg2, reg1 -## if instrname.endswith('cond'): -## return [] - return [args] - -def hexdump(s): - return ' '.join(["%02X" % ord(c) for c in s]) - - -class CodeChecker(ri386.I386CodeBuilder): - +class CodeCheckerMixin(object): def __init__(self, expected): self.expected = expected self.index = 0 @@ -246,45 +21,254 @@ def writechar(self, char): if char != self.expected[self.index:self.index+1]: print self.op - print "\x09from ri386.py:", hexdump(self.expected[self.instrindex:self.index] + char)+"..." - print "\x09from 'as': ", hexdump(self.expected[self.instrindex:self.index+1])+"..." + print "\x09from rx86.py:", hexdump(self.expected[self.instrindex:self.index] + char)+"..." + print "\x09from 'as': ", hexdump(self.expected[self.instrindex:self.index+1])+"..." raise Exception("Differs") self.index += 1 def done(self): assert len(self.expected) == self.index +def hexdump(s): + return ' '.join(["%02X" % ord(c) for c in s]) -def complete_test(methname): - if '_' in methname: - instrname, argmodes = methname.split('_') - else: - instrname, argmodes = methname, '' - print "Testing %s with argmodes=%r" % (instrname, argmodes) - ilist = make_all_tests(methname, argmodes) - oplist, as_code = run_test(methname, instrname, argmodes, ilist) - cc = CodeChecker(as_code) - for op, args in zip(oplist, ilist): - if op: - cc.begin(op) - getattr(cc, methname)(*args) - cc.done() - -def test_auto(): - import os - g = os.popen('as -version &1') - data = g.read() - g.close() - if not data.startswith('GNU assembler'): - py.test.skip("full tests require the GNU 'as' assembler") - - def do_test(name): - if name in ('CMOVPE', 'CMOVPO'): - py.test.skip("why doesn't 'as' know about CMOVPE/CMOVPO?") - if name.split('_')[0][-1].isdigit(): - print "artificial instruction: %r" % (name,) - return - complete_test(name) +# ____________________________________________________________ + +COUNT1 = 15 +COUNT2 = 25 +suffixes = {0:'', 1:'b', 2:'w', 4:'l', 8:'q'} - for name in ri386.all_instructions: - yield do_test, name + +class TestRx86_32: + WORD = 4 + TESTDIR = 'rx86_32' + X86_CodeBuilder = rx86.X86_32_CodeBuilder + REGNAMES = ['%eax', '%ecx', '%edx', '%ebx', '%esp', '%ebp', '%esi', '%edi'] + REGS = range(8) + NONSPECREGS = [rx86.eax, rx86.ecx, rx86.edx, rx86.ebx, rx86.esi, rx86.edi] + + def reg_tests(self): + return self.REGS + + def stack_tests(self, count=COUNT1): + return ([0, 4, -4, 124, 128, -128, -132] + + [random.randrange(-0x20000000, 0x20000000) * 4 + for i in range(count)]) + + def memory_tests(self): + return [rx86.reg_offset(reg, ofs) + for reg in self.NONSPECREGS + for ofs in self.stack_tests(5) + ] + + def array_tests(self): + return [rx86.reg_reg_scaleshift_offset(reg1, reg2, scaleshift, ofs) + for reg1 in self.NONSPECREGS + for reg2 in self.NONSPECREGS + for scaleshift in [0, 1, 2, 3] + for ofs in self.stack_tests(1) + ] + + def imm8_tests(self): + v = ([-128,-1,0,1,127] + + [random.randrange(-127, 127) for i in range(COUNT1)]) + return v + + def imm32_tests(self): + v = ([-0x80000000, 0x7FFFFFFF, 128, 256, -129, -255] + + [random.randrange(-32768,32768)<<16 | + random.randrange(0,65536) for i in range(COUNT1)] + + [random.randrange(128, 256) for i in range(COUNT1)]) + return self.imm8_tests() + v + + def get_all_tests(self): + return { + 'r': self.reg_tests, + 's': self.stack_tests, + 'm': self.memory_tests, + 'a': self.array_tests, + 'i': self.imm32_tests, + } + + def assembler_operand_reg(self, regnum): + return self.REGNAMES[regnum] + + def assembler_operand_stack(self, position): + return '%d(%s)' % (position, self.REGNAMES[5]) + + def assembler_operand_memory(self, reg1_offset): + reg1 = intmask(reg1_offset >> 32) + offset = intmask(reg1_offset) + if not offset: offset = '' + return '%s(%s)' % (offset, self.REGNAMES[reg1]) + + def assembler_operand_array(self, reg1_reg2_scaleshift_offset): + SIB = intmask(reg1_reg2_scaleshift_offset >> 32) + rex = SIB >> 8 + SIB = SIB & 0xFF + offset = intmask(reg1_reg2_scaleshift_offset) + if not offset: offset = '' + reg1 = SIB & 7 + reg2 = (SIB >> 3) & 7 + scaleshift = SIB >> 6 + if rex & rx86.REX_B: + reg1 |= 8 + if rex & rx86.REX_X: + reg2 |= 8 + return '%s(%s,%s,%d)' % (offset, self.REGNAMES[reg1], + self.REGNAMES[reg2], 1<=0 + j = data.find(END_TAG, i) + assert j>=0 + as_code = data[i+len(BEGIN_TAG)+1:j] + return oplist, as_code + + def make_all_tests(self, methname, modes, args=[]): + if modes: + tests = self.get_all_tests() + m = modes[0] + lst = tests[m]() + random.shuffle(lst) + result = [] + for v in lst: + result += self.make_all_tests(methname, modes[1:], args+[v]) + return result + else: + # special cases + if methname in ('ADD_ri', 'AND_ri', 'CMP_ri', 'OR_ri', + 'SUB_ri', 'XOR_ri'): + if args[0] == rx86.eax: + return [] # ADD EAX, constant: there is a special encoding + ## if methname == "MOV_": + #### if args[0] == args[1]: + #### return [] # MOV reg, same reg + ## if ((args[0][1] in (i386.eax, i386.al)) + ## and args[1][1].assembler().lstrip('-').isdigit()): + ## return [] # MOV accum, [constant-address] + ## if ((args[1][1] in (i386.eax, i386.al)) + ## and args[0][1].assembler().lstrip('-').isdigit()): + ## return [] # MOV [constant-address], accum + ## if instrname == "MOV16": + ## return [] # skipped + ## if instrname == "LEA": + ## if (args[1][1].__class__ != i386.MODRM or + ## args[1][1].is_register()): + ## return [] + ## if instrname == "INT": + ## if args[0][1].value == 3: + ## return [] + ## if instrname in ('SHL', 'SHR', 'SAR'): + ## if args[1][1].assembler() == '$1': + ## return [] + ## if instrname in ('MOVZX', 'MOVSX'): + ## if args[1][1].width == 4: + ## return [] + ## if instrname == "TEST": + ## if (args[0] != args[1] and + ## isinstance(args[0][1], i386.REG) and + ## isinstance(args[1][1], i386.REG)): + ## return [] # TEST reg1, reg2 <=> TEST reg2, reg1 + ## if instrname.endswith('cond'): + ## return [] + return [args] + + def get_code_checker_class(self): + class X86_CodeBuilder(CodeCheckerMixin, self.X86_CodeBuilder): + pass + return X86_CodeBuilder + + def complete_test(self, methname): + if methname.split('_')[0][-1].isdigit(): + print "artificial instruction: %r" % (methname,) + return + if '_' in methname: + instrname, argmodes = methname.split('_') + else: + instrname, argmodes = methname, '' + print "Testing %s with argmodes=%r" % (instrname, argmodes) + ilist = self.make_all_tests(methname, argmodes) + oplist, as_code = self.run_test(methname, instrname, argmodes, ilist) + cc = self.get_code_checker_class()(as_code) + for op, args in zip(oplist, ilist): + if op: + cc.begin(op) + getattr(cc, methname)(*args) + cc.done() + + def setup_class(cls): + import os + g = os.popen('as -version &1') + data = g.read() + g.close() + if not data.startswith('GNU assembler'): + py.test.skip("full tests require the GNU 'as' assembler") + + def test_all(self): + for name in rx86.all_instructions: + yield self.complete_test, name Added: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py ============================================================================== --- (empty file) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py Sun Oct 4 21:20:43 2009 @@ -0,0 +1,14 @@ +from pypy.jit.backend.x86 import rx86 +from pypy.jit.backend.x86.test import test_rx86_32_auto_encoding + + +class TestRx86_64(test_rx86_32_auto_encoding.TestRx86_32): + WORD = 8 + TESTDIR = 'rx86_64' + X86_CodeBuilder = rx86.X86_64_CodeBuilder + REGNAMES = ['%rax', '%rcx', '%rdx', '%rbx', '%rsp', '%rbp', '%rsi', '%rdi', + '%r8', '%r9', '%r10', '%r11', '%r12', '%r13', '%r14', '%r15'] + REGS = range(16) + NONSPECREGS = [rx86.eax, rx86.ecx, rx86.edx, rx86.ebx, rx86.esi, rx86.edi, + rx86.r8, rx86.r9, rx86.r10, rx86.r11, + rx86.r12, rx86.r13, rx86.r14, rx86.r15] From arigo at codespeak.net Sun Oct 4 21:25:07 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 4 Oct 2009 21:25:07 +0200 (CEST) Subject: [pypy-svn] r68148 - pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86 Message-ID: <20091004192507.B1E2F1683E3@codespeak.net> Author: arigo Date: Sun Oct 4 21:25:06 2009 New Revision: 68148 Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py Log: Typos. Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py Sun Oct 4 21:25:06 2009 @@ -114,7 +114,7 @@ # Emit a mod/rm referencing a memory location [reg1+offset] def reg_offset(reg, offset): - # returns a 64-bit integer encoding "reg1+offset". + # returns a 64-bits integer encoding "reg1+offset". # * 'offset' is stored as bytes 1-4 of the result; # * 'reg1' is stored as byte 5 of the result. assert reg != esp and reg != ebp @@ -159,7 +159,7 @@ # Emit a mod/rm referencing an array memory location [reg1+reg2*scale+offset] def reg_reg_scaleshift_offset(reg1, reg2, scaleshift, offset): - # returns a 64-bit integer encoding "reg1+reg2< Author: arigo Date: Sun Oct 4 22:15:00 2009 New Revision: 68149 Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py Log: Test and fix for "movq $imm64, reg". Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py Sun Oct 4 22:15:00 2009 @@ -103,6 +103,7 @@ mc.writechar(chr(0x40 | orbyte | ebp)) mc.writeimm8(offset) else: + assert fits_in_32bits(offset) mc.writechar(chr(0x80 | orbyte | ebp)) mc.writeimm32(offset) return 0 @@ -118,6 +119,7 @@ # * 'offset' is stored as bytes 1-4 of the result; # * 'reg1' is stored as byte 5 of the result. assert reg != esp and reg != ebp + assert fits_in_32bits(offset) return (r_ulonglong(reg) << 32) | r_ulonglong(rffi.r_uint(offset)) def encode_mem_reg_plus_const(mc, reg1_offset, _, orbyte): @@ -128,8 +130,8 @@ # 64-bits special cases for reg1 == r12 or r13 # (which look like esp or ebp after being truncated to 3 bits) if mc.WORD == 8: - if reg1 == esp: - SIB = (esp<<3) | esp + if reg1 == esp: # forces an SIB byte: + SIB = (esp<<3) | esp # use [r12+(no index)+offset] elif reg1 == ebp: no_offset = False # end of 64-bits special cases @@ -304,6 +306,7 @@ self.writechar(chr((imm >> 16) & 0xFF)) self.writechar(chr((imm >> 24) & 0xFF)) + MOV_ri = insn(rex_w, register(1), '\xB8', immediate(2, 'q')) #MOV_si = insn(rex_w, '\xC7', orbyte(0<<3), stack(1), immediate(2)) MOV_rr = insn(rex_w, '\x89', register(2,8), register(1), '\xC0') MOV_sr = insn(rex_w, '\x89', register(2,8), stack(1)) @@ -338,12 +341,16 @@ class X86_32_CodeBuilder(AbstractX86CodeBuilder): WORD = 4 - MOV_ri = insn(register(1), '\xB8', immediate(2)) - class X86_64_CodeBuilder(AbstractX86CodeBuilder): WORD = 8 + def writeimm64(self, imm): + self.writeimm32(intmask(rffi.cast(rffi.INT, imm))) + self.writeimm32(imm >> 32) + + # MOV_ri from the parent class is not wrong, but add a better encoding + # for the common case where the immediate bits in 32 bits _MOV_ri32 = insn(rex_w, '\xC7', register(1), '\xC0', immediate(2, 'i')) _MOV_ri64 = insn(rex_w, register(1), '\xB8', immediate(2, 'q')) Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py Sun Oct 4 22:15:00 2009 @@ -35,11 +35,10 @@ # ____________________________________________________________ COUNT1 = 15 -COUNT2 = 25 suffixes = {0:'', 1:'b', 2:'w', 4:'l', 8:'q'} -class TestRx86_32: +class TestRx86_32(object): WORD = 4 TESTDIR = 'rx86_32' X86_CodeBuilder = rx86.X86_32_CodeBuilder Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py Sun Oct 4 22:15:00 2009 @@ -1,3 +1,4 @@ +import random from pypy.jit.backend.x86 import rx86 from pypy.jit.backend.x86.test import test_rx86_32_auto_encoding @@ -12,3 +13,20 @@ NONSPECREGS = [rx86.eax, rx86.ecx, rx86.edx, rx86.ebx, rx86.esi, rx86.edi, rx86.r8, rx86.r9, rx86.r10, rx86.r11, rx86.r12, rx86.r13, rx86.r14, rx86.r15] + + def imm64_tests(self): + v = [-0x80000001, 0x80000000, + -0x8000000000000000, 0x7FFFFFFFFFFFFFFF] + for i in range(test_rx86_32_auto_encoding.COUNT1): + x = ((random.randrange(-32768,32768)<<48) | + (random.randrange(0,65536)<<32) | + (random.randrange(0,65536)<<16) | + (random.randrange(0,65536)<<0)) + if not rx86.fits_in_32bits(x): + v.append(x) + return v + self._old_imm32_tests() + + def test_extra_MOV_ri64(self): + self._old_imm32_tests = self.imm32_tests + self.imm32_tests = self.imm64_tests # patch on 'self' + self.complete_test('MOV_ri') From arigo at codespeak.net Sun Oct 4 22:42:54 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 4 Oct 2009 22:42:54 +0200 (CEST) Subject: [pypy-svn] r68150 - in pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86: . test Message-ID: <20091004204254.205071683E3@codespeak.net> Author: arigo Date: Sun Oct 4 22:42:53 2009 New Revision: 68150 Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py Log: Fix for test_rx86 and test_rx86_32_auto_encoding. Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py Sun Oct 4 22:42:53 2009 @@ -85,7 +85,7 @@ mc.writeimm8(immediate) elif width == 'h': mc.writeimm16(immediate) - elif width == 'q': + elif width == 'q' and mc.WORD == 8: mc.writeimm64(immediate) else: mc.writeimm32(immediate) @@ -350,21 +350,17 @@ self.writeimm32(imm >> 32) # MOV_ri from the parent class is not wrong, but add a better encoding - # for the common case where the immediate bits in 32 bits + # for the common case where the immediate fits in 32 bits _MOV_ri32 = insn(rex_w, '\xC7', register(1), '\xC0', immediate(2, 'i')) - _MOV_ri64 = insn(rex_w, register(1), '\xB8', immediate(2, 'q')) def MOV_ri(self, reg, immed): if fits_in_32bits(immed): self._MOV_ri32(reg, immed) else: - self._MOV_ri64(reg, immed) + AbstractX86CodeBuilder.MOV_ri(self, reg, immed) # ____________________________________________________________ all_instructions = [name for name in AbstractX86CodeBuilder.__dict__ if name.split('_')[0].isupper()] -all_instructions += [name for name in X86_32_CodeBuilder.__dict__ - if name.split('_')[0].isupper()] -all_instructions.remove('WORD') all_instructions.sort() Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86.py Sun Oct 4 22:42:53 2009 @@ -1,7 +1,7 @@ import py from pypy.jit.backend.x86.rx86 import * -class CodeBuilder(I386CodeBuilder): +class CodeBuilderMixin(object): def __init__(self): self.buffer = [] @@ -12,47 +12,45 @@ return ''.join(self.buffer) +class CodeBuilder32(CodeBuilderMixin, X86_32_CodeBuilder): + pass + def test_mov_ri(): - s = CodeBuilder() + s = CodeBuilder32() s.MOV_ri(ecx, -2) assert s.getvalue() == '\xB9\xFE\xFF\xFF\xFF' -def test_mov_si(): - s = CodeBuilder() - s.MOV_si(-36, 1<<24) - assert s.getvalue() == '\xC7\x45\xDC\x00\x00\x00\x01' - def test_mov_rr(): - s = CodeBuilder() + s = CodeBuilder32() s.MOV_rr(ebx, ebp) assert s.getvalue() == '\x89\xEB' def test_mov_sr(): - s = CodeBuilder() + s = CodeBuilder32() s.MOV_sr(-36, edx) assert s.getvalue() == '\x89\x55\xDC' def test_mov_rs(): - s = CodeBuilder() + s = CodeBuilder32() s.MOV_rs(edx, -36) assert s.getvalue() == '\x8B\x55\xDC' def test_mov_rm(): - s = CodeBuilder() + s = CodeBuilder32() s.MOV_rm(edx, reg_offset(edi, 0)) s.MOV_rm(edx, reg_offset(edi, -128)) s.MOV_rm(edx, reg_offset(edi, 128)) assert s.getvalue() == '\x8B\x17\x8B\x57\x80\x8B\x97\x80\x00\x00\x00' def test_mov_mr(): - s = CodeBuilder() + s = CodeBuilder32() s.MOV_mr(reg_offset(edi, 0), edx) s.MOV_mr(reg_offset(edi, -128), edx) s.MOV_mr(reg_offset(edi, 128), edx) assert s.getvalue() == '\x89\x17\x89\x57\x80\x89\x97\x80\x00\x00\x00' def test_mov_ra(): - s = CodeBuilder() + s = CodeBuilder32() s.MOV_ra(edx, reg_reg_scaleshift_offset(esi, edi, 2, 0)) s.MOV_ra(edx, reg_reg_scaleshift_offset(esi, edi, 2, -128)) s.MOV_ra(edx, reg_reg_scaleshift_offset(esi, edi, 2, 128)) @@ -61,7 +59,7 @@ '\x8B\x94\xBE\x80\x00\x00\x00') def test_mov_ar(): - s = CodeBuilder() + s = CodeBuilder32() s.MOV_ar(reg_reg_scaleshift_offset(esi, edi, 2, 0), edx) s.MOV_ar(reg_reg_scaleshift_offset(esi, edi, 2, -128), edx) s.MOV_ar(reg_reg_scaleshift_offset(esi, edi, 2, 128), edx) @@ -70,17 +68,28 @@ '\x89\x94\xBE\x80\x00\x00\x00') def test_nop_add_rr(): - s = CodeBuilder() + s = CodeBuilder32() s.NOP() s.ADD_rr(eax, eax) assert s.getvalue() == '\x90\x01\xC0' def test_lea_rs(): - s = CodeBuilder() + s = CodeBuilder32() s.LEA_rs(ecx, -36) assert s.getvalue() == '\x8D\x4D\xDC' def test_lea32_rs(): - s = CodeBuilder() + s = CodeBuilder32() s.LEA32_rs(ecx, -36) assert s.getvalue() == '\x8D\x8D\xDC\xFF\xFF\xFF' + + +class CodeBuilder64(CodeBuilderMixin, X86_64_CodeBuilder): + pass + +def test_mov_ri_64(): + s = CodeBuilder64() + s.MOV_ri(ecx, -2) + s.MOV_ri(r12, 0x80000042) + assert s.getvalue() == ('\x48\xC7\xC1\xFE\xFF\xFF\xFF' + + '\x49\xBC\x42\x00\x00\x80\x00\x00\x00\x00') Modified: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py Sun Oct 4 22:42:53 2009 @@ -22,8 +22,7 @@ (random.randrange(0,65536)<<32) | (random.randrange(0,65536)<<16) | (random.randrange(0,65536)<<0)) - if not rx86.fits_in_32bits(x): - v.append(x) + v.append(x) return v + self._old_imm32_tests() def test_extra_MOV_ri64(self): From fijal at codespeak.net Mon Oct 5 01:47:08 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Oct 2009 01:47:08 +0200 (CEST) Subject: [pypy-svn] r68151 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20091004234708.1337D1683D7@codespeak.net> Author: fijal Date: Mon Oct 5 01:47:07 2009 New Revision: 68151 Modified: pypy/trunk/pypy/jit/metainterp/test/test_loop.py Log: Make LoopTest a new-style class, just because we promote new-style classes Modified: pypy/trunk/pypy/jit/metainterp/test/test_loop.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_loop.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_loop.py Mon Oct 5 01:47:07 2009 @@ -7,7 +7,7 @@ from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp import history -class LoopTest: +class LoopTest(object): optimizer = OPTIMIZER_SIMPLE def meta_interp(self, f, args, policy=None): From fijal at codespeak.net Mon Oct 5 09:33:06 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Oct 2009 09:33:06 +0200 (CEST) Subject: [pypy-svn] r68152 - pypy/branch/merge-floats-via-sse2 Message-ID: <20091005073306.56B681683CF@codespeak.net> Author: fijal Date: Mon Oct 5 09:33:04 2009 New Revision: 68152 Added: pypy/branch/merge-floats-via-sse2/ - copied from r68151, pypy/trunk/ Log: A new branch to merge floats-via-sse2 From fijal at codespeak.net Mon Oct 5 10:34:08 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Oct 2009 10:34:08 +0200 (CEST) Subject: [pypy-svn] r68153 - in pypy/branch/merge-floats-via-sse2/pypy: jit/backend/llgraph jit/backend/llsupport jit/backend/llsupport/test jit/backend/test jit/backend/x86 jit/backend/x86/test jit/metainterp jit/metainterp/test module/pypyjit Message-ID: <20091005083408.3E1C01683E1@codespeak.net> Author: fijal Date: Mon Oct 5 10:34:06 2009 New Revision: 68153 Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llgraph/runner.py pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/descr.py pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/llmodel.py pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/regalloc.py pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/symbolic.py pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/test/test_descr.py pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/test/test_regalloc.py pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/test/test_runner.py pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test/runner_test.py pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/assembler.py pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/jump.py pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/regalloc.py pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/ri386.py pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/ri386setup.py pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/runner.py pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_gc_integration.py pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_jump.py pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/history.py pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/logger.py pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/oparser.py pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_executor.py pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_logger.py pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_oparser.py pypy/branch/merge-floats-via-sse2/pypy/module/pypyjit/policy.py Log: (arigo, fijal) Merge floats-via-sse2 branch. This branch adds float support to x86 backend, via SSE2 instruction set, present in Pentium IV and newer. Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llgraph/runner.py Mon Oct 5 10:34:06 2009 @@ -467,9 +467,10 @@ assert isinstance(calldescr, Descr) func = args[0].getint() for arg in args[1:]: - if (isinstance(arg, history.BoxPtr) or - isinstance(arg, history.ConstPtr)): + if arg.type == REF: llimpl.do_call_pushptr(arg.getref_base()) + elif arg.type == FLOAT: + llimpl.do_call_pushfloat(arg.getfloat()) else: llimpl.do_call_pushint(arg.getint()) if calldescr.typeinfo == REF: Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/descr.py Mon Oct 5 10:34:06 2009 @@ -1,7 +1,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.jit.backend.llsupport import symbolic from pypy.jit.metainterp.history import AbstractDescr, getkind, BoxInt, BoxPtr -from pypy.jit.metainterp.history import BasicFailDescr, LoopToken +from pypy.jit.metainterp.history import BasicFailDescr, LoopToken, BoxFloat from pypy.jit.metainterp.resoperation import ResOperation, rop # The point of the class organization in this file is to make instances @@ -67,8 +67,14 @@ def get_field_size(self, translate_support_code): raise NotImplementedError + _is_pointer_field = False # unless overridden by GcPtrFieldDescr + _is_float_field = False # unless overridden by FloatFieldDescr + def is_pointer_field(self): - return False # unless overridden by GcPtrFieldDescr + return self._is_pointer_field + + def is_float_field(self): + return self._is_float_field def repr_of_descr(self): return '<%s %s>' % (self._clsname, self.offset) @@ -81,12 +87,12 @@ class GcPtrFieldDescr(NonGcPtrFieldDescr): _clsname = 'GcPtrFieldDescr' - def is_pointer_field(self): - return True + _is_pointer_field = True def getFieldDescrClass(TYPE): return getDescrClass(TYPE, BaseFieldDescr, GcPtrFieldDescr, - NonGcPtrFieldDescr, 'Field', 'get_field_size') + NonGcPtrFieldDescr, 'Field', 'get_field_size', + '_is_float_field') def get_field_descr(gccache, STRUCT, fieldname): cache = gccache._cache_field @@ -122,8 +128,14 @@ def get_item_size(self, translate_support_code): raise NotImplementedError + _is_array_of_pointers = False # unless overridden by GcPtrArrayDescr + _is_array_of_floats = False # unless overridden by FloatArrayDescr + def is_array_of_pointers(self): - return False # unless overridden by GcPtrArrayDescr + return self._is_array_of_pointers + + def is_array_of_floats(self): + return self._is_array_of_floats def repr_of_descr(self): return '<%s>' % self._clsname @@ -136,12 +148,12 @@ class GcPtrArrayDescr(NonGcPtrArrayDescr): _clsname = 'GcPtrArrayDescr' - def is_array_of_pointers(self): - return True + _is_array_of_pointers = True def getArrayDescrClass(ARRAY): return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr, - NonGcPtrArrayDescr, 'Array', 'get_item_size') + NonGcPtrArrayDescr, 'Array', 'get_item_size', + '_is_array_of_floats') def get_array_descr(gccache, ARRAY): cache = gccache._cache_array @@ -174,13 +186,20 @@ def instantiate_arg_classes(self): result = [] for c in self.arg_classes: - if c == 'i': box = BoxInt() - else: box = BoxPtr() + if c == 'i': box = BoxInt() + elif c == 'f': box = BoxFloat() + else: box = BoxPtr() result.append(box) return result + _returns_a_pointer = False # unless overridden by GcPtrCallDescr + _returns_a_float = False # unless overridden by FloatCallDescr + def returns_a_pointer(self): - return False # unless overridden by GcPtrCallDescr + return self._returns_a_pointer + + def returns_a_float(self): + return self._returns_a_float def get_result_size(self, translate_support_code): raise NotImplementedError @@ -195,6 +214,8 @@ else: if self.returns_a_pointer(): result = BoxPtr() + elif self.returns_a_float(): + result = BoxFloat() else: result = BoxInt() result_list = [result] @@ -221,8 +242,7 @@ class GcPtrCallDescr(NonGcPtrCallDescr): _clsname = 'GcPtrCallDescr' - def returns_a_pointer(self): - return True + _returns_a_pointer = True class VoidCallDescr(NonGcPtrCallDescr): _clsname = 'VoidCallDescr' @@ -233,7 +253,8 @@ if RESULT is lltype.Void: return VoidCallDescr return getDescrClass(RESULT, BaseCallDescr, GcPtrCallDescr, - NonGcPtrCallDescr, 'Call', 'get_result_size') + NonGcPtrCallDescr, 'Call', 'get_result_size', + '_returns_a_float') def get_call_descr(gccache, ARGS, RESULT): arg_classes = [] @@ -241,6 +262,7 @@ kind = getkind(ARG) if kind == 'int': arg_classes.append('i') elif kind == 'ref': arg_classes.append('r') + elif kind == 'float': arg_classes.append('f') else: raise NotImplementedError('ARG = %r' % (ARG,)) arg_classes = ''.join(arg_classes) @@ -258,7 +280,7 @@ # ____________________________________________________________ def getDescrClass(TYPE, BaseDescr, GcPtrDescr, NonGcPtrDescr, - nameprefix, methodname, _cache={}): + nameprefix, methodname, floatattrname, _cache={}): if isinstance(TYPE, lltype.Ptr): if TYPE.TO._gckind == 'gc': return GcPtrDescr @@ -276,5 +298,8 @@ return symbolic.get_size(TYPE, translate_support_code) setattr(Descr, methodname, method) # + if TYPE is lltype.Float: + setattr(Descr, floatattrname, True) + # _cache[nameprefix, TYPE] = Descr return Descr Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/llmodel.py Mon Oct 5 10:34:06 2009 @@ -4,7 +4,8 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.rpython.annlowlevel import llhelper from pypy.rlib.objectmodel import we_are_translated, specialize -from pypy.jit.metainterp.history import BoxInt, BoxPtr, set_future_values +from pypy.jit.metainterp.history import BoxInt, BoxPtr, set_future_values,\ + BoxFloat from pypy.jit.backend.model import AbstractCPU from pypy.jit.backend.llsupport import symbolic from pypy.jit.backend.llsupport.symbolic import WORD, unroll_basic_sizes @@ -205,7 +206,8 @@ ofs = fielddescr.offset size = fielddescr.get_field_size(self.translate_support_code) ptr = fielddescr.is_pointer_field() - return ofs, size, ptr + float = fielddescr.is_float_field() + return ofs, size, ptr, float unpack_fielddescr._always_inline_ = True def arraydescrof(self, A): @@ -216,7 +218,8 @@ ofs = arraydescr.get_base_size(self.translate_support_code) size = arraydescr.get_item_size(self.translate_support_code) ptr = arraydescr.is_array_of_pointers() - return ofs, size, ptr + float = arraydescr.is_array_of_floats() + return ofs, size, ptr, float unpack_arraydescr._always_inline_ = True def calldescrof(self, FUNC, ARGS, RESULT): @@ -246,40 +249,66 @@ def do_getarrayitem_gc(self, arraybox, indexbox, arraydescr): itemindex = indexbox.getint() gcref = arraybox.getref_base() - ofs, size, ptr = self.unpack_arraydescr(arraydescr) + ofs, size, ptr, float = self.unpack_arraydescr(arraydescr) + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + # + if ptr: + items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items) + pval = self._cast_int_to_gcref(items[itemindex]) + # --- end of GC unsafe code --- + return BoxPtr(pval) + # + if float: + items = rffi.cast(rffi.CArrayPtr(lltype.Float), items) + fval = items[itemindex] + # --- end of GC unsafe code --- + return BoxFloat(fval) # for TYPE, itemsize in unroll_basic_sizes: if size == itemsize: - val = (rffi.cast(rffi.CArrayPtr(TYPE), gcref) - [ofs/itemsize + itemindex]) - val = rffi.cast(lltype.Signed, val) - break + items = rffi.cast(rffi.CArrayPtr(TYPE), items) + val = items[itemindex] + # --- end of GC unsafe code --- + return BoxInt(rffi.cast(lltype.Signed, val)) else: raise NotImplementedError("size = %d" % size) - if ptr: - return BoxPtr(self._cast_int_to_gcref(val)) - else: - return BoxInt(val) def do_setarrayitem_gc(self, arraybox, indexbox, vbox, arraydescr): itemindex = indexbox.getint() gcref = arraybox.getref_base() - ofs, size, ptr = self.unpack_arraydescr(arraydescr) + ofs, size, ptr, float = self.unpack_arraydescr(arraydescr) # if ptr: vboxptr = vbox.getref_base() self.gc_ll_descr.do_write_barrier(gcref, vboxptr) - a = rffi.cast(rffi.CArrayPtr(lltype.Signed), gcref) - a[ofs/WORD + itemindex] = self.cast_gcref_to_int(vboxptr) + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items) + items[itemindex] = self.cast_gcref_to_int(vboxptr) + # --- end of GC unsafe code --- + return + # + if float: + fval = vbox.getfloat() + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + items = rffi.cast(rffi.CArrayPtr(lltype.Float), items) + items[itemindex] = fval + # --- end of GC unsafe code --- + return + # + val = vbox.getint() + for TYPE, itemsize in unroll_basic_sizes: + if size == itemsize: + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + items = rffi.cast(rffi.CArrayPtr(TYPE), items) + items[itemindex] = rffi.cast(TYPE, val) + # --- end of GC unsafe code --- + return else: - v = vbox.getint() - for TYPE, itemsize in unroll_basic_sizes: - if size == itemsize: - a = rffi.cast(rffi.CArrayPtr(TYPE), gcref) - a[ofs/itemsize + itemindex] = rffi.cast(TYPE, v) - break - else: - raise NotImplementedError("size = %d" % size) + raise NotImplementedError("size = %d" % size) def _new_do_len(TP): def do_strlen(self, stringbox): @@ -312,18 +341,29 @@ @specialize.argtype(1) def _base_do_getfield(self, gcref, fielddescr): - ofs, size, ptr = self.unpack_fielddescr(fielddescr) + ofs, size, ptr, float = self.unpack_fielddescr(fielddescr) + # --- start of GC unsafe code (no GC operation!) --- + field = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + # + if ptr: + pval = rffi.cast(rffi.CArrayPtr(lltype.Signed), field)[0] + pval = self._cast_int_to_gcref(pval) + # --- end of GC unsafe code --- + return BoxPtr(pval) + # + if float: + fval = rffi.cast(rffi.CArrayPtr(lltype.Float), field)[0] + # --- end of GC unsafe code --- + return BoxFloat(fval) + # for TYPE, itemsize in unroll_basic_sizes: if size == itemsize: - val = rffi.cast(rffi.CArrayPtr(TYPE), gcref)[ofs/itemsize] + val = rffi.cast(rffi.CArrayPtr(TYPE), field)[0] + # --- end of GC unsafe code --- val = rffi.cast(lltype.Signed, val) - break + return BoxInt(val) else: raise NotImplementedError("size = %d" % size) - if ptr: - return BoxPtr(self._cast_int_to_gcref(val)) - else: - return BoxInt(val) def do_getfield_gc(self, structbox, fielddescr): gcref = structbox.getref_base() @@ -334,23 +374,40 @@ @specialize.argtype(1) def _base_do_setfield(self, gcref, vbox, fielddescr): - ofs, size, ptr = self.unpack_fielddescr(fielddescr) + ofs, size, ptr, float = self.unpack_fielddescr(fielddescr) + # if ptr: assert lltype.typeOf(gcref) is not lltype.Signed, ( "can't handle write barriers for setfield_raw") ptr = vbox.getref_base() self.gc_ll_descr.do_write_barrier(gcref, ptr) - a = rffi.cast(rffi.CArrayPtr(lltype.Signed), gcref) - a[ofs/WORD] = self.cast_gcref_to_int(ptr) + # --- start of GC unsafe code (no GC operation!) --- + field = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + field = rffi.cast(rffi.CArrayPtr(lltype.Signed), field) + field[0] = self.cast_gcref_to_int(ptr) + # --- end of GC unsafe code --- + return + # + if float: + fval = vbox.getfloat() + # --- start of GC unsafe code (no GC operation!) --- + field = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + field = rffi.cast(rffi.CArrayPtr(lltype.Float), field) + field[0] = fval + # --- end of GC unsafe code --- + return + # + val = vbox.getint() + for TYPE, itemsize in unroll_basic_sizes: + if size == itemsize: + # --- start of GC unsafe code (no GC operation!) --- + field = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + field = rffi.cast(rffi.CArrayPtr(TYPE), field) + field[0] = rffi.cast(TYPE, val) + # --- end of GC unsafe code --- + return else: - v = vbox.getint() - for TYPE, itemsize in unroll_basic_sizes: - if size == itemsize: - v = rffi.cast(TYPE, v) - rffi.cast(rffi.CArrayPtr(TYPE), gcref)[ofs/itemsize] = v - break - else: - raise NotImplementedError("size = %d" % size) + raise NotImplementedError("size = %d" % size) def do_setfield_gc(self, structbox, vbox, fielddescr): gcref = structbox.getref_base() @@ -416,7 +473,9 @@ # nonsense but nothing wrong (the return value should be ignored) if calldescr.returns_a_pointer(): return BoxPtr(self.get_latest_value_ref(0)) - elif calldescr.get_result_size(self.translate_support_code) != 0: + elif calldescr.returns_a_float(): + return BoxFloat(self.get_latest_value_float(0)) + elif calldescr.get_result_size(self.translate_support_code) > 0: return BoxInt(self.get_latest_value_int(0)) else: return None Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/regalloc.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/regalloc.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/regalloc.py Mon Oct 5 10:34:06 2009 @@ -22,26 +22,28 @@ def get(self, box): return self.stack_bindings.get(box, None) - def loc(self, box): + def loc(self, box, size): res = self.get(box) if res is not None: return res - newloc = self.stack_pos(self.stack_depth) + newloc = self.stack_pos(self.stack_depth, size) self.stack_bindings[box] = newloc - self.stack_depth += 1 + self.stack_depth += size return newloc # abstract methods that need to be overwritten for specific assemblers @staticmethod - def stack_pos(loc): + def stack_pos(loc, size): raise NotImplementedError("Purely abstract") class RegisterManager(object): """ Class that keeps track of register allocations """ - all_regs = [] - no_lower_byte_regs = [] + box_types = None # or a list of acceptable types + all_regs = [] + no_lower_byte_regs = [] save_around_call_regs = [] + reg_width = 1 # in terms of stack space eaten def __init__(self, longevity, stack_manager=None, assembler=None): self.free_regs = self.all_regs[:] @@ -57,11 +59,16 @@ def next_instruction(self, incr=1): self.position += incr + def _check_type(self, v): + if not we_are_translated() and self.box_types is not None: + assert isinstance(v, TempBox) or v.type in self.box_types + def possibly_free_var(self, v): """ If v is stored in a register and v is not used beyond the current position, then free it. Must be called at some point for all variables that might be in registers. """ + self._check_type(v) if isinstance(v, Const) or v not in self.reg_bindings: return if v not in self.longevity or self.longevity[v][1] <= self.position: @@ -96,6 +103,7 @@ returns allocated register or None, if not possible. """ + self._check_type(v) assert not isinstance(v, Const) if selected_reg is not None: res = self.reg_bindings.get(v, None) @@ -140,7 +148,7 @@ loc = self.reg_bindings[v_to_spill] del self.reg_bindings[v_to_spill] if self.stack_manager.get(v_to_spill) is None: - newloc = self.stack_manager.loc(v_to_spill) + newloc = self.stack_manager.loc(v_to_spill, self.reg_width) self.assembler.regalloc_mov(loc, newloc) return loc @@ -172,6 +180,7 @@ Will not spill a variable from 'forbidden_vars'. """ + self._check_type(v) if isinstance(v, TempBox): self.longevity[v] = (self.position, self.position) loc = self.try_allocate_reg(v, selected_reg, @@ -189,12 +198,13 @@ def loc(self, box): """ Return the location of 'box'. """ + self._check_type(box) if isinstance(box, Const): return self.convert_to_imm(box) try: return self.reg_bindings[box] except KeyError: - return self.stack_manager.loc(box) + return self.stack_manager.loc(box, self.reg_width) def return_constant(self, v, forbidden_vars=[], selected_reg=None, imm_fine=True): @@ -203,6 +213,7 @@ a register. See 'force_allocate_reg' for the meaning of 'selected_reg' and 'forbidden_vars'. """ + self._check_type(v) assert isinstance(v, Const) if selected_reg or not imm_fine: # this means we cannot have it in IMM, eh @@ -210,7 +221,7 @@ self.assembler.regalloc_mov(self.convert_to_imm(v), selected_reg) return selected_reg if selected_reg is None and self.free_regs: - loc = self.free_regs.pop() + loc = self.free_regs[-1] self.assembler.regalloc_mov(self.convert_to_imm(v), loc) return loc loc = self._spill_var(v, forbidden_vars, selected_reg) @@ -225,6 +236,7 @@ register. Return the register. See 'return_constant' and 'force_allocate_reg' for the meaning of the optional arguments. """ + self._check_type(v) if isinstance(v, Const): return self.return_constant(v, forbidden_vars, selected_reg, imm_fine) @@ -248,7 +260,7 @@ self.reg_bindings[v] = loc self.assembler.regalloc_mov(prev_loc, loc) else: - loc = self.stack_manager.loc(v) + loc = self.stack_manager.loc(v, self.reg_width) self.assembler.regalloc_mov(prev_loc, loc) def force_result_in_reg(self, result_v, v, forbidden_vars=[]): @@ -256,14 +268,19 @@ The variable v is copied away if it's further used. The meaning of 'forbidden_vars' is the same as in 'force_allocate_reg'. """ + self._check_type(result_v) + self._check_type(v) if isinstance(v, Const): loc = self.make_sure_var_in_reg(v, forbidden_vars, imm_fine=False) + # note that calling make_sure_var_in_reg with imm_fine=False + # will not allocate place in reg_bindings, we need to do it + # on our own self.reg_bindings[result_v] = loc self.free_regs = [reg for reg in self.free_regs if reg is not loc] return loc if v not in self.reg_bindings: - prev_loc = self.stack_manager.loc(v) + prev_loc = self.stack_manager.loc(v, self.reg_width) loc = self.force_allocate_reg(v, forbidden_vars) self.assembler.regalloc_mov(prev_loc, loc) assert v in self.reg_bindings @@ -283,7 +300,8 @@ def _sync_var(self, v): if not self.stack_manager.get(v): reg = self.reg_bindings[v] - self.assembler.regalloc_mov(reg, self.stack_manager.loc(v)) + to = self.stack_manager.loc(v, self.reg_width) + self.assembler.regalloc_mov(reg, to) # otherwise it's clean def before_call(self, force_store=[]): @@ -310,6 +328,7 @@ which is in variable v. """ if v is not None: + self._check_type(v) r = self.call_result_location(v) self.reg_bindings[v] = r self.free_regs = [fr for fr in self.free_regs if fr is not r] Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/symbolic.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/symbolic.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/symbolic.py Mon Oct 5 10:34:06 2009 @@ -61,8 +61,10 @@ SIZEOF_CHAR = get_size(lltype.Char, False) SIZEOF_SHORT = get_size(rffi.SHORT, False) SIZEOF_INT = get_size(rffi.INT, False) +SIZEOF_FLOAT = get_size(lltype.Float, False) unroll_basic_sizes = unrolling_iterable([(lltype.Signed, WORD), (lltype.Char, SIZEOF_CHAR), (rffi.SHORT, SIZEOF_SHORT), (rffi.INT, SIZEOF_INT)]) +# does not contain Float ^^^ which must be special-cased Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/test/test_descr.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/test/test_descr.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/test/test_descr.py Mon Oct 5 10:34:06 2009 @@ -26,12 +26,16 @@ T = lltype.GcStruct('T') S = lltype.GcStruct('S', ('x', lltype.Char), ('y', lltype.Ptr(T)), - ('z', lltype.Ptr(U))) + ('z', lltype.Ptr(U)), + ('f', lltype.Float)) assert getFieldDescrClass(lltype.Ptr(T)) is GcPtrFieldDescr assert getFieldDescrClass(lltype.Ptr(U)) is NonGcPtrFieldDescr cls = getFieldDescrClass(lltype.Char) assert cls != getFieldDescrClass(lltype.Signed) assert cls == getFieldDescrClass(lltype.Char) + clsf = getFieldDescrClass(lltype.Float) + assert clsf != cls + assert clsf == getFieldDescrClass(lltype.Float) # c0 = GcCache(False) c1 = GcCache(True) @@ -42,25 +46,35 @@ descr_x = get_field_descr(c2, S, 'x') descr_y = get_field_descr(c2, S, 'y') descr_z = get_field_descr(c2, S, 'z') + descr_f = get_field_descr(c2, S, 'f') assert descr_x.__class__ is cls assert descr_y.__class__ is GcPtrFieldDescr assert descr_z.__class__ is NonGcPtrFieldDescr + assert descr_f.__class__ is clsf if not tsc: assert descr_x.offset < descr_y.offset < descr_z.offset assert descr_x.sort_key() < descr_y.sort_key() < descr_z.sort_key() assert descr_x.get_field_size(False) == rffi.sizeof(lltype.Char) assert descr_y.get_field_size(False) == rffi.sizeof(lltype.Ptr(T)) assert descr_z.get_field_size(False) == rffi.sizeof(lltype.Ptr(U)) + assert descr_f.get_field_size(False) == rffi.sizeof(lltype.Float) else: assert isinstance(descr_x.offset, Symbolic) assert isinstance(descr_y.offset, Symbolic) assert isinstance(descr_z.offset, Symbolic) + assert isinstance(descr_f.offset, Symbolic) assert isinstance(descr_x.get_field_size(True), Symbolic) assert isinstance(descr_y.get_field_size(True), Symbolic) assert isinstance(descr_z.get_field_size(True), Symbolic) + assert isinstance(descr_f.get_field_size(True), Symbolic) assert not descr_x.is_pointer_field() assert descr_y.is_pointer_field() assert not descr_z.is_pointer_field() + assert not descr_f.is_pointer_field() + assert not descr_x.is_float_field() + assert not descr_y.is_float_field() + assert not descr_z.is_float_field() + assert descr_f.is_float_field() def test_get_array_descr(): @@ -69,68 +83,102 @@ A1 = lltype.GcArray(lltype.Char) A2 = lltype.GcArray(lltype.Ptr(T)) A3 = lltype.GcArray(lltype.Ptr(U)) + A4 = lltype.GcArray(lltype.Float) assert getArrayDescrClass(A2) is GcPtrArrayDescr assert getArrayDescrClass(A3) is NonGcPtrArrayDescr cls = getArrayDescrClass(A1) assert cls != getArrayDescrClass(lltype.GcArray(lltype.Signed)) assert cls == getArrayDescrClass(lltype.GcArray(lltype.Char)) + clsf = getArrayDescrClass(A4) + assert clsf != cls + assert clsf == getArrayDescrClass(lltype.GcArray(lltype.Float)) # c0 = GcCache(False) descr1 = get_array_descr(c0, A1) descr2 = get_array_descr(c0, A2) descr3 = get_array_descr(c0, A3) + descr4 = get_array_descr(c0, A4) assert descr1.__class__ is cls assert descr2.__class__ is GcPtrArrayDescr assert descr3.__class__ is NonGcPtrArrayDescr + assert descr4.__class__ is clsf assert descr1 == get_array_descr(c0, lltype.GcArray(lltype.Char)) assert not descr1.is_array_of_pointers() assert descr2.is_array_of_pointers() assert not descr3.is_array_of_pointers() + assert not descr4.is_array_of_pointers() + assert not descr1.is_array_of_floats() + assert not descr2.is_array_of_floats() + assert not descr3.is_array_of_floats() + assert descr4.is_array_of_floats() # WORD = rffi.sizeof(lltype.Signed) assert descr1.get_base_size(False) == WORD assert descr2.get_base_size(False) == WORD assert descr3.get_base_size(False) == WORD + assert descr4.get_base_size(False) == WORD assert descr1.get_ofs_length(False) == 0 assert descr2.get_ofs_length(False) == 0 assert descr3.get_ofs_length(False) == 0 + assert descr4.get_ofs_length(False) == 0 assert descr1.get_item_size(False) == rffi.sizeof(lltype.Char) assert descr2.get_item_size(False) == rffi.sizeof(lltype.Ptr(T)) assert descr3.get_item_size(False) == rffi.sizeof(lltype.Ptr(U)) + assert descr4.get_item_size(False) == rffi.sizeof(lltype.Float) # assert isinstance(descr1.get_base_size(True), Symbolic) assert isinstance(descr2.get_base_size(True), Symbolic) assert isinstance(descr3.get_base_size(True), Symbolic) + assert isinstance(descr4.get_base_size(True), Symbolic) assert isinstance(descr1.get_ofs_length(True), Symbolic) assert isinstance(descr2.get_ofs_length(True), Symbolic) assert isinstance(descr3.get_ofs_length(True), Symbolic) + assert isinstance(descr4.get_ofs_length(True), Symbolic) assert isinstance(descr1.get_item_size(True), Symbolic) assert isinstance(descr2.get_item_size(True), Symbolic) assert isinstance(descr3.get_item_size(True), Symbolic) + assert isinstance(descr4.get_item_size(True), Symbolic) -def test_get_call_descr(): +def test_get_call_descr_not_translated(): c0 = GcCache(False) descr1 = get_call_descr(c0, [lltype.Char, lltype.Signed], lltype.Char) assert descr1.get_result_size(False) == rffi.sizeof(lltype.Char) assert not descr1.returns_a_pointer() + assert not descr1.returns_a_float() assert descr1.arg_classes == "ii" # T = lltype.GcStruct('T') descr2 = get_call_descr(c0, [lltype.Ptr(T)], lltype.Ptr(T)) assert descr2.get_result_size(False) == rffi.sizeof(lltype.Ptr(T)) assert descr2.returns_a_pointer() + assert not descr2.returns_a_float() assert descr2.arg_classes == "r" # U = lltype.GcStruct('U', ('x', lltype.Signed)) assert descr2 == get_call_descr(c0, [lltype.Ptr(U)], lltype.Ptr(U)) # + descr4 = get_call_descr(c0, [lltype.Float, lltype.Float], lltype.Float) + assert descr4.get_result_size(False) == rffi.sizeof(lltype.Float) + assert not descr4.returns_a_pointer() + assert descr4.returns_a_float() + assert descr4.arg_classes == "ff" + +def test_get_call_descr_translated(): c1 = GcCache(True) + T = lltype.GcStruct('T') + U = lltype.GcStruct('U', ('x', lltype.Signed)) descr3 = get_call_descr(c1, [lltype.Ptr(T)], lltype.Ptr(U)) assert isinstance(descr3.get_result_size(True), Symbolic) assert descr3.returns_a_pointer() + assert not descr3.returns_a_float() assert descr3.arg_classes == "r" - + # + descr4 = get_call_descr(c1, [lltype.Float, lltype.Float], lltype.Float) + assert isinstance(descr4.get_result_size(True), Symbolic) + assert not descr4.returns_a_pointer() + assert descr4.returns_a_float() + assert descr4.arg_classes == "ff" def test_repr_of_descr(): c0 = GcCache(False) @@ -162,3 +210,6 @@ # descr4i = get_call_descr(c0, [lltype.Char, lltype.Ptr(S)], lltype.Char) assert 'CharCallDescr' in descr4i.repr_of_descr() + # + descr4f = get_call_descr(c0, [lltype.Char, lltype.Ptr(S)], lltype.Float) + assert 'FloatCallDescr' in descr4f.repr_of_descr() Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/test/test_regalloc.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/test/test_regalloc.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/test/test_regalloc.py Mon Oct 5 10:34:06 2009 @@ -1,5 +1,5 @@ -from pypy.jit.metainterp.history import BoxInt, ConstInt +from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat from pypy.jit.backend.llsupport.regalloc import StackManager from pypy.jit.backend.llsupport.regalloc import RegisterManager as BaseRegMan @@ -27,7 +27,7 @@ return v class TStackManager(StackManager): - def stack_pos(self, i): + def stack_pos(self, i, size): return i class MockAsm(object): @@ -146,7 +146,7 @@ rm.next_instruction() # allocate a stack position b0, b1, b2, b3, b4 = boxes - sp = sm.loc(b0) + sp = sm.loc(b0, 1) assert sp == 0 loc = rm.make_sure_var_in_reg(b0) assert isinstance(loc, FakeReg) @@ -207,7 +207,7 @@ asm = MockAsm() rm = RegisterManager(longevity, stack_manager=sm, assembler=asm) rm.next_instruction() - sm.loc(b0) + sm.loc(b0, 1) rm.force_result_in_reg(b1, b0) rm._check_invariants() loc = rm.loc(b1) @@ -233,10 +233,11 @@ assert isinstance(loc, ConstInt) for box in boxes[:-1]: rm.force_allocate_reg(box) - assert len(asm.moves) == 4 + assert len(asm.moves) == 3 loc = rm.return_constant(ConstInt(1), imm_fine=False) assert isinstance(loc, FakeReg) - assert len(asm.moves) == 6 + assert len(asm.moves) == 5 + assert len(rm.reg_bindings) == 3 def test_force_result_in_reg_const(self): boxes, longevity = boxes_and_longevity(2) @@ -276,3 +277,21 @@ rm.after_call(boxes[-1]) assert len(rm.reg_bindings) == 3 rm._check_invariants() + + def test_different_stack_width(self): + class XRegisterManager(RegisterManager): + reg_width = 2 + + sm = TStackManager() + b0 = BoxInt() + longevity = {b0: (0, 1)} + asm = MockAsm() + rm = RegisterManager(longevity, stack_manager=sm, assembler=asm) + f0 = BoxFloat() + longevity = {f0: (0, 1)} + xrm = XRegisterManager(longevity, stack_manager=sm, assembler=asm) + xrm.loc(f0) + rm.loc(b0) + assert sm.stack_depth == 3 + + Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/test/test_runner.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/test/test_runner.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llsupport/test/test_runner.py Mon Oct 5 10:34:06 2009 @@ -7,6 +7,7 @@ pass class MyLLCPU(AbstractLLCPU): + supports_floats = True def compile_loop(self, inputargs, operations, looptoken): py.test.skip("llsupport test: cannot compile operations") Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test/runner_test.py Mon Oct 5 10:34:06 2009 @@ -340,6 +340,19 @@ BoxInt(ord('A'))], calldescr) assert x.value == ord('B') + if cpu.supports_floats: + def func(f, i): + return float(i) + f + FPTR = self.Ptr(self.FuncType([lltype.Float, lltype.Signed], + lltype.Float)) + func_ptr = llhelper(FPTR, func) + FTP = deref(FPTR) + calldescr = cpu.calldescrof(FTP, FTP.ARGS, FTP.RESULT) + x = cpu.do_call( + [self.get_funcbox(cpu, func_ptr), + BoxFloat(3.5), BoxInt(42)], + calldescr) + assert x.value == 42 + 3.5 def test_call(self): @@ -367,6 +380,24 @@ 'int', descr=calldescr) assert res.value == 2 * num + if cpu.supports_floats: + def func(f0, f1, f2, f3, f4, f5, f6, i0, i1, f7, f8, f9): + return f0 + f1 + f2 + f3 + f4 + f5 + f6 + float(i0 + i1) + f7 + f8 + f9 + F = lltype.Float + I = lltype.Signed + FUNC = self.FuncType([F] * 7 + [I] * 2 + [F] * 3, F) + FPTR = self.Ptr(FUNC) + func_ptr = llhelper(FPTR, func) + calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + funcbox = self.get_funcbox(cpu, func_ptr) + args = ([BoxFloat(.1) for i in range(7)] + + [BoxInt(1), BoxInt(2), BoxFloat(.2), BoxFloat(.3), + BoxFloat(.4)]) + res = self.execute_operation(rop.CALL, + [funcbox] + args, + 'float', descr=calldescr) + assert abs(res.value - 4.6) < 0.0001 + def test_field_basic(self): t_box, T_box = self.alloc_instance(self.T) fielddescr = self.cpu.fielddescrof(self.S, 'value') @@ -416,12 +447,22 @@ res = self.execute_operation(rop.GETFIELD_GC, [t_box], 'ref', descr=fielddescr2) assert res.value == null_const.value + if self.cpu.supports_floats: + floatdescr = self.cpu.fielddescrof(self.S, 'float') + self.execute_operation(rop.SETFIELD_GC, [t_box, BoxFloat(3.4)], + 'void', descr=floatdescr) + res = self.execute_operation(rop.GETFIELD_GC, [t_box], + 'float', descr=floatdescr) + assert res.value == 3.4 + def test_passing_guards(self): - for (opname, args) in [(rop.GUARD_TRUE, [BoxInt(1)]), - (rop.GUARD_FALSE, [BoxInt(0)]), - (rop.GUARD_VALUE, [BoxInt(42), BoxInt(42)]), - ]: + all = [(rop.GUARD_TRUE, [BoxInt(1)]), + (rop.GUARD_FALSE, [BoxInt(0)]), + (rop.GUARD_VALUE, [BoxInt(42), BoxInt(42)])] + if self.cpu.supports_floats: + all.append((rop.GUARD_VALUE, [BoxFloat(3.5), BoxFloat(3.5)])) + for (opname, args) in all: assert self.execute_operation(opname, args, 'void') == None assert not self.guard_failed @@ -435,10 +476,12 @@ # 'void') def test_failing_guards(self): - for opname, args in [(rop.GUARD_TRUE, [BoxInt(0)]), - (rop.GUARD_FALSE, [BoxInt(1)]), - (rop.GUARD_VALUE, [BoxInt(42), BoxInt(41)]), - ]: + all = [(rop.GUARD_TRUE, [BoxInt(0)]), + (rop.GUARD_FALSE, [BoxInt(1)]), + (rop.GUARD_VALUE, [BoxInt(42), BoxInt(41)])] + if self.cpu.supports_floats: + all.append((rop.GUARD_VALUE, [BoxFloat(-1.0), BoxFloat(1.0)])) + for opname, args in all: assert self.execute_operation(opname, args, 'void') == None assert self.guard_failed @@ -589,6 +632,23 @@ 'int', descr=arraydescr) assert r.value == 1 + if self.cpu.supports_floats: + a_box, A = self.alloc_array_of(lltype.Float, 31) + arraydescr = self.cpu.arraydescrof(A) + self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(1), + BoxFloat(3.5)], + 'void', descr=arraydescr) + self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(2), + BoxFloat(4.5)], + 'void', descr=arraydescr) + r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(1)], + 'float', descr=arraydescr) + assert r.value == 3.5 + r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(2)], + 'float', descr=arraydescr) + assert r.value == 4.5 + + def test_string_basic(self): s_box = self.alloc_string("hello\xfe") r = self.execute_operation(rop.STRLEN, [s_box], 'int') @@ -633,6 +693,180 @@ r = self.execute_operation(rop.SAME_AS, [u_box.constbox()], 'ref') assert r.value == u_box.value + if self.cpu.supports_floats: + r = self.execute_operation(rop.SAME_AS, [ConstFloat(5.5)], 'float') + assert r.value == 5.5 + + def test_jump(self): + # this test generates small loops where the JUMP passes many + # arguments of various types, shuffling them around. + if self.cpu.supports_floats: + numkinds = 3 + else: + numkinds = 2 + seed = random.randrange(0, 10000) + print 'Seed is', seed # or choose it by changing the previous line + r = random.Random() + r.seed(seed) + for nb_args in range(50): + print 'Passing %d arguments around...' % nb_args + # + inputargs = [] + for k in range(nb_args): + kind = r.randrange(0, numkinds) + if kind == 0: + inputargs.append(BoxInt()) + elif kind == 1: + inputargs.append(BoxPtr()) + else: + inputargs.append(BoxFloat()) + jumpargs = [] + remixing = [] + for srcbox in inputargs: + n = r.randrange(0, len(inputargs)) + otherbox = inputargs[n] + if otherbox.type == srcbox.type: + remixing.append((srcbox, otherbox)) + else: + otherbox = srcbox + jumpargs.append(otherbox) + # + index_counter = r.randrange(0, len(inputargs)+1) + i0 = BoxInt() + i1 = BoxInt() + i2 = BoxInt() + inputargs.insert(index_counter, i0) + jumpargs.insert(index_counter, i1) + # + looptoken = LoopToken() + faildescr = BasicFailDescr() + operations = [ + ResOperation(rop.INT_SUB, [i0, ConstInt(1)], i1), + ResOperation(rop.INT_GE, [i1, ConstInt(0)], i2), + ResOperation(rop.GUARD_TRUE, [i2], None), + ResOperation(rop.JUMP, jumpargs, None, descr=looptoken), + ] + operations[2].fail_args = inputargs[:] + operations[2].descr = faildescr + # + self.cpu.compile_loop(inputargs, operations, looptoken) + # + values = [] + S = lltype.GcStruct('S') + for box in inputargs: + if isinstance(box, BoxInt): + values.append(r.randrange(-10000, 10000)) + elif isinstance(box, BoxPtr): + p = lltype.malloc(S) + values.append(lltype.cast_opaque_ptr(llmemory.GCREF, p)) + elif isinstance(box, BoxFloat): + values.append(r.random()) + else: + assert 0 + values[index_counter] = 11 + # + for i, (box, val) in enumerate(zip(inputargs, values)): + if isinstance(box, BoxInt): + self.cpu.set_future_value_int(i, val) + elif isinstance(box, BoxPtr): + self.cpu.set_future_value_ref(i, val) + elif isinstance(box, BoxFloat): + self.cpu.set_future_value_float(i, val) + else: + assert 0 + # + fail = self.cpu.execute_token(looptoken) + assert fail is faildescr + # + dstvalues = values[:] + for _ in range(11): + expected = dstvalues[:] + for tgtbox, srcbox in remixing: + v = dstvalues[inputargs.index(srcbox)] + expected[inputargs.index(tgtbox)] = v + dstvalues = expected + # + assert dstvalues[index_counter] == 11 + dstvalues[index_counter] = 0 + for i, (box, val) in enumerate(zip(inputargs, dstvalues)): + if isinstance(box, BoxInt): + got = self.cpu.get_latest_value_int(i) + elif isinstance(box, BoxPtr): + got = self.cpu.get_latest_value_ref(i) + elif isinstance(box, BoxFloat): + got = self.cpu.get_latest_value_float(i) + else: + assert 0 + assert type(got) == type(val) + assert got == val + + def test_compile_bridge_float(self): + if not self.cpu.supports_floats: + py.test.skip("requires floats") + fboxes = [BoxFloat() for i in range(12)] + i2 = BoxInt() + faildescr1 = BasicFailDescr() + faildescr2 = BasicFailDescr() + operations = [ + ResOperation(rop.FLOAT_LE, [fboxes[0], ConstFloat(9.2)], i2), + ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1), + ResOperation(rop.FINISH, fboxes, None, descr=faildescr2), + ] + operations[-2].fail_args = fboxes + looptoken = LoopToken() + self.cpu.compile_loop(fboxes, operations, looptoken) + + fboxes2 = [BoxFloat() for i in range(12)] + f3 = BoxFloat() + bridge = [ + ResOperation(rop.FLOAT_SUB, [fboxes2[0], ConstFloat(1.0)], f3), + ResOperation(rop.JUMP, [f3] + fboxes2[1:], None, descr=looptoken), + ] + + self.cpu.compile_bridge(faildescr1, fboxes2, bridge) + + for i in range(len(fboxes)): + self.cpu.set_future_value_float(i, 13.5 + 6.73 * i) + fail = self.cpu.execute_token(looptoken) + assert fail is faildescr2 + res = self.cpu.get_latest_value_float(0) + assert res == 8.5 + for i in range(1, len(fboxes)): + assert self.cpu.get_latest_value_float(i) == 13.5 + 6.73 * i + + def test_unused_result_float(self): + # same as test_unused_result_int, for float operations + from pypy.jit.metainterp.test.test_executor import get_float_tests + float_tests = list(get_float_tests(self.cpu)) + inputargs = [] + operations = [] + for opnum, boxargs, rettype, retvalue in float_tests: + inputargs += boxargs + if rettype == 'int': + boxres = BoxInt() + elif rettype == 'float': + boxres = BoxFloat() + else: + assert 0 + operations.append(ResOperation(opnum, boxargs, boxres)) + faildescr = BasicFailDescr() + operations.append(ResOperation(rop.FINISH, [], None, + descr=faildescr)) + looptoken = LoopToken() + # + self.cpu.compile_loop(inputargs, operations, looptoken) + # + for i, box in enumerate(inputargs): + if isinstance(box, BoxInt): + self.cpu.set_future_value_int(i, box.getint()) + elif isinstance(box, BoxFloat): + self.cpu.set_future_value_float(i, box.getfloat()) + else: + assert 0 + # + fail = self.cpu.execute_token(looptoken) + assert fail is faildescr + class LLtypeBackendTest(BaseBackendTest): @@ -656,7 +890,8 @@ ('chr1', lltype.Char), ('chr2', lltype.Char), ('short', rffi.SHORT), - ('next', lltype.Ptr(S)))) + ('next', lltype.Ptr(S)), + ('float', lltype.Float))) T = lltype.GcStruct('T', ('parent', S), ('next', lltype.Ptr(S))) U = lltype.GcStruct('U', ('parent', T), Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/assembler.py Mon Oct 5 10:34:06 2009 @@ -1,13 +1,14 @@ import sys, os import ctypes from pypy.jit.backend.llsupport import symbolic -from pypy.jit.metainterp.history import Const, Box, BoxPtr, REF +from pypy.jit.metainterp.history import Const, Box, BoxPtr, REF, FLOAT from pypy.jit.metainterp.history import AbstractFailDescr from pypy.rpython.lltypesystem import lltype, rffi, ll2ctypes, rstr, llmemory from pypy.rpython.lltypesystem.rclass import OBJECT from pypy.rpython.lltypesystem.lloperation import llop from pypy.tool.uid import fixid -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, lower_byte +from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, lower_byte,\ + X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.jit.backend.x86 import codebuf from pypy.jit.backend.x86.ri386 import * @@ -62,6 +63,12 @@ if name.upper() == name: setattr(MachineCodeBlockWrapper, name, _new_method(name)) +class ExecutableToken386(object): + _x86_loop_code = 0 + _x86_bootstrap_code = 0 + _x86_stack_depth = 0 + _x86_arglocs = (None, None) + class Assembler386(object): mc = None mc2 = None @@ -79,6 +86,8 @@ MAX_FAIL_BOXES, zero=True) self.fail_boxes_ptr = lltype.malloc(lltype.GcArray(llmemory.GCREF), MAX_FAIL_BOXES, zero=True) + self.fail_boxes_float = lltype.malloc(lltype.GcArray(lltype.Float), + MAX_FAIL_BOXES, zero=True) def leave_jitted_hook(self): fail_boxes_ptr = self.fail_boxes_ptr @@ -89,10 +98,13 @@ if self.mc is None: rffi.cast(lltype.Signed, self.fail_boxes_int) # workaround rffi.cast(lltype.Signed, self.fail_boxes_ptr) # workaround + rffi.cast(lltype.Signed, self.fail_boxes_float) # workaround self.fail_box_int_addr = rffi.cast(lltype.Signed, lltype.direct_arrayitems(self.fail_boxes_int)) self.fail_box_ptr_addr = rffi.cast(lltype.Signed, lltype.direct_arrayitems(self.fail_boxes_ptr)) + self.fail_box_float_addr = rffi.cast(lltype.Signed, + lltype.direct_arrayitems(self.fail_boxes_float)) # the address of the function called by 'new' gc_ll_descr = self.cpu.gc_ll_descr @@ -184,6 +196,7 @@ mc.done() def _assemble_bootstrap_code(self, inputargs, arglocs): + nonfloatlocs, floatlocs = arglocs self.mc.PUSH(ebp) self.mc.MOV(ebp, esp) self.mc.PUSH(ebx) @@ -192,31 +205,37 @@ # NB. exactly 4 pushes above; if this changes, fix stack_pos(). # You must also keep _get_callshape() in sync. adr_stackadjust = self._patchable_stackadjust() - for i in range(len(arglocs)): - loc = arglocs[i] - if not isinstance(loc, REG): - if inputargs[i].type == REF: - # This uses XCHG to put zeroes in fail_boxes_ptr after - # reading them - self.mc.XOR(ecx, ecx) - self.mc.XCHG(ecx, addr_add(imm(self.fail_box_ptr_addr), - imm(i*WORD))) - else: - self.mc.MOV(ecx, addr_add(imm(self.fail_box_int_addr), - imm(i*WORD))) - self.mc.MOV(loc, ecx) - for i in range(len(arglocs)): - loc = arglocs[i] + tmp = X86RegisterManager.all_regs[0] + xmmtmp = X86XMMRegisterManager.all_regs[0] + for i in range(len(nonfloatlocs)): + loc = nonfloatlocs[i] + if loc is None: + continue if isinstance(loc, REG): - if inputargs[i].type == REF: - # This uses XCHG to put zeroes in fail_boxes_ptr after - # reading them - self.mc.XOR(loc, loc) - self.mc.XCHG(loc, addr_add(imm(self.fail_box_ptr_addr), - imm(i*WORD))) - else: - self.mc.MOV(loc, addr_add(imm(self.fail_box_int_addr), + target = loc + else: + target = tmp + if inputargs[i].type == REF: + # This uses XCHG to put zeroes in fail_boxes_ptr after + # reading them + self.mc.XOR(target, target) + self.mc.XCHG(target, addr_add(imm(self.fail_box_ptr_addr), imm(i*WORD))) + else: + self.mc.MOV(target, addr_add(imm(self.fail_box_int_addr), + imm(i*WORD))) + self.mc.MOV(loc, target) + for i in range(len(floatlocs)): + loc = floatlocs[i] + if loc is None: + continue + if isinstance(loc, REG): + self.mc.MOVSD(loc, addr64_add(imm(self.fail_box_float_addr), + imm(i*WORD*2))) + else: + self.mc.MOVSD(xmmtmp, addr64_add(imm(self.fail_box_float_addr), + imm(i*WORD*2))) + self.mc.MOVSD(loc, xmmtmp) return adr_stackadjust def dump(self, text): @@ -231,14 +250,38 @@ # ------------------------------------------------------------ - def regalloc_mov(self, from_loc, to_loc): - self.mc.MOV(to_loc, from_loc) + def mov(self, from_loc, to_loc): + if isinstance(from_loc, XMMREG) or isinstance(to_loc, XMMREG): + self.mc.MOVSD(to_loc, from_loc) + else: + self.mc.MOV(to_loc, from_loc) + + regalloc_mov = mov # legacy interface + + def regalloc_fstp(self, loc): + self.mc.FSTP(loc) def regalloc_push(self, loc): - self.mc.PUSH(loc) + if isinstance(loc, XMMREG): + self.mc.SUB(esp, imm(2*WORD)) + self.mc.MOVSD(mem64(esp, 0), loc) + elif isinstance(loc, MODRM64): + # XXX evil trick + self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position))) + self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position + 1))) + else: + self.mc.PUSH(loc) def regalloc_pop(self, loc): - self.mc.POP(loc) + if isinstance(loc, XMMREG): + self.mc.MOVSD(loc, mem64(esp, 0)) + self.mc.ADD(esp, imm(2*WORD)) + elif isinstance(loc, MODRM64): + # XXX evil trick + self.mc.POP(mem(ebp, get_ebp_ofs(loc.position + 1))) + self.mc.POP(mem(ebp, get_ebp_ofs(loc.position))) + else: + self.mc.POP(loc) def regalloc_perform(self, op, arglocs, resloc): genop_list[op.opnum](self, op, arglocs, resloc) @@ -261,7 +304,7 @@ else: dispatch_opnum = op.opnum adr_jump_offset = genop_guard_list[dispatch_opnum](self, op, - guard_opnum, + guard_op, failaddr, arglocs, resloc) faildescr._x86_adr_jump_offset = adr_jump_offset @@ -296,8 +339,16 @@ getattr(self.mc, 'SET' + cond)(lower_byte(result_loc)) return genop_cmp + def _cmpop_float(cond): + def genop_cmp(self, op, arglocs, result_loc): + self.mc.UCOMISD(arglocs[0], arglocs[1]) + self.mc.MOV(result_loc, imm8(0)) + getattr(self.mc, 'SET' + cond)(lower_byte(result_loc)) + return genop_cmp + def _cmpop_guard(cond, rev_cond, false_cond, false_rev_cond): - def genop_cmp_guard(self, op, guard_opnum, addr, arglocs, result_loc): + def genop_cmp_guard(self, op, guard_op, addr, arglocs, result_loc): + guard_opnum = guard_op.opnum if isinstance(op.args[0], Const): self.mc.CMP(arglocs[1], arglocs[0]) if guard_opnum == rop.GUARD_FALSE: @@ -341,6 +392,10 @@ genop_int_and = _binaryop("AND", True) genop_int_or = _binaryop("OR", True) genop_int_xor = _binaryop("XOR", True) + genop_float_add = _binaryop("ADDSD", True) + genop_float_sub = _binaryop('SUBSD') + genop_float_mul = _binaryop('MULSD', True) + genop_float_truediv = _binaryop('DIVSD') genop_int_mul_ovf = genop_int_mul genop_int_sub_ovf = genop_int_sub @@ -355,6 +410,13 @@ genop_int_gt = _cmpop("G", "L") genop_int_ge = _cmpop("GE", "LE") + genop_float_lt = _cmpop_float('B') + genop_float_le = _cmpop_float('BE') + genop_float_eq = _cmpop_float('E') + genop_float_ne = _cmpop_float('NE') + genop_float_gt = _cmpop_float('A') + genop_float_ge = _cmpop_float('AE') + genop_uint_gt = _cmpop("A", "B") genop_uint_lt = _cmpop("B", "A") genop_uint_le = _cmpop("BE", "AE") @@ -376,6 +438,25 @@ # a difference at some point xxx_genop_char_eq = genop_int_eq + def genop_float_neg(self, op, arglocs, resloc): + self.mc.XORPD(arglocs[0], arglocs[1]) + + def genop_float_abs(self, op, arglocs, resloc): + self.mc.ANDPD(arglocs[0], arglocs[1]) + + def genop_float_is_true(self, op, arglocs, resloc): + loc0, loc1 = arglocs + self.mc.XORPD(loc0, loc0) + self.mc.UCOMISD(loc1, loc0) + self.mc.SETNE(lower_byte(resloc)) + self.mc.MOVZX(resloc, lower_byte(resloc)) + + def genop_cast_float_to_int(self, op, arglocs, resloc): + self.mc.CVTTSD2SI(resloc, arglocs[0]) + + def genop_cast_int_to_float(self, op, arglocs, resloc): + self.mc.CVTSI2SD(resloc, arglocs[0]) + def genop_bool_not(self, op, arglocs, resloc): self.mc.XOR(arglocs[0], imm8(1)) @@ -397,7 +478,8 @@ loc2 = cl self.mc.SHR(loc, loc2) - def genop_guard_oononnull(self, op, guard_opnum, addr, arglocs, resloc): + def genop_guard_oononnull(self, op, guard_op, addr, arglocs, resloc): + guard_opnum = guard_op.opnum loc = arglocs[0] self.mc.TEST(loc, loc) if guard_opnum == rop.GUARD_TRUE: @@ -405,7 +487,8 @@ else: return self.implement_guard(addr, self.mc.JNZ) - def genop_guard_ooisnull(self, op, guard_opnum, addr, arglocs, resloc): + def genop_guard_ooisnull(self, op, guard_op, addr, arglocs, resloc): + guard_opnum = guard_op.opnum loc = arglocs[0] self.mc.TEST(loc, loc) if guard_opnum == rop.GUARD_TRUE: @@ -428,7 +511,7 @@ self.mc.SETE(lower_byte(resloc)) def genop_same_as(self, op, arglocs, resloc): - self.mc.MOV(resloc, arglocs[0]) + self.mov(arglocs[0], resloc) genop_cast_ptr_to_int = genop_same_as def genop_int_mod(self, op, arglocs, resloc): @@ -474,6 +557,8 @@ self.mc.MOVZX(resloc, addr_add(base_loc, ofs_loc)) elif size == WORD: self.mc.MOV(resloc, addr_add(base_loc, ofs_loc)) + elif size == 8: + self.mc.MOVSD(resloc, addr64_add(base_loc, ofs_loc)) else: raise NotImplementedError("getfield size = %d" % size) @@ -483,15 +568,19 @@ base_loc, ofs_loc, scale, ofs = arglocs assert isinstance(ofs, IMM32) assert isinstance(scale, IMM32) - if scale.value == 0: - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc, ofs.value, - scale.value)) - elif scale.value == 2: - self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, ofs.value, - scale.value)) - else: - print "[asmgen]setarrayitem unsupported size: %d" % scale.value - raise NotImplementedError() + if op.result.type == FLOAT: + self.mc.MOVSD(resloc, addr64_add(base_loc, ofs_loc, ofs.value, + scale.value)) + else: + if scale.value == 0: + self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc, ofs.value, + scale.value)) + elif scale.value == 2: + self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, ofs.value, + scale.value)) + else: + print "[asmgen]setarrayitem unsupported size: %d" % scale.value + raise NotImplementedError() genop_getfield_raw = genop_getfield_gc genop_getarrayitem_gc_pure = genop_getarrayitem_gc @@ -500,7 +589,9 @@ base_loc, ofs_loc, size_loc, value_loc = arglocs assert isinstance(size_loc, IMM32) size = size_loc.value - if size == WORD: + if size == WORD * 2: + self.mc.MOVSD(addr64_add(base_loc, ofs_loc), value_loc) + elif size == WORD: self.mc.MOV(addr_add(base_loc, ofs_loc), value_loc) elif size == 2: self.mc.MOV16(addr_add(base_loc, ofs_loc), value_loc) @@ -514,14 +605,18 @@ base_loc, ofs_loc, value_loc, scale_loc, baseofs = arglocs assert isinstance(baseofs, IMM32) assert isinstance(scale_loc, IMM32) - if scale_loc.value == 2: - self.mc.MOV(addr_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc) - elif scale_loc.value == 0: - self.mc.MOV(addr8_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), lower_byte(value_loc)) - else: - raise NotImplementedError("scale = %d" % scale_loc.value) + if op.args[2].type == FLOAT: + self.mc.MOVSD(addr64_add(base_loc, ofs_loc, baseofs.value, + scale_loc.value), value_loc) + else: + if scale_loc.value == 2: + self.mc.MOV(addr_add(base_loc, ofs_loc, baseofs.value, + scale_loc.value), value_loc) + elif scale_loc.value == 0: + self.mc.MOV(addr8_add(base_loc, ofs_loc, baseofs.value, + scale_loc.value), lower_byte(value_loc)) + else: + raise NotImplementedError("scale = %d" % scale_loc.value) def genop_discard_strsetitem(self, op, arglocs): base_loc, ofs_loc, val_loc = arglocs @@ -580,17 +675,17 @@ else: assert 0, itemsize - def genop_guard_guard_true(self, ign_1, guard_opnum, addr, locs, ign_2): + def genop_guard_guard_true(self, ign_1, guard_op, addr, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) return self.implement_guard(addr, self.mc.JZ) - def genop_guard_guard_no_exception(self, ign_1, guard_opnum, addr, + def genop_guard_guard_no_exception(self, ign_1, guard_op, addr, locs, ign_2): self.mc.CMP(heap(self.cpu.pos_exception()), imm(0)) return self.implement_guard(addr, self.mc.JNZ) - def genop_guard_guard_exception(self, ign_1, guard_opnum, addr, + def genop_guard_guard_exception(self, ign_1, guard_op, addr, locs, resloc): loc = locs[0] loc1 = locs[1] @@ -603,57 +698,84 @@ self.mc.MOV(heap(self.cpu.pos_exc_value()), imm(0)) return addr - def genop_guard_guard_no_overflow(self, ign_1, guard_opnum, addr, + def genop_guard_guard_no_overflow(self, ign_1, guard_op, addr, locs, resloc): return self.implement_guard(addr, self.mc.JO) - def genop_guard_guard_overflow(self, ign_1, guard_opnum, addr, + def genop_guard_guard_overflow(self, ign_1, guard_op, addr, locs, resloc): return self.implement_guard(addr, self.mc.JNO) - def genop_guard_guard_false(self, ign_1, guard_opnum, addr, locs, ign_2): + def genop_guard_guard_false(self, ign_1, guard_op, addr, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) return self.implement_guard(addr, self.mc.JNZ) - def genop_guard_guard_value(self, ign_1, guard_opnum, addr, locs, ign_2): - self.mc.CMP(locs[0], locs[1]) + def genop_guard_guard_value(self, ign_1, guard_op, addr, locs, ign_2): + if guard_op.args[0].type == FLOAT: + assert guard_op.args[1].type == FLOAT + self.mc.UCOMISD(locs[0], locs[1]) + else: + self.mc.CMP(locs[0], locs[1]) return self.implement_guard(addr, self.mc.JNE) - def genop_guard_guard_class(self, ign_1, guard_opnum, addr, locs, ign_2): + def genop_guard_guard_class(self, ign_1, guard_op, addr, locs, ign_2): offset = self.cpu.vtable_offset self.mc.CMP(mem(locs[0], offset), locs[1]) return self.implement_guard(addr, self.mc.JNE) + def _no_const_locs(self, args, locs): + """ returns those locs which correspond to non-const args + """ + newlocs = [] + for i in range(len(args)): + arg = args[i] + if isinstance(arg, Box): + newlocs.append(locs[i]) + return newlocs + def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): addr = self.mc2.tell() exc = (guard_opnum == rop.GUARD_EXCEPTION or guard_opnum == rop.GUARD_NO_EXCEPTION) - faildescr._x86_faillocs = fail_locs + faildescr._x86_faillocs = self._no_const_locs(failargs, fail_locs) self.generate_failure(self.mc2, faildescr, failargs, fail_locs, exc) return addr def generate_failure(self, mc, faildescr, failargs, locs, exc): assert len(failargs) < MAX_FAIL_BOXES pos = mc.tell() - for i in range(len(locs)): + for i in range(len(failargs)): + arg = failargs[i] loc = locs[i] - if isinstance(loc, REG): - if failargs[i].type == REF: - base = self.fail_box_ptr_addr - else: - base = self.fail_box_int_addr - mc.MOV(addr_add(imm(base), imm(i*WORD)), loc) - for i in range(len(locs)): + if arg.type == FLOAT: + if isinstance(loc, REG): + mc.MOVSD(addr64_add(imm(self.fail_box_float_addr), + imm(i*WORD*2)), loc) + else: + if isinstance(loc, REG): + if arg.type == REF: + base = self.fail_box_ptr_addr + else: + base = self.fail_box_int_addr + mc.MOV(addr_add(imm(base), imm(i*WORD)), loc) + for i in range(len(failargs)): + arg = failargs[i] loc = locs[i] - if not isinstance(loc, REG): - if failargs[i].type == REF: - base = self.fail_box_ptr_addr - else: - base = self.fail_box_int_addr - mc.MOV(eax, loc) - mc.MOV(addr_add(imm(base), imm(i*WORD)), eax) + if arg.type == FLOAT: + if not isinstance(loc, REG): + mc.MOVSD(xmm0, loc) + mc.MOVSD(addr64_add(imm(self.fail_box_float_addr), + imm(i*WORD*2)), xmm0) + else: + if not isinstance(loc, REG): + if arg.type == REF: + base = self.fail_box_ptr_addr + else: + base = self.fail_box_int_addr + mc.MOV(eax, loc) + mc.MOV(addr_add(imm(base), imm(i*WORD)), eax) if self.debug_markers: mc.MOV(eax, imm(pos)) mc.MOV(addr_add(imm(self.fail_box_int_addr), @@ -690,16 +812,42 @@ assert isinstance(sizeloc, IMM32) size = sizeloc.value nargs = len(op.args)-1 - extra_on_stack = self.align_stack_for_call(nargs) - for i in range(nargs+1, 1, -1): - self.mc.PUSH(arglocs[i]) + extra_on_stack = 0 + for arg in range(2, nargs + 2): + extra_on_stack += round_up_to_4(arglocs[arg].width) + extra_on_stack = self.align_stack_for_call(extra_on_stack) + self.mc.SUB(esp, imm(extra_on_stack)) if isinstance(op.args[0], Const): x = rel32(op.args[0].getint()) else: x = arglocs[1] + if x is eax: + tmp = ecx + else: + tmp = eax + p = 0 + for i in range(2, nargs + 2): + loc = arglocs[i] + if isinstance(loc, REG): + if isinstance(loc, XMMREG): + self.mc.MOVSD(mem64(esp, p), loc) + else: + self.mc.MOV(mem(esp, p), loc) + p += round_up_to_4(loc.width) + p = 0 + for i in range(2, nargs + 2): + loc = arglocs[i] + if not isinstance(loc, REG): + if isinstance(loc, MODRM64): + self.mc.MOVSD(xmm0, loc) + self.mc.MOVSD(mem64(esp, p), xmm0) + else: + self.mc.MOV(tmp, loc) + self.mc.MOV(mem(esp, p), tmp) + p += round_up_to_4(loc.width) self.mc.CALL(x) self.mark_gc_roots() - self.mc.ADD(esp, imm(WORD * extra_on_stack)) + self.mc.ADD(esp, imm(extra_on_stack)) if size == 1: self.mc.AND(eax, imm(0xff)) elif size == 2: @@ -737,16 +885,19 @@ mc.overwrite(jz_location-1, chr(offset)) def not_implemented_op_discard(self, op, arglocs): - print "not implemented operation: %s" % op.getopname() - raise NotImplementedError + msg = "not implemented operation: %s" % op.getopname() + print msg + raise NotImplementedError(msg) def not_implemented_op(self, op, arglocs, resloc): - print "not implemented operation with res: %s" % op.getopname() - raise NotImplementedError + msg = "not implemented operation with res: %s" % op.getopname() + print msg + raise NotImplementedError(msg) def not_implemented_op_guard(self, op, regalloc, arglocs, resloc, descr): - print "not implemented operation (guard): %s" % op.getopname() - raise NotImplementedError + msg = "not implemented operation (guard): %s" % op.getopname() + print msg + raise NotImplementedError(msg) def mark_gc_roots(self): gcrootmap = self.cpu.gc_ll_descr.gcrootmap @@ -779,34 +930,32 @@ num = getattr(rop, opname.upper()) genop_list[num] = value -def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): - if isinstance(reg_or_imm1, IMM32): - if isinstance(reg_or_imm2, IMM32): - return heap(reg_or_imm1.value + offset + - (reg_or_imm2.value << scale)) - else: - return memSIB(None, reg_or_imm2, scale, reg_or_imm1.value + offset) - else: - if isinstance(reg_or_imm2, IMM32): - return mem(reg_or_imm1, offset + (reg_or_imm2.value << scale)) +def new_addr_add(heap, mem, memsib): + def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): + if isinstance(reg_or_imm1, IMM32): + if isinstance(reg_or_imm2, IMM32): + return heap(reg_or_imm1.value + offset + + (reg_or_imm2.value << scale)) + else: + return memsib(None, reg_or_imm2, scale, reg_or_imm1.value + offset) else: - return memSIB(reg_or_imm1, reg_or_imm2, scale, offset) + if isinstance(reg_or_imm2, IMM32): + return mem(reg_or_imm1, offset + (reg_or_imm2.value << scale)) + else: + return memsib(reg_or_imm1, reg_or_imm2, scale, offset) + return addr_add -def addr8_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): - if isinstance(reg_or_imm1, IMM32): - if isinstance(reg_or_imm2, IMM32): - return heap8(reg_or_imm1.value + (offset << scale) + - reg_or_imm2.value) - else: - return memSIB8(None, reg_or_imm2, scale, reg_or_imm1.value + offset) - else: - if isinstance(reg_or_imm2, IMM32): - return mem8(reg_or_imm1, (offset << scale) + reg_or_imm2.value) - else: - return memSIB8(reg_or_imm1, reg_or_imm2, scale, offset) +addr8_add = new_addr_add(heap8, mem8, memSIB8) +addr_add = new_addr_add(heap, mem, memSIB) +addr64_add = new_addr_add(heap64, mem64, memSIB64) def addr_add_const(reg_or_imm1, offset): if isinstance(reg_or_imm1, IMM32): return heap(reg_or_imm1.value + offset) else: return mem(reg_or_imm1, offset) + +def round_up_to_4(size): + if size < 4: + return 4 + return size Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/jump.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/jump.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/jump.py Mon Oct 5 10:34:06 2009 @@ -80,10 +80,7 @@ assert pending_dests == 0 def _move(assembler, src, dst, tmpreg): - if isinstance(dst, MODRM): - if isinstance(src, MODRM): - assembler.regalloc_mov(src, tmpreg) - src = tmpreg - assembler.regalloc_mov(src, dst) - else: - assembler.regalloc_mov(src, dst) + if isinstance(dst, MODRM) and isinstance(src, MODRM): + assembler.regalloc_mov(src, tmpreg) + src = tmpreg + assembler.regalloc_mov(src, dst) Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/regalloc.py Mon Oct 5 10:34:06 2009 @@ -4,7 +4,7 @@ from pypy.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr, ResOperation, ConstAddr, BoxPtr, - LoopToken) + LoopToken, INT, REF, FLOAT) from pypy.jit.backend.x86.ri386 import * from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr from pypy.rlib.objectmodel import we_are_translated @@ -20,8 +20,15 @@ WORD = 4 +width_of_type = { + INT : 1, + REF : 1, + FLOAT : 2, + } + class X86RegisterManager(RegisterManager): + box_types = [INT, REF] all_regs = [eax, ecx, edx, ebx, esi, edi] no_lower_byte_regs = [esi, edi] save_around_call_regs = [eax, edx, ecx] @@ -43,11 +50,70 @@ print "convert_to_imm: got a %s" % c raise AssertionError +BASE_CONSTANT_SIZE = 1000 + +# cheat cheat cheat.... +# why not -0.0? People tell me it's platform-dependent +# nan is not portable +import struct +NEG_ZERO, = struct.unpack('d', struct.pack('ll', 0, -2147483648)) +NAN, = struct.unpack('d', struct.pack('ll', -1, 2147483647)) +# XXX These are actually masks for float_neg and float_abs. +# They should not be converted to 'double' and given +# names that reflect their float value. + +class X86XMMRegisterManager(RegisterManager): + + box_types = [FLOAT] + all_regs = [xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7] + # we never need lower byte I hope + save_around_call_regs = all_regs + reg_width = 2 + + def new_const_array(self): + return lltype.malloc(rffi.CArray(lltype.Float), BASE_CONSTANT_SIZE, + flavor='raw') + + def __init__(self, longevity, stack_manager=None, assembler=None): + RegisterManager.__init__(self, longevity, stack_manager=stack_manager, + assembler=assembler) + self.constant_arrays = [self.new_const_array()] + self.constant_arrays[-1][0] = NEG_ZERO + self.constant_arrays[-1][1] = NAN + self.constant_array_counter = 2 + + def convert_to_imm(self, c): + if self.constant_array_counter >= BASE_CONSTANT_SIZE: + self.constant_arrays.append(self.new_const_array()) + self.constant_array_counter = 0 + res = self.constant_array_counter + self.constant_array_counter += 1 + arr = self.constant_arrays[-1] + arr[res] = c.getfloat() + return self.get_addr_of_const_float(-1, res) + + def get_addr_of_const_float(self, num_arr, num_pos): + arr = self.constant_arrays[num_arr] + return heap64(rffi.cast(lltype.Signed, arr) + num_pos * WORD * 2) + + def after_call(self, v): + # the result is stored in st0, but we don't have this around, + # so we move it to some stack location + if v is not None: + loc = self.stack_manager.loc(v, 2) + self.assembler.regalloc_fstp(loc) + class X86StackManager(StackManager): @staticmethod - def stack_pos(i): - res = mem(ebp, get_ebp_ofs(i)) + def stack_pos(i, size): + if size == 1: + res = mem(ebp, get_ebp_ofs(i)) + elif size == 2: + res = mem64(ebp, get_ebp_ofs(i + 1)) + else: + print "Unimplemented size %d" % i + raise NotImplementedError("unimplemented size %d" % i) res.position = i return res @@ -68,9 +134,12 @@ cpu.gc_ll_descr.rewrite_assembler(cpu, operations) # compute longevity of variables longevity = self._compute_vars_longevity(inputargs, operations) + self.longevity = longevity self.rm = X86RegisterManager(longevity, stack_manager = self.sm, assembler = self.assembler) + self.xrm = X86XMMRegisterManager(longevity, stack_manager = self.sm, + assembler = self.assembler) def prepare_loop(self, inputargs, operations, looptoken): self._prepare(inputargs, operations) @@ -88,28 +157,72 @@ def _process_inputargs(self, inputargs): # XXX we can sort out here by longevity if we need something # more optimal - locs = [None] * len(inputargs) + floatlocs = [None] * len(inputargs) + nonfloatlocs = [None] * len(inputargs) # Don't use all_regs[0] for passing arguments around a loop. # Must be kept in sync with consider_jump(). # XXX this should probably go to llsupport/regalloc.py + xmmtmp = self.xrm.free_regs.pop(0) tmpreg = self.rm.free_regs.pop(0) assert tmpreg == X86RegisterManager.all_regs[0] + assert xmmtmp == X86XMMRegisterManager.all_regs[0] for i in range(len(inputargs)): arg = inputargs[i] assert not isinstance(arg, Const) reg = None - if arg not in self.loop_consts and self.rm.longevity[arg][1] > -1: - reg = self.rm.try_allocate_reg(arg) + if arg not in self.loop_consts and self.longevity[arg][1] > -1: + if arg.type == FLOAT: + # xxx is it really a good idea? at the first CALL they + # will all be flushed anyway + reg = self.xrm.try_allocate_reg(arg) + else: + reg = self.rm.try_allocate_reg(arg) if reg: - locs[i] = reg + loc = reg else: - loc = self.sm.loc(arg) - locs[i] = loc + loc = self.sm.loc(arg, width_of_type[arg.type]) + if arg.type == FLOAT: + floatlocs[i] = loc + else: + nonfloatlocs[i] = loc # otherwise we have it saved on stack, so no worry self.rm.free_regs.insert(0, tmpreg) - assert tmpreg not in locs - self.rm.possibly_free_vars(inputargs) - return locs + self.xrm.free_regs.insert(0, xmmtmp) + assert tmpreg not in nonfloatlocs + assert xmmtmp not in floatlocs + self.possibly_free_vars(inputargs) + return nonfloatlocs, floatlocs + + def possibly_free_var(self, var): + if var.type == FLOAT: + self.xrm.possibly_free_var(var) + else: + self.rm.possibly_free_var(var) + + def possibly_free_vars(self, vars): + for var in vars: + self.possibly_free_var(var) + + def make_sure_var_in_reg(self, var, forbidden_vars=[], + selected_reg=None, imm_fine=True, + need_lower_byte=False): + if var.type == FLOAT: + return self.xrm.make_sure_var_in_reg(var, forbidden_vars, + selected_reg, imm_fine, + need_lower_byte) + else: + return self.rm.make_sure_var_in_reg(var, forbidden_vars, + selected_reg, imm_fine, + need_lower_byte) + + def force_allocate_reg(self, var, forbidden_vars=[], selected_reg=None, + need_lower_byte=False): + if var.type == FLOAT: + return self.xrm.force_allocate_reg(var, forbidden_vars, + selected_reg, need_lower_byte) + else: + return self.rm.force_allocate_reg(var, forbidden_vars, + selected_reg, need_lower_byte) def _compute_loop_consts(self, inputargs, jump, looptoken): if jump.opnum != rop.JUMP or jump.descr is not looptoken: @@ -123,26 +236,33 @@ def _update_bindings(self, locs, args): # XXX this should probably go to llsupport/regalloc.py - newlocs = [] - for loc in locs: - if not isinstance(loc, IMM8) and not isinstance(loc, IMM32): - newlocs.append(loc) - locs = newlocs - assert len(locs) == len(args) used = {} - for i in range(len(locs)): - v = args[i] + for i in range(len(args)): + arg = args[i] loc = locs[i] - if isinstance(loc, REG) and self.rm.longevity[v][1] > -1: - self.rm.reg_bindings[v] = loc - used[loc] = None + if arg.type == FLOAT: + if isinstance(loc, REG): + self.xrm.reg_bindings[arg] = loc + used[loc] = None + else: + self.sm.stack_bindings[arg] = loc else: - self.sm.stack_bindings[v] = loc + if isinstance(loc, REG): + self.rm.reg_bindings[arg] = loc + used[loc] = None + else: + self.sm.stack_bindings[arg] = loc self.rm.free_regs = [] for reg in X86RegisterManager.all_regs: if reg not in used: self.rm.free_regs.append(reg) + self.xrm.free_regs = [] + for reg in X86XMMRegisterManager.all_regs: + if reg not in used: + self.xrm.free_regs.append(reg) + self.possibly_free_vars(args) self.rm._check_invariants() + self.xrm._check_invariants() def Perform(self, op, arglocs, result_loc): if not we_are_translated(): @@ -155,11 +275,12 @@ def perform_with_guard(self, op, guard_op, arglocs, result_loc): faillocs = self.locs_for_fail(guard_op) self.rm.position += 1 + self.xrm.position += 1 self.assembler.regalloc_perform_with_guard(op, guard_op, faillocs, arglocs, result_loc, self.sm.stack_depth) self.rm.possibly_free_var(op.result) - self.rm.possibly_free_vars(guard_op.fail_args) + self.possibly_free_vars(guard_op.fail_args) def perform_guard(self, guard_op, arglocs, result_loc): faillocs = self.locs_for_fail(guard_op) @@ -172,7 +293,7 @@ self.assembler.regalloc_perform_guard(guard_op, faillocs, arglocs, result_loc, self.sm.stack_depth) - self.rm.possibly_free_vars(guard_op.fail_args) + self.possibly_free_vars(guard_op.fail_args) def PerformDiscard(self, op, arglocs): if not we_are_translated(): @@ -188,7 +309,7 @@ return False if operations[i + 1].args[0] is not op.result: return False - if (self.rm.longevity[op.result][1] > i + 1 or + if (self.longevity[op.result][1] > i + 1 or op.result in operations[i + 1].fail_args): return False return True @@ -199,19 +320,23 @@ while i < len(operations): op = operations[i] self.rm.position = i - if op.has_no_side_effect() and op.result not in self.rm.longevity: + self.xrm.position = i + if op.has_no_side_effect() and op.result not in self.longevity: i += 1 - self.rm.possibly_free_vars(op.args) + self.possibly_free_vars(op.args) continue if self.can_optimize_cmp_op(op, i, operations): oplist[op.opnum](self, op, operations[i + 1]) i += 1 else: oplist[op.opnum](self, op, None) - self.rm.possibly_free_var(op.result) + if op.result is not None: + self.possibly_free_var(op.result) self.rm._check_invariants() + self.xrm._check_invariants() i += 1 assert not self.rm.reg_bindings + assert not self.xrm.reg_bindings def _compute_vars_longevity(self, inputargs, operations): # compute a dictionary that maps variables to index in @@ -245,6 +370,8 @@ return longevity def loc(self, v): + if v.type == FLOAT: + return self.xrm.loc(v) return self.rm.loc(v) def _consider_guard(self, op, ignored): @@ -256,10 +383,10 @@ consider_guard_false = _consider_guard def consider_finish(self, op, ignored): - locs = [self.loc(arg) for arg in op.args] + locs = [self.loc(v) for v in op.args] self.assembler.generate_failure(self.assembler.mc, op.descr, op.args, locs, self.exc) - self.rm.possibly_free_vars(op.args) + self.possibly_free_vars(op.args) def consider_guard_no_exception(self, op, ignored): self.perform_guard(op, [], None) @@ -268,7 +395,7 @@ loc = self.rm.make_sure_var_in_reg(op.args[0]) box = TempBox() loc1 = self.rm.force_allocate_reg(box, op.args) - if op.result in self.rm.longevity: + if op.result in self.longevity: # this means, is it ever used resloc = self.rm.force_allocate_reg(op.result, op.args + [box]) else: @@ -281,10 +408,10 @@ consider_guard_overflow = consider_guard_no_exception def consider_guard_value(self, op, ignored): - x = self.rm.make_sure_var_in_reg(op.args[0]) + x = self.make_sure_var_in_reg(op.args[0]) y = self.loc(op.args[1]) self.perform_guard(op, [x, y], None) - self.rm.possibly_free_vars(op.args) + self.possibly_free_vars(op.args) def consider_guard_class(self, op, ignored): assert isinstance(op.args[0], Box) @@ -385,10 +512,89 @@ consider_oois = _consider_compop consider_ooisnot = _consider_compop + def _consider_float_op(self, op, ignored): + loc0 = self.xrm.force_result_in_reg(op.result, op.args[0], op.args) + loc1 = self.xrm.loc(op.args[1]) + self.Perform(op, [loc0, loc1], loc0) + self.xrm.possibly_free_vars(op.args) + + consider_float_add = _consider_float_op + consider_float_sub = _consider_float_op + consider_float_mul = _consider_float_op + consider_float_truediv = _consider_float_op + + def _consider_float_cmp(self, op, ignored): + assert ignored is None + # XXX so far we don't have guards here, but we want them + loc0 = self.xrm.make_sure_var_in_reg(op.args[0], op.args, + imm_fine=False) + loc1 = self.xrm.loc(op.args[1]) + res = self.rm.force_allocate_reg(op.result, need_lower_byte=True) + self.Perform(op, [loc0, loc1], res) + self.xrm.possibly_free_vars(op.args) + + consider_float_lt = _consider_float_cmp + consider_float_le = _consider_float_cmp + consider_float_eq = _consider_float_cmp + consider_float_ne = _consider_float_cmp + consider_float_gt = _consider_float_cmp + consider_float_ge = _consider_float_cmp + + def consider_float_neg(self, op, ignored): + # Following what gcc does... + # XXX we can ignore having constant in a reg, but we need + # to be careful with 128-bit alignment + loc0 = self.xrm.force_result_in_reg(op.result, op.args[0]) + constloc = self.xrm.get_addr_of_const_float(0, 0) + tmpbox = TempBox() + loc1 = self.xrm.force_allocate_reg(tmpbox, op.args) + self.assembler.regalloc_mov(constloc, loc1) + self.Perform(op, [loc0, loc1], loc0) + self.xrm.possibly_free_var(tmpbox) + self.xrm.possibly_free_var(op.args[0]) + + def consider_float_abs(self, op, ignored): + # XXX we can ignore having constant in a reg, but we need + # to be careful with 128-bit alignment + loc0 = self.xrm.force_result_in_reg(op.result, op.args[0]) + constloc = self.xrm.get_addr_of_const_float(0, 1) + tmpbox = TempBox() + loc1 = self.xrm.force_allocate_reg(tmpbox, op.args) + self.assembler.regalloc_mov(constloc, loc1) + self.Perform(op, [loc0, loc1], loc0) + self.xrm.possibly_free_var(tmpbox) + self.xrm.possibly_free_var(op.args[0]) + + def consider_float_is_true(self, op, ignored): + tmpbox0 = TempBox() + loc0 = self.xrm.force_allocate_reg(tmpbox0) + loc1 = self.xrm.loc(op.args[0]) + loc2 = self.rm.force_allocate_reg(op.result, need_lower_byte=True) + self.Perform(op, [loc0, loc1], loc2) + self.xrm.possibly_free_var(op.args[0]) + self.xrm.possibly_free_var(tmpbox0) + + def consider_cast_float_to_int(self, op, ignored): + loc0 = self.xrm.make_sure_var_in_reg(op.args[0], imm_fine=False) + loc1 = self.rm.force_allocate_reg(op.result) + self.Perform(op, [loc0], loc1) + self.xrm.possibly_free_var(op.args[0]) + + def consider_cast_int_to_float(self, op, ignored): + loc0 = self.rm.loc(op.args[0]) + loc1 = self.xrm.force_allocate_reg(op.result) + self.Perform(op, [loc0], loc1) + self.rm.possibly_free_var(op.args[0]) + def _call(self, op, arglocs, force_store=[]): self.rm.before_call(force_store) + self.xrm.before_call(force_store) self.Perform(op, arglocs, eax) - self.rm.after_call(op.result) + if op.result is not None: + if op.result.type == FLOAT: + self.xrm.after_call(op.result) + else: + self.rm.after_call(op.result) def consider_call(self, op, ignored): calldescr = op.descr @@ -514,9 +720,9 @@ else: need_lower_byte = False base_loc = self.rm.make_sure_var_in_reg(op.args[0], op.args) - value_loc = self.rm.make_sure_var_in_reg(op.args[1], op.args, + value_loc = self.make_sure_var_in_reg(op.args[1], op.args, need_lower_byte=need_lower_byte) - self.rm.possibly_free_vars(op.args) + self.possibly_free_vars(op.args) self.PerformDiscard(op, [base_loc, ofs_loc, size_loc, value_loc]) consider_setfield_raw = consider_setfield_gc @@ -538,10 +744,10 @@ need_lower_byte = True else: need_lower_byte = False - value_loc = self.rm.make_sure_var_in_reg(op.args[2], op.args, - need_lower_byte=need_lower_byte) + value_loc = self.make_sure_var_in_reg(op.args[2], op.args, + need_lower_byte=need_lower_byte) ofs_loc = self.rm.make_sure_var_in_reg(op.args[1], op.args) - self.rm.possibly_free_vars(op.args) + self.possibly_free_vars(op.args) self.PerformDiscard(op, [base_loc, ofs_loc, value_loc, imm(scale), imm(ofs)]) @@ -551,7 +757,7 @@ ofs_loc, size_loc, _ = self._unpack_fielddescr(op.descr) base_loc = self.rm.make_sure_var_in_reg(op.args[0], op.args) self.rm.possibly_free_vars(op.args) - result_loc = self.rm.force_allocate_reg(op.result) + result_loc = self.force_allocate_reg(op.result) self.Perform(op, [base_loc, ofs_loc, size_loc], result_loc) consider_getfield_gc_pure = consider_getfield_gc @@ -561,7 +767,7 @@ base_loc = self.rm.make_sure_var_in_reg(op.args[0], op.args) ofs_loc = self.rm.make_sure_var_in_reg(op.args[1], op.args) self.rm.possibly_free_vars(op.args) - result_loc = self.rm.force_allocate_reg(op.result) + result_loc = self.force_allocate_reg(op.result) self.Perform(op, [base_loc, ofs_loc, imm(scale), imm(ofs)], result_loc) consider_getfield_raw = consider_getfield_gc @@ -586,8 +792,8 @@ def consider_same_as(self, op, ignored): argloc = self.loc(op.args[0]) - self.rm.possibly_free_var(op.args[0]) - resloc = self.rm.force_allocate_reg(op.result) + self.possibly_free_var(op.args[0]) + resloc = self.force_allocate_reg(op.result) self.Perform(op, [argloc], resloc) consider_cast_ptr_to_int = consider_same_as @@ -623,17 +829,26 @@ descr = op.descr assert isinstance(descr, LoopToken) self.jump_target_descr = descr - arglocs = assembler.target_arglocs(self.jump_target_descr) + nonfloatlocs, floatlocs = assembler.target_arglocs(self.jump_target_descr) # compute 'tmploc' to be all_regs[0] by spilling what is there box = TempBox() + box1 = TempBox() tmpreg = X86RegisterManager.all_regs[0] - tmploc = self.rm.force_allocate_reg(box, [], selected_reg=tmpreg) - src_locations = [self.rm.loc(arg) for arg in op.args] - dst_locations = arglocs - assert tmploc not in dst_locations + tmploc = self.rm.force_allocate_reg(box, selected_reg=tmpreg) + xmmtmp = X86XMMRegisterManager.all_regs[0] + xmmtmploc = self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp) + # Part about non-floats + src_locations = [self.loc(arg) for arg in op.args if arg.type != FLOAT] + assert tmploc not in nonfloatlocs + dst_locations = [loc for loc in nonfloatlocs if loc is not None] remap_stack_layout(assembler, src_locations, dst_locations, tmploc) + # Part about floats + src_locations = [self.loc(arg) for arg in op.args if arg.type == FLOAT] + dst_locations = [loc for loc in floatlocs if loc is not None] + remap_stack_layout(assembler, src_locations, dst_locations, xmmtmp) self.rm.possibly_free_var(box) - self.rm.possibly_free_vars(op.args) + self.xrm.possibly_free_var(box1) + self.possibly_free_vars(op.args) assembler.closing_jump(self.jump_target_descr) def consider_debug_merge_point(self, op, ignored): @@ -658,8 +873,9 @@ return gcrootmap.compress_callshape(shape) def not_implemented_op(self, op, ignored): - print "[regalloc] Not implemented operation: %s" % op.getopname() - raise NotImplementedError + msg = "[regalloc] Not implemented operation: %s" % op.getopname() + print msg + raise NotImplementedError(msg) oplist = [RegAlloc.not_implemented_op] * rop._LAST Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/ri386.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/ri386.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/ri386.py Mon Oct 5 10:34:06 2009 @@ -27,7 +27,7 @@ def assembler(self): raise TypeError("Float registers should not appear in assembler") -class XMMREG(OPERAND): +class XMMREG(REG): width = 8 def __repr__(self): @@ -309,6 +309,10 @@ assert register.width == 1 return MODRM8(0xC0 | register.op, '') +def memregister64(register): + assert register.width == 8 + return MODRM64(0xC0 | register.op, '') + def mem8(basereg, offset=0): return memSIB8(basereg, None, 0, offset) Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/ri386setup.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/ri386setup.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/ri386setup.py Mon Oct 5 10:34:06 2009 @@ -11,6 +11,9 @@ def reg2modrm(builder, reg): return memregister(reg) +def reg2modrm64(builder, reg): + return memregister64(reg) + def reg2modrm8(builder, reg): return memregister8(reg) @@ -45,7 +48,7 @@ MODRM: [(MODRM, None)], MODRM8: [(MODRM8, None)], MODRM64: [(MODRM64, None)], - XMMREG: [(XMMREG, None)], + XMMREG: [(XMMREG, None), (MODRM64, reg2modrm64)], MISSING: [(MISSING, None)], # missing operands } @@ -486,10 +489,10 @@ FUCOMPP = Instruction() FUCOMPP.mode0(['\xDA\xE9']) -FSTPL = Instruction() -FSTPL.mode1(MODRM64, ['\xDD', orbyte(3<<3), modrm(1)]) -FSTL = Instruction() -FSTL.mode1(MODRM64, ['\xDD', orbyte(2<<3), modrm(1)]) +FSTP = Instruction() +FSTP.mode1(MODRM64, ['\xDD', orbyte(3<<3), modrm(1)]) +FST = Instruction() +FST.mode1(MODRM64, ['\xDD', orbyte(2<<3), modrm(1)]) FISTP = Instruction() FISTP.mode1(MODRM, ['\xDB', orbyte(3<<3), modrm(1)]) @@ -522,6 +525,24 @@ DIVSD = Instruction() DIVSD.mode2(XMMREG, MODRM64, ['\xF2\x0F\x5E', register(1, 8), modrm(2)]) +UCOMISD = Instruction() +UCOMISD.mode2(XMMREG, MODRM64, ['\x66\x0F\x2E', register(1, 8), modrm(2)]) + +XORPD = Instruction() +XORPD.mode2(XMMREG, XMMREG, ['\x66\x0f\x57', register(1, 8), register(2), + '\xC0']) + +ANDPD = Instruction() +ANDPD.mode2(XMMREG, XMMREG, ['\x66\x0F\x54', register(1, 8), register(2), + '\xC0']) + +CVTTSD2SI = Instruction() +CVTTSD2SI.mode2(REG, XMMREG, ['\xF2\x0F\x2C', register(1, 8), register(2), + '\xC0']) + +CVTSI2SD = Instruction() +CVTSI2SD.mode2(XMMREG, MODRM, ['\xF2\x0F\x2A', register(1, 8), modrm(2)]) + # ------------------------------ end of SSE2 ----------------------------- UD2 = Instruction() # reserved as an illegal instruction Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/runner.py Mon Oct 5 10:34:06 2009 @@ -11,6 +11,7 @@ class CPU386(AbstractLLCPU): debug = True + supports_floats = True BOOTSTRAP_TP = lltype.FuncType([], lltype.Signed) dont_keepalive_stuff = False # for tests @@ -48,6 +49,10 @@ assert index < MAX_FAIL_BOXES, "overflow!" self.assembler.fail_boxes_int[index] = intvalue + def set_future_value_float(self, index, floatvalue): + assert index < MAX_FAIL_BOXES, "overflow!" + self.assembler.fail_boxes_float[index] = floatvalue + def set_future_value_ref(self, index, ptrvalue): assert index < MAX_FAIL_BOXES, "overflow!" self.assembler.fail_boxes_ptr[index] = ptrvalue @@ -55,6 +60,9 @@ def get_latest_value_int(self, index): return self.assembler.fail_boxes_int[index] + def get_latest_value_float(self, index): + return self.assembler.fail_boxes_float[index] + def get_latest_value_ref(self, index): ptrvalue = self.assembler.fail_boxes_ptr[index] # clear after reading Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_gc_integration.py Mon Oct 5 10:34:06 2009 @@ -18,7 +18,8 @@ from pypy.jit.backend.x86.test.test_regalloc import MockAssembler from pypy.jit.backend.x86.test.test_regalloc import BaseTestRegalloc -from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86StackManager +from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86StackManager,\ + X86XMMRegisterManager class MockGcRootMap(object): def get_basic_shape(self): @@ -64,6 +65,8 @@ regalloc.sm = X86StackManager() regalloc.rm = X86RegisterManager(longevity, regalloc.sm, assembler=regalloc.assembler) + regalloc.xrm = X86XMMRegisterManager(longevity, regalloc.sm, + assembler=regalloc.assembler) cpu = regalloc.assembler.cpu for box in boxes: regalloc.rm.try_allocate_reg(box) Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_jump.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_jump.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_jump.py Mon Oct 5 10:34:06 2009 @@ -41,9 +41,9 @@ remap_stack_layout(assembler, [eax, ebx, ecx, edx, esi, edi], [eax, ebx, ecx, edx, esi, edi], '?') assert assembler.ops == [] - s8 = stack_pos(1) - s12 = stack_pos(31) - s20 = stack_pos(6) + s8 = stack_pos(1, 1) + s12 = stack_pos(31, 1) + s20 = stack_pos(6, 1) remap_stack_layout(assembler, [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], '?') @@ -58,10 +58,10 @@ def test_simple_stacklocs(): assembler = MockAssembler() - s8 = stack_pos(0) - s12 = stack_pos(13) - s20 = stack_pos(20) - s24 = stack_pos(221) + s8 = stack_pos(0, 1) + s12 = stack_pos(13, 1) + s20 = stack_pos(20, 1) + s24 = stack_pos(221, 1) remap_stack_layout(assembler, [s8, eax, s12], [s20, s24, edi], edx) assert assembler.ops == [('mov', s8, edx), ('mov', edx, s20), @@ -70,10 +70,10 @@ def test_reordering(): assembler = MockAssembler() - s8 = stack_pos(8) - s12 = stack_pos(12) - s20 = stack_pos(19) - s24 = stack_pos(1) + s8 = stack_pos(8, 1) + s12 = stack_pos(12, 1) + s20 = stack_pos(19, 1) + s24 = stack_pos(1, 1) remap_stack_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, edi], '?') assert assembler.got([('mov', ebx, edi), @@ -83,10 +83,10 @@ def test_cycle(): assembler = MockAssembler() - s8 = stack_pos(8) - s12 = stack_pos(12) - s20 = stack_pos(19) - s24 = stack_pos(1) + s8 = stack_pos(8, 1) + s12 = stack_pos(12, 1) + s20 = stack_pos(19, 1) + s24 = stack_pos(1, 1) remap_stack_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, s20], '?') assert assembler.got([('push', s8), @@ -97,12 +97,12 @@ def test_cycle_2(): assembler = MockAssembler() - s8 = stack_pos(8) - s12 = stack_pos(12) - s20 = stack_pos(19) - s24 = stack_pos(1) - s2 = stack_pos(2) - s3 = stack_pos(3) + s8 = stack_pos(8, 1) + s12 = stack_pos(12, 1) + s20 = stack_pos(19, 1) + s24 = stack_pos(1, 1) + s2 = stack_pos(2, 1) + s3 = stack_pos(3, 1) remap_stack_layout(assembler, [eax, s8, edi, s20, eax, s20, s24, esi, s2, s3], [s8, s20, edi, eax, edx, s24, ebx, s12, s3, s2], @@ -127,14 +127,14 @@ remap_stack_layout(assembler, [c3], [eax], '?') assert assembler.ops == [('mov', c3, eax)] assembler = MockAssembler() - s12 = stack_pos(12) + s12 = stack_pos(12, 1) remap_stack_layout(assembler, [c3], [s12], '?') assert assembler.ops == [('mov', c3, s12)] def test_constants_and_cycle(): assembler = MockAssembler() c3 = imm(3) - s12 = stack_pos(13) + s12 = stack_pos(13, 1) remap_stack_layout(assembler, [ebx, c3, s12], [s12, eax, ebx], edi) assert assembler.ops == [('mov', c3, eax), Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py Mon Oct 5 10:34:06 2009 @@ -8,7 +8,8 @@ from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.backend.llsupport.descr import GcCache from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, X86RegisterManager +from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, X86RegisterManager,\ + BASE_CONSTANT_SIZE from pypy.jit.metainterp.test.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.annlowlevel import llhelper @@ -94,6 +95,8 @@ for i, arg in enumerate(args): if isinstance(arg, int): self.cpu.set_future_value_int(i, arg) + elif isinstance(arg, float): + self.cpu.set_future_value_float(i, arg) else: assert isinstance(lltype.typeOf(arg), lltype.Ptr) llgcref = lltype.cast_opaque_ptr(llmemory.GCREF, arg) @@ -105,10 +108,17 @@ def getint(self, index): return self.cpu.get_latest_value_int(index) + def getfloat(self, index): + return self.cpu.get_latest_value_float(index) + def getints(self, end): return [self.cpu.get_latest_value_int(index) for index in range(0, end)] + def getfloats(self, end): + return [self.cpu.get_latest_value_float(index) for + index in range(0, end)] + def getptr(self, index, T): gcref = self.cpu.get_latest_value_ref(index) return lltype.cast_opaque_ptr(T, gcref) @@ -464,3 +474,61 @@ s = lltype.malloc(self.A, 3) self.interpret(ops, [s, ord('a')]) assert s[1] == 'a' + +class TestRegallocFloats(BaseTestRegalloc): + def test_float_add(self): + ops = ''' + [f0, f1] + f2 = float_add(f0, f1) + finish(f2, f0, f1) + ''' + self.interpret(ops, [3.0, 1.5]) + assert self.getfloats(3) == [4.5, 3.0, 1.5] + + def test_float_adds_stack(self): + ops = ''' + [f0, f1, f2, f3, f4, f5, f6, f7, f8] + f9 = float_add(f0, f1) + f10 = float_add(f8, 3.5) + finish(f9, f10, f2, f3, f4, f5, f6, f7, f8) + ''' + self.interpret(ops, [0.1, .2, .3, .4, .5, .6, .7, .8, .9]) + assert self.getfloats(9) == [.1+.2, .9+3.5, .3, .4, .5, .6, .7, .8, .9] + + def test_float_overflow_const_list(self): + ops = ['[f0]'] + for i in range(BASE_CONSTANT_SIZE * 2): + ops.append('f%d = float_add(f%d, 3.5)' % (i + 1, i)) + ops.append('finish(f%d)' % (BASE_CONSTANT_SIZE * 2)) + ops = "\n".join(ops) + self.interpret(ops, [0.1]) + assert abs(self.getfloat(0) - (BASE_CONSTANT_SIZE * 2) * 3.5 - 0.1) < 0.00001 + + def test_lt_const(self): + ops = ''' + [f0] + i1 = float_lt(3.5, f0) + finish(i1) + ''' + self.interpret(ops, [0.1]) + assert self.getint(0) == 0 + + def test_bug_wrong_stack_adj(self): + ops = ''' + [i0, i1, i2, i3, i4, i5, i6, i7, i8] + guard_true(i0) [i0, i1, i2, i3, i4, i5, i6, i7, i8] + finish(4.5, i0, i1, i2, i3, i4, i5, i6, i7, i8) + ''' + loop = self.interpret(ops, [0, 1, 2, 3, 4, 5, 6, 7, 8]) + assert self.getint(0) == 0 + bridge_ops = ''' + [i0, i1, i2, i3, i4, i5, i6, i7, i8] + call(ConstClass(raising_fptr), 0, descr=raising_calldescr) + finish(i0, i1, i2, i3, i4, i5, i6, i7, i8) + ''' + self.attach_bridge(bridge_ops, loop, 0) + for i in range(9): + self.cpu.set_future_value_int(i, i) + self.run(loop) + assert self.getints(9) == range(9) + Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Mon Oct 5 10:34:06 2009 @@ -141,8 +141,10 @@ all = instr.as_all_suffixes for m, extra in args: if m in (i386.MODRM, i386.MODRM8) or all: - if not instrname == 'FNSTCW': + if instrname != 'FNSTCW': suffix = suffixes[sizes[m]] + suffix + if m is i386.MODRM64 and instrname in ['FST', 'FSTP']: + suffix = 'l' following = "" if instr.indirect: suffix = "" Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/history.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/history.py Mon Oct 5 10:34:06 2009 @@ -463,6 +463,8 @@ try: if self.type == INT: t = 'i' + elif self.type == FLOAT: + t = 'f' else: t = 'p' except AttributeError: Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/logger.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/logger.py Mon Oct 5 10:34:06 2009 @@ -2,7 +2,7 @@ from pypy.rlib.objectmodel import compute_unique_id from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.history import Const, ConstInt, Box, \ - BoxInt, ConstAddr + BoxInt, ConstAddr, ConstFloat, BoxFloat from pypy.rlib.streamio import open_file_as_stream class Logger(object): @@ -45,10 +45,14 @@ return 'ConstPtr(ptr' + str(mv) + ')' elif isinstance(arg, self.ts.BoxRef): return 'p' + str(mv) + elif isinstance(arg, ConstFloat): + return str(arg.value) + elif isinstance(arg, BoxFloat): + return 'f' + str(mv) elif isinstance(arg, self.ts.ConstAddr): return 'ConstClass(cls' + str(mv) + ')' else: - raise NotImplementedError + return '?' def log_operations(self, inputargs, operations, memo, indent=0): if self.log_stream is None: Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/oparser.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/oparser.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/oparser.py Mon Oct 5 10:34:06 2009 @@ -4,7 +4,8 @@ """ from pypy.jit.metainterp.history import TreeLoop, BoxInt, ConstInt,\ - ConstAddr, ConstObj, ConstPtr, Box, BasicFailDescr, LoopToken + ConstAddr, ConstObj, ConstPtr, Box, BasicFailDescr, BoxFloat, ConstFloat,\ + LoopToken from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.metainterp.typesystem import llhelper from pypy.rpython.lltypesystem import lltype, llmemory @@ -70,6 +71,9 @@ # integer box = BoxInt() _box_counter_more_than(elem[1:]) + elif elem.startswith('f'): + box = BoxFloat() + _box_counter_more_than(elem[1:]) elif elem.startswith('p'): # pointer ts = getattr(self.cpu, 'ts', llhelper) @@ -96,12 +100,21 @@ self.vars[elem] = box return vars + def is_float(self, arg): + try: + float(arg) + return True + except ValueError: + return False + def getvar(self, arg): if not arg: return ConstInt(0) try: return ConstInt(int(arg)) except ValueError: + if self.is_float(arg): + return ConstFloat(float(arg)) if arg.startswith('"') or arg.startswith("'"): # XXX ootype info = arg.strip("'\"") Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_executor.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_executor.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_executor.py Mon Oct 5 10:34:06 2009 @@ -196,6 +196,7 @@ yield (rop.FLOAT_NE, [10.125, y], 'int', 10.125 != y) yield (rop.FLOAT_GT, [10.125, y], 'int', 10.125 > y) yield (rop.FLOAT_GE, [10.125, y], 'int', 10.125 >= y) + yield (rop.FLOAT_EQ, [0.0, -0.0], 'int', 0.0 == -0.0) def _float_unary_operations(): yield (rop.FLOAT_NEG, [-5.9], 'float', 5.9) @@ -204,6 +205,7 @@ yield (rop.FLOAT_ABS, [15.9], 'float', 15.9) yield (rop.FLOAT_IS_TRUE, [-5.9], 'int', 1) yield (rop.FLOAT_IS_TRUE, [0.0], 'int', 0) + yield (rop.FLOAT_IS_TRUE, [-0.0], 'int', 0) yield (rop.CAST_FLOAT_TO_INT, [-5.9], 'int', -5) yield (rop.CAST_FLOAT_TO_INT, [5.9], 'int', 5) yield (rop.CAST_INT_TO_FLOAT, [123], 'float', 123.0) Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_logger.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_logger.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_logger.py Mon Oct 5 10:34:06 2009 @@ -74,3 +74,12 @@ loop, oloop = self.reparse(inp, check_equal=False) assert loop.operations[0].args[0]._get_str() == 'info' assert oloop.operations[0].args[0]._get_str() == 'info' + + def test_floats(self): + inp = ''' + [f0] + f1 = float_add(3.5, f0) + ''' + loop, oloop = self.reparse(inp) + equaloplists(loop.operations, oloop.operations) + Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_oparser.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_oparser.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_oparser.py Mon Oct 5 10:34:06 2009 @@ -3,7 +3,8 @@ from pypy.jit.metainterp.test.oparser import parse from pypy.jit.metainterp.resoperation import rop -from pypy.jit.metainterp.history import AbstractDescr, BoxInt, LoopToken +from pypy.jit.metainterp.history import AbstractDescr, BoxInt, LoopToken,\ + BoxFloat def test_basic_parse(): x = """ @@ -129,6 +130,14 @@ loop = parse(x, namespace=locals()) assert loop.operations[0].descr is looptoken +def test_floats(): + x = ''' + [f0] + f1 = float_add(f0, 3.5) + ''' + loop = parse(x) + assert isinstance(loop.operations[0].args[0], BoxFloat) + def test_debug_merge_point(): x = ''' [] Modified: pypy/branch/merge-floats-via-sse2/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/module/pypyjit/policy.py Mon Oct 5 10:34:06 2009 @@ -10,12 +10,6 @@ if (func.__name__.startswith('_mm_') or func.__name__.startswith('__mm_')): # multimethods - name = func.__name__.lstrip('_') - if (name.startswith('mm_truediv') or - name.startswith('mm_inplace_truediv') or - name.startswith('mm_float')): - # floats - return False return True if '_mth_mm_' in func.__name__: # e.g. str_mth_mm_join_xxx return True @@ -27,18 +21,15 @@ return False if mod.startswith('pypy.objspace.'): - # we don't support floats - if 'float' in mod or 'complex' in mod: - return False - if func.__name__ == 'format_float': - return False # gc_id operation if func.__name__ == 'id__ANY': return False - # floats if mod == 'pypy.rlib.rbigint': #if func.__name__ == '_bigint_true_divide': return False + if mod == 'pypy.rpython.lltypesystem.module.ll_math': + # XXX temporary, contains force_cast + return False if '_geninterp_' in func.func_globals: # skip all geninterped stuff return False if mod.startswith('pypy.interpreter.astcompiler.'): From arigo at codespeak.net Mon Oct 5 10:40:00 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 5 Oct 2009 10:40:00 +0200 (CEST) Subject: [pypy-svn] r68154 - pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test Message-ID: <20091005084000.970D91683E3@codespeak.net> Author: arigo Date: Mon Oct 5 10:39:59 2009 New Revision: 68154 Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test/runner_test.py Log: Re-add test_unused_result_int. Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test/runner_test.py Mon Oct 5 10:39:59 2009 @@ -834,13 +834,24 @@ for i in range(1, len(fboxes)): assert self.cpu.get_latest_value_float(i) == 13.5 + 6.73 * i + def test_unused_result_int(self): + # test pure operations on integers whose result is not used + from pypy.jit.metainterp.test.test_executor import get_int_tests + int_tests = list(get_int_tests()) + int_tests = [(opnum, boxargs, 'int', retvalue) + for opnum, boxargs, retvalue in int_tests] + self._test_unused_result(int_tests) + def test_unused_result_float(self): # same as test_unused_result_int, for float operations from pypy.jit.metainterp.test.test_executor import get_float_tests float_tests = list(get_float_tests(self.cpu)) + self._test_unused_result(float_tests) + + def _test_unused_result(self, tests): inputargs = [] operations = [] - for opnum, boxargs, rettype, retvalue in float_tests: + for opnum, boxargs, rettype, retvalue in tests: inputargs += boxargs if rettype == 'int': boxres = BoxInt() From arigo at codespeak.net Mon Oct 5 10:49:07 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 5 Oct 2009 10:49:07 +0200 (CEST) Subject: [pypy-svn] r68155 - pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test Message-ID: <20091005084907.CA4601683DC@codespeak.net> Author: arigo Date: Mon Oct 5 10:49:07 2009 New Revision: 68155 Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test/runner_test.py Log: Missing bits from a hand-made imperfect merge. Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test/runner_test.py Mon Oct 5 10:49:07 2009 @@ -1179,7 +1179,20 @@ descr_B) assert isinstance(x, BoxPtr) assert x.getref(lltype.Ptr(A)) == a - # + if self.cpu.supports_floats: + C = lltype.GcArray(lltype.Float) + c = lltype.malloc(C, 6) + c[3] = 3.5 + descr_C = cpu.arraydescrof(C) + x = cpu.do_getarrayitem_gc( + BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, c)), BoxInt(3), + descr_C) + assert isinstance(x, BoxFloat) + assert x.getfloat() == 3.5 + cpu.do_setarrayitem_gc( + BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, c)), BoxInt(4), + BoxFloat(4.5), descr_C) + assert c[4] == 4.5 s = rstr.mallocstr(6) x = cpu.do_strlen( BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s))) @@ -1190,7 +1203,8 @@ BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s)), BoxInt(3)) assert x.value == ord('X') # - S = lltype.GcStruct('S', ('x', lltype.Char), ('y', lltype.Ptr(A))) + S = lltype.GcStruct('S', ('x', lltype.Char), ('y', lltype.Ptr(A)), + ('z', lltype.Float)) descrfld_x = cpu.fielddescrof(S, 'x') s = lltype.malloc(S) s.x = 'Z' @@ -1234,6 +1248,19 @@ descrfld_rx) assert rs.x == '!' # + + if self.cpu.supports_floats: + descrfld_z = cpu.fielddescrof(S, 'z') + cpu.do_setfield_gc( + BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s)), + BoxFloat(3.5), + descrfld_z) + assert s.z == 3.5 + s.z = 3.2 + x = cpu.do_getfield_gc( + BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s)), + descrfld_z) + assert x.getfloat() == 3.2 ### we don't support in the JIT for now GC pointers ### stored inside non-GC structs. #descrfld_ry = cpu.fielddescrof(RS, 'y') From arigo at codespeak.net Mon Oct 5 10:56:22 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 5 Oct 2009 10:56:22 +0200 (CEST) Subject: [pypy-svn] r68156 - pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test Message-ID: <20091005085622.A67FA1683DC@codespeak.net> Author: arigo Date: Mon Oct 5 10:56:21 2009 New Revision: 68156 Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py Log: Correctly merge this test. It fails now. :-/ Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py Mon Oct 5 10:56:21 2009 @@ -516,7 +516,8 @@ def test_bug_wrong_stack_adj(self): ops = ''' [i0, i1, i2, i3, i4, i5, i6, i7, i8] - guard_true(i0) [i0, i1, i2, i3, i4, i5, i6, i7, i8] + f0 = same_as(3.5) + guard_true(i0) [f0, i0, i1, i2, i3, i4, i5, i6, i7, i8] finish(4.5, i0, i1, i2, i3, i4, i5, i6, i7, i8) ''' loop = self.interpret(ops, [0, 1, 2, 3, 4, 5, 6, 7, 8]) @@ -526,9 +527,9 @@ call(ConstClass(raising_fptr), 0, descr=raising_calldescr) finish(i0, i1, i2, i3, i4, i5, i6, i7, i8) ''' - self.attach_bridge(bridge_ops, loop, 0) + self.attach_bridge(bridge_ops, loop, 1) for i in range(9): self.cpu.set_future_value_int(i, i) self.run(loop) assert self.getints(9) == range(9) - + assert self.getfloat(0) == 3.5 From arigo at codespeak.net Mon Oct 5 11:15:50 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 5 Oct 2009 11:15:50 +0200 (CEST) Subject: [pypy-svn] r68157 - pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test Message-ID: <20091005091550.1336F1683E1@codespeak.net> Author: arigo Date: Mon Oct 5 11:15:49 2009 New Revision: 68157 Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py Log: Revert the test to its original state and mark it as "xfail", to avoid blocking the merge to trunk. Fijal, please review when you are around :-) Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py Mon Oct 5 11:15:49 2009 @@ -130,6 +130,8 @@ guard_op = loop.operations[guard_op_index] assert guard_op.is_guard() bridge = self.parse(ops, **kwds) + assert ([box.type for box in bridge.inputargs] == + [box.type for box in guard_op.fail_args]) faildescr = guard_op.descr self.cpu.compile_bridge(faildescr, bridge.inputargs, bridge.operations) return bridge @@ -513,21 +515,22 @@ self.interpret(ops, [0.1]) assert self.getint(0) == 0 + @py.test.mark.xfail # this test is bogus in the branch floats-via-sse2 def test_bug_wrong_stack_adj(self): ops = ''' [i0, i1, i2, i3, i4, i5, i6, i7, i8] - f0 = same_as(3.5) - guard_true(i0) [f0, i0, i1, i2, i3, i4, i5, i6, i7, i8] - finish(4.5, i0, i1, i2, i3, i4, i5, i6, i7, i8) + guard_true(i0) + fail(3.5, i0, i1, i2, i3, i4, i5, i6, i7, i8) + fail(4.5, i0, i1, i2, i3, i4, i5, i6, i7, i8) ''' loop = self.interpret(ops, [0, 1, 2, 3, 4, 5, 6, 7, 8]) assert self.getint(0) == 0 bridge_ops = ''' [i0, i1, i2, i3, i4, i5, i6, i7, i8] call(ConstClass(raising_fptr), 0, descr=raising_calldescr) - finish(i0, i1, i2, i3, i4, i5, i6, i7, i8) + fail(i0, i1, i2, i3, i4, i5, i6, i7, i8) ''' - self.attach_bridge(bridge_ops, loop, 1) + self.attach_bridge(bridge_ops, loop, 0) for i in range(9): self.cpu.set_future_value_int(i, i) self.run(loop) From pedronis at codespeak.net Mon Oct 5 11:18:24 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 5 Oct 2009 11:18:24 +0200 (CEST) Subject: [pypy-svn] r68158 - pypy/trunk/pypy/translator/c/test Message-ID: <20091005091824.030871683E1@codespeak.net> Author: pedronis Date: Mon Oct 5 11:18:24 2009 New Revision: 68158 Added: pypy/trunk/pypy/translator/c/test/test_refcount.py (contents, props changed) - copied, changed from r68152, pypy/trunk/pypy/translator/c/test/test_newgc.py Modified: pypy/trunk/pypy/translator/c/test/test_newgc.py Log: (cfbolz, pedronis) split out the refcounting tests, garbage collect imports, kill some silly tests Modified: pypy/trunk/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_newgc.py (original) +++ pypy/trunk/pypy/translator/c/test/test_newgc.py Mon Oct 5 11:18:24 2009 @@ -1,286 +1,16 @@ -import autopath import sys import py -from py.test import raises import os from pypy.objspace.flow.model import summary from pypy.translator.translator import TranslationContext -from pypy.translator.backendopt.stat import print_statistics -from pypy.translator.c import genc, gc -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.translator.c import genc +from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.memory.test import snippet from pypy.rlib.objectmodel import keepalive_until_here from pypy.rlib.rstring import StringBuilder, UnicodeBuilder -from pypy import conftest from pypy.tool.udir import udir - -def compile_func(fn, inputtypes, t=None, gcpolicy="ref"): - from pypy.config.pypyoption import get_pypy_config - config = get_pypy_config(translating=True) - config.translation.gc = gcpolicy - config.translation.countmallocs = True - if t is None: - t = TranslationContext(config=config) - if inputtypes is not None: - t.buildannotator().build_types(fn, inputtypes) - t.buildrtyper().specialize() - builder = genc.CExtModuleBuilder(t, fn, config=config) - builder.generate_source() - builder.compile() - if conftest.option.view: - t.view() - compiled_fn = builder.get_entry_point() - malloc_counters = builder.get_malloc_counters() - def checking_fn(*args, **kwds): - try: - return compiled_fn(*args, **kwds) - finally: - mallocs, frees = malloc_counters() - assert mallocs == frees - return checking_fn - -def test_something(): - def f(): - return 1 - fn = compile_func(f, []) - assert fn() == 1 - -def test_something_more(): - S = lltype.GcStruct("S", ('x', lltype.Signed)) - def f(x): - s = lltype.malloc(S) - s.x = x - return s.x - fn = compile_func(f, [int]) - assert fn(1) == 1 - -def test_call_function(): - class C: - pass - def f(): - c = C() - c.x = 1 - return c - def g(): - return f().x - fn = compile_func(g, []) - assert fn() == 1 - -def test_multiple_exits(): - S = lltype.GcStruct("S", ('x', lltype.Signed)) - T = lltype.GcStruct("T", ('y', lltype.Signed)) - def f(n): - c = lltype.malloc(S) - d = lltype.malloc(T) - d.y = 1 - e = lltype.malloc(T) - e.y = 2 - if n: - x = d - else: - x = e - return x.y - fn = compile_func(f, [int]) - assert fn(1) == 1 - assert fn(0) == 2 - - -def test_cleanup_vars_on_call(): - S = lltype.GcStruct("S", ('x', lltype.Signed)) - def f(): - return lltype.malloc(S) - def g(): - s1 = f() - s1.x = 42 - s2 = f() - s3 = f() - return s1.x - fn = compile_func(g, []) - assert fn() == 42 - -def test_multiply_passed_var(): - S = lltype.GcStruct("S", ('x', lltype.Signed)) - def f(x): - if x: - a = lltype.malloc(S) - a.x = 1 - b = a - else: - a = lltype.malloc(S) - a.x = 1 - b = lltype.malloc(S) - b.x = 2 - return a.x + b.x - fn = compile_func(f, [int]) - fn(1) == 2 - fn(0) == 3 - -def test_pyobj(): - py.test.skip("unsupported") - def f(x): - if x: - a = 1 - else: - a = "1" - return int(a) - fn = compile_func(f, [int]) - assert fn(1) == 1 -# assert fn(0) == 0 #XXX this should work but it's not my fault - -def test_write_barrier(): - S = lltype.GcStruct("S", ('x', lltype.Signed)) - T = lltype.GcStruct("T", ('s', lltype.Ptr(S))) - def f(x): - s = lltype.malloc(S) - s.x = 0 - s1 = lltype.malloc(S) - s1.x = 1 - s2 = lltype.malloc(S) - s2.x = 2 - t = lltype.malloc(T) - t.s = s - if x: - t.s = s1 - else: - t.s = s2 - return t.s.x + s.x + s1.x + s2.x - fn = compile_func(f, [int]) - assert fn(1) == 4 - assert fn(0) == 5 - -def test_del_basic(): - for gcpolicy in ["ref"]: #, "framework"]: - S = lltype.GcStruct('S', ('x', lltype.Signed)) - TRASH = lltype.GcStruct('TRASH', ('x', lltype.Signed)) - lltype.attachRuntimeTypeInfo(S) - GLOBAL = lltype.Struct('GLOBAL', ('x', lltype.Signed)) - glob = lltype.malloc(GLOBAL, immortal=True) - def destructor(s): - glob.x = s.x + 1 - def type_info_S(s): - return lltype.getRuntimeTypeInfo(S) - - def g(n): - s = lltype.malloc(S) - s.x = n - # now 's' should go away - def entrypoint(n): - g(n) - # llop.gc__collect(lltype.Void) - return glob.x - - t = TranslationContext() - t.buildannotator().build_types(entrypoint, [int]) - rtyper = t.buildrtyper() - destrptr = rtyper.annotate_helper_fn(destructor, [lltype.Ptr(S)]) - rtyper.attachRuntimeTypeInfoFunc(S, type_info_S, destrptr=destrptr) - rtyper.specialize() - fn = compile_func(entrypoint, None, t, gcpolicy=gcpolicy) - - res = fn(123) - assert res == 124 - -def test_del_catches(): - import os - def g(): - pass - class A(object): - def __del__(self): - try: - g() - except: - os.write(1, "hallo") - def f1(i): - if i: - raise TypeError - def f(i): - a = A() - f1(i) - a.b = 1 - return a.b - fn = compile_func(f, [int]) - assert fn(0) == 1 - assert py.test.raises(TypeError, fn, 1) - -def test_del_raises(): - class B(object): - def __del__(self): - raise TypeError - def func(): - b = B() - fn = compile_func(func, []) - # does not crash - fn() - -def test_wrong_order_setitem(): - import os - class A(object): - pass - a = A() - a.b = None - class B(object): - def __del__(self): - a.freed += 1 - a.b = None - def f(n): - a.freed = 0 - a.b = B() - if n: - a.b = None - return a.freed - fn = compile_func(f, [int]) - res = fn(1) - assert res == 1 - -def test_wrong_startblock_incref(): - class B(object): - pass - def g(b): - while True: - b.x -= 10 - if b.x < 0: - return b.x - def f(n): - b = B() - b.x = n - return g(b) - - # XXX obscure: remove the first empty block in the graph of 'g' - t = TranslationContext() - graph = t.buildflowgraph(g) - assert graph.startblock.operations == [] - graph.startblock = graph.startblock.exits[0].target - graph.startblock.isstartblock = True - from pypy.objspace.flow.model import checkgraph - checkgraph(graph) - t._prebuilt_graphs[g] = graph - - fn = compile_func(f, [int], t) - res = fn(112) - assert res == -8 - -def test_gc_x_operations(): - t = TranslationContext() - from pypy.rlib.rgc import gc_clone, gc_swap_pool - S = lltype.GcStruct("S", ('x', lltype.Signed)) - def f(): - s = lltype.malloc(S) - gc_swap_pool(None) - try: - t = gc_clone(s, None) - except RuntimeError: - return 1 - else: - return 0 - fn = compile_func(f, [], t=t) - res = fn() - assert res == 1 - -# _______________________________________________________________ -# test framework - from pypy.translator.c.test.test_boehm import AbstractGCTestClass class TestUsingFramework(AbstractGCTestClass): Copied: pypy/trunk/pypy/translator/c/test/test_refcount.py (from r68152, pypy/trunk/pypy/translator/c/test/test_newgc.py) ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_newgc.py (original) +++ pypy/trunk/pypy/translator/c/test/test_refcount.py Mon Oct 5 11:18:24 2009 @@ -1,20 +1,10 @@ -import autopath -import sys import py -from py.test import raises import os -from pypy.objspace.flow.model import summary from pypy.translator.translator import TranslationContext -from pypy.translator.backendopt.stat import print_statistics -from pypy.translator.c import genc, gc -from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rpython.memory.test import snippet -from pypy.rlib.objectmodel import keepalive_until_here -from pypy.rlib.rstring import StringBuilder, UnicodeBuilder +from pypy.translator.c import genc +from pypy.rpython.lltypesystem import lltype from pypy import conftest -from pypy.tool.udir import udir def compile_func(fn, inputtypes, t=None, gcpolicy="ref"): from pypy.config.pypyoption import get_pypy_config @@ -117,18 +107,6 @@ fn(1) == 2 fn(0) == 3 -def test_pyobj(): - py.test.skip("unsupported") - def f(x): - if x: - a = 1 - else: - a = "1" - return int(a) - fn = compile_func(f, [int]) - assert fn(1) == 1 -# assert fn(0) == 0 #XXX this should work but it's not my fault - def test_write_barrier(): S = lltype.GcStruct("S", ('x', lltype.Signed)) T = lltype.GcStruct("T", ('s', lltype.Ptr(S))) @@ -260,770 +238,3 @@ fn = compile_func(f, [int], t) res = fn(112) assert res == -8 - -def test_gc_x_operations(): - t = TranslationContext() - from pypy.rlib.rgc import gc_clone, gc_swap_pool - S = lltype.GcStruct("S", ('x', lltype.Signed)) - def f(): - s = lltype.malloc(S) - gc_swap_pool(None) - try: - t = gc_clone(s, None) - except RuntimeError: - return 1 - else: - return 0 - fn = compile_func(f, [], t=t) - res = fn() - assert res == 1 - -# _______________________________________________________________ -# test framework - -from pypy.translator.c.test.test_boehm import AbstractGCTestClass - -class TestUsingFramework(AbstractGCTestClass): - gcpolicy = "marksweep" - should_be_moving = False - GC_CAN_MOVE = False - GC_CANNOT_MALLOC_NONMOVABLE = False - - # interface for snippet.py - large_tests_ok = True - def run(self, func): - fn = self.getcompiled(func) - return fn() - - def test_empty_collect(self): - def f(): - llop.gc__collect(lltype.Void) - return 41 - fn = self.getcompiled(f) - res = fn() - assert res == 41 - - def test_framework_simple(self): - def g(x): # cannot cause a collect - return x + 1 - class A(object): - pass - def make(): - a = A() - a.b = g(1) - return a - make._dont_inline_ = True - def f(): - a = make() - llop.gc__collect(lltype.Void) - return a.b - fn = self.getcompiled(f) - res = fn() - assert res == 2 - insns = summary(self.t.graphs[0]) - assert ('gc_reload_possibly_moved' in insns) == self.should_be_moving - - def test_framework_safe_pushpop(self): - class A(object): - pass - class B(object): - pass - def g(x): # cause a collect - llop.gc__collect(lltype.Void) - g._dont_inline_ = True - global_a = A() - global_a.b = B() - global_a.b.a = A() - global_a.b.a.b = B() - global_a.b.a.b.c = 1 - def make(): - global_a.b.a.b.c = 40 - a = global_a.b.a - b = a.b - b.c = 41 - g(1) - b0 = a.b - b0.c = b.c = 42 - make._dont_inline_ = True - def f(): - make() - llop.gc__collect(lltype.Void) - return global_a.b.a.b.c - fn = self.getcompiled(f) - res = fn() - assert res == 42 - insns = summary(self.t.graphs[0]) - assert 'gc_reload_possibly_moved' not in insns - - def test_framework_protect_getfield(self): - class A(object): - pass - class B(object): - pass - def prepare(b, n): - a = A() - a.value = n - b.a = a - b.othervalue = 5 - def g(a): - llop.gc__collect(lltype.Void) - for i in range(1000): - prepare(B(), -1) # probably overwrites collected memory - return a.value - g._dont_inline_ = True - def f(): - b = B() - prepare(b, 123) - a = b.a - b.a = None - return g(a) + b.othervalue - fn = self.getcompiled(f) - res = fn() - assert res == 128 - - def test_framework_varsized(self): - S = lltype.GcStruct("S", ('x', lltype.Signed)) - T = lltype.GcStruct("T", ('y', lltype.Signed), - ('s', lltype.Ptr(S))) - ARRAY_Ts = lltype.GcArray(lltype.Ptr(T)) - - def f(): - r = 0 - for i in range(30): - a = lltype.malloc(ARRAY_Ts, i) - for j in range(i): - a[j] = lltype.malloc(T) - a[j].y = i - a[j].s = lltype.malloc(S) - a[j].s.x = 2*i - r += a[j].y + a[j].s.x - a[j].s = lltype.malloc(S) - a[j].s.x = 3*i - r -= a[j].s.x - for j in range(i): - r += a[j].y - return r - fn = self.getcompiled(f) - res = fn() - assert res == f() - - - def test_framework_using_lists(self): - class A(object): - pass - N = 1000 - def f(): - static_list = [] - for i in range(N): - a = A() - a.x = i - static_list.append(a) - r = 0 - for a in static_list: - r += a.x - return r - fn = self.getcompiled(f) - res = fn() - assert res == N*(N - 1)/2 - - def test_framework_static_roots(self): - class A(object): - def __init__(self, y): - self.y = y - a = A(0) - a.x = None - def make(): - a.x = A(42) - make._dont_inline_ = True - def f(): - make() - llop.gc__collect(lltype.Void) - return a.x.y - fn = self.getcompiled(f) - res = fn() - assert res == 42 - - def test_framework_nongc_static_root(self): - S = lltype.GcStruct("S", ('x', lltype.Signed)) - T = lltype.Struct("T", ('p', lltype.Ptr(S))) - t = lltype.malloc(T, immortal=True) - def f(): - t.p = lltype.malloc(S) - t.p.x = 43 - for i in range(2500000): - s = lltype.malloc(S) - s.x = i - return t.p.x - fn = self.getcompiled(f) - res = fn() - assert res == 43 - - def test_framework_void_array(self): - A = lltype.GcArray(lltype.Void) - a = lltype.malloc(A, 44) - def f(): - return len(a) - fn = self.getcompiled(f) - res = fn() - assert res == 44 - - - def test_framework_malloc_failure(self): - def f(): - a = [1] * (sys.maxint//2) - return len(a) + a[0] - fn = self.getcompiled(f) - py.test.raises(MemoryError, fn) - - def test_framework_array_of_void(self): - def f(): - a = [None] * 43 - b = [] - for i in range(1000000): - a.append(None) - b.append(len(a)) - return b[-1] - fn = self.getcompiled(f) - res = fn() - assert res == 43 + 1000000 - - def test_framework_opaque(self): - A = lltype.GcStruct('A', ('value', lltype.Signed)) - O = lltype.GcOpaqueType('test.framework') - - def gethidden(n): - a = lltype.malloc(A) - a.value = -n * 7 - return lltype.cast_opaque_ptr(lltype.Ptr(O), a) - gethidden._dont_inline_ = True - def reveal(o): - return lltype.cast_opaque_ptr(lltype.Ptr(A), o) - def overwrite(a, i): - a.value = i - overwrite._dont_inline_ = True - def f(): - o = gethidden(10) - llop.gc__collect(lltype.Void) - for i in range(1000): # overwrite freed memory - overwrite(lltype.malloc(A), i) - a = reveal(o) - return a.value - fn = self.getcompiled(f) - res = fn() - assert res == -70 - - def test_framework_finalizer(self): - class B(object): - pass - b = B() - b.nextid = 0 - b.num_deleted = 0 - class A(object): - def __init__(self): - self.id = b.nextid - b.nextid += 1 - def __del__(self): - b.num_deleted += 1 - def f(): - a = A() - i = 0 - while i < 5: - i += 1 - a = A() - llop.gc__collect(lltype.Void) - llop.gc__collect(lltype.Void) - return b.num_deleted - run = self.getcompiled(f) - res = run() - assert res == 6 - - def test_del_catches(self): - import os - def g(): - pass - class A(object): - def __del__(self): - try: - g() - except: - os.write(1, "hallo") - def f1(i): - if i: - raise TypeError - def f(i=int): - a = A() - f1(i) - a.b = 1 - llop.gc__collect(lltype.Void) - return a.b - def f_0(): - try: - return f(0) - except TypeError: - return 42 - def f_1(): - try: - return f(1) - except TypeError: - return 42 - fn = self.getcompiled(f_0) - assert fn() == 1 - fn = self.getcompiled(f_1) - assert fn() == 42 - - def test_del_raises(self): - class B(object): - def __del__(self): - raise TypeError - def func(): - b = B() - return 0 - fn = self.getcompiled(func) - # does not crash - fn() - - def test_weakref(self): - import weakref - from pypy.rlib import rgc - - class A: - pass - - keepalive = [] - def fn(): - n = 7000 - weakrefs = [] - a = None - for i in range(n): - if i & 1 == 0: - a = A() - a.index = i - assert a is not None - weakrefs.append(weakref.ref(a)) - if i % 7 == 6: - keepalive.append(a) - rgc.collect() - count_free = 0 - for i in range(n): - a = weakrefs[i]() - if i % 7 == 6: - assert a is not None - if a is not None: - assert a.index == i & ~1 - else: - count_free += 1 - return count_free - c_fn = self.getcompiled(fn) - res = c_fn() - # more than half of them should have been freed, ideally up to 6000 - assert 3500 <= res <= 6000 - - def test_prebuilt_weakref(self): - import weakref - from pypy.rlib import rgc - class A: - pass - a = A() - a.hello = 42 - refs = [weakref.ref(a), weakref.ref(A())] - rgc.collect() - def fn(): - result = 0 - for i in range(2): - a = refs[i]() - rgc.collect() - if a is None: - result += (i+1) - else: - result += a.hello * (i+1) - return result - c_fn = self.getcompiled(fn) - res = c_fn() - assert res == fn() - - def test_framework_malloc_raw(self): - A = lltype.Struct('A', ('value', lltype.Signed)) - - def f(): - p = lltype.malloc(A, flavor='raw') - p.value = 123 - llop.gc__collect(lltype.Void) - res = p.value - lltype.free(p, flavor='raw') - return res - fn = self.getcompiled(f) - res = fn() - assert res == 123 - - def test_framework_del_seeing_new_types(self): - class B(object): - pass - class A(object): - def __del__(self): - B() - def f(): - A() - return 42 - fn = self.getcompiled(f) - res = fn() - assert res == 42 - - def test_memory_error_varsize(self): - py.test.skip("Needs lots (>2GB) of memory.") - import gc - import pypy.rlib.rgc - from pypy.rpython.lltypesystem import lltype - N = sys.maxint / 4 + 4 - A = lltype.GcArray(lltype.Signed) - def alloc(n): - return lltype.malloc(A, n) - def f(): - try: - try: - x = alloc(N) - except MemoryError: - y = alloc(10) - return len(y) - return -1 - finally: - gc.collect() - - fn = self.getcompiled(f) - res = fn() - assert res == 10 - N = sys.maxint / 4 - fn = self.getcompiled(f) - res = fn() - assert res == 10 - - N = sys.maxint / 4 - 1 - fn = self.getcompiled(f) - res = fn() - assert res == 10 - - N = sys.maxint / 8 + 1000 - def f(): - try: - x0 = alloc(N) - try: - x1 = alloc(N) - return len(x0) + len(x1) - except MemoryError: - y = alloc(10) - return len(y) - return -1 - finally: - gc.collect() - - fn = self.getcompiled(f) - res = fn() - assert res == 10 - - def test_framework_late_filling_pointers(self): - A = lltype.GcStruct('A', ('x', lltype.Signed)) - B = lltype.GcStruct('B', ('a', lltype.Ptr(A))) - - def f(): - p = lltype.malloc(B) - llop.gc__collect(lltype.Void) - p.a = lltype.malloc(A) - return p.a.x - fn = self.getcompiled(f) - # the point is just not to segfault - res = fn() - - def test_dict_segfault(self): - " was segfaulting at one point see rev 39665 for fix and details " - py.test.skip("Takes about 30 minutes in nightly test run, see rev 39665 for a minimal test that does the same") - - class Element: - pass - - elements = [Element() for ii in range(10000)] - - def dostuff(): - reverse = {} - l = elements[:] - - for ii in elements: - reverse[ii] = ii - - for jj in range(100): - e = l[-1] - del reverse[e] - l.remove(e) - - def f(): - for ii in range(100): - dostuff() - return 0 - - fn = self.getcompiled(f) - # the point is just not to segfault - res = fn() - - def test_zero_raw_malloc(self): - S = lltype.Struct('S', ('x', lltype.Signed), ('y', lltype.Signed)) - def f(): - for i in range(100): - p = lltype.malloc(S, flavor='raw', zero=True) - if p.x != 0 or p.y != 0: - return -1 - p.x = i - p.y = i - lltype.free(p, flavor='raw') - return 42 - - fn = self.getcompiled(f) - res = fn() - assert res == 42 - - def test_object_alignment(self): - # all objects returned by the GC should be properly aligned. - from pypy.rpython.lltypesystem import rffi - from pypy.rpython.tool import rffi_platform - mylist = ['a', 'bc', '84139871', 'ajkdh', '876'] - def f(): - result = 0 - buffer = "" - for j in range(100): - for s in mylist: - buffer += s - addr = rffi.cast(lltype.Signed, buffer) - result |= addr - return result - - fn = self.getcompiled(f) - res = fn() - expected_alignment = rffi_platform.memory_alignment() - assert (res & (expected_alignment-1)) == 0 - - def test_void_list(self): - class E: - def __init__(self): - self.l = [] - def f(): - e = E() - return len(e.l) - c_fn = self.getcompiled(f) - assert c_fn() == 0 - - def test_open_read_write_seek_close(self): - filename = str(udir.join('test_open_read_write_close.txt')) - def does_stuff(): - fd = os.open(filename, os.O_WRONLY | os.O_CREAT, 0777) - count = os.write(fd, "hello world\n") - assert count == len("hello world\n") - os.close(fd) - fd = os.open(filename, os.O_RDONLY, 0777) - result = os.lseek(fd, 1, 0) - assert result == 1 - data = os.read(fd, 500) - assert data == "ello world\n" - os.close(fd) - - f1 = self.getcompiled(does_stuff) - f1() - assert open(filename, 'r').read() == "hello world\n" - os.unlink(filename) - - def test_callback_with_collect(self): - from pypy.rlib.libffi import ffi_type_pointer, cast_type_to_ffitype,\ - CDLL, ffi_type_void, CallbackFuncPtr, ffi_type_sint - from pypy.rpython.lltypesystem import rffi, ll2ctypes - from pypy.rlib import rgc - import gc - slong = cast_type_to_ffitype(rffi.LONG) - - from pypy.rpython.lltypesystem.ll2ctypes import libc_name - - def callback(ll_args, ll_res, stuff): - gc.collect() - p_a1 = rffi.cast(rffi.VOIDPP, ll_args[0])[0] - p_a2 = rffi.cast(rffi.VOIDPP, ll_args[1])[0] - a1 = rffi.cast(rffi.INTP, p_a1)[0] - a2 = rffi.cast(rffi.INTP, p_a2)[0] - res = rffi.cast(rffi.INTP, ll_res) - if a1 > a2: - res[0] = 1 - else: - res[0] = -1 - - def f(): - libc = CDLL(libc_name) - qsort = libc.getpointer('qsort', [ffi_type_pointer, slong, - slong, ffi_type_pointer], - ffi_type_void) - - ptr = CallbackFuncPtr([ffi_type_pointer, ffi_type_pointer], - ffi_type_sint, callback) - - TP = rffi.CArray(rffi.INT) - to_sort = lltype.malloc(TP, 4, flavor='raw') - to_sort[0] = 4 - to_sort[1] = 3 - to_sort[2] = 1 - to_sort[3] = 2 - qsort.push_arg(rffi.cast(rffi.VOIDP, to_sort)) - qsort.push_arg(rffi.sizeof(rffi.INT)) - qsort.push_arg(4) - qsort.push_arg(rffi.cast(rffi.VOIDP, ptr.ll_closure)) - qsort.call(lltype.Void) - result = [to_sort[i] for i in range(4)] == [1,2,3,4] - lltype.free(to_sort, flavor='raw') - keepalive_until_here(ptr) - return int(result) - - c_fn = self.getcompiled(f) - assert c_fn() == 1 - - def test_can_move(self): - from pypy.rlib import rgc - class A: - pass - def fn(): - return rgc.can_move(A()) - - c_fn = self.getcompiled(fn) - assert c_fn() == self.GC_CAN_MOVE - - def test_malloc_nonmovable(self): - TP = lltype.GcArray(lltype.Char) - def func(): - try: - from pypy.rlib import rgc - a = rgc.malloc_nonmovable(TP, 3) - rgc.collect() - if a: - assert not rgc.can_move(a) - return 0 - return 1 - except Exception, e: - return 2 - - run = self.getcompiled(func) - assert run() == self.GC_CANNOT_MALLOC_NONMOVABLE - - def test_resizable_buffer(self): - from pypy.rpython.lltypesystem.rstr import STR - from pypy.rpython.annlowlevel import hlstr - from pypy.rlib import rgc - - def f(): - ptr = rgc.resizable_buffer_of_shape(STR, 2) - ptr.chars[0] = 'a' - ptr = rgc.resize_buffer(ptr, 1, 200) - ptr.chars[1] = 'b' - return hlstr(rgc.finish_building_buffer(ptr, 2)) == "ab" - - run = self.getcompiled(f) - assert run() == True - -class TestSemiSpaceGC(TestUsingFramework, snippet.SemiSpaceGCTests): - gcpolicy = "semispace" - should_be_moving = True - GC_CAN_MOVE = True - GC_CANNOT_MALLOC_NONMOVABLE = True - - def test_many_ids(self): - from pypy.rlib.objectmodel import compute_unique_id - class A(object): - pass - def f(): - from pypy.rpython.lltypesystem import lltype, rffi - alist = [A() for i in range(50000)] - idarray = lltype.malloc(rffi.INTP.TO, len(alist), flavor='raw') - # Compute the id of all elements of the list. The goal is - # to not allocate memory, so that if the GC needs memory to - # remember the ids, it will trigger some collections itself - i = 0 - while i < len(alist): - idarray[i] = compute_unique_id(alist[i]) - i += 1 - j = 0 - while j < 2: - if j == 1: # allocate some stuff between the two iterations - [A() for i in range(20000)] - i = 0 - while i < len(alist): - if idarray[i] != compute_unique_id(alist[i]): - return j * 1000000 + i - i += 1 - j += 1 - lltype.free(idarray, flavor='raw') - return -2 - fn = self.getcompiled(f) - res = fn() - assert res == -2 - - def test_gc_set_max_heap_size(self): - def g(n): - return 'x' * n - def fn(): - # the semispace size starts at 8MB for now, so setting a - # smaller limit has no effect - from pypy.rlib import rgc - # set to more than 32MB -- which should be rounded down to 32MB - rgc.set_max_heap_size(32*1024*1024 + 20000) - s1 = s2 = s3 = None - try: - s1 = g(400000) # ~ 400 KB - s2 = g(4000000) # ~ 4 MB - s3 = g(40000000) # ~ 40 MB - except MemoryError: - pass - return (s1 is not None) + (s2 is not None) + (s3 is not None) - c_fn = self.getcompiled(fn) - res = c_fn() - assert res == 2 - - def test_string_builder(self): - def fn(): - s = StringBuilder() - s.append("a") - s.append("abc") - s.append_slice("abc", 1, 2) - s.append_multiple_char('d', 4) - return s.build() - c_fn = self.getcompiled(fn) - res = c_fn() - assert res == "aabcbdddd" - - def test_string_builder_over_allocation(self): - import gc - def fn(): - s = StringBuilder(4) - s.append("abcd") - s.append("defg") - s.append("rty") - s.append_multiple_char('y', 1000) - gc.collect() - s.append_multiple_char('y', 1000) - res = s.build() - gc.collect() - return res - c_fn = self.getcompiled(fn) - res = c_fn() - assert res[1000] == 'y' - -class TestGenerationalGC(TestSemiSpaceGC): - gcpolicy = "generation" - should_be_moving = True - -class TestHybridGC(TestGenerationalGC): - gcpolicy = "hybrid" - should_be_moving = True - GC_CANNOT_MALLOC_NONMOVABLE = False - - def test_gc_set_max_heap_size(self): - py.test.skip("not implemented") - -class TestMarkCompactGC(TestSemiSpaceGC): - gcpolicy = "markcompact" - should_be_moving = True - - def test_gc_set_max_heap_size(self): - py.test.skip("not implemented") - - def test_finalizer_order(self): - py.test.skip("not implemented") From arigo at codespeak.net Mon Oct 5 11:37:02 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 5 Oct 2009 11:37:02 +0200 (CEST) Subject: [pypy-svn] r68159 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091005093702.394AE1683C6@codespeak.net> Author: arigo Date: Mon Oct 5 11:37:01 2009 New Revision: 68159 Modified: pypy/trunk/pypy/jit/metainterp/resume.py Log: Improve dump_storage(). Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Mon Oct 5 11:37:01 2009 @@ -179,7 +179,7 @@ storage.rd_consts = self.consts[:] storage.rd_snapshot = None if debug: - dump_storage(storage) + dump_storage(storage, liveboxes) return liveboxes def _getboxindex(self, box): @@ -314,7 +314,7 @@ # ____________________________________________________________ -def dump_storage(storage): +def dump_storage(storage, liveboxes): "For profiling only." import os from pypy.rlib import objectmodel @@ -327,6 +327,9 @@ for const in storage.rd_consts: os.write(fd, '\t"%s",\n' % (const.repr_rpython(),)) os.write(fd, '\t], [\n') + for box in liveboxes: + os.write(fd, '\t"%s",\n' % (box.repr_rpython(),)) + os.write(fd, '\t], [\n') if storage.rd_virtuals is not None: for virtual in storage.rd_virtuals: os.write(fd, '\t%s,\n' % (virtual.repr_rpython(),)) From arigo at codespeak.net Mon Oct 5 12:12:03 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 5 Oct 2009 12:12:03 +0200 (CEST) Subject: [pypy-svn] r68160 - pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86 Message-ID: <20091005101203.9A3401683E1@codespeak.net> Author: arigo Date: Mon Oct 5 12:12:02 2009 New Revision: 68160 Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/regalloc.py Log: Systematize the TempBox() usage -- it was missing a possibly_free_var() in a corner case, but I don't feel like writing a test for it, mostly because it's too obvious and boring. Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/regalloc.py Mon Oct 5 12:12:02 2009 @@ -472,7 +472,8 @@ assert l0 is eax assert l1 is ecx assert l2 is resultreg - self.rm.possibly_free_vars(op.args + [tmpvar]) + self.rm.possibly_free_vars(op.args) + self.rm.possibly_free_var(tmpvar) def consider_int_mod(self, op, ignored): self._consider_int_div_or_mod(op, edx, eax) @@ -668,9 +669,11 @@ # XXX kill this function at some point if isinstance(v, Box): loc = self.rm.make_sure_var_in_reg(v, [v]) - other_loc = self.rm.force_allocate_reg(TempBox(), [v]) + tempbox = TempBox() + other_loc = self.rm.force_allocate_reg(tempbox, [v]) self.assembler.load_effective_addr(loc, ofs_items,scale, other_loc) else: + tempbox = None other_loc = imm(ofs_items + (v.getint() << scale)) self._call(ResOperation(rop.NEW, [v], res_v), [other_loc], [v]) @@ -678,6 +681,8 @@ assert self.loc(res_v) == eax # now we have to reload length to some reasonable place self.rm.possibly_free_var(v) + if tempbox is not None: + self.rm.possibly_free_var(tempbox) self.PerformDiscard(ResOperation(rop.SETFIELD_GC, [], None), [eax, imm(ofs_length), imm(WORD), loc]) From arigo at codespeak.net Mon Oct 5 12:25:26 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 5 Oct 2009 12:25:26 +0200 (CEST) Subject: [pypy-svn] r68161 - in pypy/branch/merge-floats-via-sse2/pypy/jit: backend/test metainterp/test Message-ID: <20091005102526.991091683E3@codespeak.net> Author: arigo Date: Mon Oct 5 12:25:25 2009 New Revision: 68161 Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test/runner_test.py pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_executor.py Log: Add tests with Consts in addition to tests with Boxes only. The idea is that the pure operations should not have all-Consts arguments, but operations with more than one argument can take some Consts. This crashes the x86 backend. Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test/runner_test.py Mon Oct 5 12:25:25 2009 @@ -852,7 +852,7 @@ inputargs = [] operations = [] for opnum, boxargs, rettype, retvalue in tests: - inputargs += boxargs + inputargs += [box for box in boxargs if isinstance(box, Box)] if rettype == 'int': boxres = BoxInt() elif rettype == 'float': Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_executor.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_executor.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_executor.py Mon Oct 5 12:25:25 2009 @@ -168,6 +168,10 @@ list(_int_comparison_operations()) + list(_int_unary_operations())): yield opnum, [BoxInt(x) for x in args], retvalue + if len(args) > 1: + assert len(args) == 2 + yield opnum, [BoxInt(args[0]), ConstInt(args[1])], retvalue + yield opnum, [ConstInt(args[0]), BoxInt(args[1])], retvalue def test_int_ops(): @@ -224,6 +228,10 @@ else: boxargs.append(BoxInt(x)) yield opnum, boxargs, rettype, retvalue + if len(args) > 1: + assert len(args) == 2 + yield opnum, [boxargs[0], boxargs[1].constbox()], rettype, retvalue + yield opnum, [boxargs[0].constbox(), boxargs[1]], rettype, retvalue def test_float_ops(): cpu = FakeCPU() From arigo at codespeak.net Mon Oct 5 12:49:10 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 5 Oct 2009 12:49:10 +0200 (CEST) Subject: [pypy-svn] r68162 - pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test Message-ID: <20091005104910.CEE5716801F@codespeak.net> Author: arigo Date: Mon Oct 5 12:49:09 2009 New Revision: 68162 Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_executor.py Log: Uh? Nonsense. Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_executor.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_executor.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_executor.py Mon Oct 5 12:49:09 2009 @@ -147,7 +147,6 @@ for i in range(20): x = pick() y = pick() - res = execute_nonspec(cpu, opnum, [BoxInt(x), BoxInt(y)]) z = int(operation(x, y)) yield opnum, [x, y], z From arigo at codespeak.net Mon Oct 5 12:57:44 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 5 Oct 2009 12:57:44 +0200 (CEST) Subject: [pypy-svn] r68163 - pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test Message-ID: <20091005105744.4685616801F@codespeak.net> Author: arigo Date: Mon Oct 5 12:57:43 2009 New Revision: 68163 Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test/runner_test.py Log: Oups, there was already code in runner_test about inserting some Consts in some cases. Remove it. Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/test/runner_test.py Mon Oct 5 12:57:43 2009 @@ -247,28 +247,14 @@ def test_int_operations(self): from pypy.jit.metainterp.test.test_executor import get_int_tests for opnum, boxargs, retvalue in get_int_tests(): - if len(boxargs) == 2: - args_variants = [(boxargs[0], boxargs[1]), - (boxargs[0], boxargs[1].constbox()), - (boxargs[0].constbox(), boxargs[1])] - else: - args_variants = [boxargs] - for argboxes in args_variants: - res = self.execute_operation(opnum, argboxes, 'int') - assert res.value == retvalue + res = self.execute_operation(opnum, boxargs, 'int') + assert res.value == retvalue def test_float_operations(self): from pypy.jit.metainterp.test.test_executor import get_float_tests for opnum, boxargs, rettype, retvalue in get_float_tests(self.cpu): - if len(boxargs) == 2: - args_variants = [(boxargs[0], boxargs[1]), - (boxargs[0], boxargs[1].constbox()), - (boxargs[0].constbox(), boxargs[1])] - else: - args_variants = [boxargs] - for argboxes in args_variants: - res = self.execute_operation(opnum, argboxes, rettype) - assert res.value == retvalue + res = self.execute_operation(opnum, boxargs, rettype) + assert res.value == retvalue def test_ovf_operations(self, reversed=False): minint = -sys.maxint-1 From fijal at codespeak.net Mon Oct 5 13:17:59 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Oct 2009 13:17:59 +0200 (CEST) Subject: [pypy-svn] r68164 - in pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86: . test Message-ID: <20091005111759.528D61683BF@codespeak.net> Author: fijal Date: Mon Oct 5 13:17:58 2009 New Revision: 68164 Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/assembler.py pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py Log: A bug and a fix Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/assembler.py Mon Oct 5 13:17:58 2009 @@ -447,7 +447,7 @@ def genop_float_is_true(self, op, arglocs, resloc): loc0, loc1 = arglocs self.mc.XORPD(loc0, loc0) - self.mc.UCOMISD(loc1, loc0) + self.mc.UCOMISD(loc0, loc1) self.mc.SETNE(lower_byte(resloc)) self.mc.MOVZX(resloc, lower_byte(resloc)) Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py Mon Oct 5 13:17:58 2009 @@ -536,3 +536,21 @@ self.run(loop) assert self.getints(9) == range(9) assert self.getfloat(0) == 3.5 + + def test_bug_float_is_true_stack(self): + ops = ''' + [f0, f1, f2, f3, f4, f5, f6, f7, f8, f9] + i0 = float_is_true(f0) + i1 = float_is_true(f1) + i2 = float_is_true(f2) + i3 = float_is_true(f3) + i4 = float_is_true(f4) + i5 = float_is_true(f5) + i6 = float_is_true(f6) + i7 = float_is_true(f7) + i8 = float_is_true(f8) + i9 = float_is_true(f9) + finish(i0, i1, i2, i3, i4, i5, i6, i7, i8, i9) + ''' + loop = self.interpret(ops, [0.0, .1, .2, .3, .4, .5, .6, .7, .8, .9]) + assert self.getints(9) == [0, 1, 1, 1, 1, 1, 1, 1, 1] From fijal at codespeak.net Mon Oct 5 13:22:07 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Oct 2009 13:22:07 +0200 (CEST) Subject: [pypy-svn] r68165 - pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test Message-ID: <20091005112207.7DBFC1683BF@codespeak.net> Author: fijal Date: Mon Oct 5 13:22:07 2009 New Revision: 68165 Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py Log: Remove outdated test Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/test/test_regalloc.py Mon Oct 5 13:22:07 2009 @@ -515,28 +515,6 @@ self.interpret(ops, [0.1]) assert self.getint(0) == 0 - @py.test.mark.xfail # this test is bogus in the branch floats-via-sse2 - def test_bug_wrong_stack_adj(self): - ops = ''' - [i0, i1, i2, i3, i4, i5, i6, i7, i8] - guard_true(i0) - fail(3.5, i0, i1, i2, i3, i4, i5, i6, i7, i8) - fail(4.5, i0, i1, i2, i3, i4, i5, i6, i7, i8) - ''' - loop = self.interpret(ops, [0, 1, 2, 3, 4, 5, 6, 7, 8]) - assert self.getint(0) == 0 - bridge_ops = ''' - [i0, i1, i2, i3, i4, i5, i6, i7, i8] - call(ConstClass(raising_fptr), 0, descr=raising_calldescr) - fail(i0, i1, i2, i3, i4, i5, i6, i7, i8) - ''' - self.attach_bridge(bridge_ops, loop, 0) - for i in range(9): - self.cpu.set_future_value_int(i, i) - self.run(loop) - assert self.getints(9) == range(9) - assert self.getfloat(0) == 3.5 - def test_bug_float_is_true_stack(self): ops = ''' [f0, f1, f2, f3, f4, f5, f6, f7, f8, f9] From fijal at codespeak.net Mon Oct 5 13:39:54 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Oct 2009 13:39:54 +0200 (CEST) Subject: [pypy-svn] r68166 - pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test Message-ID: <20091005113954.EFE9D1683CF@codespeak.net> Author: fijal Date: Mon Oct 5 13:39:54 2009 New Revision: 68166 Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizeopt.py Log: Add a failing test for optimizer, there is more than one to explode like this by returning BoxInt, needs careful look all over the place Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizefindnode.py Mon Oct 5 13:39:54 2009 @@ -49,6 +49,7 @@ NODE = lltype.GcForwardReference() NODE.become(lltype.GcStruct('NODE', ('parent', OBJECT), ('value', lltype.Signed), + ('floatval', lltype.Float), ('next', lltype.Ptr(NODE)))) NODE2 = lltype.GcStruct('NODE2', ('parent', NODE), ('other', lltype.Ptr(NODE))) @@ -61,6 +62,7 @@ nodesize = cpu.sizeof(NODE) nodesize2 = cpu.sizeof(NODE2) valuedescr = cpu.fielddescrof(NODE, 'value') + floatdescr = cpu.fielddescrof(NODE, 'floatval') nextdescr = cpu.fielddescrof(NODE, 'next') otherdescr = cpu.fielddescrof(NODE2, 'other') Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizeopt.py Mon Oct 5 13:39:54 2009 @@ -584,6 +584,23 @@ self.optimize_loop(ops, 'Not, Virtual(node_vtable, valuedescr=Not)', expected, checkspecnodes=False) + @py.test.mark.xfail() + def test_virtual_float(self): + ops = """ + [f, p0] + f0 = getfield_gc(p0, descr=floatdescr) + f1 = float_add(f0, f) + setfield_gc(p0, f1, descr=floatdescr) + jump(f, p0) + """ + expected = """ + [f, f2] + f1 = int_add(f2, f) + jump(f, f1) + """ + self.optimize_loop(ops, 'Not, Virtual(node_vtable, floatdescr=Not)', + expected, checkspecnodes=False) + def test_virtual_2(self): ops = """ [i, p0] From fijal at codespeak.net Mon Oct 5 14:23:18 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Oct 2009 14:23:18 +0200 (CEST) Subject: [pypy-svn] r68167 - in pypy/branch/merge-floats-via-sse2/pypy/jit: backend/llgraph metainterp metainterp/test Message-ID: <20091005122318.611C11683DC@codespeak.net> Author: fijal Date: Mon Oct 5 14:23:17 2009 New Revision: 68167 Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llgraph/runner.py pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/optimizeopt.py pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizeopt.py Log: A couple of fixes for optimizeopt Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llgraph/runner.py Mon Oct 5 14:23:17 2009 @@ -47,9 +47,15 @@ def is_pointer_field(self): return self.typeinfo == REF + def is_float_field(self): + return self.typeinfo == FLOAT + def is_array_of_pointers(self): return self.typeinfo == REF + def is_array_of_floats(self): + return self.typeinfo == FLOAT + def equals(self, other): if not isinstance(other, Descr): return False @@ -775,6 +781,7 @@ self.getfield = getfield self.setfield = setfield self._is_pointer_field = (history.getkind(T) == 'ref') + self._is_float_field = (history.getkind(T) == 'float') def sort_key(self): return self._keys.getkey((self.TYPE, self.fieldname)) @@ -782,6 +789,9 @@ def is_pointer_field(self): return self._is_pointer_field + def is_float_field(self): + return self._is_float_field + def equals(self, other): return self.TYPE == other.TYPE and \ self.fieldname == other.fieldname Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/optimizeopt.py Mon Oct 5 14:23:17 2009 @@ -1,4 +1,5 @@ -from pypy.jit.metainterp.history import Box, BoxInt, LoopToken +from pypy.jit.metainterp.history import Box, BoxInt, LoopToken, BoxFloat,\ + ConstFloat from pypy.jit.metainterp.history import Const, ConstInt, ConstPtr, ConstObj, REF from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.metainterp.executor import execute_nonspec @@ -143,6 +144,7 @@ CONST_0 = ConstInt(0) CONST_1 = ConstInt(1) CVAL_ZERO = ConstantValue(CONST_0) +CVAL_ZERO_FLOAT = ConstantValue(ConstFloat(0.0)) llhelper.CONST_NULL = ConstPtr(ConstPtr.value) llhelper.CVAL_NULLREF = ConstantValue(llhelper.CONST_NULL) oohelper.CONST_NULL = ConstObj(ConstObj.value) @@ -438,24 +440,32 @@ def new_box(self, fieldofs): if fieldofs.is_pointer_field(): return self.new_ptr_box() + elif fieldofs.is_float_field(): + return BoxFloat() else: return BoxInt() def new_const(self, fieldofs): if fieldofs.is_pointer_field(): return self.cpu.ts.CVAL_NULLREF + elif fieldofs.is_float_field(): + return CVAL_ZERO_FLOAT else: return CVAL_ZERO def new_box_item(self, arraydescr): if arraydescr.is_array_of_pointers(): return self.new_ptr_box() + elif arraydescr.is_array_of_floats(): + return BoxFloat() else: return BoxInt() def new_const_item(self, arraydescr): if arraydescr.is_array_of_pointers(): return self.cpu.ts.CVAL_NULLREF + elif arraydescr.is_array_of_floats(): + return CVAL_ZERO_FLOAT else: return CVAL_ZERO Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizefindnode.py Mon Oct 5 14:23:17 2009 @@ -107,6 +107,7 @@ cpu = runner.OOtypeCPU(None) NODE = ootype.Instance('NODE', ootype.ROOT, {}) NODE._add_fields({'value': ootype.Signed, + 'floatval' : ootype.Float, 'next': NODE}) NODE2 = ootype.Instance('NODE2', NODE, {'other': NODE}) @@ -121,6 +122,7 @@ myptr2 = ootype.cast_to_object(ootype.new(NODE)) nodebox2 = BoxObj(ootype.cast_to_object(node)) valuedescr = cpu.fielddescrof(NODE, 'value') + floatdescr = cpu.fielddescrof(NODE, 'floatval') nextdescr = cpu.fielddescrof(NODE, 'next') otherdescr = cpu.fielddescrof(NODE2, 'other') nodesize = cpu.typedescrof(NODE) Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizeopt.py Mon Oct 5 14:23:17 2009 @@ -584,7 +584,6 @@ self.optimize_loop(ops, 'Not, Virtual(node_vtable, valuedescr=Not)', expected, checkspecnodes=False) - @py.test.mark.xfail() def test_virtual_float(self): ops = """ [f, p0] @@ -595,7 +594,7 @@ """ expected = """ [f, f2] - f1 = int_add(f2, f) + f1 = float_add(f2, f) jump(f, f1) """ self.optimize_loop(ops, 'Not, Virtual(node_vtable, floatdescr=Not)', From fijal at codespeak.net Mon Oct 5 14:26:15 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Oct 2009 14:26:15 +0200 (CEST) Subject: [pypy-svn] r68168 - in pypy/branch/merge-floats-via-sse2/pypy/jit: backend/llgraph metainterp/test Message-ID: <20091005122615.3F2CB1683DC@codespeak.net> Author: fijal Date: Mon Oct 5 14:26:14 2009 New Revision: 68168 Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llgraph/runner.py pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizeopt.py Log: More tests & fixes Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/llgraph/runner.py Mon Oct 5 14:26:14 2009 @@ -751,11 +751,16 @@ self.getarraylength = getarraylength self.instanceof = instanceof self._is_array_of_pointers = (history.getkind(TYPE) == 'ref') + self._is_array_of_floats = (history.getkind(TYPE) == 'float') def is_array_of_pointers(self): # for arrays, TYPE is the type of the array item. return self._is_array_of_pointers + def is_array_of_floats(self): + # for arrays, TYPE is the type of the array item. + return self._is_array_of_floats + def __repr__(self): return '' % self.TYPE._short_name() Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizefindnode.py Mon Oct 5 14:26:14 2009 @@ -67,6 +67,7 @@ otherdescr = cpu.fielddescrof(NODE2, 'other') arraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Signed)) + floatarraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Float)) # a GcStruct not inheriting from OBJECT S = lltype.GcStruct('TUPLE', ('a', lltype.Signed), ('b', lltype.Ptr(NODE))) @@ -129,6 +130,7 @@ nodesize2 = cpu.typedescrof(NODE2) arraydescr = cpu.arraydescrof(ootype.Array(ootype.Signed)) + floatarraydescr = cpu.arraydescrof(ootype.Array(ootype.Float)) # a plain Record S = ootype.Record({'a': ootype.Signed, 'b': NODE}) Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_optimizeopt.py Mon Oct 5 14:26:14 2009 @@ -902,6 +902,23 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_varray_float(self): + ops = """ + [f1] + p1 = new_array(3, descr=floatarraydescr) + i3 = arraylen_gc(p1, descr=floatarraydescr) + guard_value(i3, 3) [] + setarrayitem_gc(p1, 1, f1, descr=floatarraydescr) + setarrayitem_gc(p1, 0, 3.5, descr=floatarraydescr) + f2 = getarrayitem_gc(p1, 1, descr=floatarraydescr) + jump(f2) + """ + expected = """ + [f1] + jump(f1) + """ + self.optimize_loop(ops, 'Not', expected) + def test_array_non_optimized(self): ops = """ [i1, p0] From fijal at codespeak.net Mon Oct 5 14:51:07 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Oct 2009 14:51:07 +0200 (CEST) Subject: [pypy-svn] r68169 - pypy/extradoc/talk/pycon2010 Message-ID: <20091005125107.117D31683CF@codespeak.net> Author: fijal Date: Mon Oct 5 14:51:07 2009 New Revision: 68169 Modified: pypy/extradoc/talk/pycon2010/jit_abstract.txt Log: Try to expand timeline Modified: pypy/extradoc/talk/pycon2010/jit_abstract.txt ============================================================================== --- pypy/extradoc/talk/pycon2010/jit_abstract.txt (original) +++ pypy/extradoc/talk/pycon2010/jit_abstract.txt Mon Oct 5 14:51:07 2009 @@ -21,14 +21,28 @@ Proposed talk agenda: -* Show various benchmarking data and compare different Python interpreters +* Show various benchmarking data for comparison of different + Python interpreters (5 minutes) + +* Describe what is JIT with some examples (10 minutes) * A bit of introduction into how PyPy's JIT work (it's a generated tracing JIT, - like the tracemonkey JS interpreter, unlike Psyco or JVM JITs). + like the tracemonkey JS interpreter, unlike Psyco or JVM JITs). (5 minutes) * Tracing JITs focus on often executed loops. Which kind of programs will the PyPy JIT speed-up, which constructs we know it is currently poor at. + (15 minutes) * Future plans for PyPy, especially JIT-wise. + (5 minutes) + +* Q&A (5 minutes) .. _`faster than Psyco`: http://morepypy.blogspot.com/2009/09/first-results-of-jit.html + +The main part of the talk would focus on a detailed explanation what +information JIT can use (at the time of talk) or at all and what +constructs it'll be poor at. + +Also, the talk will be focused on intermediate audience, hence prolonged +timeline for explanation of what is JIT and how it works. From arigo at codespeak.net Mon Oct 5 15:19:06 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 5 Oct 2009 15:19:06 +0200 (CEST) Subject: [pypy-svn] r68170 - pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86 Message-ID: <20091005131906.B04591683CF@codespeak.net> Author: arigo Date: Mon Oct 5 15:19:05 2009 New Revision: 68170 Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/assembler.py Log: This _no_const_locs() makes zero sense at all (to take words from Samuele). Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/assembler.py Mon Oct 5 15:19:05 2009 @@ -63,12 +63,6 @@ if name.upper() == name: setattr(MachineCodeBlockWrapper, name, _new_method(name)) -class ExecutableToken386(object): - _x86_loop_code = 0 - _x86_bootstrap_code = 0 - _x86_stack_depth = 0 - _x86_arglocs = (None, None) - class Assembler386(object): mc = None mc2 = None @@ -724,22 +718,19 @@ self.mc.CMP(mem(locs[0], offset), locs[1]) return self.implement_guard(addr, self.mc.JNE) - def _no_const_locs(self, args, locs): - """ returns those locs which correspond to non-const args + def _no_const_locs(self, args): + """ assert that all args are actually Boxes """ - newlocs = [] - for i in range(len(args)): - arg = args[i] - if isinstance(arg, Box): - newlocs.append(locs[i]) - return newlocs + for arg in args: + assert isinstance(arg, Box) def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): + self._no_const_locs(failargs) addr = self.mc2.tell() exc = (guard_opnum == rop.GUARD_EXCEPTION or guard_opnum == rop.GUARD_NO_EXCEPTION) - faildescr._x86_faillocs = self._no_const_locs(failargs, fail_locs) + faildescr._x86_faillocs = fail_locs self.generate_failure(self.mc2, faildescr, failargs, fail_locs, exc) return addr @@ -749,12 +740,11 @@ for i in range(len(failargs)): arg = failargs[i] loc = locs[i] - if arg.type == FLOAT: - if isinstance(loc, REG): + if isinstance(loc, REG): + if arg.type == FLOAT: mc.MOVSD(addr64_add(imm(self.fail_box_float_addr), imm(i*WORD*2)), loc) - else: - if isinstance(loc, REG): + else: if arg.type == REF: base = self.fail_box_ptr_addr else: @@ -763,13 +753,12 @@ for i in range(len(failargs)): arg = failargs[i] loc = locs[i] - if arg.type == FLOAT: - if not isinstance(loc, REG): + if not isinstance(loc, REG): + if arg.type == FLOAT: mc.MOVSD(xmm0, loc) mc.MOVSD(addr64_add(imm(self.fail_box_float_addr), imm(i*WORD*2)), xmm0) - else: - if not isinstance(loc, REG): + else: if arg.type == REF: base = self.fail_box_ptr_addr else: From fijal at codespeak.net Mon Oct 5 15:38:18 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Oct 2009 15:38:18 +0200 (CEST) Subject: [pypy-svn] r68171 - in pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp: . test Message-ID: <20091005133818.288BD1683DD@codespeak.net> Author: fijal Date: Mon Oct 5 15:38:17 2009 New Revision: 68171 Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_virtual.py pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/warmspot.py Log: A test and fix a bug Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_virtual.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_virtual.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/test/test_virtual.py Mon Oct 5 15:38:17 2009 @@ -33,6 +33,23 @@ self.check_loops(new=0, new_with_vtable=0, getfield_gc=0, setfield_gc=0) + def test_virtualized_float(self): + myjitdriver = JitDriver(greens = [], reds = ['n', 'node']) + def f(n): + node = self._new() + node.floatval = 0.0 + while n > 0: + myjitdriver.can_enter_jit(n=n, node=node) + myjitdriver.jit_merge_point(n=n, node=node) + next = self._new() + next.floatval = node.floatval + .5 + n -= 1 + return node.floatval + res = self.meta_interp(f, [10]) + assert res == f(10) + self.check_loop_count(1) + self.check_loops(new=0, float_add=1) + def test_virtualized_2(self): myjitdriver = JitDriver(greens = [], reds = ['n', 'node']) def f(n): @@ -339,6 +356,7 @@ # Run 2: all the tests use lltype.malloc to make a NODE NODE = lltype.GcStruct('NODE', ('value', lltype.Signed), + ('floatval', lltype.Float), ('extra', lltype.Signed)) class TestLLtype_NotObject(VirtualTests, LLJitMixin): @@ -352,6 +370,7 @@ OONODE = ootype.Instance('NODE', ootype.ROOT, {}) OONODE._add_fields({'value': ootype.Signed, + 'floatval' : ootype.Float, 'extra': ootype.Signed}) class TestOOtype_NotObject(VirtualTests, OOJitMixin): @@ -367,6 +386,7 @@ # (same as Run 2 but it is part of the OBJECT hierarchy) NODE2 = lltype.GcStruct('NODE2', ('parent', rclass.OBJECT), + ('floatval', lltype.Float), ('value', lltype.Signed), ('extra', lltype.Signed)) Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/metainterp/warmspot.py Mon Oct 5 15:38:17 2009 @@ -434,7 +434,7 @@ return 'DoneWithThisFrameRef(%s)' % (self.result,) class DoneWithThisFrameFloat(JitException): - def __init__(self, cpu, result): + def __init__(self, result): assert lltype.typeOf(result) is lltype.Float self.result = result def __str__(self): From arigo at codespeak.net Mon Oct 5 15:45:48 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 5 Oct 2009 15:45:48 +0200 (CEST) Subject: [pypy-svn] r68172 - pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86 Message-ID: <20091005134548.65BAA1683E8@codespeak.net> Author: arigo Date: Mon Oct 5 15:45:48 2009 New Revision: 68172 Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/assembler.py Log: Comment out the stack alignment code. Mac OS/X support is incomplete anyway, and this logic was now confused about what type of argument it received. Modified: pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/merge-floats-via-sse2/pypy/jit/backend/x86/assembler.py Mon Oct 5 15:45:48 2009 @@ -360,17 +360,18 @@ name = 'J' + false_cond return self.implement_guard(addr, getattr(self.mc, name)) return genop_cmp_guard - - def align_stack_for_call(self, nargs): - # xxx do something when we don't use push anymore for calls - extra_on_stack = align_stack_words(nargs) - for i in range(extra_on_stack-nargs): - self.mc.PUSH(imm(0)) - return extra_on_stack + +## XXX redo me +## def align_stack_for_call(self, nargs): +## # xxx do something when we don't use push anymore for calls +## extra_on_stack = align_stack_words(nargs) +## for i in range(extra_on_stack-nargs): +## self.mc.PUSH(imm(0)) --- or just use a single SUB(esp, imm) +## return extra_on_stack def call(self, addr, args, res): nargs = len(args) - extra_on_stack = self.align_stack_for_call(nargs) + extra_on_stack = nargs #self.align_stack_for_call(nargs) for i in range(nargs-1, -1, -1): self.mc.PUSH(args[i]) self.mc.CALL(rel32(addr)) @@ -804,7 +805,7 @@ extra_on_stack = 0 for arg in range(2, nargs + 2): extra_on_stack += round_up_to_4(arglocs[arg].width) - extra_on_stack = self.align_stack_for_call(extra_on_stack) + #extra_on_stack = self.align_stack_for_call(extra_on_stack) self.mc.SUB(esp, imm(extra_on_stack)) if isinstance(op.args[0], Const): x = rel32(op.args[0].getint()) From fijal at codespeak.net Mon Oct 5 16:43:59 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Oct 2009 16:43:59 +0200 (CEST) Subject: [pypy-svn] r68173 - in pypy/trunk/pypy: jit/backend/llgraph jit/backend/llsupport jit/backend/llsupport/test jit/backend/test jit/backend/x86 jit/backend/x86/test jit/metainterp jit/metainterp/test module/pypyjit Message-ID: <20091005144359.338791683E2@codespeak.net> Author: fijal Date: Mon Oct 5 16:43:56 2009 New Revision: 68173 Modified: pypy/trunk/pypy/jit/backend/llgraph/runner.py pypy/trunk/pypy/jit/backend/llsupport/descr.py pypy/trunk/pypy/jit/backend/llsupport/llmodel.py pypy/trunk/pypy/jit/backend/llsupport/regalloc.py pypy/trunk/pypy/jit/backend/llsupport/symbolic.py pypy/trunk/pypy/jit/backend/llsupport/test/test_descr.py pypy/trunk/pypy/jit/backend/llsupport/test/test_regalloc.py pypy/trunk/pypy/jit/backend/llsupport/test/test_runner.py pypy/trunk/pypy/jit/backend/test/runner_test.py pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/jump.py pypy/trunk/pypy/jit/backend/x86/regalloc.py pypy/trunk/pypy/jit/backend/x86/ri386.py pypy/trunk/pypy/jit/backend/x86/ri386setup.py pypy/trunk/pypy/jit/backend/x86/runner.py pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py pypy/trunk/pypy/jit/backend/x86/test/test_jump.py pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py pypy/trunk/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py pypy/trunk/pypy/jit/metainterp/history.py pypy/trunk/pypy/jit/metainterp/logger.py pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/oparser.py pypy/trunk/pypy/jit/metainterp/test/test_executor.py pypy/trunk/pypy/jit/metainterp/test/test_logger.py pypy/trunk/pypy/jit/metainterp/test/test_oparser.py pypy/trunk/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_virtual.py pypy/trunk/pypy/jit/metainterp/warmspot.py pypy/trunk/pypy/module/pypyjit/policy.py Log: (arigo, fijal) Merge floats-via-sse2 branch. This branch adds float support to x86 backend, via SSE2 instruction set, present in Pentium IV and newer. Modified: pypy/trunk/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/runner.py Mon Oct 5 16:43:56 2009 @@ -47,9 +47,15 @@ def is_pointer_field(self): return self.typeinfo == REF + def is_float_field(self): + return self.typeinfo == FLOAT + def is_array_of_pointers(self): return self.typeinfo == REF + def is_array_of_floats(self): + return self.typeinfo == FLOAT + def equals(self, other): if not isinstance(other, Descr): return False @@ -467,9 +473,10 @@ assert isinstance(calldescr, Descr) func = args[0].getint() for arg in args[1:]: - if (isinstance(arg, history.BoxPtr) or - isinstance(arg, history.ConstPtr)): + if arg.type == REF: llimpl.do_call_pushptr(arg.getref_base()) + elif arg.type == FLOAT: + llimpl.do_call_pushfloat(arg.getfloat()) else: llimpl.do_call_pushint(arg.getint()) if calldescr.typeinfo == REF: @@ -744,11 +751,16 @@ self.getarraylength = getarraylength self.instanceof = instanceof self._is_array_of_pointers = (history.getkind(TYPE) == 'ref') + self._is_array_of_floats = (history.getkind(TYPE) == 'float') def is_array_of_pointers(self): # for arrays, TYPE is the type of the array item. return self._is_array_of_pointers + def is_array_of_floats(self): + # for arrays, TYPE is the type of the array item. + return self._is_array_of_floats + def __repr__(self): return '' % self.TYPE._short_name() @@ -774,6 +786,7 @@ self.getfield = getfield self.setfield = setfield self._is_pointer_field = (history.getkind(T) == 'ref') + self._is_float_field = (history.getkind(T) == 'float') def sort_key(self): return self._keys.getkey((self.TYPE, self.fieldname)) @@ -781,6 +794,9 @@ def is_pointer_field(self): return self._is_pointer_field + def is_float_field(self): + return self._is_float_field + def equals(self, other): return self.TYPE == other.TYPE and \ self.fieldname == other.fieldname Modified: pypy/trunk/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/descr.py Mon Oct 5 16:43:56 2009 @@ -1,7 +1,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.jit.backend.llsupport import symbolic from pypy.jit.metainterp.history import AbstractDescr, getkind, BoxInt, BoxPtr -from pypy.jit.metainterp.history import BasicFailDescr, LoopToken +from pypy.jit.metainterp.history import BasicFailDescr, LoopToken, BoxFloat from pypy.jit.metainterp.resoperation import ResOperation, rop # The point of the class organization in this file is to make instances @@ -67,8 +67,14 @@ def get_field_size(self, translate_support_code): raise NotImplementedError + _is_pointer_field = False # unless overridden by GcPtrFieldDescr + _is_float_field = False # unless overridden by FloatFieldDescr + def is_pointer_field(self): - return False # unless overridden by GcPtrFieldDescr + return self._is_pointer_field + + def is_float_field(self): + return self._is_float_field def repr_of_descr(self): return '<%s %s>' % (self._clsname, self.offset) @@ -81,12 +87,12 @@ class GcPtrFieldDescr(NonGcPtrFieldDescr): _clsname = 'GcPtrFieldDescr' - def is_pointer_field(self): - return True + _is_pointer_field = True def getFieldDescrClass(TYPE): return getDescrClass(TYPE, BaseFieldDescr, GcPtrFieldDescr, - NonGcPtrFieldDescr, 'Field', 'get_field_size') + NonGcPtrFieldDescr, 'Field', 'get_field_size', + '_is_float_field') def get_field_descr(gccache, STRUCT, fieldname): cache = gccache._cache_field @@ -122,8 +128,14 @@ def get_item_size(self, translate_support_code): raise NotImplementedError + _is_array_of_pointers = False # unless overridden by GcPtrArrayDescr + _is_array_of_floats = False # unless overridden by FloatArrayDescr + def is_array_of_pointers(self): - return False # unless overridden by GcPtrArrayDescr + return self._is_array_of_pointers + + def is_array_of_floats(self): + return self._is_array_of_floats def repr_of_descr(self): return '<%s>' % self._clsname @@ -136,12 +148,12 @@ class GcPtrArrayDescr(NonGcPtrArrayDescr): _clsname = 'GcPtrArrayDescr' - def is_array_of_pointers(self): - return True + _is_array_of_pointers = True def getArrayDescrClass(ARRAY): return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr, - NonGcPtrArrayDescr, 'Array', 'get_item_size') + NonGcPtrArrayDescr, 'Array', 'get_item_size', + '_is_array_of_floats') def get_array_descr(gccache, ARRAY): cache = gccache._cache_array @@ -174,13 +186,20 @@ def instantiate_arg_classes(self): result = [] for c in self.arg_classes: - if c == 'i': box = BoxInt() - else: box = BoxPtr() + if c == 'i': box = BoxInt() + elif c == 'f': box = BoxFloat() + else: box = BoxPtr() result.append(box) return result + _returns_a_pointer = False # unless overridden by GcPtrCallDescr + _returns_a_float = False # unless overridden by FloatCallDescr + def returns_a_pointer(self): - return False # unless overridden by GcPtrCallDescr + return self._returns_a_pointer + + def returns_a_float(self): + return self._returns_a_float def get_result_size(self, translate_support_code): raise NotImplementedError @@ -195,6 +214,8 @@ else: if self.returns_a_pointer(): result = BoxPtr() + elif self.returns_a_float(): + result = BoxFloat() else: result = BoxInt() result_list = [result] @@ -221,8 +242,7 @@ class GcPtrCallDescr(NonGcPtrCallDescr): _clsname = 'GcPtrCallDescr' - def returns_a_pointer(self): - return True + _returns_a_pointer = True class VoidCallDescr(NonGcPtrCallDescr): _clsname = 'VoidCallDescr' @@ -233,7 +253,8 @@ if RESULT is lltype.Void: return VoidCallDescr return getDescrClass(RESULT, BaseCallDescr, GcPtrCallDescr, - NonGcPtrCallDescr, 'Call', 'get_result_size') + NonGcPtrCallDescr, 'Call', 'get_result_size', + '_returns_a_float') def get_call_descr(gccache, ARGS, RESULT): arg_classes = [] @@ -241,6 +262,7 @@ kind = getkind(ARG) if kind == 'int': arg_classes.append('i') elif kind == 'ref': arg_classes.append('r') + elif kind == 'float': arg_classes.append('f') else: raise NotImplementedError('ARG = %r' % (ARG,)) arg_classes = ''.join(arg_classes) @@ -258,7 +280,7 @@ # ____________________________________________________________ def getDescrClass(TYPE, BaseDescr, GcPtrDescr, NonGcPtrDescr, - nameprefix, methodname, _cache={}): + nameprefix, methodname, floatattrname, _cache={}): if isinstance(TYPE, lltype.Ptr): if TYPE.TO._gckind == 'gc': return GcPtrDescr @@ -276,5 +298,8 @@ return symbolic.get_size(TYPE, translate_support_code) setattr(Descr, methodname, method) # + if TYPE is lltype.Float: + setattr(Descr, floatattrname, True) + # _cache[nameprefix, TYPE] = Descr return Descr Modified: pypy/trunk/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/llmodel.py Mon Oct 5 16:43:56 2009 @@ -4,7 +4,8 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.rpython.annlowlevel import llhelper from pypy.rlib.objectmodel import we_are_translated, specialize -from pypy.jit.metainterp.history import BoxInt, BoxPtr, set_future_values +from pypy.jit.metainterp.history import BoxInt, BoxPtr, set_future_values,\ + BoxFloat from pypy.jit.backend.model import AbstractCPU from pypy.jit.backend.llsupport import symbolic from pypy.jit.backend.llsupport.symbolic import WORD, unroll_basic_sizes @@ -205,7 +206,8 @@ ofs = fielddescr.offset size = fielddescr.get_field_size(self.translate_support_code) ptr = fielddescr.is_pointer_field() - return ofs, size, ptr + float = fielddescr.is_float_field() + return ofs, size, ptr, float unpack_fielddescr._always_inline_ = True def arraydescrof(self, A): @@ -216,7 +218,8 @@ ofs = arraydescr.get_base_size(self.translate_support_code) size = arraydescr.get_item_size(self.translate_support_code) ptr = arraydescr.is_array_of_pointers() - return ofs, size, ptr + float = arraydescr.is_array_of_floats() + return ofs, size, ptr, float unpack_arraydescr._always_inline_ = True def calldescrof(self, FUNC, ARGS, RESULT): @@ -246,40 +249,66 @@ def do_getarrayitem_gc(self, arraybox, indexbox, arraydescr): itemindex = indexbox.getint() gcref = arraybox.getref_base() - ofs, size, ptr = self.unpack_arraydescr(arraydescr) + ofs, size, ptr, float = self.unpack_arraydescr(arraydescr) + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + # + if ptr: + items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items) + pval = self._cast_int_to_gcref(items[itemindex]) + # --- end of GC unsafe code --- + return BoxPtr(pval) + # + if float: + items = rffi.cast(rffi.CArrayPtr(lltype.Float), items) + fval = items[itemindex] + # --- end of GC unsafe code --- + return BoxFloat(fval) # for TYPE, itemsize in unroll_basic_sizes: if size == itemsize: - val = (rffi.cast(rffi.CArrayPtr(TYPE), gcref) - [ofs/itemsize + itemindex]) - val = rffi.cast(lltype.Signed, val) - break + items = rffi.cast(rffi.CArrayPtr(TYPE), items) + val = items[itemindex] + # --- end of GC unsafe code --- + return BoxInt(rffi.cast(lltype.Signed, val)) else: raise NotImplementedError("size = %d" % size) - if ptr: - return BoxPtr(self._cast_int_to_gcref(val)) - else: - return BoxInt(val) def do_setarrayitem_gc(self, arraybox, indexbox, vbox, arraydescr): itemindex = indexbox.getint() gcref = arraybox.getref_base() - ofs, size, ptr = self.unpack_arraydescr(arraydescr) + ofs, size, ptr, float = self.unpack_arraydescr(arraydescr) # if ptr: vboxptr = vbox.getref_base() self.gc_ll_descr.do_write_barrier(gcref, vboxptr) - a = rffi.cast(rffi.CArrayPtr(lltype.Signed), gcref) - a[ofs/WORD + itemindex] = self.cast_gcref_to_int(vboxptr) + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items) + items[itemindex] = self.cast_gcref_to_int(vboxptr) + # --- end of GC unsafe code --- + return + # + if float: + fval = vbox.getfloat() + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + items = rffi.cast(rffi.CArrayPtr(lltype.Float), items) + items[itemindex] = fval + # --- end of GC unsafe code --- + return + # + val = vbox.getint() + for TYPE, itemsize in unroll_basic_sizes: + if size == itemsize: + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + items = rffi.cast(rffi.CArrayPtr(TYPE), items) + items[itemindex] = rffi.cast(TYPE, val) + # --- end of GC unsafe code --- + return else: - v = vbox.getint() - for TYPE, itemsize in unroll_basic_sizes: - if size == itemsize: - a = rffi.cast(rffi.CArrayPtr(TYPE), gcref) - a[ofs/itemsize + itemindex] = rffi.cast(TYPE, v) - break - else: - raise NotImplementedError("size = %d" % size) + raise NotImplementedError("size = %d" % size) def _new_do_len(TP): def do_strlen(self, stringbox): @@ -312,18 +341,29 @@ @specialize.argtype(1) def _base_do_getfield(self, gcref, fielddescr): - ofs, size, ptr = self.unpack_fielddescr(fielddescr) + ofs, size, ptr, float = self.unpack_fielddescr(fielddescr) + # --- start of GC unsafe code (no GC operation!) --- + field = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + # + if ptr: + pval = rffi.cast(rffi.CArrayPtr(lltype.Signed), field)[0] + pval = self._cast_int_to_gcref(pval) + # --- end of GC unsafe code --- + return BoxPtr(pval) + # + if float: + fval = rffi.cast(rffi.CArrayPtr(lltype.Float), field)[0] + # --- end of GC unsafe code --- + return BoxFloat(fval) + # for TYPE, itemsize in unroll_basic_sizes: if size == itemsize: - val = rffi.cast(rffi.CArrayPtr(TYPE), gcref)[ofs/itemsize] + val = rffi.cast(rffi.CArrayPtr(TYPE), field)[0] + # --- end of GC unsafe code --- val = rffi.cast(lltype.Signed, val) - break + return BoxInt(val) else: raise NotImplementedError("size = %d" % size) - if ptr: - return BoxPtr(self._cast_int_to_gcref(val)) - else: - return BoxInt(val) def do_getfield_gc(self, structbox, fielddescr): gcref = structbox.getref_base() @@ -334,23 +374,40 @@ @specialize.argtype(1) def _base_do_setfield(self, gcref, vbox, fielddescr): - ofs, size, ptr = self.unpack_fielddescr(fielddescr) + ofs, size, ptr, float = self.unpack_fielddescr(fielddescr) + # if ptr: assert lltype.typeOf(gcref) is not lltype.Signed, ( "can't handle write barriers for setfield_raw") ptr = vbox.getref_base() self.gc_ll_descr.do_write_barrier(gcref, ptr) - a = rffi.cast(rffi.CArrayPtr(lltype.Signed), gcref) - a[ofs/WORD] = self.cast_gcref_to_int(ptr) + # --- start of GC unsafe code (no GC operation!) --- + field = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + field = rffi.cast(rffi.CArrayPtr(lltype.Signed), field) + field[0] = self.cast_gcref_to_int(ptr) + # --- end of GC unsafe code --- + return + # + if float: + fval = vbox.getfloat() + # --- start of GC unsafe code (no GC operation!) --- + field = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + field = rffi.cast(rffi.CArrayPtr(lltype.Float), field) + field[0] = fval + # --- end of GC unsafe code --- + return + # + val = vbox.getint() + for TYPE, itemsize in unroll_basic_sizes: + if size == itemsize: + # --- start of GC unsafe code (no GC operation!) --- + field = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + field = rffi.cast(rffi.CArrayPtr(TYPE), field) + field[0] = rffi.cast(TYPE, val) + # --- end of GC unsafe code --- + return else: - v = vbox.getint() - for TYPE, itemsize in unroll_basic_sizes: - if size == itemsize: - v = rffi.cast(TYPE, v) - rffi.cast(rffi.CArrayPtr(TYPE), gcref)[ofs/itemsize] = v - break - else: - raise NotImplementedError("size = %d" % size) + raise NotImplementedError("size = %d" % size) def do_setfield_gc(self, structbox, vbox, fielddescr): gcref = structbox.getref_base() @@ -416,7 +473,9 @@ # nonsense but nothing wrong (the return value should be ignored) if calldescr.returns_a_pointer(): return BoxPtr(self.get_latest_value_ref(0)) - elif calldescr.get_result_size(self.translate_support_code) != 0: + elif calldescr.returns_a_float(): + return BoxFloat(self.get_latest_value_float(0)) + elif calldescr.get_result_size(self.translate_support_code) > 0: return BoxInt(self.get_latest_value_int(0)) else: return None Modified: pypy/trunk/pypy/jit/backend/llsupport/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/regalloc.py Mon Oct 5 16:43:56 2009 @@ -22,26 +22,28 @@ def get(self, box): return self.stack_bindings.get(box, None) - def loc(self, box): + def loc(self, box, size): res = self.get(box) if res is not None: return res - newloc = self.stack_pos(self.stack_depth) + newloc = self.stack_pos(self.stack_depth, size) self.stack_bindings[box] = newloc - self.stack_depth += 1 + self.stack_depth += size return newloc # abstract methods that need to be overwritten for specific assemblers @staticmethod - def stack_pos(loc): + def stack_pos(loc, size): raise NotImplementedError("Purely abstract") class RegisterManager(object): """ Class that keeps track of register allocations """ - all_regs = [] - no_lower_byte_regs = [] + box_types = None # or a list of acceptable types + all_regs = [] + no_lower_byte_regs = [] save_around_call_regs = [] + reg_width = 1 # in terms of stack space eaten def __init__(self, longevity, stack_manager=None, assembler=None): self.free_regs = self.all_regs[:] @@ -57,11 +59,16 @@ def next_instruction(self, incr=1): self.position += incr + def _check_type(self, v): + if not we_are_translated() and self.box_types is not None: + assert isinstance(v, TempBox) or v.type in self.box_types + def possibly_free_var(self, v): """ If v is stored in a register and v is not used beyond the current position, then free it. Must be called at some point for all variables that might be in registers. """ + self._check_type(v) if isinstance(v, Const) or v not in self.reg_bindings: return if v not in self.longevity or self.longevity[v][1] <= self.position: @@ -96,6 +103,7 @@ returns allocated register or None, if not possible. """ + self._check_type(v) assert not isinstance(v, Const) if selected_reg is not None: res = self.reg_bindings.get(v, None) @@ -140,7 +148,7 @@ loc = self.reg_bindings[v_to_spill] del self.reg_bindings[v_to_spill] if self.stack_manager.get(v_to_spill) is None: - newloc = self.stack_manager.loc(v_to_spill) + newloc = self.stack_manager.loc(v_to_spill, self.reg_width) self.assembler.regalloc_mov(loc, newloc) return loc @@ -172,6 +180,7 @@ Will not spill a variable from 'forbidden_vars'. """ + self._check_type(v) if isinstance(v, TempBox): self.longevity[v] = (self.position, self.position) loc = self.try_allocate_reg(v, selected_reg, @@ -189,12 +198,13 @@ def loc(self, box): """ Return the location of 'box'. """ + self._check_type(box) if isinstance(box, Const): return self.convert_to_imm(box) try: return self.reg_bindings[box] except KeyError: - return self.stack_manager.loc(box) + return self.stack_manager.loc(box, self.reg_width) def return_constant(self, v, forbidden_vars=[], selected_reg=None, imm_fine=True): @@ -203,6 +213,7 @@ a register. See 'force_allocate_reg' for the meaning of 'selected_reg' and 'forbidden_vars'. """ + self._check_type(v) assert isinstance(v, Const) if selected_reg or not imm_fine: # this means we cannot have it in IMM, eh @@ -210,7 +221,7 @@ self.assembler.regalloc_mov(self.convert_to_imm(v), selected_reg) return selected_reg if selected_reg is None and self.free_regs: - loc = self.free_regs.pop() + loc = self.free_regs[-1] self.assembler.regalloc_mov(self.convert_to_imm(v), loc) return loc loc = self._spill_var(v, forbidden_vars, selected_reg) @@ -225,6 +236,7 @@ register. Return the register. See 'return_constant' and 'force_allocate_reg' for the meaning of the optional arguments. """ + self._check_type(v) if isinstance(v, Const): return self.return_constant(v, forbidden_vars, selected_reg, imm_fine) @@ -248,7 +260,7 @@ self.reg_bindings[v] = loc self.assembler.regalloc_mov(prev_loc, loc) else: - loc = self.stack_manager.loc(v) + loc = self.stack_manager.loc(v, self.reg_width) self.assembler.regalloc_mov(prev_loc, loc) def force_result_in_reg(self, result_v, v, forbidden_vars=[]): @@ -256,14 +268,19 @@ The variable v is copied away if it's further used. The meaning of 'forbidden_vars' is the same as in 'force_allocate_reg'. """ + self._check_type(result_v) + self._check_type(v) if isinstance(v, Const): loc = self.make_sure_var_in_reg(v, forbidden_vars, imm_fine=False) + # note that calling make_sure_var_in_reg with imm_fine=False + # will not allocate place in reg_bindings, we need to do it + # on our own self.reg_bindings[result_v] = loc self.free_regs = [reg for reg in self.free_regs if reg is not loc] return loc if v not in self.reg_bindings: - prev_loc = self.stack_manager.loc(v) + prev_loc = self.stack_manager.loc(v, self.reg_width) loc = self.force_allocate_reg(v, forbidden_vars) self.assembler.regalloc_mov(prev_loc, loc) assert v in self.reg_bindings @@ -283,7 +300,8 @@ def _sync_var(self, v): if not self.stack_manager.get(v): reg = self.reg_bindings[v] - self.assembler.regalloc_mov(reg, self.stack_manager.loc(v)) + to = self.stack_manager.loc(v, self.reg_width) + self.assembler.regalloc_mov(reg, to) # otherwise it's clean def before_call(self, force_store=[]): @@ -310,6 +328,7 @@ which is in variable v. """ if v is not None: + self._check_type(v) r = self.call_result_location(v) self.reg_bindings[v] = r self.free_regs = [fr for fr in self.free_regs if fr is not r] Modified: pypy/trunk/pypy/jit/backend/llsupport/symbolic.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/symbolic.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/symbolic.py Mon Oct 5 16:43:56 2009 @@ -61,8 +61,10 @@ SIZEOF_CHAR = get_size(lltype.Char, False) SIZEOF_SHORT = get_size(rffi.SHORT, False) SIZEOF_INT = get_size(rffi.INT, False) +SIZEOF_FLOAT = get_size(lltype.Float, False) unroll_basic_sizes = unrolling_iterable([(lltype.Signed, WORD), (lltype.Char, SIZEOF_CHAR), (rffi.SHORT, SIZEOF_SHORT), (rffi.INT, SIZEOF_INT)]) +# does not contain Float ^^^ which must be special-cased Modified: pypy/trunk/pypy/jit/backend/llsupport/test/test_descr.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/test/test_descr.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/test/test_descr.py Mon Oct 5 16:43:56 2009 @@ -26,12 +26,16 @@ T = lltype.GcStruct('T') S = lltype.GcStruct('S', ('x', lltype.Char), ('y', lltype.Ptr(T)), - ('z', lltype.Ptr(U))) + ('z', lltype.Ptr(U)), + ('f', lltype.Float)) assert getFieldDescrClass(lltype.Ptr(T)) is GcPtrFieldDescr assert getFieldDescrClass(lltype.Ptr(U)) is NonGcPtrFieldDescr cls = getFieldDescrClass(lltype.Char) assert cls != getFieldDescrClass(lltype.Signed) assert cls == getFieldDescrClass(lltype.Char) + clsf = getFieldDescrClass(lltype.Float) + assert clsf != cls + assert clsf == getFieldDescrClass(lltype.Float) # c0 = GcCache(False) c1 = GcCache(True) @@ -42,25 +46,35 @@ descr_x = get_field_descr(c2, S, 'x') descr_y = get_field_descr(c2, S, 'y') descr_z = get_field_descr(c2, S, 'z') + descr_f = get_field_descr(c2, S, 'f') assert descr_x.__class__ is cls assert descr_y.__class__ is GcPtrFieldDescr assert descr_z.__class__ is NonGcPtrFieldDescr + assert descr_f.__class__ is clsf if not tsc: assert descr_x.offset < descr_y.offset < descr_z.offset assert descr_x.sort_key() < descr_y.sort_key() < descr_z.sort_key() assert descr_x.get_field_size(False) == rffi.sizeof(lltype.Char) assert descr_y.get_field_size(False) == rffi.sizeof(lltype.Ptr(T)) assert descr_z.get_field_size(False) == rffi.sizeof(lltype.Ptr(U)) + assert descr_f.get_field_size(False) == rffi.sizeof(lltype.Float) else: assert isinstance(descr_x.offset, Symbolic) assert isinstance(descr_y.offset, Symbolic) assert isinstance(descr_z.offset, Symbolic) + assert isinstance(descr_f.offset, Symbolic) assert isinstance(descr_x.get_field_size(True), Symbolic) assert isinstance(descr_y.get_field_size(True), Symbolic) assert isinstance(descr_z.get_field_size(True), Symbolic) + assert isinstance(descr_f.get_field_size(True), Symbolic) assert not descr_x.is_pointer_field() assert descr_y.is_pointer_field() assert not descr_z.is_pointer_field() + assert not descr_f.is_pointer_field() + assert not descr_x.is_float_field() + assert not descr_y.is_float_field() + assert not descr_z.is_float_field() + assert descr_f.is_float_field() def test_get_array_descr(): @@ -69,68 +83,102 @@ A1 = lltype.GcArray(lltype.Char) A2 = lltype.GcArray(lltype.Ptr(T)) A3 = lltype.GcArray(lltype.Ptr(U)) + A4 = lltype.GcArray(lltype.Float) assert getArrayDescrClass(A2) is GcPtrArrayDescr assert getArrayDescrClass(A3) is NonGcPtrArrayDescr cls = getArrayDescrClass(A1) assert cls != getArrayDescrClass(lltype.GcArray(lltype.Signed)) assert cls == getArrayDescrClass(lltype.GcArray(lltype.Char)) + clsf = getArrayDescrClass(A4) + assert clsf != cls + assert clsf == getArrayDescrClass(lltype.GcArray(lltype.Float)) # c0 = GcCache(False) descr1 = get_array_descr(c0, A1) descr2 = get_array_descr(c0, A2) descr3 = get_array_descr(c0, A3) + descr4 = get_array_descr(c0, A4) assert descr1.__class__ is cls assert descr2.__class__ is GcPtrArrayDescr assert descr3.__class__ is NonGcPtrArrayDescr + assert descr4.__class__ is clsf assert descr1 == get_array_descr(c0, lltype.GcArray(lltype.Char)) assert not descr1.is_array_of_pointers() assert descr2.is_array_of_pointers() assert not descr3.is_array_of_pointers() + assert not descr4.is_array_of_pointers() + assert not descr1.is_array_of_floats() + assert not descr2.is_array_of_floats() + assert not descr3.is_array_of_floats() + assert descr4.is_array_of_floats() # WORD = rffi.sizeof(lltype.Signed) assert descr1.get_base_size(False) == WORD assert descr2.get_base_size(False) == WORD assert descr3.get_base_size(False) == WORD + assert descr4.get_base_size(False) == WORD assert descr1.get_ofs_length(False) == 0 assert descr2.get_ofs_length(False) == 0 assert descr3.get_ofs_length(False) == 0 + assert descr4.get_ofs_length(False) == 0 assert descr1.get_item_size(False) == rffi.sizeof(lltype.Char) assert descr2.get_item_size(False) == rffi.sizeof(lltype.Ptr(T)) assert descr3.get_item_size(False) == rffi.sizeof(lltype.Ptr(U)) + assert descr4.get_item_size(False) == rffi.sizeof(lltype.Float) # assert isinstance(descr1.get_base_size(True), Symbolic) assert isinstance(descr2.get_base_size(True), Symbolic) assert isinstance(descr3.get_base_size(True), Symbolic) + assert isinstance(descr4.get_base_size(True), Symbolic) assert isinstance(descr1.get_ofs_length(True), Symbolic) assert isinstance(descr2.get_ofs_length(True), Symbolic) assert isinstance(descr3.get_ofs_length(True), Symbolic) + assert isinstance(descr4.get_ofs_length(True), Symbolic) assert isinstance(descr1.get_item_size(True), Symbolic) assert isinstance(descr2.get_item_size(True), Symbolic) assert isinstance(descr3.get_item_size(True), Symbolic) + assert isinstance(descr4.get_item_size(True), Symbolic) -def test_get_call_descr(): +def test_get_call_descr_not_translated(): c0 = GcCache(False) descr1 = get_call_descr(c0, [lltype.Char, lltype.Signed], lltype.Char) assert descr1.get_result_size(False) == rffi.sizeof(lltype.Char) assert not descr1.returns_a_pointer() + assert not descr1.returns_a_float() assert descr1.arg_classes == "ii" # T = lltype.GcStruct('T') descr2 = get_call_descr(c0, [lltype.Ptr(T)], lltype.Ptr(T)) assert descr2.get_result_size(False) == rffi.sizeof(lltype.Ptr(T)) assert descr2.returns_a_pointer() + assert not descr2.returns_a_float() assert descr2.arg_classes == "r" # U = lltype.GcStruct('U', ('x', lltype.Signed)) assert descr2 == get_call_descr(c0, [lltype.Ptr(U)], lltype.Ptr(U)) # + descr4 = get_call_descr(c0, [lltype.Float, lltype.Float], lltype.Float) + assert descr4.get_result_size(False) == rffi.sizeof(lltype.Float) + assert not descr4.returns_a_pointer() + assert descr4.returns_a_float() + assert descr4.arg_classes == "ff" + +def test_get_call_descr_translated(): c1 = GcCache(True) + T = lltype.GcStruct('T') + U = lltype.GcStruct('U', ('x', lltype.Signed)) descr3 = get_call_descr(c1, [lltype.Ptr(T)], lltype.Ptr(U)) assert isinstance(descr3.get_result_size(True), Symbolic) assert descr3.returns_a_pointer() + assert not descr3.returns_a_float() assert descr3.arg_classes == "r" - + # + descr4 = get_call_descr(c1, [lltype.Float, lltype.Float], lltype.Float) + assert isinstance(descr4.get_result_size(True), Symbolic) + assert not descr4.returns_a_pointer() + assert descr4.returns_a_float() + assert descr4.arg_classes == "ff" def test_repr_of_descr(): c0 = GcCache(False) @@ -162,3 +210,6 @@ # descr4i = get_call_descr(c0, [lltype.Char, lltype.Ptr(S)], lltype.Char) assert 'CharCallDescr' in descr4i.repr_of_descr() + # + descr4f = get_call_descr(c0, [lltype.Char, lltype.Ptr(S)], lltype.Float) + assert 'FloatCallDescr' in descr4f.repr_of_descr() Modified: pypy/trunk/pypy/jit/backend/llsupport/test/test_regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/test/test_regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/test/test_regalloc.py Mon Oct 5 16:43:56 2009 @@ -1,5 +1,5 @@ -from pypy.jit.metainterp.history import BoxInt, ConstInt +from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat from pypy.jit.backend.llsupport.regalloc import StackManager from pypy.jit.backend.llsupport.regalloc import RegisterManager as BaseRegMan @@ -27,7 +27,7 @@ return v class TStackManager(StackManager): - def stack_pos(self, i): + def stack_pos(self, i, size): return i class MockAsm(object): @@ -146,7 +146,7 @@ rm.next_instruction() # allocate a stack position b0, b1, b2, b3, b4 = boxes - sp = sm.loc(b0) + sp = sm.loc(b0, 1) assert sp == 0 loc = rm.make_sure_var_in_reg(b0) assert isinstance(loc, FakeReg) @@ -207,7 +207,7 @@ asm = MockAsm() rm = RegisterManager(longevity, stack_manager=sm, assembler=asm) rm.next_instruction() - sm.loc(b0) + sm.loc(b0, 1) rm.force_result_in_reg(b1, b0) rm._check_invariants() loc = rm.loc(b1) @@ -233,10 +233,11 @@ assert isinstance(loc, ConstInt) for box in boxes[:-1]: rm.force_allocate_reg(box) - assert len(asm.moves) == 4 + assert len(asm.moves) == 3 loc = rm.return_constant(ConstInt(1), imm_fine=False) assert isinstance(loc, FakeReg) - assert len(asm.moves) == 6 + assert len(asm.moves) == 5 + assert len(rm.reg_bindings) == 3 def test_force_result_in_reg_const(self): boxes, longevity = boxes_and_longevity(2) @@ -276,3 +277,21 @@ rm.after_call(boxes[-1]) assert len(rm.reg_bindings) == 3 rm._check_invariants() + + def test_different_stack_width(self): + class XRegisterManager(RegisterManager): + reg_width = 2 + + sm = TStackManager() + b0 = BoxInt() + longevity = {b0: (0, 1)} + asm = MockAsm() + rm = RegisterManager(longevity, stack_manager=sm, assembler=asm) + f0 = BoxFloat() + longevity = {f0: (0, 1)} + xrm = XRegisterManager(longevity, stack_manager=sm, assembler=asm) + xrm.loc(f0) + rm.loc(b0) + assert sm.stack_depth == 3 + + Modified: pypy/trunk/pypy/jit/backend/llsupport/test/test_runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/test/test_runner.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/test/test_runner.py Mon Oct 5 16:43:56 2009 @@ -7,6 +7,7 @@ pass class MyLLCPU(AbstractLLCPU): + supports_floats = True def compile_loop(self, inputargs, operations, looptoken): py.test.skip("llsupport test: cannot compile operations") Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Mon Oct 5 16:43:56 2009 @@ -247,28 +247,14 @@ def test_int_operations(self): from pypy.jit.metainterp.test.test_executor import get_int_tests for opnum, boxargs, retvalue in get_int_tests(): - if len(boxargs) == 2: - args_variants = [(boxargs[0], boxargs[1]), - (boxargs[0], boxargs[1].constbox()), - (boxargs[0].constbox(), boxargs[1])] - else: - args_variants = [boxargs] - for argboxes in args_variants: - res = self.execute_operation(opnum, argboxes, 'int') - assert res.value == retvalue + res = self.execute_operation(opnum, boxargs, 'int') + assert res.value == retvalue def test_float_operations(self): from pypy.jit.metainterp.test.test_executor import get_float_tests for opnum, boxargs, rettype, retvalue in get_float_tests(self.cpu): - if len(boxargs) == 2: - args_variants = [(boxargs[0], boxargs[1]), - (boxargs[0], boxargs[1].constbox()), - (boxargs[0].constbox(), boxargs[1])] - else: - args_variants = [boxargs] - for argboxes in args_variants: - res = self.execute_operation(opnum, argboxes, rettype) - assert res.value == retvalue + res = self.execute_operation(opnum, boxargs, rettype) + assert res.value == retvalue def test_ovf_operations(self, reversed=False): minint = -sys.maxint-1 @@ -340,6 +326,19 @@ BoxInt(ord('A'))], calldescr) assert x.value == ord('B') + if cpu.supports_floats: + def func(f, i): + return float(i) + f + FPTR = self.Ptr(self.FuncType([lltype.Float, lltype.Signed], + lltype.Float)) + func_ptr = llhelper(FPTR, func) + FTP = deref(FPTR) + calldescr = cpu.calldescrof(FTP, FTP.ARGS, FTP.RESULT) + x = cpu.do_call( + [self.get_funcbox(cpu, func_ptr), + BoxFloat(3.5), BoxInt(42)], + calldescr) + assert x.value == 42 + 3.5 def test_call(self): @@ -367,6 +366,24 @@ 'int', descr=calldescr) assert res.value == 2 * num + if cpu.supports_floats: + def func(f0, f1, f2, f3, f4, f5, f6, i0, i1, f7, f8, f9): + return f0 + f1 + f2 + f3 + f4 + f5 + f6 + float(i0 + i1) + f7 + f8 + f9 + F = lltype.Float + I = lltype.Signed + FUNC = self.FuncType([F] * 7 + [I] * 2 + [F] * 3, F) + FPTR = self.Ptr(FUNC) + func_ptr = llhelper(FPTR, func) + calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + funcbox = self.get_funcbox(cpu, func_ptr) + args = ([BoxFloat(.1) for i in range(7)] + + [BoxInt(1), BoxInt(2), BoxFloat(.2), BoxFloat(.3), + BoxFloat(.4)]) + res = self.execute_operation(rop.CALL, + [funcbox] + args, + 'float', descr=calldescr) + assert abs(res.value - 4.6) < 0.0001 + def test_field_basic(self): t_box, T_box = self.alloc_instance(self.T) fielddescr = self.cpu.fielddescrof(self.S, 'value') @@ -416,12 +433,22 @@ res = self.execute_operation(rop.GETFIELD_GC, [t_box], 'ref', descr=fielddescr2) assert res.value == null_const.value + if self.cpu.supports_floats: + floatdescr = self.cpu.fielddescrof(self.S, 'float') + self.execute_operation(rop.SETFIELD_GC, [t_box, BoxFloat(3.4)], + 'void', descr=floatdescr) + res = self.execute_operation(rop.GETFIELD_GC, [t_box], + 'float', descr=floatdescr) + assert res.value == 3.4 + def test_passing_guards(self): - for (opname, args) in [(rop.GUARD_TRUE, [BoxInt(1)]), - (rop.GUARD_FALSE, [BoxInt(0)]), - (rop.GUARD_VALUE, [BoxInt(42), BoxInt(42)]), - ]: + all = [(rop.GUARD_TRUE, [BoxInt(1)]), + (rop.GUARD_FALSE, [BoxInt(0)]), + (rop.GUARD_VALUE, [BoxInt(42), BoxInt(42)])] + if self.cpu.supports_floats: + all.append((rop.GUARD_VALUE, [BoxFloat(3.5), BoxFloat(3.5)])) + for (opname, args) in all: assert self.execute_operation(opname, args, 'void') == None assert not self.guard_failed @@ -435,10 +462,12 @@ # 'void') def test_failing_guards(self): - for opname, args in [(rop.GUARD_TRUE, [BoxInt(0)]), - (rop.GUARD_FALSE, [BoxInt(1)]), - (rop.GUARD_VALUE, [BoxInt(42), BoxInt(41)]), - ]: + all = [(rop.GUARD_TRUE, [BoxInt(0)]), + (rop.GUARD_FALSE, [BoxInt(1)]), + (rop.GUARD_VALUE, [BoxInt(42), BoxInt(41)])] + if self.cpu.supports_floats: + all.append((rop.GUARD_VALUE, [BoxFloat(-1.0), BoxFloat(1.0)])) + for opname, args in all: assert self.execute_operation(opname, args, 'void') == None assert self.guard_failed @@ -589,6 +618,23 @@ 'int', descr=arraydescr) assert r.value == 1 + if self.cpu.supports_floats: + a_box, A = self.alloc_array_of(lltype.Float, 31) + arraydescr = self.cpu.arraydescrof(A) + self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(1), + BoxFloat(3.5)], + 'void', descr=arraydescr) + self.execute_operation(rop.SETARRAYITEM_GC, [a_box, BoxInt(2), + BoxFloat(4.5)], + 'void', descr=arraydescr) + r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(1)], + 'float', descr=arraydescr) + assert r.value == 3.5 + r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(2)], + 'float', descr=arraydescr) + assert r.value == 4.5 + + def test_string_basic(self): s_box = self.alloc_string("hello\xfe") r = self.execute_operation(rop.STRLEN, [s_box], 'int') @@ -633,6 +679,191 @@ r = self.execute_operation(rop.SAME_AS, [u_box.constbox()], 'ref') assert r.value == u_box.value + if self.cpu.supports_floats: + r = self.execute_operation(rop.SAME_AS, [ConstFloat(5.5)], 'float') + assert r.value == 5.5 + + def test_jump(self): + # this test generates small loops where the JUMP passes many + # arguments of various types, shuffling them around. + if self.cpu.supports_floats: + numkinds = 3 + else: + numkinds = 2 + seed = random.randrange(0, 10000) + print 'Seed is', seed # or choose it by changing the previous line + r = random.Random() + r.seed(seed) + for nb_args in range(50): + print 'Passing %d arguments around...' % nb_args + # + inputargs = [] + for k in range(nb_args): + kind = r.randrange(0, numkinds) + if kind == 0: + inputargs.append(BoxInt()) + elif kind == 1: + inputargs.append(BoxPtr()) + else: + inputargs.append(BoxFloat()) + jumpargs = [] + remixing = [] + for srcbox in inputargs: + n = r.randrange(0, len(inputargs)) + otherbox = inputargs[n] + if otherbox.type == srcbox.type: + remixing.append((srcbox, otherbox)) + else: + otherbox = srcbox + jumpargs.append(otherbox) + # + index_counter = r.randrange(0, len(inputargs)+1) + i0 = BoxInt() + i1 = BoxInt() + i2 = BoxInt() + inputargs.insert(index_counter, i0) + jumpargs.insert(index_counter, i1) + # + looptoken = LoopToken() + faildescr = BasicFailDescr() + operations = [ + ResOperation(rop.INT_SUB, [i0, ConstInt(1)], i1), + ResOperation(rop.INT_GE, [i1, ConstInt(0)], i2), + ResOperation(rop.GUARD_TRUE, [i2], None), + ResOperation(rop.JUMP, jumpargs, None, descr=looptoken), + ] + operations[2].fail_args = inputargs[:] + operations[2].descr = faildescr + # + self.cpu.compile_loop(inputargs, operations, looptoken) + # + values = [] + S = lltype.GcStruct('S') + for box in inputargs: + if isinstance(box, BoxInt): + values.append(r.randrange(-10000, 10000)) + elif isinstance(box, BoxPtr): + p = lltype.malloc(S) + values.append(lltype.cast_opaque_ptr(llmemory.GCREF, p)) + elif isinstance(box, BoxFloat): + values.append(r.random()) + else: + assert 0 + values[index_counter] = 11 + # + for i, (box, val) in enumerate(zip(inputargs, values)): + if isinstance(box, BoxInt): + self.cpu.set_future_value_int(i, val) + elif isinstance(box, BoxPtr): + self.cpu.set_future_value_ref(i, val) + elif isinstance(box, BoxFloat): + self.cpu.set_future_value_float(i, val) + else: + assert 0 + # + fail = self.cpu.execute_token(looptoken) + assert fail is faildescr + # + dstvalues = values[:] + for _ in range(11): + expected = dstvalues[:] + for tgtbox, srcbox in remixing: + v = dstvalues[inputargs.index(srcbox)] + expected[inputargs.index(tgtbox)] = v + dstvalues = expected + # + assert dstvalues[index_counter] == 11 + dstvalues[index_counter] = 0 + for i, (box, val) in enumerate(zip(inputargs, dstvalues)): + if isinstance(box, BoxInt): + got = self.cpu.get_latest_value_int(i) + elif isinstance(box, BoxPtr): + got = self.cpu.get_latest_value_ref(i) + elif isinstance(box, BoxFloat): + got = self.cpu.get_latest_value_float(i) + else: + assert 0 + assert type(got) == type(val) + assert got == val + + def test_compile_bridge_float(self): + if not self.cpu.supports_floats: + py.test.skip("requires floats") + fboxes = [BoxFloat() for i in range(12)] + i2 = BoxInt() + faildescr1 = BasicFailDescr() + faildescr2 = BasicFailDescr() + operations = [ + ResOperation(rop.FLOAT_LE, [fboxes[0], ConstFloat(9.2)], i2), + ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1), + ResOperation(rop.FINISH, fboxes, None, descr=faildescr2), + ] + operations[-2].fail_args = fboxes + looptoken = LoopToken() + self.cpu.compile_loop(fboxes, operations, looptoken) + + fboxes2 = [BoxFloat() for i in range(12)] + f3 = BoxFloat() + bridge = [ + ResOperation(rop.FLOAT_SUB, [fboxes2[0], ConstFloat(1.0)], f3), + ResOperation(rop.JUMP, [f3] + fboxes2[1:], None, descr=looptoken), + ] + + self.cpu.compile_bridge(faildescr1, fboxes2, bridge) + + for i in range(len(fboxes)): + self.cpu.set_future_value_float(i, 13.5 + 6.73 * i) + fail = self.cpu.execute_token(looptoken) + assert fail is faildescr2 + res = self.cpu.get_latest_value_float(0) + assert res == 8.5 + for i in range(1, len(fboxes)): + assert self.cpu.get_latest_value_float(i) == 13.5 + 6.73 * i + + def test_unused_result_int(self): + # test pure operations on integers whose result is not used + from pypy.jit.metainterp.test.test_executor import get_int_tests + int_tests = list(get_int_tests()) + int_tests = [(opnum, boxargs, 'int', retvalue) + for opnum, boxargs, retvalue in int_tests] + self._test_unused_result(int_tests) + + def test_unused_result_float(self): + # same as test_unused_result_int, for float operations + from pypy.jit.metainterp.test.test_executor import get_float_tests + float_tests = list(get_float_tests(self.cpu)) + self._test_unused_result(float_tests) + + def _test_unused_result(self, tests): + inputargs = [] + operations = [] + for opnum, boxargs, rettype, retvalue in tests: + inputargs += [box for box in boxargs if isinstance(box, Box)] + if rettype == 'int': + boxres = BoxInt() + elif rettype == 'float': + boxres = BoxFloat() + else: + assert 0 + operations.append(ResOperation(opnum, boxargs, boxres)) + faildescr = BasicFailDescr() + operations.append(ResOperation(rop.FINISH, [], None, + descr=faildescr)) + looptoken = LoopToken() + # + self.cpu.compile_loop(inputargs, operations, looptoken) + # + for i, box in enumerate(inputargs): + if isinstance(box, BoxInt): + self.cpu.set_future_value_int(i, box.getint()) + elif isinstance(box, BoxFloat): + self.cpu.set_future_value_float(i, box.getfloat()) + else: + assert 0 + # + fail = self.cpu.execute_token(looptoken) + assert fail is faildescr + class LLtypeBackendTest(BaseBackendTest): @@ -656,7 +887,8 @@ ('chr1', lltype.Char), ('chr2', lltype.Char), ('short', rffi.SHORT), - ('next', lltype.Ptr(S)))) + ('next', lltype.Ptr(S)), + ('float', lltype.Float))) T = lltype.GcStruct('T', ('parent', S), ('next', lltype.Ptr(S))) U = lltype.GcStruct('U', ('parent', T), @@ -933,7 +1165,20 @@ descr_B) assert isinstance(x, BoxPtr) assert x.getref(lltype.Ptr(A)) == a - # + if self.cpu.supports_floats: + C = lltype.GcArray(lltype.Float) + c = lltype.malloc(C, 6) + c[3] = 3.5 + descr_C = cpu.arraydescrof(C) + x = cpu.do_getarrayitem_gc( + BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, c)), BoxInt(3), + descr_C) + assert isinstance(x, BoxFloat) + assert x.getfloat() == 3.5 + cpu.do_setarrayitem_gc( + BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, c)), BoxInt(4), + BoxFloat(4.5), descr_C) + assert c[4] == 4.5 s = rstr.mallocstr(6) x = cpu.do_strlen( BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s))) @@ -944,7 +1189,8 @@ BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s)), BoxInt(3)) assert x.value == ord('X') # - S = lltype.GcStruct('S', ('x', lltype.Char), ('y', lltype.Ptr(A))) + S = lltype.GcStruct('S', ('x', lltype.Char), ('y', lltype.Ptr(A)), + ('z', lltype.Float)) descrfld_x = cpu.fielddescrof(S, 'x') s = lltype.malloc(S) s.x = 'Z' @@ -988,6 +1234,19 @@ descrfld_rx) assert rs.x == '!' # + + if self.cpu.supports_floats: + descrfld_z = cpu.fielddescrof(S, 'z') + cpu.do_setfield_gc( + BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s)), + BoxFloat(3.5), + descrfld_z) + assert s.z == 3.5 + s.z = 3.2 + x = cpu.do_getfield_gc( + BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, s)), + descrfld_z) + assert x.getfloat() == 3.2 ### we don't support in the JIT for now GC pointers ### stored inside non-GC structs. #descrfld_ry = cpu.fielddescrof(RS, 'y') Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Mon Oct 5 16:43:56 2009 @@ -1,13 +1,14 @@ import sys, os import ctypes from pypy.jit.backend.llsupport import symbolic -from pypy.jit.metainterp.history import Const, Box, BoxPtr, REF +from pypy.jit.metainterp.history import Const, Box, BoxPtr, REF, FLOAT from pypy.jit.metainterp.history import AbstractFailDescr from pypy.rpython.lltypesystem import lltype, rffi, ll2ctypes, rstr, llmemory from pypy.rpython.lltypesystem.rclass import OBJECT from pypy.rpython.lltypesystem.lloperation import llop from pypy.tool.uid import fixid -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, lower_byte +from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, lower_byte,\ + X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.jit.backend.x86 import codebuf from pypy.jit.backend.x86.ri386 import * @@ -79,6 +80,8 @@ MAX_FAIL_BOXES, zero=True) self.fail_boxes_ptr = lltype.malloc(lltype.GcArray(llmemory.GCREF), MAX_FAIL_BOXES, zero=True) + self.fail_boxes_float = lltype.malloc(lltype.GcArray(lltype.Float), + MAX_FAIL_BOXES, zero=True) def leave_jitted_hook(self): fail_boxes_ptr = self.fail_boxes_ptr @@ -89,10 +92,13 @@ if self.mc is None: rffi.cast(lltype.Signed, self.fail_boxes_int) # workaround rffi.cast(lltype.Signed, self.fail_boxes_ptr) # workaround + rffi.cast(lltype.Signed, self.fail_boxes_float) # workaround self.fail_box_int_addr = rffi.cast(lltype.Signed, lltype.direct_arrayitems(self.fail_boxes_int)) self.fail_box_ptr_addr = rffi.cast(lltype.Signed, lltype.direct_arrayitems(self.fail_boxes_ptr)) + self.fail_box_float_addr = rffi.cast(lltype.Signed, + lltype.direct_arrayitems(self.fail_boxes_float)) # the address of the function called by 'new' gc_ll_descr = self.cpu.gc_ll_descr @@ -184,6 +190,7 @@ mc.done() def _assemble_bootstrap_code(self, inputargs, arglocs): + nonfloatlocs, floatlocs = arglocs self.mc.PUSH(ebp) self.mc.MOV(ebp, esp) self.mc.PUSH(ebx) @@ -192,31 +199,37 @@ # NB. exactly 4 pushes above; if this changes, fix stack_pos(). # You must also keep _get_callshape() in sync. adr_stackadjust = self._patchable_stackadjust() - for i in range(len(arglocs)): - loc = arglocs[i] - if not isinstance(loc, REG): - if inputargs[i].type == REF: - # This uses XCHG to put zeroes in fail_boxes_ptr after - # reading them - self.mc.XOR(ecx, ecx) - self.mc.XCHG(ecx, addr_add(imm(self.fail_box_ptr_addr), - imm(i*WORD))) - else: - self.mc.MOV(ecx, addr_add(imm(self.fail_box_int_addr), - imm(i*WORD))) - self.mc.MOV(loc, ecx) - for i in range(len(arglocs)): - loc = arglocs[i] + tmp = X86RegisterManager.all_regs[0] + xmmtmp = X86XMMRegisterManager.all_regs[0] + for i in range(len(nonfloatlocs)): + loc = nonfloatlocs[i] + if loc is None: + continue if isinstance(loc, REG): - if inputargs[i].type == REF: - # This uses XCHG to put zeroes in fail_boxes_ptr after - # reading them - self.mc.XOR(loc, loc) - self.mc.XCHG(loc, addr_add(imm(self.fail_box_ptr_addr), - imm(i*WORD))) - else: - self.mc.MOV(loc, addr_add(imm(self.fail_box_int_addr), + target = loc + else: + target = tmp + if inputargs[i].type == REF: + # This uses XCHG to put zeroes in fail_boxes_ptr after + # reading them + self.mc.XOR(target, target) + self.mc.XCHG(target, addr_add(imm(self.fail_box_ptr_addr), imm(i*WORD))) + else: + self.mc.MOV(target, addr_add(imm(self.fail_box_int_addr), + imm(i*WORD))) + self.mc.MOV(loc, target) + for i in range(len(floatlocs)): + loc = floatlocs[i] + if loc is None: + continue + if isinstance(loc, REG): + self.mc.MOVSD(loc, addr64_add(imm(self.fail_box_float_addr), + imm(i*WORD*2))) + else: + self.mc.MOVSD(xmmtmp, addr64_add(imm(self.fail_box_float_addr), + imm(i*WORD*2))) + self.mc.MOVSD(loc, xmmtmp) return adr_stackadjust def dump(self, text): @@ -231,14 +244,38 @@ # ------------------------------------------------------------ - def regalloc_mov(self, from_loc, to_loc): - self.mc.MOV(to_loc, from_loc) + def mov(self, from_loc, to_loc): + if isinstance(from_loc, XMMREG) or isinstance(to_loc, XMMREG): + self.mc.MOVSD(to_loc, from_loc) + else: + self.mc.MOV(to_loc, from_loc) + + regalloc_mov = mov # legacy interface + + def regalloc_fstp(self, loc): + self.mc.FSTP(loc) def regalloc_push(self, loc): - self.mc.PUSH(loc) + if isinstance(loc, XMMREG): + self.mc.SUB(esp, imm(2*WORD)) + self.mc.MOVSD(mem64(esp, 0), loc) + elif isinstance(loc, MODRM64): + # XXX evil trick + self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position))) + self.mc.PUSH(mem(ebp, get_ebp_ofs(loc.position + 1))) + else: + self.mc.PUSH(loc) def regalloc_pop(self, loc): - self.mc.POP(loc) + if isinstance(loc, XMMREG): + self.mc.MOVSD(loc, mem64(esp, 0)) + self.mc.ADD(esp, imm(2*WORD)) + elif isinstance(loc, MODRM64): + # XXX evil trick + self.mc.POP(mem(ebp, get_ebp_ofs(loc.position + 1))) + self.mc.POP(mem(ebp, get_ebp_ofs(loc.position))) + else: + self.mc.POP(loc) def regalloc_perform(self, op, arglocs, resloc): genop_list[op.opnum](self, op, arglocs, resloc) @@ -261,7 +298,7 @@ else: dispatch_opnum = op.opnum adr_jump_offset = genop_guard_list[dispatch_opnum](self, op, - guard_opnum, + guard_op, failaddr, arglocs, resloc) faildescr._x86_adr_jump_offset = adr_jump_offset @@ -296,8 +333,16 @@ getattr(self.mc, 'SET' + cond)(lower_byte(result_loc)) return genop_cmp + def _cmpop_float(cond): + def genop_cmp(self, op, arglocs, result_loc): + self.mc.UCOMISD(arglocs[0], arglocs[1]) + self.mc.MOV(result_loc, imm8(0)) + getattr(self.mc, 'SET' + cond)(lower_byte(result_loc)) + return genop_cmp + def _cmpop_guard(cond, rev_cond, false_cond, false_rev_cond): - def genop_cmp_guard(self, op, guard_opnum, addr, arglocs, result_loc): + def genop_cmp_guard(self, op, guard_op, addr, arglocs, result_loc): + guard_opnum = guard_op.opnum if isinstance(op.args[0], Const): self.mc.CMP(arglocs[1], arglocs[0]) if guard_opnum == rop.GUARD_FALSE: @@ -315,17 +360,18 @@ name = 'J' + false_cond return self.implement_guard(addr, getattr(self.mc, name)) return genop_cmp_guard - - def align_stack_for_call(self, nargs): - # xxx do something when we don't use push anymore for calls - extra_on_stack = align_stack_words(nargs) - for i in range(extra_on_stack-nargs): - self.mc.PUSH(imm(0)) - return extra_on_stack + +## XXX redo me +## def align_stack_for_call(self, nargs): +## # xxx do something when we don't use push anymore for calls +## extra_on_stack = align_stack_words(nargs) +## for i in range(extra_on_stack-nargs): +## self.mc.PUSH(imm(0)) --- or just use a single SUB(esp, imm) +## return extra_on_stack def call(self, addr, args, res): nargs = len(args) - extra_on_stack = self.align_stack_for_call(nargs) + extra_on_stack = nargs #self.align_stack_for_call(nargs) for i in range(nargs-1, -1, -1): self.mc.PUSH(args[i]) self.mc.CALL(rel32(addr)) @@ -341,6 +387,10 @@ genop_int_and = _binaryop("AND", True) genop_int_or = _binaryop("OR", True) genop_int_xor = _binaryop("XOR", True) + genop_float_add = _binaryop("ADDSD", True) + genop_float_sub = _binaryop('SUBSD') + genop_float_mul = _binaryop('MULSD', True) + genop_float_truediv = _binaryop('DIVSD') genop_int_mul_ovf = genop_int_mul genop_int_sub_ovf = genop_int_sub @@ -355,6 +405,13 @@ genop_int_gt = _cmpop("G", "L") genop_int_ge = _cmpop("GE", "LE") + genop_float_lt = _cmpop_float('B') + genop_float_le = _cmpop_float('BE') + genop_float_eq = _cmpop_float('E') + genop_float_ne = _cmpop_float('NE') + genop_float_gt = _cmpop_float('A') + genop_float_ge = _cmpop_float('AE') + genop_uint_gt = _cmpop("A", "B") genop_uint_lt = _cmpop("B", "A") genop_uint_le = _cmpop("BE", "AE") @@ -376,6 +433,25 @@ # a difference at some point xxx_genop_char_eq = genop_int_eq + def genop_float_neg(self, op, arglocs, resloc): + self.mc.XORPD(arglocs[0], arglocs[1]) + + def genop_float_abs(self, op, arglocs, resloc): + self.mc.ANDPD(arglocs[0], arglocs[1]) + + def genop_float_is_true(self, op, arglocs, resloc): + loc0, loc1 = arglocs + self.mc.XORPD(loc0, loc0) + self.mc.UCOMISD(loc0, loc1) + self.mc.SETNE(lower_byte(resloc)) + self.mc.MOVZX(resloc, lower_byte(resloc)) + + def genop_cast_float_to_int(self, op, arglocs, resloc): + self.mc.CVTTSD2SI(resloc, arglocs[0]) + + def genop_cast_int_to_float(self, op, arglocs, resloc): + self.mc.CVTSI2SD(resloc, arglocs[0]) + def genop_bool_not(self, op, arglocs, resloc): self.mc.XOR(arglocs[0], imm8(1)) @@ -397,7 +473,8 @@ loc2 = cl self.mc.SHR(loc, loc2) - def genop_guard_oononnull(self, op, guard_opnum, addr, arglocs, resloc): + def genop_guard_oononnull(self, op, guard_op, addr, arglocs, resloc): + guard_opnum = guard_op.opnum loc = arglocs[0] self.mc.TEST(loc, loc) if guard_opnum == rop.GUARD_TRUE: @@ -405,7 +482,8 @@ else: return self.implement_guard(addr, self.mc.JNZ) - def genop_guard_ooisnull(self, op, guard_opnum, addr, arglocs, resloc): + def genop_guard_ooisnull(self, op, guard_op, addr, arglocs, resloc): + guard_opnum = guard_op.opnum loc = arglocs[0] self.mc.TEST(loc, loc) if guard_opnum == rop.GUARD_TRUE: @@ -428,7 +506,7 @@ self.mc.SETE(lower_byte(resloc)) def genop_same_as(self, op, arglocs, resloc): - self.mc.MOV(resloc, arglocs[0]) + self.mov(arglocs[0], resloc) genop_cast_ptr_to_int = genop_same_as def genop_int_mod(self, op, arglocs, resloc): @@ -474,6 +552,8 @@ self.mc.MOVZX(resloc, addr_add(base_loc, ofs_loc)) elif size == WORD: self.mc.MOV(resloc, addr_add(base_loc, ofs_loc)) + elif size == 8: + self.mc.MOVSD(resloc, addr64_add(base_loc, ofs_loc)) else: raise NotImplementedError("getfield size = %d" % size) @@ -483,15 +563,19 @@ base_loc, ofs_loc, scale, ofs = arglocs assert isinstance(ofs, IMM32) assert isinstance(scale, IMM32) - if scale.value == 0: - self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc, ofs.value, - scale.value)) - elif scale.value == 2: - self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, ofs.value, - scale.value)) - else: - print "[asmgen]setarrayitem unsupported size: %d" % scale.value - raise NotImplementedError() + if op.result.type == FLOAT: + self.mc.MOVSD(resloc, addr64_add(base_loc, ofs_loc, ofs.value, + scale.value)) + else: + if scale.value == 0: + self.mc.MOVZX(resloc, addr8_add(base_loc, ofs_loc, ofs.value, + scale.value)) + elif scale.value == 2: + self.mc.MOV(resloc, addr_add(base_loc, ofs_loc, ofs.value, + scale.value)) + else: + print "[asmgen]setarrayitem unsupported size: %d" % scale.value + raise NotImplementedError() genop_getfield_raw = genop_getfield_gc genop_getarrayitem_gc_pure = genop_getarrayitem_gc @@ -500,7 +584,9 @@ base_loc, ofs_loc, size_loc, value_loc = arglocs assert isinstance(size_loc, IMM32) size = size_loc.value - if size == WORD: + if size == WORD * 2: + self.mc.MOVSD(addr64_add(base_loc, ofs_loc), value_loc) + elif size == WORD: self.mc.MOV(addr_add(base_loc, ofs_loc), value_loc) elif size == 2: self.mc.MOV16(addr_add(base_loc, ofs_loc), value_loc) @@ -514,14 +600,18 @@ base_loc, ofs_loc, value_loc, scale_loc, baseofs = arglocs assert isinstance(baseofs, IMM32) assert isinstance(scale_loc, IMM32) - if scale_loc.value == 2: - self.mc.MOV(addr_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), value_loc) - elif scale_loc.value == 0: - self.mc.MOV(addr8_add(base_loc, ofs_loc, baseofs.value, - scale_loc.value), lower_byte(value_loc)) - else: - raise NotImplementedError("scale = %d" % scale_loc.value) + if op.args[2].type == FLOAT: + self.mc.MOVSD(addr64_add(base_loc, ofs_loc, baseofs.value, + scale_loc.value), value_loc) + else: + if scale_loc.value == 2: + self.mc.MOV(addr_add(base_loc, ofs_loc, baseofs.value, + scale_loc.value), value_loc) + elif scale_loc.value == 0: + self.mc.MOV(addr8_add(base_loc, ofs_loc, baseofs.value, + scale_loc.value), lower_byte(value_loc)) + else: + raise NotImplementedError("scale = %d" % scale_loc.value) def genop_discard_strsetitem(self, op, arglocs): base_loc, ofs_loc, val_loc = arglocs @@ -580,17 +670,17 @@ else: assert 0, itemsize - def genop_guard_guard_true(self, ign_1, guard_opnum, addr, locs, ign_2): + def genop_guard_guard_true(self, ign_1, guard_op, addr, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) return self.implement_guard(addr, self.mc.JZ) - def genop_guard_guard_no_exception(self, ign_1, guard_opnum, addr, + def genop_guard_guard_no_exception(self, ign_1, guard_op, addr, locs, ign_2): self.mc.CMP(heap(self.cpu.pos_exception()), imm(0)) return self.implement_guard(addr, self.mc.JNZ) - def genop_guard_guard_exception(self, ign_1, guard_opnum, addr, + def genop_guard_guard_exception(self, ign_1, guard_op, addr, locs, resloc): loc = locs[0] loc1 = locs[1] @@ -603,30 +693,41 @@ self.mc.MOV(heap(self.cpu.pos_exc_value()), imm(0)) return addr - def genop_guard_guard_no_overflow(self, ign_1, guard_opnum, addr, + def genop_guard_guard_no_overflow(self, ign_1, guard_op, addr, locs, resloc): return self.implement_guard(addr, self.mc.JO) - def genop_guard_guard_overflow(self, ign_1, guard_opnum, addr, + def genop_guard_guard_overflow(self, ign_1, guard_op, addr, locs, resloc): return self.implement_guard(addr, self.mc.JNO) - def genop_guard_guard_false(self, ign_1, guard_opnum, addr, locs, ign_2): + def genop_guard_guard_false(self, ign_1, guard_op, addr, locs, ign_2): loc = locs[0] self.mc.TEST(loc, loc) return self.implement_guard(addr, self.mc.JNZ) - def genop_guard_guard_value(self, ign_1, guard_opnum, addr, locs, ign_2): - self.mc.CMP(locs[0], locs[1]) + def genop_guard_guard_value(self, ign_1, guard_op, addr, locs, ign_2): + if guard_op.args[0].type == FLOAT: + assert guard_op.args[1].type == FLOAT + self.mc.UCOMISD(locs[0], locs[1]) + else: + self.mc.CMP(locs[0], locs[1]) return self.implement_guard(addr, self.mc.JNE) - def genop_guard_guard_class(self, ign_1, guard_opnum, addr, locs, ign_2): + def genop_guard_guard_class(self, ign_1, guard_op, addr, locs, ign_2): offset = self.cpu.vtable_offset self.mc.CMP(mem(locs[0], offset), locs[1]) return self.implement_guard(addr, self.mc.JNE) + def _no_const_locs(self, args): + """ assert that all args are actually Boxes + """ + for arg in args: + assert isinstance(arg, Box) + def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): + self._no_const_locs(failargs) addr = self.mc2.tell() exc = (guard_opnum == rop.GUARD_EXCEPTION or guard_opnum == rop.GUARD_NO_EXCEPTION) @@ -637,23 +738,34 @@ def generate_failure(self, mc, faildescr, failargs, locs, exc): assert len(failargs) < MAX_FAIL_BOXES pos = mc.tell() - for i in range(len(locs)): + for i in range(len(failargs)): + arg = failargs[i] loc = locs[i] if isinstance(loc, REG): - if failargs[i].type == REF: - base = self.fail_box_ptr_addr + if arg.type == FLOAT: + mc.MOVSD(addr64_add(imm(self.fail_box_float_addr), + imm(i*WORD*2)), loc) else: - base = self.fail_box_int_addr - mc.MOV(addr_add(imm(base), imm(i*WORD)), loc) - for i in range(len(locs)): + if arg.type == REF: + base = self.fail_box_ptr_addr + else: + base = self.fail_box_int_addr + mc.MOV(addr_add(imm(base), imm(i*WORD)), loc) + for i in range(len(failargs)): + arg = failargs[i] loc = locs[i] if not isinstance(loc, REG): - if failargs[i].type == REF: - base = self.fail_box_ptr_addr + if arg.type == FLOAT: + mc.MOVSD(xmm0, loc) + mc.MOVSD(addr64_add(imm(self.fail_box_float_addr), + imm(i*WORD*2)), xmm0) else: - base = self.fail_box_int_addr - mc.MOV(eax, loc) - mc.MOV(addr_add(imm(base), imm(i*WORD)), eax) + if arg.type == REF: + base = self.fail_box_ptr_addr + else: + base = self.fail_box_int_addr + mc.MOV(eax, loc) + mc.MOV(addr_add(imm(base), imm(i*WORD)), eax) if self.debug_markers: mc.MOV(eax, imm(pos)) mc.MOV(addr_add(imm(self.fail_box_int_addr), @@ -690,16 +802,42 @@ assert isinstance(sizeloc, IMM32) size = sizeloc.value nargs = len(op.args)-1 - extra_on_stack = self.align_stack_for_call(nargs) - for i in range(nargs+1, 1, -1): - self.mc.PUSH(arglocs[i]) + extra_on_stack = 0 + for arg in range(2, nargs + 2): + extra_on_stack += round_up_to_4(arglocs[arg].width) + #extra_on_stack = self.align_stack_for_call(extra_on_stack) + self.mc.SUB(esp, imm(extra_on_stack)) if isinstance(op.args[0], Const): x = rel32(op.args[0].getint()) else: x = arglocs[1] + if x is eax: + tmp = ecx + else: + tmp = eax + p = 0 + for i in range(2, nargs + 2): + loc = arglocs[i] + if isinstance(loc, REG): + if isinstance(loc, XMMREG): + self.mc.MOVSD(mem64(esp, p), loc) + else: + self.mc.MOV(mem(esp, p), loc) + p += round_up_to_4(loc.width) + p = 0 + for i in range(2, nargs + 2): + loc = arglocs[i] + if not isinstance(loc, REG): + if isinstance(loc, MODRM64): + self.mc.MOVSD(xmm0, loc) + self.mc.MOVSD(mem64(esp, p), xmm0) + else: + self.mc.MOV(tmp, loc) + self.mc.MOV(mem(esp, p), tmp) + p += round_up_to_4(loc.width) self.mc.CALL(x) self.mark_gc_roots() - self.mc.ADD(esp, imm(WORD * extra_on_stack)) + self.mc.ADD(esp, imm(extra_on_stack)) if size == 1: self.mc.AND(eax, imm(0xff)) elif size == 2: @@ -737,16 +875,19 @@ mc.overwrite(jz_location-1, chr(offset)) def not_implemented_op_discard(self, op, arglocs): - print "not implemented operation: %s" % op.getopname() - raise NotImplementedError + msg = "not implemented operation: %s" % op.getopname() + print msg + raise NotImplementedError(msg) def not_implemented_op(self, op, arglocs, resloc): - print "not implemented operation with res: %s" % op.getopname() - raise NotImplementedError + msg = "not implemented operation with res: %s" % op.getopname() + print msg + raise NotImplementedError(msg) def not_implemented_op_guard(self, op, regalloc, arglocs, resloc, descr): - print "not implemented operation (guard): %s" % op.getopname() - raise NotImplementedError + msg = "not implemented operation (guard): %s" % op.getopname() + print msg + raise NotImplementedError(msg) def mark_gc_roots(self): gcrootmap = self.cpu.gc_ll_descr.gcrootmap @@ -779,34 +920,32 @@ num = getattr(rop, opname.upper()) genop_list[num] = value -def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): - if isinstance(reg_or_imm1, IMM32): - if isinstance(reg_or_imm2, IMM32): - return heap(reg_or_imm1.value + offset + - (reg_or_imm2.value << scale)) - else: - return memSIB(None, reg_or_imm2, scale, reg_or_imm1.value + offset) - else: - if isinstance(reg_or_imm2, IMM32): - return mem(reg_or_imm1, offset + (reg_or_imm2.value << scale)) +def new_addr_add(heap, mem, memsib): + def addr_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): + if isinstance(reg_or_imm1, IMM32): + if isinstance(reg_or_imm2, IMM32): + return heap(reg_or_imm1.value + offset + + (reg_or_imm2.value << scale)) + else: + return memsib(None, reg_or_imm2, scale, reg_or_imm1.value + offset) else: - return memSIB(reg_or_imm1, reg_or_imm2, scale, offset) + if isinstance(reg_or_imm2, IMM32): + return mem(reg_or_imm1, offset + (reg_or_imm2.value << scale)) + else: + return memsib(reg_or_imm1, reg_or_imm2, scale, offset) + return addr_add -def addr8_add(reg_or_imm1, reg_or_imm2, offset=0, scale=0): - if isinstance(reg_or_imm1, IMM32): - if isinstance(reg_or_imm2, IMM32): - return heap8(reg_or_imm1.value + (offset << scale) + - reg_or_imm2.value) - else: - return memSIB8(None, reg_or_imm2, scale, reg_or_imm1.value + offset) - else: - if isinstance(reg_or_imm2, IMM32): - return mem8(reg_or_imm1, (offset << scale) + reg_or_imm2.value) - else: - return memSIB8(reg_or_imm1, reg_or_imm2, scale, offset) +addr8_add = new_addr_add(heap8, mem8, memSIB8) +addr_add = new_addr_add(heap, mem, memSIB) +addr64_add = new_addr_add(heap64, mem64, memSIB64) def addr_add_const(reg_or_imm1, offset): if isinstance(reg_or_imm1, IMM32): return heap(reg_or_imm1.value + offset) else: return mem(reg_or_imm1, offset) + +def round_up_to_4(size): + if size < 4: + return 4 + return size Modified: pypy/trunk/pypy/jit/backend/x86/jump.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/jump.py (original) +++ pypy/trunk/pypy/jit/backend/x86/jump.py Mon Oct 5 16:43:56 2009 @@ -80,10 +80,7 @@ assert pending_dests == 0 def _move(assembler, src, dst, tmpreg): - if isinstance(dst, MODRM): - if isinstance(src, MODRM): - assembler.regalloc_mov(src, tmpreg) - src = tmpreg - assembler.regalloc_mov(src, dst) - else: - assembler.regalloc_mov(src, dst) + if isinstance(dst, MODRM) and isinstance(src, MODRM): + assembler.regalloc_mov(src, tmpreg) + src = tmpreg + assembler.regalloc_mov(src, dst) Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Mon Oct 5 16:43:56 2009 @@ -4,7 +4,7 @@ from pypy.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr, ResOperation, ConstAddr, BoxPtr, - LoopToken) + LoopToken, INT, REF, FLOAT) from pypy.jit.backend.x86.ri386 import * from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr from pypy.rlib.objectmodel import we_are_translated @@ -20,8 +20,15 @@ WORD = 4 +width_of_type = { + INT : 1, + REF : 1, + FLOAT : 2, + } + class X86RegisterManager(RegisterManager): + box_types = [INT, REF] all_regs = [eax, ecx, edx, ebx, esi, edi] no_lower_byte_regs = [esi, edi] save_around_call_regs = [eax, edx, ecx] @@ -43,11 +50,70 @@ print "convert_to_imm: got a %s" % c raise AssertionError +BASE_CONSTANT_SIZE = 1000 + +# cheat cheat cheat.... +# why not -0.0? People tell me it's platform-dependent +# nan is not portable +import struct +NEG_ZERO, = struct.unpack('d', struct.pack('ll', 0, -2147483648)) +NAN, = struct.unpack('d', struct.pack('ll', -1, 2147483647)) +# XXX These are actually masks for float_neg and float_abs. +# They should not be converted to 'double' and given +# names that reflect their float value. + +class X86XMMRegisterManager(RegisterManager): + + box_types = [FLOAT] + all_regs = [xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7] + # we never need lower byte I hope + save_around_call_regs = all_regs + reg_width = 2 + + def new_const_array(self): + return lltype.malloc(rffi.CArray(lltype.Float), BASE_CONSTANT_SIZE, + flavor='raw') + + def __init__(self, longevity, stack_manager=None, assembler=None): + RegisterManager.__init__(self, longevity, stack_manager=stack_manager, + assembler=assembler) + self.constant_arrays = [self.new_const_array()] + self.constant_arrays[-1][0] = NEG_ZERO + self.constant_arrays[-1][1] = NAN + self.constant_array_counter = 2 + + def convert_to_imm(self, c): + if self.constant_array_counter >= BASE_CONSTANT_SIZE: + self.constant_arrays.append(self.new_const_array()) + self.constant_array_counter = 0 + res = self.constant_array_counter + self.constant_array_counter += 1 + arr = self.constant_arrays[-1] + arr[res] = c.getfloat() + return self.get_addr_of_const_float(-1, res) + + def get_addr_of_const_float(self, num_arr, num_pos): + arr = self.constant_arrays[num_arr] + return heap64(rffi.cast(lltype.Signed, arr) + num_pos * WORD * 2) + + def after_call(self, v): + # the result is stored in st0, but we don't have this around, + # so we move it to some stack location + if v is not None: + loc = self.stack_manager.loc(v, 2) + self.assembler.regalloc_fstp(loc) + class X86StackManager(StackManager): @staticmethod - def stack_pos(i): - res = mem(ebp, get_ebp_ofs(i)) + def stack_pos(i, size): + if size == 1: + res = mem(ebp, get_ebp_ofs(i)) + elif size == 2: + res = mem64(ebp, get_ebp_ofs(i + 1)) + else: + print "Unimplemented size %d" % i + raise NotImplementedError("unimplemented size %d" % i) res.position = i return res @@ -68,9 +134,12 @@ cpu.gc_ll_descr.rewrite_assembler(cpu, operations) # compute longevity of variables longevity = self._compute_vars_longevity(inputargs, operations) + self.longevity = longevity self.rm = X86RegisterManager(longevity, stack_manager = self.sm, assembler = self.assembler) + self.xrm = X86XMMRegisterManager(longevity, stack_manager = self.sm, + assembler = self.assembler) def prepare_loop(self, inputargs, operations, looptoken): self._prepare(inputargs, operations) @@ -88,28 +157,72 @@ def _process_inputargs(self, inputargs): # XXX we can sort out here by longevity if we need something # more optimal - locs = [None] * len(inputargs) + floatlocs = [None] * len(inputargs) + nonfloatlocs = [None] * len(inputargs) # Don't use all_regs[0] for passing arguments around a loop. # Must be kept in sync with consider_jump(). # XXX this should probably go to llsupport/regalloc.py + xmmtmp = self.xrm.free_regs.pop(0) tmpreg = self.rm.free_regs.pop(0) assert tmpreg == X86RegisterManager.all_regs[0] + assert xmmtmp == X86XMMRegisterManager.all_regs[0] for i in range(len(inputargs)): arg = inputargs[i] assert not isinstance(arg, Const) reg = None - if arg not in self.loop_consts and self.rm.longevity[arg][1] > -1: - reg = self.rm.try_allocate_reg(arg) + if arg not in self.loop_consts and self.longevity[arg][1] > -1: + if arg.type == FLOAT: + # xxx is it really a good idea? at the first CALL they + # will all be flushed anyway + reg = self.xrm.try_allocate_reg(arg) + else: + reg = self.rm.try_allocate_reg(arg) if reg: - locs[i] = reg + loc = reg else: - loc = self.sm.loc(arg) - locs[i] = loc + loc = self.sm.loc(arg, width_of_type[arg.type]) + if arg.type == FLOAT: + floatlocs[i] = loc + else: + nonfloatlocs[i] = loc # otherwise we have it saved on stack, so no worry self.rm.free_regs.insert(0, tmpreg) - assert tmpreg not in locs - self.rm.possibly_free_vars(inputargs) - return locs + self.xrm.free_regs.insert(0, xmmtmp) + assert tmpreg not in nonfloatlocs + assert xmmtmp not in floatlocs + self.possibly_free_vars(inputargs) + return nonfloatlocs, floatlocs + + def possibly_free_var(self, var): + if var.type == FLOAT: + self.xrm.possibly_free_var(var) + else: + self.rm.possibly_free_var(var) + + def possibly_free_vars(self, vars): + for var in vars: + self.possibly_free_var(var) + + def make_sure_var_in_reg(self, var, forbidden_vars=[], + selected_reg=None, imm_fine=True, + need_lower_byte=False): + if var.type == FLOAT: + return self.xrm.make_sure_var_in_reg(var, forbidden_vars, + selected_reg, imm_fine, + need_lower_byte) + else: + return self.rm.make_sure_var_in_reg(var, forbidden_vars, + selected_reg, imm_fine, + need_lower_byte) + + def force_allocate_reg(self, var, forbidden_vars=[], selected_reg=None, + need_lower_byte=False): + if var.type == FLOAT: + return self.xrm.force_allocate_reg(var, forbidden_vars, + selected_reg, need_lower_byte) + else: + return self.rm.force_allocate_reg(var, forbidden_vars, + selected_reg, need_lower_byte) def _compute_loop_consts(self, inputargs, jump, looptoken): if jump.opnum != rop.JUMP or jump.descr is not looptoken: @@ -123,26 +236,33 @@ def _update_bindings(self, locs, args): # XXX this should probably go to llsupport/regalloc.py - newlocs = [] - for loc in locs: - if not isinstance(loc, IMM8) and not isinstance(loc, IMM32): - newlocs.append(loc) - locs = newlocs - assert len(locs) == len(args) used = {} - for i in range(len(locs)): - v = args[i] + for i in range(len(args)): + arg = args[i] loc = locs[i] - if isinstance(loc, REG) and self.rm.longevity[v][1] > -1: - self.rm.reg_bindings[v] = loc - used[loc] = None + if arg.type == FLOAT: + if isinstance(loc, REG): + self.xrm.reg_bindings[arg] = loc + used[loc] = None + else: + self.sm.stack_bindings[arg] = loc else: - self.sm.stack_bindings[v] = loc + if isinstance(loc, REG): + self.rm.reg_bindings[arg] = loc + used[loc] = None + else: + self.sm.stack_bindings[arg] = loc self.rm.free_regs = [] for reg in X86RegisterManager.all_regs: if reg not in used: self.rm.free_regs.append(reg) + self.xrm.free_regs = [] + for reg in X86XMMRegisterManager.all_regs: + if reg not in used: + self.xrm.free_regs.append(reg) + self.possibly_free_vars(args) self.rm._check_invariants() + self.xrm._check_invariants() def Perform(self, op, arglocs, result_loc): if not we_are_translated(): @@ -155,11 +275,12 @@ def perform_with_guard(self, op, guard_op, arglocs, result_loc): faillocs = self.locs_for_fail(guard_op) self.rm.position += 1 + self.xrm.position += 1 self.assembler.regalloc_perform_with_guard(op, guard_op, faillocs, arglocs, result_loc, self.sm.stack_depth) self.rm.possibly_free_var(op.result) - self.rm.possibly_free_vars(guard_op.fail_args) + self.possibly_free_vars(guard_op.fail_args) def perform_guard(self, guard_op, arglocs, result_loc): faillocs = self.locs_for_fail(guard_op) @@ -172,7 +293,7 @@ self.assembler.regalloc_perform_guard(guard_op, faillocs, arglocs, result_loc, self.sm.stack_depth) - self.rm.possibly_free_vars(guard_op.fail_args) + self.possibly_free_vars(guard_op.fail_args) def PerformDiscard(self, op, arglocs): if not we_are_translated(): @@ -188,7 +309,7 @@ return False if operations[i + 1].args[0] is not op.result: return False - if (self.rm.longevity[op.result][1] > i + 1 or + if (self.longevity[op.result][1] > i + 1 or op.result in operations[i + 1].fail_args): return False return True @@ -199,19 +320,23 @@ while i < len(operations): op = operations[i] self.rm.position = i - if op.has_no_side_effect() and op.result not in self.rm.longevity: + self.xrm.position = i + if op.has_no_side_effect() and op.result not in self.longevity: i += 1 - self.rm.possibly_free_vars(op.args) + self.possibly_free_vars(op.args) continue if self.can_optimize_cmp_op(op, i, operations): oplist[op.opnum](self, op, operations[i + 1]) i += 1 else: oplist[op.opnum](self, op, None) - self.rm.possibly_free_var(op.result) + if op.result is not None: + self.possibly_free_var(op.result) self.rm._check_invariants() + self.xrm._check_invariants() i += 1 assert not self.rm.reg_bindings + assert not self.xrm.reg_bindings def _compute_vars_longevity(self, inputargs, operations): # compute a dictionary that maps variables to index in @@ -245,6 +370,8 @@ return longevity def loc(self, v): + if v.type == FLOAT: + return self.xrm.loc(v) return self.rm.loc(v) def _consider_guard(self, op, ignored): @@ -256,10 +383,10 @@ consider_guard_false = _consider_guard def consider_finish(self, op, ignored): - locs = [self.loc(arg) for arg in op.args] + locs = [self.loc(v) for v in op.args] self.assembler.generate_failure(self.assembler.mc, op.descr, op.args, locs, self.exc) - self.rm.possibly_free_vars(op.args) + self.possibly_free_vars(op.args) def consider_guard_no_exception(self, op, ignored): self.perform_guard(op, [], None) @@ -268,7 +395,7 @@ loc = self.rm.make_sure_var_in_reg(op.args[0]) box = TempBox() loc1 = self.rm.force_allocate_reg(box, op.args) - if op.result in self.rm.longevity: + if op.result in self.longevity: # this means, is it ever used resloc = self.rm.force_allocate_reg(op.result, op.args + [box]) else: @@ -281,10 +408,10 @@ consider_guard_overflow = consider_guard_no_exception def consider_guard_value(self, op, ignored): - x = self.rm.make_sure_var_in_reg(op.args[0]) + x = self.make_sure_var_in_reg(op.args[0]) y = self.loc(op.args[1]) self.perform_guard(op, [x, y], None) - self.rm.possibly_free_vars(op.args) + self.possibly_free_vars(op.args) def consider_guard_class(self, op, ignored): assert isinstance(op.args[0], Box) @@ -345,7 +472,8 @@ assert l0 is eax assert l1 is ecx assert l2 is resultreg - self.rm.possibly_free_vars(op.args + [tmpvar]) + self.rm.possibly_free_vars(op.args) + self.rm.possibly_free_var(tmpvar) def consider_int_mod(self, op, ignored): self._consider_int_div_or_mod(op, edx, eax) @@ -385,10 +513,89 @@ consider_oois = _consider_compop consider_ooisnot = _consider_compop + def _consider_float_op(self, op, ignored): + loc0 = self.xrm.force_result_in_reg(op.result, op.args[0], op.args) + loc1 = self.xrm.loc(op.args[1]) + self.Perform(op, [loc0, loc1], loc0) + self.xrm.possibly_free_vars(op.args) + + consider_float_add = _consider_float_op + consider_float_sub = _consider_float_op + consider_float_mul = _consider_float_op + consider_float_truediv = _consider_float_op + + def _consider_float_cmp(self, op, ignored): + assert ignored is None + # XXX so far we don't have guards here, but we want them + loc0 = self.xrm.make_sure_var_in_reg(op.args[0], op.args, + imm_fine=False) + loc1 = self.xrm.loc(op.args[1]) + res = self.rm.force_allocate_reg(op.result, need_lower_byte=True) + self.Perform(op, [loc0, loc1], res) + self.xrm.possibly_free_vars(op.args) + + consider_float_lt = _consider_float_cmp + consider_float_le = _consider_float_cmp + consider_float_eq = _consider_float_cmp + consider_float_ne = _consider_float_cmp + consider_float_gt = _consider_float_cmp + consider_float_ge = _consider_float_cmp + + def consider_float_neg(self, op, ignored): + # Following what gcc does... + # XXX we can ignore having constant in a reg, but we need + # to be careful with 128-bit alignment + loc0 = self.xrm.force_result_in_reg(op.result, op.args[0]) + constloc = self.xrm.get_addr_of_const_float(0, 0) + tmpbox = TempBox() + loc1 = self.xrm.force_allocate_reg(tmpbox, op.args) + self.assembler.regalloc_mov(constloc, loc1) + self.Perform(op, [loc0, loc1], loc0) + self.xrm.possibly_free_var(tmpbox) + self.xrm.possibly_free_var(op.args[0]) + + def consider_float_abs(self, op, ignored): + # XXX we can ignore having constant in a reg, but we need + # to be careful with 128-bit alignment + loc0 = self.xrm.force_result_in_reg(op.result, op.args[0]) + constloc = self.xrm.get_addr_of_const_float(0, 1) + tmpbox = TempBox() + loc1 = self.xrm.force_allocate_reg(tmpbox, op.args) + self.assembler.regalloc_mov(constloc, loc1) + self.Perform(op, [loc0, loc1], loc0) + self.xrm.possibly_free_var(tmpbox) + self.xrm.possibly_free_var(op.args[0]) + + def consider_float_is_true(self, op, ignored): + tmpbox0 = TempBox() + loc0 = self.xrm.force_allocate_reg(tmpbox0) + loc1 = self.xrm.loc(op.args[0]) + loc2 = self.rm.force_allocate_reg(op.result, need_lower_byte=True) + self.Perform(op, [loc0, loc1], loc2) + self.xrm.possibly_free_var(op.args[0]) + self.xrm.possibly_free_var(tmpbox0) + + def consider_cast_float_to_int(self, op, ignored): + loc0 = self.xrm.make_sure_var_in_reg(op.args[0], imm_fine=False) + loc1 = self.rm.force_allocate_reg(op.result) + self.Perform(op, [loc0], loc1) + self.xrm.possibly_free_var(op.args[0]) + + def consider_cast_int_to_float(self, op, ignored): + loc0 = self.rm.loc(op.args[0]) + loc1 = self.xrm.force_allocate_reg(op.result) + self.Perform(op, [loc0], loc1) + self.rm.possibly_free_var(op.args[0]) + def _call(self, op, arglocs, force_store=[]): self.rm.before_call(force_store) + self.xrm.before_call(force_store) self.Perform(op, arglocs, eax) - self.rm.after_call(op.result) + if op.result is not None: + if op.result.type == FLOAT: + self.xrm.after_call(op.result) + else: + self.rm.after_call(op.result) def consider_call(self, op, ignored): calldescr = op.descr @@ -462,9 +669,11 @@ # XXX kill this function at some point if isinstance(v, Box): loc = self.rm.make_sure_var_in_reg(v, [v]) - other_loc = self.rm.force_allocate_reg(TempBox(), [v]) + tempbox = TempBox() + other_loc = self.rm.force_allocate_reg(tempbox, [v]) self.assembler.load_effective_addr(loc, ofs_items,scale, other_loc) else: + tempbox = None other_loc = imm(ofs_items + (v.getint() << scale)) self._call(ResOperation(rop.NEW, [v], res_v), [other_loc], [v]) @@ -472,6 +681,8 @@ assert self.loc(res_v) == eax # now we have to reload length to some reasonable place self.rm.possibly_free_var(v) + if tempbox is not None: + self.rm.possibly_free_var(tempbox) self.PerformDiscard(ResOperation(rop.SETFIELD_GC, [], None), [eax, imm(ofs_length), imm(WORD), loc]) @@ -514,9 +725,9 @@ else: need_lower_byte = False base_loc = self.rm.make_sure_var_in_reg(op.args[0], op.args) - value_loc = self.rm.make_sure_var_in_reg(op.args[1], op.args, + value_loc = self.make_sure_var_in_reg(op.args[1], op.args, need_lower_byte=need_lower_byte) - self.rm.possibly_free_vars(op.args) + self.possibly_free_vars(op.args) self.PerformDiscard(op, [base_loc, ofs_loc, size_loc, value_loc]) consider_setfield_raw = consider_setfield_gc @@ -538,10 +749,10 @@ need_lower_byte = True else: need_lower_byte = False - value_loc = self.rm.make_sure_var_in_reg(op.args[2], op.args, - need_lower_byte=need_lower_byte) + value_loc = self.make_sure_var_in_reg(op.args[2], op.args, + need_lower_byte=need_lower_byte) ofs_loc = self.rm.make_sure_var_in_reg(op.args[1], op.args) - self.rm.possibly_free_vars(op.args) + self.possibly_free_vars(op.args) self.PerformDiscard(op, [base_loc, ofs_loc, value_loc, imm(scale), imm(ofs)]) @@ -551,7 +762,7 @@ ofs_loc, size_loc, _ = self._unpack_fielddescr(op.descr) base_loc = self.rm.make_sure_var_in_reg(op.args[0], op.args) self.rm.possibly_free_vars(op.args) - result_loc = self.rm.force_allocate_reg(op.result) + result_loc = self.force_allocate_reg(op.result) self.Perform(op, [base_loc, ofs_loc, size_loc], result_loc) consider_getfield_gc_pure = consider_getfield_gc @@ -561,7 +772,7 @@ base_loc = self.rm.make_sure_var_in_reg(op.args[0], op.args) ofs_loc = self.rm.make_sure_var_in_reg(op.args[1], op.args) self.rm.possibly_free_vars(op.args) - result_loc = self.rm.force_allocate_reg(op.result) + result_loc = self.force_allocate_reg(op.result) self.Perform(op, [base_loc, ofs_loc, imm(scale), imm(ofs)], result_loc) consider_getfield_raw = consider_getfield_gc @@ -586,8 +797,8 @@ def consider_same_as(self, op, ignored): argloc = self.loc(op.args[0]) - self.rm.possibly_free_var(op.args[0]) - resloc = self.rm.force_allocate_reg(op.result) + self.possibly_free_var(op.args[0]) + resloc = self.force_allocate_reg(op.result) self.Perform(op, [argloc], resloc) consider_cast_ptr_to_int = consider_same_as @@ -623,17 +834,26 @@ descr = op.descr assert isinstance(descr, LoopToken) self.jump_target_descr = descr - arglocs = assembler.target_arglocs(self.jump_target_descr) + nonfloatlocs, floatlocs = assembler.target_arglocs(self.jump_target_descr) # compute 'tmploc' to be all_regs[0] by spilling what is there box = TempBox() + box1 = TempBox() tmpreg = X86RegisterManager.all_regs[0] - tmploc = self.rm.force_allocate_reg(box, [], selected_reg=tmpreg) - src_locations = [self.rm.loc(arg) for arg in op.args] - dst_locations = arglocs - assert tmploc not in dst_locations + tmploc = self.rm.force_allocate_reg(box, selected_reg=tmpreg) + xmmtmp = X86XMMRegisterManager.all_regs[0] + xmmtmploc = self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp) + # Part about non-floats + src_locations = [self.loc(arg) for arg in op.args if arg.type != FLOAT] + assert tmploc not in nonfloatlocs + dst_locations = [loc for loc in nonfloatlocs if loc is not None] remap_stack_layout(assembler, src_locations, dst_locations, tmploc) + # Part about floats + src_locations = [self.loc(arg) for arg in op.args if arg.type == FLOAT] + dst_locations = [loc for loc in floatlocs if loc is not None] + remap_stack_layout(assembler, src_locations, dst_locations, xmmtmp) self.rm.possibly_free_var(box) - self.rm.possibly_free_vars(op.args) + self.xrm.possibly_free_var(box1) + self.possibly_free_vars(op.args) assembler.closing_jump(self.jump_target_descr) def consider_debug_merge_point(self, op, ignored): @@ -658,8 +878,9 @@ return gcrootmap.compress_callshape(shape) def not_implemented_op(self, op, ignored): - print "[regalloc] Not implemented operation: %s" % op.getopname() - raise NotImplementedError + msg = "[regalloc] Not implemented operation: %s" % op.getopname() + print msg + raise NotImplementedError(msg) oplist = [RegAlloc.not_implemented_op] * rop._LAST Modified: pypy/trunk/pypy/jit/backend/x86/ri386.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/ri386.py (original) +++ pypy/trunk/pypy/jit/backend/x86/ri386.py Mon Oct 5 16:43:56 2009 @@ -27,7 +27,7 @@ def assembler(self): raise TypeError("Float registers should not appear in assembler") -class XMMREG(OPERAND): +class XMMREG(REG): width = 8 def __repr__(self): @@ -309,6 +309,10 @@ assert register.width == 1 return MODRM8(0xC0 | register.op, '') +def memregister64(register): + assert register.width == 8 + return MODRM64(0xC0 | register.op, '') + def mem8(basereg, offset=0): return memSIB8(basereg, None, 0, offset) Modified: pypy/trunk/pypy/jit/backend/x86/ri386setup.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/ri386setup.py (original) +++ pypy/trunk/pypy/jit/backend/x86/ri386setup.py Mon Oct 5 16:43:56 2009 @@ -11,6 +11,9 @@ def reg2modrm(builder, reg): return memregister(reg) +def reg2modrm64(builder, reg): + return memregister64(reg) + def reg2modrm8(builder, reg): return memregister8(reg) @@ -45,7 +48,7 @@ MODRM: [(MODRM, None)], MODRM8: [(MODRM8, None)], MODRM64: [(MODRM64, None)], - XMMREG: [(XMMREG, None)], + XMMREG: [(XMMREG, None), (MODRM64, reg2modrm64)], MISSING: [(MISSING, None)], # missing operands } @@ -486,10 +489,10 @@ FUCOMPP = Instruction() FUCOMPP.mode0(['\xDA\xE9']) -FSTPL = Instruction() -FSTPL.mode1(MODRM64, ['\xDD', orbyte(3<<3), modrm(1)]) -FSTL = Instruction() -FSTL.mode1(MODRM64, ['\xDD', orbyte(2<<3), modrm(1)]) +FSTP = Instruction() +FSTP.mode1(MODRM64, ['\xDD', orbyte(3<<3), modrm(1)]) +FST = Instruction() +FST.mode1(MODRM64, ['\xDD', orbyte(2<<3), modrm(1)]) FISTP = Instruction() FISTP.mode1(MODRM, ['\xDB', orbyte(3<<3), modrm(1)]) @@ -522,6 +525,24 @@ DIVSD = Instruction() DIVSD.mode2(XMMREG, MODRM64, ['\xF2\x0F\x5E', register(1, 8), modrm(2)]) +UCOMISD = Instruction() +UCOMISD.mode2(XMMREG, MODRM64, ['\x66\x0F\x2E', register(1, 8), modrm(2)]) + +XORPD = Instruction() +XORPD.mode2(XMMREG, XMMREG, ['\x66\x0f\x57', register(1, 8), register(2), + '\xC0']) + +ANDPD = Instruction() +ANDPD.mode2(XMMREG, XMMREG, ['\x66\x0F\x54', register(1, 8), register(2), + '\xC0']) + +CVTTSD2SI = Instruction() +CVTTSD2SI.mode2(REG, XMMREG, ['\xF2\x0F\x2C', register(1, 8), register(2), + '\xC0']) + +CVTSI2SD = Instruction() +CVTSI2SD.mode2(XMMREG, MODRM, ['\xF2\x0F\x2A', register(1, 8), modrm(2)]) + # ------------------------------ end of SSE2 ----------------------------- UD2 = Instruction() # reserved as an illegal instruction Modified: pypy/trunk/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/runner.py Mon Oct 5 16:43:56 2009 @@ -11,6 +11,7 @@ class CPU386(AbstractLLCPU): debug = True + supports_floats = False # XXX fix bugs and changeme BOOTSTRAP_TP = lltype.FuncType([], lltype.Signed) dont_keepalive_stuff = False # for tests @@ -48,6 +49,10 @@ assert index < MAX_FAIL_BOXES, "overflow!" self.assembler.fail_boxes_int[index] = intvalue + def set_future_value_float(self, index, floatvalue): + assert index < MAX_FAIL_BOXES, "overflow!" + self.assembler.fail_boxes_float[index] = floatvalue + def set_future_value_ref(self, index, ptrvalue): assert index < MAX_FAIL_BOXES, "overflow!" self.assembler.fail_boxes_ptr[index] = ptrvalue @@ -55,6 +60,9 @@ def get_latest_value_int(self, index): return self.assembler.fail_boxes_int[index] + def get_latest_value_float(self, index): + return self.assembler.fail_boxes_float[index] + def get_latest_value_ref(self, index): ptrvalue = self.assembler.fail_boxes_ptr[index] # clear after reading Modified: pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py Mon Oct 5 16:43:56 2009 @@ -18,7 +18,8 @@ from pypy.jit.backend.x86.test.test_regalloc import MockAssembler from pypy.jit.backend.x86.test.test_regalloc import BaseTestRegalloc -from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86StackManager +from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86StackManager,\ + X86XMMRegisterManager class MockGcRootMap(object): def get_basic_shape(self): @@ -64,6 +65,8 @@ regalloc.sm = X86StackManager() regalloc.rm = X86RegisterManager(longevity, regalloc.sm, assembler=regalloc.assembler) + regalloc.xrm = X86XMMRegisterManager(longevity, regalloc.sm, + assembler=regalloc.assembler) cpu = regalloc.assembler.cpu for box in boxes: regalloc.rm.try_allocate_reg(box) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_jump.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_jump.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_jump.py Mon Oct 5 16:43:56 2009 @@ -41,9 +41,9 @@ remap_stack_layout(assembler, [eax, ebx, ecx, edx, esi, edi], [eax, ebx, ecx, edx, esi, edi], '?') assert assembler.ops == [] - s8 = stack_pos(1) - s12 = stack_pos(31) - s20 = stack_pos(6) + s8 = stack_pos(1, 1) + s12 = stack_pos(31, 1) + s20 = stack_pos(6, 1) remap_stack_layout(assembler, [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], [eax, ebx, ecx, s20, s8, edx, s12, esi, edi], '?') @@ -58,10 +58,10 @@ def test_simple_stacklocs(): assembler = MockAssembler() - s8 = stack_pos(0) - s12 = stack_pos(13) - s20 = stack_pos(20) - s24 = stack_pos(221) + s8 = stack_pos(0, 1) + s12 = stack_pos(13, 1) + s20 = stack_pos(20, 1) + s24 = stack_pos(221, 1) remap_stack_layout(assembler, [s8, eax, s12], [s20, s24, edi], edx) assert assembler.ops == [('mov', s8, edx), ('mov', edx, s20), @@ -70,10 +70,10 @@ def test_reordering(): assembler = MockAssembler() - s8 = stack_pos(8) - s12 = stack_pos(12) - s20 = stack_pos(19) - s24 = stack_pos(1) + s8 = stack_pos(8, 1) + s12 = stack_pos(12, 1) + s20 = stack_pos(19, 1) + s24 = stack_pos(1, 1) remap_stack_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, edi], '?') assert assembler.got([('mov', ebx, edi), @@ -83,10 +83,10 @@ def test_cycle(): assembler = MockAssembler() - s8 = stack_pos(8) - s12 = stack_pos(12) - s20 = stack_pos(19) - s24 = stack_pos(1) + s8 = stack_pos(8, 1) + s12 = stack_pos(12, 1) + s20 = stack_pos(19, 1) + s24 = stack_pos(1, 1) remap_stack_layout(assembler, [eax, s8, s20, ebx], [s8, ebx, eax, s20], '?') assert assembler.got([('push', s8), @@ -97,12 +97,12 @@ def test_cycle_2(): assembler = MockAssembler() - s8 = stack_pos(8) - s12 = stack_pos(12) - s20 = stack_pos(19) - s24 = stack_pos(1) - s2 = stack_pos(2) - s3 = stack_pos(3) + s8 = stack_pos(8, 1) + s12 = stack_pos(12, 1) + s20 = stack_pos(19, 1) + s24 = stack_pos(1, 1) + s2 = stack_pos(2, 1) + s3 = stack_pos(3, 1) remap_stack_layout(assembler, [eax, s8, edi, s20, eax, s20, s24, esi, s2, s3], [s8, s20, edi, eax, edx, s24, ebx, s12, s3, s2], @@ -127,14 +127,14 @@ remap_stack_layout(assembler, [c3], [eax], '?') assert assembler.ops == [('mov', c3, eax)] assembler = MockAssembler() - s12 = stack_pos(12) + s12 = stack_pos(12, 1) remap_stack_layout(assembler, [c3], [s12], '?') assert assembler.ops == [('mov', c3, s12)] def test_constants_and_cycle(): assembler = MockAssembler() c3 = imm(3) - s12 = stack_pos(13) + s12 = stack_pos(13, 1) remap_stack_layout(assembler, [ebx, c3, s12], [s12, eax, ebx], edi) assert assembler.ops == [('mov', c3, eax), Modified: pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py Mon Oct 5 16:43:56 2009 @@ -8,7 +8,8 @@ from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.backend.llsupport.descr import GcCache from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, X86RegisterManager +from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, X86RegisterManager,\ + BASE_CONSTANT_SIZE from pypy.jit.metainterp.test.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.annlowlevel import llhelper @@ -94,6 +95,8 @@ for i, arg in enumerate(args): if isinstance(arg, int): self.cpu.set_future_value_int(i, arg) + elif isinstance(arg, float): + self.cpu.set_future_value_float(i, arg) else: assert isinstance(lltype.typeOf(arg), lltype.Ptr) llgcref = lltype.cast_opaque_ptr(llmemory.GCREF, arg) @@ -105,10 +108,17 @@ def getint(self, index): return self.cpu.get_latest_value_int(index) + def getfloat(self, index): + return self.cpu.get_latest_value_float(index) + def getints(self, end): return [self.cpu.get_latest_value_int(index) for index in range(0, end)] + def getfloats(self, end): + return [self.cpu.get_latest_value_float(index) for + index in range(0, end)] + def getptr(self, index, T): gcref = self.cpu.get_latest_value_ref(index) return lltype.cast_opaque_ptr(T, gcref) @@ -120,6 +130,8 @@ guard_op = loop.operations[guard_op_index] assert guard_op.is_guard() bridge = self.parse(ops, **kwds) + assert ([box.type for box in bridge.inputargs] == + [box.type for box in guard_op.fail_args]) faildescr = guard_op.descr self.cpu.compile_bridge(faildescr, bridge.inputargs, bridge.operations) return bridge @@ -464,3 +476,59 @@ s = lltype.malloc(self.A, 3) self.interpret(ops, [s, ord('a')]) assert s[1] == 'a' + +class TestRegallocFloats(BaseTestRegalloc): + def test_float_add(self): + ops = ''' + [f0, f1] + f2 = float_add(f0, f1) + finish(f2, f0, f1) + ''' + self.interpret(ops, [3.0, 1.5]) + assert self.getfloats(3) == [4.5, 3.0, 1.5] + + def test_float_adds_stack(self): + ops = ''' + [f0, f1, f2, f3, f4, f5, f6, f7, f8] + f9 = float_add(f0, f1) + f10 = float_add(f8, 3.5) + finish(f9, f10, f2, f3, f4, f5, f6, f7, f8) + ''' + self.interpret(ops, [0.1, .2, .3, .4, .5, .6, .7, .8, .9]) + assert self.getfloats(9) == [.1+.2, .9+3.5, .3, .4, .5, .6, .7, .8, .9] + + def test_float_overflow_const_list(self): + ops = ['[f0]'] + for i in range(BASE_CONSTANT_SIZE * 2): + ops.append('f%d = float_add(f%d, 3.5)' % (i + 1, i)) + ops.append('finish(f%d)' % (BASE_CONSTANT_SIZE * 2)) + ops = "\n".join(ops) + self.interpret(ops, [0.1]) + assert abs(self.getfloat(0) - (BASE_CONSTANT_SIZE * 2) * 3.5 - 0.1) < 0.00001 + + def test_lt_const(self): + ops = ''' + [f0] + i1 = float_lt(3.5, f0) + finish(i1) + ''' + self.interpret(ops, [0.1]) + assert self.getint(0) == 0 + + def test_bug_float_is_true_stack(self): + ops = ''' + [f0, f1, f2, f3, f4, f5, f6, f7, f8, f9] + i0 = float_is_true(f0) + i1 = float_is_true(f1) + i2 = float_is_true(f2) + i3 = float_is_true(f3) + i4 = float_is_true(f4) + i5 = float_is_true(f5) + i6 = float_is_true(f6) + i7 = float_is_true(f7) + i8 = float_is_true(f8) + i9 = float_is_true(f9) + finish(i0, i1, i2, i3, i4, i5, i6, i7, i8, i9) + ''' + loop = self.interpret(ops, [0.0, .1, .2, .3, .4, .5, .6, .7, .8, .9]) + assert self.getints(9) == [0, 1, 1, 1, 1, 1, 1, 1, 1] Modified: pypy/trunk/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Mon Oct 5 16:43:56 2009 @@ -141,8 +141,10 @@ all = instr.as_all_suffixes for m, extra in args: if m in (i386.MODRM, i386.MODRM8) or all: - if not instrname == 'FNSTCW': + if instrname != 'FNSTCW': suffix = suffixes[sizes[m]] + suffix + if m is i386.MODRM64 and instrname in ['FST', 'FSTP']: + suffix = 'l' following = "" if instr.indirect: suffix = "" Modified: pypy/trunk/pypy/jit/metainterp/history.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/history.py (original) +++ pypy/trunk/pypy/jit/metainterp/history.py Mon Oct 5 16:43:56 2009 @@ -463,6 +463,8 @@ try: if self.type == INT: t = 'i' + elif self.type == FLOAT: + t = 'f' else: t = 'p' except AttributeError: Modified: pypy/trunk/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/logger.py (original) +++ pypy/trunk/pypy/jit/metainterp/logger.py Mon Oct 5 16:43:56 2009 @@ -2,7 +2,7 @@ from pypy.rlib.objectmodel import compute_unique_id from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.history import Const, ConstInt, Box, \ - BoxInt, ConstAddr + BoxInt, ConstAddr, ConstFloat, BoxFloat from pypy.rlib.streamio import open_file_as_stream class Logger(object): @@ -45,10 +45,14 @@ return 'ConstPtr(ptr' + str(mv) + ')' elif isinstance(arg, self.ts.BoxRef): return 'p' + str(mv) + elif isinstance(arg, ConstFloat): + return str(arg.value) + elif isinstance(arg, BoxFloat): + return 'f' + str(mv) elif isinstance(arg, self.ts.ConstAddr): return 'ConstClass(cls' + str(mv) + ')' else: - raise NotImplementedError + return '?' def log_operations(self, inputargs, operations, memo, indent=0): if self.log_stream is None: Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Mon Oct 5 16:43:56 2009 @@ -1,4 +1,5 @@ -from pypy.jit.metainterp.history import Box, BoxInt, LoopToken +from pypy.jit.metainterp.history import Box, BoxInt, LoopToken, BoxFloat,\ + ConstFloat from pypy.jit.metainterp.history import Const, ConstInt, ConstPtr, ConstObj, REF from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.metainterp.executor import execute_nonspec @@ -143,6 +144,7 @@ CONST_0 = ConstInt(0) CONST_1 = ConstInt(1) CVAL_ZERO = ConstantValue(CONST_0) +CVAL_ZERO_FLOAT = ConstantValue(ConstFloat(0.0)) llhelper.CONST_NULL = ConstPtr(ConstPtr.value) llhelper.CVAL_NULLREF = ConstantValue(llhelper.CONST_NULL) oohelper.CONST_NULL = ConstObj(ConstObj.value) @@ -438,24 +440,32 @@ def new_box(self, fieldofs): if fieldofs.is_pointer_field(): return self.new_ptr_box() + elif fieldofs.is_float_field(): + return BoxFloat() else: return BoxInt() def new_const(self, fieldofs): if fieldofs.is_pointer_field(): return self.cpu.ts.CVAL_NULLREF + elif fieldofs.is_float_field(): + return CVAL_ZERO_FLOAT else: return CVAL_ZERO def new_box_item(self, arraydescr): if arraydescr.is_array_of_pointers(): return self.new_ptr_box() + elif arraydescr.is_array_of_floats(): + return BoxFloat() else: return BoxInt() def new_const_item(self, arraydescr): if arraydescr.is_array_of_pointers(): return self.cpu.ts.CVAL_NULLREF + elif arraydescr.is_array_of_floats(): + return CVAL_ZERO_FLOAT else: return CVAL_ZERO Modified: pypy/trunk/pypy/jit/metainterp/test/oparser.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/oparser.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/oparser.py Mon Oct 5 16:43:56 2009 @@ -4,7 +4,8 @@ """ from pypy.jit.metainterp.history import TreeLoop, BoxInt, ConstInt,\ - ConstAddr, ConstObj, ConstPtr, Box, BasicFailDescr, LoopToken + ConstAddr, ConstObj, ConstPtr, Box, BasicFailDescr, BoxFloat, ConstFloat,\ + LoopToken from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.metainterp.typesystem import llhelper from pypy.rpython.lltypesystem import lltype, llmemory @@ -70,6 +71,9 @@ # integer box = BoxInt() _box_counter_more_than(elem[1:]) + elif elem.startswith('f'): + box = BoxFloat() + _box_counter_more_than(elem[1:]) elif elem.startswith('p'): # pointer ts = getattr(self.cpu, 'ts', llhelper) @@ -96,12 +100,21 @@ self.vars[elem] = box return vars + def is_float(self, arg): + try: + float(arg) + return True + except ValueError: + return False + def getvar(self, arg): if not arg: return ConstInt(0) try: return ConstInt(int(arg)) except ValueError: + if self.is_float(arg): + return ConstFloat(float(arg)) if arg.startswith('"') or arg.startswith("'"): # XXX ootype info = arg.strip("'\"") Modified: pypy/trunk/pypy/jit/metainterp/test/test_executor.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_executor.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_executor.py Mon Oct 5 16:43:56 2009 @@ -147,7 +147,6 @@ for i in range(20): x = pick() y = pick() - res = execute_nonspec(cpu, opnum, [BoxInt(x), BoxInt(y)]) z = int(operation(x, y)) yield opnum, [x, y], z @@ -168,6 +167,10 @@ list(_int_comparison_operations()) + list(_int_unary_operations())): yield opnum, [BoxInt(x) for x in args], retvalue + if len(args) > 1: + assert len(args) == 2 + yield opnum, [BoxInt(args[0]), ConstInt(args[1])], retvalue + yield opnum, [ConstInt(args[0]), BoxInt(args[1])], retvalue def test_int_ops(): @@ -196,6 +199,7 @@ yield (rop.FLOAT_NE, [10.125, y], 'int', 10.125 != y) yield (rop.FLOAT_GT, [10.125, y], 'int', 10.125 > y) yield (rop.FLOAT_GE, [10.125, y], 'int', 10.125 >= y) + yield (rop.FLOAT_EQ, [0.0, -0.0], 'int', 0.0 == -0.0) def _float_unary_operations(): yield (rop.FLOAT_NEG, [-5.9], 'float', 5.9) @@ -204,6 +208,7 @@ yield (rop.FLOAT_ABS, [15.9], 'float', 15.9) yield (rop.FLOAT_IS_TRUE, [-5.9], 'int', 1) yield (rop.FLOAT_IS_TRUE, [0.0], 'int', 0) + yield (rop.FLOAT_IS_TRUE, [-0.0], 'int', 0) yield (rop.CAST_FLOAT_TO_INT, [-5.9], 'int', -5) yield (rop.CAST_FLOAT_TO_INT, [5.9], 'int', 5) yield (rop.CAST_INT_TO_FLOAT, [123], 'float', 123.0) @@ -222,6 +227,10 @@ else: boxargs.append(BoxInt(x)) yield opnum, boxargs, rettype, retvalue + if len(args) > 1: + assert len(args) == 2 + yield opnum, [boxargs[0], boxargs[1].constbox()], rettype, retvalue + yield opnum, [boxargs[0].constbox(), boxargs[1]], rettype, retvalue def test_float_ops(): cpu = FakeCPU() Modified: pypy/trunk/pypy/jit/metainterp/test/test_logger.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_logger.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_logger.py Mon Oct 5 16:43:56 2009 @@ -74,3 +74,12 @@ loop, oloop = self.reparse(inp, check_equal=False) assert loop.operations[0].args[0]._get_str() == 'info' assert oloop.operations[0].args[0]._get_str() == 'info' + + def test_floats(self): + inp = ''' + [f0] + f1 = float_add(3.5, f0) + ''' + loop, oloop = self.reparse(inp) + equaloplists(loop.operations, oloop.operations) + Modified: pypy/trunk/pypy/jit/metainterp/test/test_oparser.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_oparser.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_oparser.py Mon Oct 5 16:43:56 2009 @@ -3,7 +3,8 @@ from pypy.jit.metainterp.test.oparser import parse from pypy.jit.metainterp.resoperation import rop -from pypy.jit.metainterp.history import AbstractDescr, BoxInt, LoopToken +from pypy.jit.metainterp.history import AbstractDescr, BoxInt, LoopToken,\ + BoxFloat def test_basic_parse(): x = """ @@ -129,6 +130,14 @@ loop = parse(x, namespace=locals()) assert loop.operations[0].descr is looptoken +def test_floats(): + x = ''' + [f0] + f1 = float_add(f0, 3.5) + ''' + loop = parse(x) + assert isinstance(loop.operations[0].args[0], BoxFloat) + def test_debug_merge_point(): x = ''' [] Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizefindnode.py Mon Oct 5 16:43:56 2009 @@ -49,6 +49,7 @@ NODE = lltype.GcForwardReference() NODE.become(lltype.GcStruct('NODE', ('parent', OBJECT), ('value', lltype.Signed), + ('floatval', lltype.Float), ('next', lltype.Ptr(NODE)))) NODE2 = lltype.GcStruct('NODE2', ('parent', NODE), ('other', lltype.Ptr(NODE))) @@ -61,10 +62,12 @@ nodesize = cpu.sizeof(NODE) nodesize2 = cpu.sizeof(NODE2) valuedescr = cpu.fielddescrof(NODE, 'value') + floatdescr = cpu.fielddescrof(NODE, 'floatval') nextdescr = cpu.fielddescrof(NODE, 'next') otherdescr = cpu.fielddescrof(NODE2, 'other') arraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Signed)) + floatarraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Float)) # a GcStruct not inheriting from OBJECT S = lltype.GcStruct('TUPLE', ('a', lltype.Signed), ('b', lltype.Ptr(NODE))) @@ -105,6 +108,7 @@ cpu = runner.OOtypeCPU(None) NODE = ootype.Instance('NODE', ootype.ROOT, {}) NODE._add_fields({'value': ootype.Signed, + 'floatval' : ootype.Float, 'next': NODE}) NODE2 = ootype.Instance('NODE2', NODE, {'other': NODE}) @@ -119,12 +123,14 @@ myptr2 = ootype.cast_to_object(ootype.new(NODE)) nodebox2 = BoxObj(ootype.cast_to_object(node)) valuedescr = cpu.fielddescrof(NODE, 'value') + floatdescr = cpu.fielddescrof(NODE, 'floatval') nextdescr = cpu.fielddescrof(NODE, 'next') otherdescr = cpu.fielddescrof(NODE2, 'other') nodesize = cpu.typedescrof(NODE) nodesize2 = cpu.typedescrof(NODE2) arraydescr = cpu.arraydescrof(ootype.Array(ootype.Signed)) + floatarraydescr = cpu.arraydescrof(ootype.Array(ootype.Float)) # a plain Record S = ootype.Record({'a': ootype.Signed, 'b': NODE}) Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Mon Oct 5 16:43:56 2009 @@ -584,6 +584,22 @@ self.optimize_loop(ops, 'Not, Virtual(node_vtable, valuedescr=Not)', expected, checkspecnodes=False) + def test_virtual_float(self): + ops = """ + [f, p0] + f0 = getfield_gc(p0, descr=floatdescr) + f1 = float_add(f0, f) + setfield_gc(p0, f1, descr=floatdescr) + jump(f, p0) + """ + expected = """ + [f, f2] + f1 = float_add(f2, f) + jump(f, f1) + """ + self.optimize_loop(ops, 'Not, Virtual(node_vtable, floatdescr=Not)', + expected, checkspecnodes=False) + def test_virtual_2(self): ops = """ [i, p0] @@ -886,6 +902,23 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_varray_float(self): + ops = """ + [f1] + p1 = new_array(3, descr=floatarraydescr) + i3 = arraylen_gc(p1, descr=floatarraydescr) + guard_value(i3, 3) [] + setarrayitem_gc(p1, 1, f1, descr=floatarraydescr) + setarrayitem_gc(p1, 0, 3.5, descr=floatarraydescr) + f2 = getarrayitem_gc(p1, 1, descr=floatarraydescr) + jump(f2) + """ + expected = """ + [f1] + jump(f1) + """ + self.optimize_loop(ops, 'Not', expected) + def test_array_non_optimized(self): ops = """ [i1, p0] Modified: pypy/trunk/pypy/jit/metainterp/test/test_virtual.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_virtual.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_virtual.py Mon Oct 5 16:43:56 2009 @@ -33,6 +33,23 @@ self.check_loops(new=0, new_with_vtable=0, getfield_gc=0, setfield_gc=0) + def test_virtualized_float(self): + myjitdriver = JitDriver(greens = [], reds = ['n', 'node']) + def f(n): + node = self._new() + node.floatval = 0.0 + while n > 0: + myjitdriver.can_enter_jit(n=n, node=node) + myjitdriver.jit_merge_point(n=n, node=node) + next = self._new() + next.floatval = node.floatval + .5 + n -= 1 + return node.floatval + res = self.meta_interp(f, [10]) + assert res == f(10) + self.check_loop_count(1) + self.check_loops(new=0, float_add=1) + def test_virtualized_2(self): myjitdriver = JitDriver(greens = [], reds = ['n', 'node']) def f(n): @@ -339,6 +356,7 @@ # Run 2: all the tests use lltype.malloc to make a NODE NODE = lltype.GcStruct('NODE', ('value', lltype.Signed), + ('floatval', lltype.Float), ('extra', lltype.Signed)) class TestLLtype_NotObject(VirtualTests, LLJitMixin): @@ -352,6 +370,7 @@ OONODE = ootype.Instance('NODE', ootype.ROOT, {}) OONODE._add_fields({'value': ootype.Signed, + 'floatval' : ootype.Float, 'extra': ootype.Signed}) class TestOOtype_NotObject(VirtualTests, OOJitMixin): @@ -367,6 +386,7 @@ # (same as Run 2 but it is part of the OBJECT hierarchy) NODE2 = lltype.GcStruct('NODE2', ('parent', rclass.OBJECT), + ('floatval', lltype.Float), ('value', lltype.Signed), ('extra', lltype.Signed)) Modified: pypy/trunk/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmspot.py Mon Oct 5 16:43:56 2009 @@ -434,7 +434,7 @@ return 'DoneWithThisFrameRef(%s)' % (self.result,) class DoneWithThisFrameFloat(JitException): - def __init__(self, cpu, result): + def __init__(self, result): assert lltype.typeOf(result) is lltype.Float self.result = result def __str__(self): Modified: pypy/trunk/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/policy.py (original) +++ pypy/trunk/pypy/module/pypyjit/policy.py Mon Oct 5 16:43:56 2009 @@ -10,12 +10,6 @@ if (func.__name__.startswith('_mm_') or func.__name__.startswith('__mm_')): # multimethods - name = func.__name__.lstrip('_') - if (name.startswith('mm_truediv') or - name.startswith('mm_inplace_truediv') or - name.startswith('mm_float')): - # floats - return False return True if '_mth_mm_' in func.__name__: # e.g. str_mth_mm_join_xxx return True @@ -27,18 +21,15 @@ return False if mod.startswith('pypy.objspace.'): - # we don't support floats - if 'float' in mod or 'complex' in mod: - return False - if func.__name__ == 'format_float': - return False # gc_id operation if func.__name__ == 'id__ANY': return False - # floats if mod == 'pypy.rlib.rbigint': #if func.__name__ == '_bigint_true_divide': return False + if mod == 'pypy.rpython.lltypesystem.module.ll_math': + # XXX temporary, contains force_cast + return False if '_geninterp_' in func.func_globals: # skip all geninterped stuff return False if mod.startswith('pypy.interpreter.astcompiler.'): From fijal at codespeak.net Mon Oct 5 16:45:48 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Oct 2009 16:45:48 +0200 (CEST) Subject: [pypy-svn] r68174 - pypy/branch/floats-via-sse2 Message-ID: <20091005144548.2F8811683E2@codespeak.net> Author: fijal Date: Mon Oct 5 16:45:47 2009 New Revision: 68174 Removed: pypy/branch/floats-via-sse2/ Log: Kill merged branch From fijal at codespeak.net Mon Oct 5 16:46:01 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Oct 2009 16:46:01 +0200 (CEST) Subject: [pypy-svn] r68175 - pypy/branch/merge-floats-via-sse2 Message-ID: <20091005144601.6DE201683E2@codespeak.net> Author: fijal Date: Mon Oct 5 16:46:01 2009 New Revision: 68175 Removed: pypy/branch/merge-floats-via-sse2/ Log: Kill merged branch From arigo at codespeak.net Mon Oct 5 17:10:15 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 5 Oct 2009 17:10:15 +0200 (CEST) Subject: [pypy-svn] r68176 - pypy/trunk/pypy/jit/backend/test Message-ID: <20091005151015.3AA7F1683DD@codespeak.net> Author: arigo Date: Mon Oct 5 17:10:14 2009 New Revision: 68176 Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py Log: Add a new test, which fails on the x86 backend. Will be refactored, but I check it in anyway temporarily. Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Mon Oct 5 17:10:14 2009 @@ -256,6 +256,19 @@ res = self.execute_operation(opnum, boxargs, rettype) assert res.value == retvalue + box = BoxFloat() + zbox = BoxFloat() + operations = [ + ResOperation(rop.FLOAT_MUL, [box, box], zbox), + ResOperation(rop.FINISH, [zbox], None, + descr=BasicFailDescr()), + ] + looptoken = LoopToken() + self.cpu.compile_loop([box], operations, looptoken) + self.cpu.set_future_value_float(0, 1.5) + res = self.cpu.execute_token(looptoken) + assert self.cpu.get_latest_value_float(0) == 2.25 + def test_ovf_operations(self, reversed=False): minint = -sys.maxint-1 boom = 'boom' From fijal at codespeak.net Mon Oct 5 17:29:58 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Oct 2009 17:29:58 +0200 (CEST) Subject: [pypy-svn] r68177 - pypy/trunk/pypy/jit/backend/x86 Message-ID: <20091005152958.494251683E2@codespeak.net> Author: fijal Date: Mon Oct 5 17:29:49 2009 New Revision: 68177 Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py Log: A fix for the test, same as binop_part Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Mon Oct 5 17:29:49 2009 @@ -514,8 +514,8 @@ consider_ooisnot = _consider_compop def _consider_float_op(self, op, ignored): - loc0 = self.xrm.force_result_in_reg(op.result, op.args[0], op.args) loc1 = self.xrm.loc(op.args[1]) + loc0 = self.xrm.force_result_in_reg(op.result, op.args[0], op.args) self.Perform(op, [loc0, loc1], loc0) self.xrm.possibly_free_vars(op.args) From arigo at codespeak.net Mon Oct 5 17:30:26 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 5 Oct 2009 17:30:26 +0200 (CEST) Subject: [pypy-svn] r68178 - in pypy/trunk/pypy/jit: backend/test metainterp/test Message-ID: <20091005153026.004B81683E7@codespeak.net> Author: arigo Date: Mon Oct 5 17:30:26 2009 New Revision: 68178 Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py pypy/trunk/pypy/jit/metainterp/test/test_executor.py Log: Update the test to cover all operations. Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Mon Oct 5 17:30:26 2009 @@ -26,7 +26,7 @@ looptoken = LoopToken() self.cpu.compile_loop(inputargs, operations, looptoken) j = 0 - for box in valueboxes: + for box in inputargs: if isinstance(box, BoxInt): self.cpu.set_future_value_int(j, box.getint()) j += 1 @@ -37,7 +37,7 @@ self.cpu.set_future_value_float(j, box.getfloat()) j += 1 else: - assert isinstance(box, Const) + raise NotImplementedError(box) res = self.cpu.execute_token(looptoken) if res is operations[-1].descr: self.guard_failed = False @@ -80,8 +80,7 @@ operations[0].descr = descr inputargs = [] for box in valueboxes: - if isinstance(box, Box): - assert box not in inputargs, "repeated box!" + if isinstance(box, Box) and box not in inputargs: inputargs.append(box) return inputargs, operations @@ -256,19 +255,6 @@ res = self.execute_operation(opnum, boxargs, rettype) assert res.value == retvalue - box = BoxFloat() - zbox = BoxFloat() - operations = [ - ResOperation(rop.FLOAT_MUL, [box, box], zbox), - ResOperation(rop.FINISH, [zbox], None, - descr=BasicFailDescr()), - ] - looptoken = LoopToken() - self.cpu.compile_loop([box], operations, looptoken) - self.cpu.set_future_value_float(0, 1.5) - res = self.cpu.execute_token(looptoken) - assert self.cpu.get_latest_value_float(0) == 2.25 - def test_ovf_operations(self, reversed=False): minint = -sys.maxint-1 boom = 'boom' Modified: pypy/trunk/pypy/jit/metainterp/test/test_executor.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_executor.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_executor.py Mon Oct 5 17:30:26 2009 @@ -94,30 +94,43 @@ def _int_binary_operations(): minint = -sys.maxint-1 + # Test cases. Note that for each operation there should be at least + # one case in which the two input arguments are equal. for opnum, testcases in [ - (rop.INT_ADD, [(10, -2, 8)]), - (rop.INT_SUB, [(10, -2, 12)]), - (rop.INT_MUL, [(-6, -3, 18)]), + (rop.INT_ADD, [(10, -2, 8), + (-60, -60, -120)]), + (rop.INT_SUB, [(10, -2, 12), + (133, 133, 0)]), + (rop.INT_MUL, [(-6, -3, 18), + (15, 15, 225)]), (rop.INT_FLOORDIV, [(110, 3, 36), (-110, 3, -36), (110, -3, -36), (-110, -3, 36), (-110, -1, 110), - (minint, 1, minint)]), + (minint, 1, minint), + (-87, -87, 1)]), (rop.INT_MOD, [(11, 3, 2), (-11, 3, -2), (11, -3, 2), - (-11, -3, -2)]), - (rop.INT_AND, [(0xFF00, 0x0FF0, 0x0F00)]), - (rop.INT_OR, [(0xFF00, 0x0FF0, 0xFFF0)]), - (rop.INT_XOR, [(0xFF00, 0x0FF0, 0xF0F0)]), + (-11, -3, -2), + (-87, -87, 0)]), + (rop.INT_AND, [(0xFF00, 0x0FF0, 0x0F00), + (-111, -111, -111)]), + (rop.INT_OR, [(0xFF00, 0x0FF0, 0xFFF0), + (-111, -111, -111)]), + (rop.INT_XOR, [(0xFF00, 0x0FF0, 0xF0F0), + (-111, -111, 0)]), (rop.INT_LSHIFT, [(10, 4, 10<<4), (-5, 2, -20), - (-5, 0, -5)]), + (-5, 0, -5), + (3, 3, 24)]), (rop.INT_RSHIFT, [(-17, 2, -5), - (19, 1, 9)]), + (19, 1, 9), + (3, 3, 0)]), (rop.UINT_RSHIFT, [(-1, 4, intmask(r_uint(-1) >> r_uint(4))), - ( 1, 4, intmask(r_uint(1) >> r_uint(4)))]) + ( 1, 4, intmask(r_uint(1) >> r_uint(4))), + ( 3, 3, 0)]) ]: for x, y, z in testcases: yield opnum, [x, y], z @@ -146,7 +159,10 @@ ]: for i in range(20): x = pick() - y = pick() + if i == 1: # there should be at least one case + y = x # where the two arguments are equal + else: + y = pick() z = int(operation(x, y)) yield opnum, [x, y], z @@ -171,6 +187,9 @@ assert len(args) == 2 yield opnum, [BoxInt(args[0]), ConstInt(args[1])], retvalue yield opnum, [ConstInt(args[0]), BoxInt(args[1])], retvalue + if args[0] == args[1]: + commonbox = BoxInt(args[0]) + yield opnum, [commonbox, commonbox], retvalue def test_int_ops(): @@ -182,16 +201,24 @@ # floats def _float_binary_operations(): + # Test cases. Note that for each operation there should be at least + # one case in which the two input arguments are equal. for opnum, testcases in [ - (rop.FLOAT_ADD, [(10.5, -2.25, 8.25)]), - (rop.FLOAT_SUB, [(10.5, -2.25, 12.75)]), - (rop.FLOAT_MUL, [(-6.5, -3.5, 22.75)]), - (rop.FLOAT_TRUEDIV, [(118.75, 12.5, 9.5)]), + (rop.FLOAT_ADD, [(10.5, -2.25, 8.25), + (5.25, 5.25, 10.5)]), + (rop.FLOAT_SUB, [(10.5, -2.25, 12.75), + (5.25, 5.25, 0.0)]), + (rop.FLOAT_MUL, [(-6.5, -3.5, 22.75), + (1.5, 1.5, 2.25)]), + (rop.FLOAT_TRUEDIV, [(118.75, 12.5, 9.5), + (-6.5, -6.5, 1.0)]), ]: for x, y, z in testcases: yield (opnum, [x, y], 'float', z) def _float_comparison_operations(): + # Test cases. Note that for each operation there should be at least + # one case in which the two input arguments are equal. for y in [-522.25, 10.125, 22.6]: yield (rop.FLOAT_LT, [10.125, y], 'int', 10.125 < y) yield (rop.FLOAT_LE, [10.125, y], 'int', 10.125 <= y) @@ -231,6 +258,11 @@ assert len(args) == 2 yield opnum, [boxargs[0], boxargs[1].constbox()], rettype, retvalue yield opnum, [boxargs[0].constbox(), boxargs[1]], rettype, retvalue + if (isinstance(args[0], float) and + isinstance(args[1], float) and + args[0] == args[1]): + commonbox = BoxFloat(args[0]) + yield opnum, [commonbox, commonbox], rettype, retvalue def test_float_ops(): cpu = FakeCPU() From pedronis at codespeak.net Mon Oct 5 17:30:32 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 5 Oct 2009 17:30:32 +0200 (CEST) Subject: [pypy-svn] r68179 - in pypy/trunk/pypy: tool translator translator/test Message-ID: <20091005153032.A492A1683ED@codespeak.net> Author: pedronis Date: Mon Oct 5 17:30:31 2009 New Revision: 68179 Modified: pypy/trunk/pypy/tool/isolate.py pypy/trunk/pypy/translator/driver.py pypy/trunk/pypy/translator/interactive.py pypy/trunk/pypy/translator/test/test_interactive.py Log: (cfbolz, pedronis) - add a way to control isolated and DEBUG_DEFINES for the c backend and in theory other backend specific options - add a shortcut to cleanup isolates on the invokers Modified: pypy/trunk/pypy/tool/isolate.py ============================================================================== --- pypy/trunk/pypy/tool/isolate.py (original) +++ pypy/trunk/pypy/tool/isolate.py Mon Oct 5 17:30:31 2009 @@ -17,6 +17,9 @@ def __repr__(self): return "" % (self.isolate.module, self.name) + def close_isolate(self): + self.isolate._close() + class Isolate(object): """ Isolate lets load a module in a different process, Modified: pypy/trunk/pypy/translator/driver.py ============================================================================== --- pypy/trunk/pypy/translator/driver.py (original) +++ pypy/trunk/pypy/translator/driver.py Mon Oct 5 17:30:31 2009 @@ -73,6 +73,7 @@ class TranslationDriver(SimpleTaskEngine): + _backend_extra_options = {} def __init__(self, setopts=None, default_goal=None, disable=[], @@ -147,6 +148,9 @@ def set_extra_goals(self, goals): self.extra_goals = goals + def set_backend_extra_options(self, extra_options): + self._backend_extra_options = extra_options + def get_info(self): # XXX more? d = {'backend': self.config.translation.backend} return d @@ -464,7 +468,11 @@ translator = self.translator cbuilder = self.cbuilder database = self.database - c_source_filename = cbuilder.generate_source(database) + if self._backend_extra_options.get('c_debug_defines', False): + defines = cbuilder.DEBUG_DEFINES + else: + defines = {} + c_source_filename = cbuilder.generate_source(database, defines) self.log.info("written: %s" % (c_source_filename,)) if self.config.translation.dump_static_data_info: from pypy.translator.tool.staticsizereport import dump_static_data_info @@ -499,7 +507,8 @@ self.c_entryp = cbuilder.executable_name self.create_exe() else: - self.c_entryp = cbuilder.get_entry_point() + isolated = self._backend_extra_options.get('c_isolated', False) + self.c_entryp = cbuilder.get_entry_point(isolated=isolated) # task_compile_c = taskdef(task_compile_c, ['source_c'], "Compiling c source") Modified: pypy/trunk/pypy/translator/interactive.py ============================================================================== --- pypy/trunk/pypy/translator/interactive.py (original) +++ pypy/trunk/pypy/translator/interactive.py Mon Oct 5 17:30:31 2009 @@ -104,6 +104,12 @@ def disable(self, to_disable): self.driver.disable(to_disable) + def set_backend_extra_options(self, **extra_options): + for name in extra_options: + backend, option = name.split('_', 1) + self.ensure_backend(backend) + self.driver.set_backend_extra_options(extra_options) + # backend independent def annotate(self, argtypes=None, **kwds): Modified: pypy/trunk/pypy/translator/test/test_interactive.py ============================================================================== --- pypy/trunk/pypy/translator/test/test_interactive.py (original) +++ pypy/trunk/pypy/translator/test/test_interactive.py Mon Oct 5 17:30:31 2009 @@ -100,6 +100,24 @@ res = t_f(2,3) assert res == 5 +def test_simple_compile_c_isolate(): + from pypy.tool import isolate + + def f(x,y): + return x+y + + t = Translation(f, [int, int]) + t.set_backend_extra_options(c_isolated=True) + t_f = t.compile() + + assert isinstance(t_f, isolate.IsolateInvoker) + + res = t_f(2,3) + assert res == 5 + + # cleanup + t_f.close_isolate() + def test_simple_rtype_with_type_system(): def f(x,y): From pedronis at codespeak.net Mon Oct 5 17:32:29 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 5 Oct 2009 17:32:29 +0200 (CEST) Subject: [pypy-svn] r68180 - in pypy/trunk/pypy: rpython/memory/test translator/c/gcc/test translator/c/test Message-ID: <20091005153229.ACF801683E3@codespeak.net> Author: pedronis Date: Mon Oct 5 17:32:28 2009 New Revision: 68180 Modified: pypy/trunk/pypy/rpython/memory/test/snippet.py pypy/trunk/pypy/rpython/memory/test/test_gc.py pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py pypy/trunk/pypy/translator/c/test/test_boehm.py pypy/trunk/pypy/translator/c/test/test_newgc.py Log: (cfbolz, pedronis) - restructure test_newgc and memory.test.snippet for speed - kill some tests that were skipped since forever Modified: pypy/trunk/pypy/rpython/memory/test/snippet.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/test/snippet.py (original) +++ pypy/trunk/pypy/rpython/memory/test/snippet.py Mon Oct 5 17:32:28 2009 @@ -3,19 +3,16 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem.lloperation import llop -class SemiSpaceGCTests: - large_tests_ok = False - def run_ok(self, f): - res = self.run(f) - assert res == 'ok' +class SemiSpaceGCTestDefines: + large_tests_ok = False - def test_finalizer_order(self): + def definestr_finalizer_order(cls): import random from pypy.tool.algo import graphlib - examples = [] - if self.large_tests_ok: + cls.finalizer_order_examples = examples = [] + if cls.large_tests_ok: letters = 'abcdefghijklmnopqrstuvwxyz' COUNT = 20 else: @@ -69,7 +66,7 @@ for c, d in input: vertices[c].refs.append(vertices[d]) - def f(): + def f(_): i = 0 while i < len(examples): input, components, strict = examples[i] @@ -114,22 +111,25 @@ def error(i, summary, msg): return '%d\n%s\n%s' % (i, summary, msg) - res = self.run(f) + return f + + def test_finalizer_order(self): + res = self.run('finalizer_order') if res != "ok": i, summary, msg = res.split('\n') i = int(i) import pprint print 'Example:' - pprint.pprint(examples[i]) + pprint.pprint(self.finalizer_order_examples[i]) print 'Finalization ages:' print summary print msg py.test.fail(msg) - def test_from_objwithfinalizer_to_youngobj(self): + def define_from_objwithfinalizer_to_youngobj(cls): import gc - if self.large_tests_ok: + if cls.large_tests_ok: MAX = 500000 else: MAX = 150 @@ -154,7 +154,25 @@ def f(): b, lst = g() gc.collect() - return str(b.count) + return b.count + return f + + def test_from_objwithfinalizer_to_youngobj(self): + res = self.run('from_objwithfinalizer_to_youngobj') + assert res == 1 + +class SemiSpaceGCTests(SemiSpaceGCTestDefines): + # xxx messy + + def run(self, name): # for test_gc.py + if name == 'finalizer_order': + func = self.definestr_finalizer_order() + res = self.interpret(func, [-1]) + return ''.join(res.chars) + elif name == 'from_objwithfinalizer_to_youngobj': + func = self.define_from_objwithfinalizer_to_youngobj() + return self.interpret(func, []) + else: + assert 0, "don't know what to do with that" - res = self.run(f) - assert res == '1' + Modified: pypy/trunk/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/trunk/pypy/rpython/memory/test/test_gc.py Mon Oct 5 17:32:28 2009 @@ -39,12 +39,6 @@ self.GC_PARAMS) return interp.eval_graph(graph, values) - def run(self, func): # for snippet.py - res = self.interpret(func, []) - if lltype.typeOf(res) == lltype.Ptr(STR): - res = ''.join(res.chars) - return res - def test_llinterp_lists(self): #curr = simulator.current_size def malloc_a_lot(): Modified: pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py Mon Oct 5 17:32:28 2009 @@ -17,10 +17,13 @@ # instructions: should_be_moving = False - def getcompiled(self, func): + @classmethod + def _makefunc2(cls, func): def main(argv): + arg0 = int(argv[1]) + arg1 = int(argv[2]) try: - res = func() + res = func(arg0, arg1) except MemoryError: print 'Result: MemoryError' else: @@ -31,12 +34,11 @@ return 0 from pypy.config.pypyoption import get_pypy_config config = get_pypy_config(translating=True) - config.translation.gc = self.gcpolicy + config.translation.gc = cls.gcpolicy config.translation.gcrootfinder = "asmgcc" if sys.platform == 'win32': config.translation.cc = 'mingw32' t = TranslationContext(config=config) - self.t = t a = t.buildannotator() a.build_types(main, [s_list_of_strings]) t.buildrtyper().specialize() @@ -45,15 +47,15 @@ cbuilder = CStandaloneBuilder(t, main, config=config) c_source_filename = cbuilder.generate_source( defines = cbuilder.DEBUG_DEFINES) - self.patch_makefile(cbuilder.targetdir) + cls._patch_makefile(cbuilder.targetdir) if conftest.option.view: t.view() exe_name = cbuilder.compile() - def run(): + def run(arg0, arg1): lines = [] print >> sys.stderr, 'RUN: starting', exe_name - g = os.popen('"%s"' % (exe_name,), 'r') + g = os.popen('"%s" %d %d' % (exe_name, arg0, arg1), 'r') for line in g: print >> sys.stderr, 'RUN:', line.rstrip() lines.append(line) @@ -71,7 +73,8 @@ return int(result) return run - def patch_makefile(self, targetdir): + @classmethod + def _patch_makefile(cls, targetdir): # for testing, patch the Makefile to add the -r option to # trackgcroot.py. makefile = targetdir.join('Makefile') @@ -89,7 +92,12 @@ f.writelines(lines) f.close() - def test_large_function(self): +class TestAsmGCRootWithSemiSpaceGC(AbstractTestAsmGCRoot, + test_newgc.TestSemiSpaceGC): + # for the individual tests see + # ====> ../../test/test_newgc.py + + def define_large_function(cls): class A(object): def __init__(self): self.x = 0 @@ -103,17 +111,13 @@ a = A() g(a) return a.x - c_fn = self.getcompiled(f) - assert c_fn() == 1000 - + return f -class TestAsmGCRootWithSemiSpaceGC(AbstractTestAsmGCRoot, - test_newgc.TestSemiSpaceGC): - pass - # for the individual tests see - # ====> ../../test/test_newgc.py + def test_large_function(self): + res = self.run('large_function') + assert res == 1000 - def test_callback_simple(self): + def define_callback_simple(cls): import gc from pypy.rpython.lltypesystem import lltype, rffi from pypy.rpython.annlowlevel import llhelper @@ -142,8 +146,12 @@ result = z(mycallback) return result * p.x - c_fn = self.getcompiled(f) - assert c_fn() == 4900 + return f + + + def test_callback_simple(self): + res = self.run('callback_simple') + assert res == 4900 if sys.platform == 'win32': def test_callback_with_collect(self): Modified: pypy/trunk/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_boehm.py (original) +++ pypy/trunk/pypy/translator/c/test/test_boehm.py Mon Oct 5 17:32:28 2009 @@ -422,13 +422,3 @@ return True run = self.getcompiled(f) assert run() == True - - # reusing some tests from pypy.rpython.memory.test.snippet - large_tests_ok = True - - def run_ok(self, f): - def wrapper(): - return int(f() == 'ok') - c_fn = self.getcompiled(wrapper, []) - res = c_fn() - assert res == 1 Modified: pypy/trunk/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_newgc.py (original) +++ pypy/trunk/pypy/translator/c/test/test_newgc.py Mon Oct 5 17:32:28 2009 @@ -1,39 +1,116 @@ -import sys import py -import os +import sys, os, inspect from pypy.objspace.flow.model import summary -from pypy.translator.translator import TranslationContext -from pypy.translator.c import genc from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.memory.test import snippet +from pypy.rlib import rgc from pypy.rlib.objectmodel import keepalive_until_here from pypy.rlib.rstring import StringBuilder, UnicodeBuilder from pypy.tool.udir import udir -from pypy.translator.c.test.test_boehm import AbstractGCTestClass +from pypy.translator.interactive import Translation +from pypy.annotation import policy as annpolicy +from pypy import conftest -class TestUsingFramework(AbstractGCTestClass): +class TestUsingFramework(object): gcpolicy = "marksweep" should_be_moving = False GC_CAN_MOVE = False GC_CANNOT_MALLOC_NONMOVABLE = False - # interface for snippet.py - large_tests_ok = True - def run(self, func): - fn = self.getcompiled(func) - return fn() + _isolated_func = None - def test_empty_collect(self): + @classmethod + def _makefunc2(cls, f): + t = Translation(f, [int, int], gc=cls.gcpolicy, + policy=annpolicy.StrictAnnotatorPolicy()) + t.disable(['backendopt']) + t.set_backend_extra_options(c_isolated=True, c_debug_defines=True) + t.rtype() + if conftest.option.view: + t.viewcg() + isolated_func = t.compile() + return isolated_func + + def setup_class(cls): + funcs0 = [] + funcs1 = [] + funcsstr = [] + name_to_func = {} + for fullname in dir(cls): + if not fullname.startswith('define'): + continue + prefix, name = fullname.split('_', 1) + definefunc = getattr(cls, fullname) + func = definefunc.im_func(cls) + func.func_name = 'f_'+name + if prefix == 'definestr': + funcsstr.append(func) + funcs0.append(None) + funcs1.append(None) + else: + numargs = len(inspect.getargspec(func)[0]) + funcsstr.append(None) + if numargs == 0: + funcs0.append(func) + funcs1.append(None) + else: + assert numargs == 1 + funcs0.append(None) + funcs1.append(func) + assert name not in name_to_func + name_to_func[name] = len(name_to_func) + def allfuncs(num, arg): + rgc.collect() + func0 = funcs0[num] + if func0: + return str(func0()) + func1 = funcs1[num] + if func1: + return str(func1(arg)) + funcstr = funcsstr[num] + if funcstr: + return funcstr(arg) + assert 0, 'unreachable' + cls.funcsstr = funcsstr + cls.c_allfuncs = staticmethod(cls._makefunc2(allfuncs)) + cls.allfuncs = staticmethod(allfuncs) + cls.name_to_func = name_to_func + + def teardown_class(cls): + if hasattr(cls.c_allfuncs, 'close_isolate'): + cls.c_allfuncs.close_isolate() + + def run(self, name, *args): + if not args: + args = (-1, ) + num = self.name_to_func[name] + res = self.c_allfuncs(num, *args) + if self.funcsstr[num]: + return res + return int(res) + + def run_orig(self, name, *args): + if not args: + args = (-1, ) + num = self.name_to_func[name] + res = self.allfuncs(num, *args) + if self.funcsstr[num]: + return res + return int(res) + + def define_empty_collect(cls): def f(): llop.gc__collect(lltype.Void) return 41 - fn = self.getcompiled(f) - res = fn() + return f + + def test_empty_collect(self): + res = self.run('empty_collect') assert res == 41 - def test_framework_simple(self): + def define_framework_simple(cls): def g(x): # cannot cause a collect return x + 1 class A(object): @@ -47,13 +124,13 @@ a = make() llop.gc__collect(lltype.Void) return a.b - fn = self.getcompiled(f) - res = fn() + return f + + def test_framework_simple(self): + res = self.run('framework_simple') assert res == 2 - insns = summary(self.t.graphs[0]) - assert ('gc_reload_possibly_moved' in insns) == self.should_be_moving - def test_framework_safe_pushpop(self): + def define_framework_safe_pushpop(cls): class A(object): pass class B(object): @@ -79,13 +156,13 @@ make() llop.gc__collect(lltype.Void) return global_a.b.a.b.c - fn = self.getcompiled(f) - res = fn() + return f + + def test_framework_safe_pushpop(self): + res = self.run('framework_safe_pushpop') assert res == 42 - insns = summary(self.t.graphs[0]) - assert 'gc_reload_possibly_moved' not in insns - def test_framework_protect_getfield(self): + def define_framework_protect_getfield(cls): class A(object): pass class B(object): @@ -107,11 +184,13 @@ a = b.a b.a = None return g(a) + b.othervalue - fn = self.getcompiled(f) - res = fn() + return f + + def test_framework_protect_getfield(self): + res = self.run('framework_protect_getfield') assert res == 128 - def test_framework_varsized(self): + def define_framework_varsized(cls): S = lltype.GcStruct("S", ('x', lltype.Signed)) T = lltype.GcStruct("T", ('y', lltype.Signed), ('s', lltype.Ptr(S))) @@ -133,12 +212,13 @@ for j in range(i): r += a[j].y return r - fn = self.getcompiled(f) - res = fn() - assert res == f() - + return f - def test_framework_using_lists(self): + def test_framework_varsized(self): + res = self.run('framework_varsized') + assert res == self.run_orig('framework_varsized') + + def define_framework_using_lists(cls): class A(object): pass N = 1000 @@ -152,11 +232,14 @@ for a in static_list: r += a.x return r - fn = self.getcompiled(f) - res = fn() + return f + + def test_framework_using_lists(self): + N = 1000 + res = self.run('framework_using_lists') assert res == N*(N - 1)/2 - def test_framework_static_roots(self): + def define_framework_static_roots(cls): class A(object): def __init__(self, y): self.y = y @@ -169,11 +252,13 @@ make() llop.gc__collect(lltype.Void) return a.x.y - fn = self.getcompiled(f) - res = fn() + return f + + def test_framework_static_roots(self): + res = self.run('framework_static_roots') assert res == 42 - def test_framework_nongc_static_root(self): + def define_framework_nongc_static_root(cls): S = lltype.GcStruct("S", ('x', lltype.Signed)) T = lltype.Struct("T", ('p', lltype.Ptr(S))) t = lltype.malloc(T, immortal=True) @@ -184,28 +269,34 @@ s = lltype.malloc(S) s.x = i return t.p.x - fn = self.getcompiled(f) - res = fn() + return f + + def test_framework_nongc_static_root(self): + res = self.run('framework_nongc_static_root') assert res == 43 - def test_framework_void_array(self): + def define_framework_void_array(cls): A = lltype.GcArray(lltype.Void) a = lltype.malloc(A, 44) def f(): return len(a) - fn = self.getcompiled(f) - res = fn() + return f + + def test_framework_void_array(self): + res = self.run('framework_void_array') assert res == 44 - def test_framework_malloc_failure(self): + def define_framework_malloc_failure(cls): def f(): a = [1] * (sys.maxint//2) return len(a) + a[0] - fn = self.getcompiled(f) - py.test.raises(MemoryError, fn) + return f - def test_framework_array_of_void(self): + def test_framework_malloc_failure(self): + py.test.raises(MemoryError, self.run, 'framework_malloc_failure') + + def define_framework_array_of_void(cls): def f(): a = [None] * 43 b = [] @@ -213,11 +304,13 @@ a.append(None) b.append(len(a)) return b[-1] - fn = self.getcompiled(f) - res = fn() + return f + + def test_framework_array_of_void(self): + res = self.run('framework_array_of_void') assert res == 43 + 1000000 - def test_framework_opaque(self): + def define_framework_opaque(cls): A = lltype.GcStruct('A', ('value', lltype.Signed)) O = lltype.GcOpaqueType('test.framework') @@ -238,11 +331,13 @@ overwrite(lltype.malloc(A), i) a = reveal(o) return a.value - fn = self.getcompiled(f) - res = fn() + return f + + def test_framework_opaque(self): + res = self.run('framework_opaque') assert res == -70 - def test_framework_finalizer(self): + def define_framework_finalizer(cls): class B(object): pass b = B() @@ -263,11 +358,13 @@ llop.gc__collect(lltype.Void) llop.gc__collect(lltype.Void) return b.num_deleted - run = self.getcompiled(f) - res = run() + return f + + def test_framework_finalizer(self): + res = self.run('framework_finalizer') assert res == 6 - def test_del_catches(self): + def define_del_catches(cls): import os def g(): pass @@ -280,41 +377,39 @@ def f1(i): if i: raise TypeError - def f(i=int): + def f(i): a = A() f1(i) a.b = 1 llop.gc__collect(lltype.Void) return a.b - def f_0(): + def h(x): try: - return f(0) + return f(x) except TypeError: return 42 - def f_1(): - try: - return f(1) - except TypeError: - return 42 - fn = self.getcompiled(f_0) - assert fn() == 1 - fn = self.getcompiled(f_1) - assert fn() == 42 + return h - def test_del_raises(self): + def test_del_catches(self): + res = self.run('del_catches', 0) + assert res == 1 + res = self.run('del_catches', 1) + assert res == 42 + + def define_del_raises(cls): class B(object): def __del__(self): raise TypeError def func(): b = B() return 0 - fn = self.getcompiled(func) - # does not crash - fn() + return func + + def test_del_raises(self): + self.run('del_raises') # does not raise - def test_weakref(self): + def define_weakref(cls): import weakref - from pypy.rlib import rgc class A: pass @@ -343,14 +438,15 @@ else: count_free += 1 return count_free - c_fn = self.getcompiled(fn) - res = c_fn() + return fn + + def test_weakref(self): + res = self.run('weakref') # more than half of them should have been freed, ideally up to 6000 assert 3500 <= res <= 6000 - def test_prebuilt_weakref(self): + def define_prebuilt_weakref(cls): import weakref - from pypy.rlib import rgc class A: pass a = A() @@ -367,11 +463,13 @@ else: result += a.hello * (i+1) return result - c_fn = self.getcompiled(fn) - res = c_fn() - assert res == fn() + return fn - def test_framework_malloc_raw(self): + def test_prebuilt_weakref(self): + res = self.run('prebuilt_weakref') + assert res == self.run_orig('prebuilt_weakref') + + def define_framework_malloc_raw(cls): A = lltype.Struct('A', ('value', lltype.Signed)) def f(): @@ -381,11 +479,13 @@ res = p.value lltype.free(p, flavor='raw') return res - fn = self.getcompiled(f) - res = fn() + return f + + def test_framework_malloc_raw(self): + res = self.run('framework_malloc_raw') assert res == 123 - def test_framework_del_seeing_new_types(self): + def define_framework_del_seeing_new_types(cls): class B(object): pass class A(object): @@ -394,62 +494,13 @@ def f(): A() return 42 - fn = self.getcompiled(f) - res = fn() - assert res == 42 + return f - def test_memory_error_varsize(self): - py.test.skip("Needs lots (>2GB) of memory.") - import gc - import pypy.rlib.rgc - from pypy.rpython.lltypesystem import lltype - N = sys.maxint / 4 + 4 - A = lltype.GcArray(lltype.Signed) - def alloc(n): - return lltype.malloc(A, n) - def f(): - try: - try: - x = alloc(N) - except MemoryError: - y = alloc(10) - return len(y) - return -1 - finally: - gc.collect() - - fn = self.getcompiled(f) - res = fn() - assert res == 10 - N = sys.maxint / 4 - fn = self.getcompiled(f) - res = fn() - assert res == 10 - - N = sys.maxint / 4 - 1 - fn = self.getcompiled(f) - res = fn() - assert res == 10 - - N = sys.maxint / 8 + 1000 - def f(): - try: - x0 = alloc(N) - try: - x1 = alloc(N) - return len(x0) + len(x1) - except MemoryError: - y = alloc(10) - return len(y) - return -1 - finally: - gc.collect() - - fn = self.getcompiled(f) - res = fn() - assert res == 10 + def test_framework_del_seeing_new_types(self): + res = self.run('framework_del_seeing_new_types') + assert res == 42 - def test_framework_late_filling_pointers(self): + def define_framework_late_filling_pointers(cls): A = lltype.GcStruct('A', ('x', lltype.Signed)) B = lltype.GcStruct('B', ('a', lltype.Ptr(A))) @@ -458,41 +509,13 @@ llop.gc__collect(lltype.Void) p.a = lltype.malloc(A) return p.a.x - fn = self.getcompiled(f) - # the point is just not to segfault - res = fn() - - def test_dict_segfault(self): - " was segfaulting at one point see rev 39665 for fix and details " - py.test.skip("Takes about 30 minutes in nightly test run, see rev 39665 for a minimal test that does the same") - - class Element: - pass + return f - elements = [Element() for ii in range(10000)] - - def dostuff(): - reverse = {} - l = elements[:] - - for ii in elements: - reverse[ii] = ii - - for jj in range(100): - e = l[-1] - del reverse[e] - l.remove(e) - - def f(): - for ii in range(100): - dostuff() - return 0 - - fn = self.getcompiled(f) + def test_framework_late_filling_pointers(self): # the point is just not to segfault - res = fn() + self.run('framework_late_filling_pointers') - def test_zero_raw_malloc(self): + def define_zero_raw_malloc(cls): S = lltype.Struct('S', ('x', lltype.Signed), ('y', lltype.Signed)) def f(): for i in range(100): @@ -504,14 +527,15 @@ lltype.free(p, flavor='raw') return 42 - fn = self.getcompiled(f) - res = fn() + return f + + def test_zero_raw_malloc(self): + res = self.run('zero_raw_malloc') assert res == 42 - def test_object_alignment(self): + def define_object_alignment(cls): # all objects returned by the GC should be properly aligned. from pypy.rpython.lltypesystem import rffi - from pypy.rpython.tool import rffi_platform mylist = ['a', 'bc', '84139871', 'ajkdh', '876'] def f(): result = 0 @@ -523,23 +547,29 @@ result |= addr return result - fn = self.getcompiled(f) - res = fn() + return f + + def test_object_alignment(self): + res = self.run('object_alignment') + from pypy.rpython.tool import rffi_platform expected_alignment = rffi_platform.memory_alignment() assert (res & (expected_alignment-1)) == 0 - def test_void_list(self): + def define_void_list(cls): class E: def __init__(self): self.l = [] def f(): e = E() return len(e.l) - c_fn = self.getcompiled(f) - assert c_fn() == 0 + return f - def test_open_read_write_seek_close(self): - filename = str(udir.join('test_open_read_write_close.txt')) + def test_void_list(self): + assert self.run('void_list') == 0 + + filename = str(udir.join('test_open_read_write_close.txt')) + def define_open_read_write_seek_close(cls): + filename = cls.filename def does_stuff(): fd = os.open(filename, os.O_WRONLY | os.O_CREAT, 0777) count = os.write(fd, "hello world\n") @@ -551,17 +581,19 @@ data = os.read(fd, 500) assert data == "ello world\n" os.close(fd) + return 0 - f1 = self.getcompiled(does_stuff) - f1() - assert open(filename, 'r').read() == "hello world\n" - os.unlink(filename) + return does_stuff - def test_callback_with_collect(self): + def test_open_read_write_seek_close(self): + self.run('open_read_write_seek_close') + assert open(self.filename, 'r').read() == "hello world\n" + os.unlink(self.filename) + + def define_callback_with_collect(cls): from pypy.rlib.libffi import ffi_type_pointer, cast_type_to_ffitype,\ CDLL, ffi_type_void, CallbackFuncPtr, ffi_type_sint from pypy.rpython.lltypesystem import rffi, ll2ctypes - from pypy.rlib import rgc import gc slong = cast_type_to_ffitype(rffi.LONG) @@ -604,20 +636,22 @@ keepalive_until_here(ptr) return int(result) - c_fn = self.getcompiled(f) - assert c_fn() == 1 + return f + + def test_callback_with_collect(self): + assert self.run('callback_with_collect') - def test_can_move(self): - from pypy.rlib import rgc + def define_can_move(cls): class A: pass def fn(): return rgc.can_move(A()) + return fn - c_fn = self.getcompiled(fn) - assert c_fn() == self.GC_CAN_MOVE + def test_can_move(self): + assert self.run('can_move') == self.GC_CAN_MOVE - def test_malloc_nonmovable(self): + def define_malloc_nonmovable(cls): TP = lltype.GcArray(lltype.Char) def func(): try: @@ -631,13 +665,15 @@ except Exception, e: return 2 - run = self.getcompiled(func) - assert run() == self.GC_CANNOT_MALLOC_NONMOVABLE + return func - def test_resizable_buffer(self): + def test_malloc_nonmovable(self): + res = self.run('malloc_nonmovable') + assert res == self.GC_CANNOT_MALLOC_NONMOVABLE + + def define_resizable_buffer(cls): from pypy.rpython.lltypesystem.rstr import STR from pypy.rpython.annlowlevel import hlstr - from pypy.rlib import rgc def f(): ptr = rgc.resizable_buffer_of_shape(STR, 2) @@ -646,16 +682,21 @@ ptr.chars[1] = 'b' return hlstr(rgc.finish_building_buffer(ptr, 2)) == "ab" - run = self.getcompiled(f) - assert run() == True + return f + + def test_resizable_buffer(self): + assert self.run('resizable_buffer') -class TestSemiSpaceGC(TestUsingFramework, snippet.SemiSpaceGCTests): +class TestSemiSpaceGC(TestUsingFramework, snippet.SemiSpaceGCTestDefines): gcpolicy = "semispace" should_be_moving = True GC_CAN_MOVE = True GC_CANNOT_MALLOC_NONMOVABLE = True - def test_many_ids(self): + # for snippets + large_tests_ok = True + + def define_many_ids(cls): from pypy.rlib.objectmodel import compute_unique_id class A(object): pass @@ -682,17 +723,18 @@ j += 1 lltype.free(idarray, flavor='raw') return -2 - fn = self.getcompiled(f) - res = fn() + return f + + def test_many_ids(self): + res = self.run('many_ids') assert res == -2 - def test_gc_set_max_heap_size(self): + def define_gc_set_max_heap_size(cls): def g(n): return 'x' * n def fn(): # the semispace size starts at 8MB for now, so setting a # smaller limit has no effect - from pypy.rlib import rgc # set to more than 32MB -- which should be rounded down to 32MB rgc.set_max_heap_size(32*1024*1024 + 20000) s1 = s2 = s3 = None @@ -703,25 +745,29 @@ except MemoryError: pass return (s1 is not None) + (s2 is not None) + (s3 is not None) - c_fn = self.getcompiled(fn) - res = c_fn() + return fn + + def test_gc_set_max_heap_size(self): + res = self.run('gc_set_max_heap_size') assert res == 2 - def test_string_builder(self): - def fn(): + def definestr_string_builder(cls): + def fn(_): s = StringBuilder() s.append("a") s.append("abc") s.append_slice("abc", 1, 2) s.append_multiple_char('d', 4) return s.build() - c_fn = self.getcompiled(fn) - res = c_fn() + return fn + + def test_string_builder(self): + res = self.run('string_builder') assert res == "aabcbdddd" - def test_string_builder_over_allocation(self): + def definestr_string_builder_over_allocation(cls): import gc - def fn(): + def fn(_): s = StringBuilder(4) s.append("abcd") s.append("defg") @@ -732,8 +778,10 @@ res = s.build() gc.collect() return res - c_fn = self.getcompiled(fn) - res = c_fn() + return fn + + def test_string_builder_over_allocation(self): + res = self.run('string_builder_over_allocation') assert res[1000] == 'y' class TestGenerationalGC(TestSemiSpaceGC): From pedronis at codespeak.net Mon Oct 5 17:33:17 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 5 Oct 2009 17:33:17 +0200 (CEST) Subject: [pypy-svn] r68181 - pypy/trunk/pypy/annotation Message-ID: <20091005153317.C445A1683E3@codespeak.net> Author: pedronis Date: Mon Oct 5 17:33:17 2009 New Revision: 68181 Modified: pypy/trunk/pypy/annotation/policy.py Log: (cfbolz, pedronis) oops, should have gone with the last commit Modified: pypy/trunk/pypy/annotation/policy.py ============================================================================== --- pypy/trunk/pypy/annotation/policy.py (original) +++ pypy/trunk/pypy/annotation/policy.py Mon Oct 5 17:33:17 2009 @@ -96,3 +96,5 @@ bk = getbookkeeper() return bk.immutablevalue(None) +class StrictAnnotatorPolicy(AnnotatorPolicy): + allow_someobjects = False From fijal at codespeak.net Mon Oct 5 18:06:46 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Oct 2009 18:06:46 +0200 (CEST) Subject: [pypy-svn] r68182 - pypy/trunk/pypy/jit/backend/x86 Message-ID: <20091005160646.8B9D11683E7@codespeak.net> Author: fijal Date: Mon Oct 5 18:06:46 2009 New Revision: 68182 Modified: pypy/trunk/pypy/jit/backend/x86/runner.py Log: Good, now it works Modified: pypy/trunk/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/runner.py Mon Oct 5 18:06:46 2009 @@ -11,7 +11,7 @@ class CPU386(AbstractLLCPU): debug = True - supports_floats = False # XXX fix bugs and changeme + supports_floats = True BOOTSTRAP_TP = lltype.FuncType([], lltype.Signed) dont_keepalive_stuff = False # for tests From fijal at codespeak.net Mon Oct 5 18:09:54 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Oct 2009 18:09:54 +0200 (CEST) Subject: [pypy-svn] r68183 - pypy/trunk/pypy/module/pypyjit Message-ID: <20091005160954.B510B1683E7@codespeak.net> Author: fijal Date: Mon Oct 5 18:09:53 2009 New Revision: 68183 Modified: pypy/trunk/pypy/module/pypyjit/policy.py Log: See math module Modified: pypy/trunk/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/policy.py (original) +++ pypy/trunk/pypy/module/pypyjit/policy.py Mon Oct 5 18:09:53 2009 @@ -41,10 +41,10 @@ if mod.endswith('operation') or mod.endswith('abstractinst'): return True - if (not mod.startswith('pypy.module.pypyjit.') and - not mod.startswith('pypy.module.signal.') and - not mod.startswith('pypy.module.micronumpy.')): - return False + modname = mod.split('.')[2] + if mod in ['pypyjit', 'signal', 'micronumpy', 'math']: + return True + return False if mod.startswith('pypy.translator.'): return False From fijal at codespeak.net Mon Oct 5 18:13:03 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 5 Oct 2009 18:13:03 +0200 (CEST) Subject: [pypy-svn] r68184 - pypy/trunk/pypy/module/pypyjit Message-ID: <20091005161303.2D4E21683E8@codespeak.net> Author: fijal Date: Mon Oct 5 18:13:02 2009 New Revision: 68184 Modified: pypy/trunk/pypy/module/pypyjit/policy.py Log: oops, typo typo Modified: pypy/trunk/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/policy.py (original) +++ pypy/trunk/pypy/module/pypyjit/policy.py Mon Oct 5 18:13:02 2009 @@ -42,7 +42,7 @@ return True modname = mod.split('.')[2] - if mod in ['pypyjit', 'signal', 'micronumpy', 'math']: + if modname in ['pypyjit', 'signal', 'micronumpy', 'math']: return True return False From arigo at codespeak.net Mon Oct 5 18:14:41 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 5 Oct 2009 18:14:41 +0200 (CEST) Subject: [pypy-svn] r68185 - pypy/trunk/pypy/jit/backend/test Message-ID: <20091005161441.DF6DF1683E8@codespeak.net> Author: arigo Date: Mon Oct 5 18:14:41 2009 New Revision: 68185 Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py Log: Only run up to 50 tests at once, to avoid hitting the x86's limit of the number of inputargs for now. Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Mon Oct 5 18:14:41 2009 @@ -834,6 +834,9 @@ self._test_unused_result(float_tests) def _test_unused_result(self, tests): + while len(tests) > 50: # only up to 50 tests at once + self._test_unused_result(tests[:50]) + tests = tests[50:] inputargs = [] operations = [] for opnum, boxargs, rettype, retvalue in tests: From arigo at codespeak.net Mon Oct 5 18:50:08 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 5 Oct 2009 18:50:08 +0200 (CEST) Subject: [pypy-svn] r68186 - pypy/trunk/pypy/jit/backend/test Message-ID: <20091005165008.20A181683F0@codespeak.net> Author: arigo Date: Mon Oct 5 18:50:07 2009 New Revision: 68186 Modified: pypy/trunk/pypy/jit/backend/test/test_random.py Log: Add float support to test_random. Fix it (partially) to not crash when given the --output option. Modified: pypy/trunk/pypy/jit/backend/test/test_random.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/test_random.py (original) +++ pypy/trunk/pypy/jit/backend/test/test_random.py Mon Oct 5 18:50:07 2009 @@ -5,6 +5,7 @@ from pypy.jit.metainterp.history import BasicFailDescr, TreeLoop from pypy.jit.metainterp.history import BoxInt, ConstInt, LoopToken from pypy.jit.metainterp.history import BoxPtr, ConstPtr, ConstAddr +from pypy.jit.metainterp.history import BoxFloat, ConstFloat, Const from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.metainterp.executor import execute_nonspec from pypy.jit.metainterp.resoperation import opname @@ -20,12 +21,18 @@ def __init__(self, cpu, loop, vars): self.cpu = cpu self.loop = loop - self.intvars = vars + self.intvars = [box for box in vars if isinstance(box, BoxInt)] self.boolvars = [] # subset of self.intvars self.ptrvars = [] self.prebuilt_ptr_consts = [] + floatvars = [box for box in vars if isinstance(box, BoxFloat)] + if cpu.supports_floats: + self.floatvars = floatvars + else: + assert floatvars == [] self.should_fail_by = None self.counter = 0 + assert len(self.intvars) == len(dict.fromkeys(self.intvars)) def fork(self, cpu, loop, vars): fork = self.__class__(cpu, loop, vars) @@ -34,8 +41,8 @@ def do(self, opnum, argboxes, descr=None): v_result = execute_nonspec(self.cpu, opnum, argboxes, descr) - if isinstance(v_result, ConstInt): - v_result = BoxInt(v_result.value) + if isinstance(v_result, Const): + v_result = v_result.clonebox() self.loop.operations.append(ResOperation(opnum, argboxes, v_result, descr)) return v_result @@ -68,8 +75,12 @@ subset = [] k = r.random() num = int(k * len(self.intvars)) + seen = {} for i in range(num): - subset.append(r.choice(self.intvars)) + v = r.choice(self.intvars) + if v not in seen: + subset.append(v) + seen[v] = True return subset def process_operation(self, s, op, names, subops): @@ -87,8 +98,12 @@ args.append( 'ConstAddr(llmemory.cast_ptr_to_adr(%s_vtable), cpu)' % name) - else: + elif isinstance(v, ConstFloat): + args.append('ConstFloat(%r)' % v.value) + elif isinstance(v, ConstInt): args.append('ConstInt(%d)' % v.value) + else: + raise NotImplementedError(v) if op.descr is None: descrstr = '' else: @@ -98,8 +113,8 @@ descrstr = ', descr=...' print >>s, ' ResOperation(rop.%s, [%s], %s%s),' % ( opname[op.opnum], ', '.join(args), names[op.result], descrstr) - if getattr(op, 'suboperations', None) is not None: - subops.append(op) + #if getattr(op, 'suboperations', None) is not None: + # subops.append(op) def print_loop(self): #raise PleaseRewriteMe() @@ -108,8 +123,8 @@ v = op.result if v not in names: writevar(v, 'tmp') - if getattr(op, 'suboperations', None) is not None: - update_names(op.suboperations) + #if getattr(op, 'suboperations', None) is not None: + # update_names(op.suboperations) def print_loop_prebuilt(ops): for op in ops: @@ -117,8 +132,8 @@ if isinstance(arg, ConstPtr): if arg not in names: writevar(arg, 'const_ptr') - if getattr(op, 'suboperations', None) is not None: - print_loop_prebuilt(op.suboperations) + #if getattr(op, 'suboperations', None) is not None: + # print_loop_prebuilt(op.suboperations) if demo_conftest.option.output: s = open(demo_conftest.option.output, "w") @@ -134,6 +149,8 @@ # for v in self.intvars: writevar(v, 'v') + for v in self.floatvars: + writevar(v, 'f') for v, S in self.ptrvars: writevar(v, 'p') update_names(self.loop.operations) @@ -149,8 +166,8 @@ print >>s, ' ]' while subops: next = subops.pop(0) - for op in next.suboperations: - self.process_operation(s, op, names, subops) + #for op in next.suboperations: + # self.process_operation(s, op, names, subops) # XXX think what to do about the one below #if len(op.suboperations) > 1: # continue # XXX @@ -163,18 +180,24 @@ print >>s, ' cpu.compile_loop(inputargs, operations, looptoken)' if hasattr(self.loop, 'inputargs'): for i, v in enumerate(self.loop.inputargs): - print >>s, ' cpu.set_future_value_int(%d, %d)' % (i, - v.value) + if isinstance(v, (BoxFloat, ConstFloat)): + print >>s, ' cpu.set_future_value_float(%d, %r)' % (i, + v.value) + else: + print >>s, ' cpu.set_future_value_int(%d, %d)' % (i, + v.value) print >>s, ' op = cpu.execute_token(looptoken)' if self.should_fail_by is None: - for i, v in enumerate(self.loop.operations[-1].args): - print >>s, ' assert cpu.get_latest_value_int(%d) == %d' % ( - i, v.value) - else: - #print >>s, ' assert op is loop.operations[%d].suboperations[0]' % self.should_fail_by_num - for i, v in enumerate(self.should_fail_by.fail_args): - print >>s, ' assert cpu.get_latest_value_int(%d) == %d' % ( - i, v.value) + fail_args = self.loop.operations[-1].args + else: + fail_args = self.should_fail_by.fail_args + for i, v in enumerate(fail_args): + if isinstance(v, (BoxFloat, ConstFloat)): + print >>s, (' assert cpu.get_latest_value_float(%d) == %r' + % (i, v.value)) + else: + print >>s, (' assert cpu.get_latest_value_int(%d) == %d' + % (i, v.value)) self.names = names if demo_conftest.option.output: s.close() @@ -186,15 +209,23 @@ def __init__(self, opnum, boolres=False): self.opnum = opnum self.boolres = boolres + def filter(self, builder): + pass def put(self, builder, args, descr=None): v_result = builder.do(self.opnum, args, descr=descr) if v_result is not None: - builder.intvars.append(v_result) - boolres = self.boolres - if boolres == 'sometimes': - boolres = v_result.value in [0, 1] - if boolres: - builder.boolvars.append(v_result) + if isinstance(v_result, BoxInt): + builder.intvars.append(v_result) + boolres = self.boolres + if boolres == 'sometimes': + boolres = v_result.value in [0, 1] + if boolres: + builder.boolvars.append(v_result) + elif isinstance(v_result, BoxFloat): + builder.floatvars.append(v_result) + assert self.boolres != True + else: + raise NotImplementedError(v_result) class UnaryOperation(AbstractOperation): def produce_into(self, builder, r): @@ -207,7 +238,10 @@ class ConstUnaryOperation(UnaryOperation): def produce_into(self, builder, r): - self.put(builder, [ConstInt(r.random_integer())]) + if r.random() < 0.75 or not builder.cpu.supports_floats: + self.put(builder, [ConstInt(r.random_integer())]) + else: + self.put(builder, [ConstFloat(r.random_float())]) class BinaryOperation(AbstractOperation): def __init__(self, opnum, and_mask=-1, or_mask=0, boolres=False): @@ -254,6 +288,49 @@ class BinaryOvfOperation(AbstractOvfOperation, BinaryOperation): pass +class AbstractFloatOperation(AbstractOperation): + def filter(self, builder): + if not builder.cpu.supports_floats: + raise CannotProduceOperation + +class BinaryFloatOperation(AbstractFloatOperation): + def produce_into(self, builder, r): + if not builder.floatvars: + raise CannotProduceOperation + k = r.random() + if k < 0.18: + v_first = ConstFloat(r.random_float()) + else: + v_first = r.choice(builder.floatvars) + if k > 0.82: + v_second = ConstFloat(r.random_float()) + else: + v_second = r.choice(builder.floatvars) + if abs(v_first.value) > 1E100 or abs(v_second.value) > 1E100: + raise CannotProduceOperation # avoid infinities + if abs(v_second.value) < 1E-100: + raise CannotProduceOperation # e.g. division by zero error + self.put(builder, [v_first, v_second]) + +class UnaryFloatOperation(AbstractFloatOperation): + def produce_into(self, builder, r): + if not builder.floatvars: + raise CannotProduceOperation + self.put(builder, [r.choice(builder.floatvars)]) + +class CastIntToFloatOperation(AbstractFloatOperation): + def produce_into(self, builder, r): + self.put(builder, [r.choice(builder.intvars)]) + +class CastFloatToIntOperation(AbstractFloatOperation): + def produce_into(self, builder, r): + if not builder.floatvars: + raise CannotProduceOperation + box = r.choice(builder.floatvars) + if not (-sys.maxint-1 <= box.value <= sys.maxint): + raise CannotProduceOperation # would give an overflow + self.put(builder, [box]) + class GuardOperation(AbstractOperation): def gen_guard(self, builder, r): v = builder.get_bool_var(r) @@ -340,6 +417,22 @@ ]: OPERATIONS.append(BinaryOvfOperation(_op)) +for _op in [rop.FLOAT_ADD, + rop.FLOAT_SUB, + rop.FLOAT_MUL, + rop.FLOAT_TRUEDIV, + ]: + OPERATIONS.append(BinaryFloatOperation(_op)) + +for _op in [rop.FLOAT_NEG, + rop.FLOAT_ABS, + rop.FLOAT_IS_TRUE, + ]: + OPERATIONS.append(UnaryFloatOperation(_op)) + +OPERATIONS.append(CastFloatToIntOperation(rop.CAST_FLOAT_TO_INT)) +OPERATIONS.append(CastIntToFloatOperation(rop.CAST_INT_TO_FLOAT)) + OperationBuilder.OPERATIONS = OPERATIONS # ____________________________________________________________ @@ -367,8 +460,15 @@ return result def get_random_char(): return chr(get_random_integer() % 256) + def get_random_float(): + x = float(get_random_integer()) + k = r.random() * 1.2 + if k < 1.0: + x += k + return x r.random_integer = get_random_integer r.random_char = get_random_char + r.random_float = get_random_float return r def get_cpu(): @@ -389,10 +489,24 @@ def __init__(self, cpu, builder_factory, r, startvars=None): self.cpu = cpu if startvars is None: - startvars = [BoxInt(r.random_integer()) - for i in range(demo_conftest.option.n_vars)] + startvars = [] + if cpu.supports_floats: + # pick up a single threshold for the whole 'inputargs', so + # that some loops have no or mostly no BoxFloat while others + # have a lot of them + k = r.random() + # but make sure there is at least one BoxInt + at_least_once = r.randrange(0, demo_conftest.option.n_vars) + else: + k = -1 + at_least_once = 0 + for i in range(demo_conftest.option.n_vars): + if r.random() < k and i != at_least_once: + startvars.append(BoxFloat(r.random_float())) + else: + startvars.append(BoxInt(r.random_integer())) + assert len(dict.fromkeys(startvars)) == len(startvars) self.startvars = startvars - self.values = [var.value for var in startvars] self.prebuilt_ptr_consts = [] self.r = r self.build_random_loop(cpu, builder_factory, r, startvars) @@ -415,7 +529,9 @@ for i in range(block_length): try: - r.choice(builder.OPERATIONS).produce_into(builder, r) + op = r.choice(builder.OPERATIONS) + op.filter(builder) + op.produce_into(builder, r) except CannotProduceOperation: pass if builder.should_fail_by is not None: @@ -468,14 +584,22 @@ assert not cpu.get_exception() assert not cpu.get_exc_value() - for i, v in enumerate(self.values): - cpu.set_future_value_int(i, v) + for i, box in enumerate(self.startvars): + if isinstance(box, BoxInt): + cpu.set_future_value_int(i, box.value) + elif isinstance(box, BoxFloat): + cpu.set_future_value_float(i, box.value) + else: + raise NotImplementedError(box) fail = cpu.execute_token(self.loop.token) assert fail is self.should_fail_by.descr for i, v in enumerate(self.get_fail_args()): - value = cpu.get_latest_value_int(i) + if isinstance(v, (BoxFloat, ConstFloat)): + value = cpu.get_latest_value_float(i) + else: + value = cpu.get_latest_value_int(i) assert value == self.expected[v], ( - "Got %d, expected %d for value #%d" % (value, + "Got %r, expected %r for value #%d" % (value, self.expected[v], i) ) From arigo at codespeak.net Mon Oct 5 19:13:21 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 5 Oct 2009 19:13:21 +0200 (CEST) Subject: [pypy-svn] r68187 - pypy/branch/remove-ri386-multimethod Message-ID: <20091005171321.D71971683F7@codespeak.net> Author: arigo Date: Mon Oct 5 19:13:20 2009 New Revision: 68187 Removed: pypy/branch/remove-ri386-multimethod/ Log: Remove the branch, to be added again freshly from thunk. From arigo at codespeak.net Mon Oct 5 19:15:06 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 5 Oct 2009 19:15:06 +0200 (CEST) Subject: [pypy-svn] r68188 - pypy/branch/remove-ri386-multimethod Message-ID: <20091005171506.1C4E61683F7@codespeak.net> Author: arigo Date: Mon Oct 5 19:15:05 2009 New Revision: 68188 Added: pypy/branch/remove-ri386-multimethod/ - copied from r68187, pypy/trunk/ Log: Restart a branch in which to remove multimethods in the x86 backend. For fun and profit, that branch also contains support for AMD64. From arigo at codespeak.net Mon Oct 5 19:19:26 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 5 Oct 2009 19:19:26 +0200 (CEST) Subject: [pypy-svn] r68189 - in pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86: . test Message-ID: <20091005171926.EABAC1683F9@codespeak.net> Author: arigo Date: Mon Oct 5 19:19:26 2009 New Revision: 68189 Added: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py - copied, changed from r68186, pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86.py - copied unchanged from r68186, pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py - copied unchanged from r68186, pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py - copied unchanged from r68186, pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_rx86_64_auto_encoding.py Removed: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/ri386.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/ri386setup.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386.py pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Log: Copy rx86.py and the tests from the previous branch. Copied: pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py (from r68186, pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py) ============================================================================== --- pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/remove-ri386-multimethod/pypy/jit/backend/x86/rx86.py Mon Oct 5 19:19:26 2009 @@ -349,7 +349,7 @@ self.writeimm32(intmask(rffi.cast(rffi.INT, imm))) self.writeimm32(imm >> 32) - # MOV_ri from the parent class is not wrong, but add a better encoding + # MOV_ri from the parent class is not wrong, but here is a better encoding # for the common case where the immediate fits in 32 bits _MOV_ri32 = insn(rex_w, '\xC7', register(1), '\xC0', immediate(2, 'i')) From pedronis at codespeak.net Mon Oct 5 19:20:59 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 5 Oct 2009 19:20:59 +0200 (CEST) Subject: [pypy-svn] r68190 - in pypy/trunk/pypy: jit/metainterp jit/metainterp/test rpython rpython/test Message-ID: <20091005172059.4E3AD1683F9@codespeak.net> Author: pedronis Date: Mon Oct 5 19:20:58 2009 New Revision: 68190 Modified: pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_resume.py pypy/trunk/pypy/rpython/rlist.py pypy/trunk/pypy/rpython/test/test_rlist.py Log: (cfbolz, pedronis) - start moving to a tagging scheme and using shorts to encode resume information - some help from the rtyper Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Mon Oct 5 19:20:58 2009 @@ -1,6 +1,8 @@ import sys from pypy.jit.metainterp.history import Box, Const, ConstInt from pypy.jit.metainterp.resoperation import rop +from pypy.rpython.lltypesystem import rffi +from pypy.rlib import rarithmetic # Logic to encode the chain of frames and the state of the boxes at a # guard operation, and to decode it again. This is a bit advanced, @@ -58,6 +60,32 @@ snapshot = Snapshot(snapshot, virtualizable_boxes[:]) # xxx for now storage.rd_snapshot = snapshot +TAGMASK = 3 + +def tag(value, tagbits): + if tagbits >> 2: + raise ValueError + sx = value >> 13 + if sx != 0 and sx != -1: + raise ValueError + return rffi.r_short(value<<2|tagbits) + +def untag(value): + value = rarithmetic.widen(value) + tagbits = value&TAGMASK + return value>>2, tagbits + +def tagged_eq(x, y): + # please rpython :( + return rarithmetic.widen(x) == rarithmetic.widen(y) + +TAGCONST = 0 +TAGBOX = 1 +TAGVIRTUAL = 2 + +NEXTFRAME = tag(-1, TAGVIRTUAL) +UNASSIGNED = tag(-1, TAGBOX) + VIRTUAL_FLAG = int((sys.maxint+1) // 2) assert not (VIRTUAL_FLAG & (VIRTUAL_FLAG-1)) # a power of two @@ -93,8 +121,8 @@ def make_constant(self, box, const): # this part of the interface is not used so far by optimizeopt.py - if self.liveboxes[box] == 0: - self.liveboxes[box] = self._getconstindex(const) + if tagged_eq(self.liveboxes[box], UNASSIGNED): + self.liveboxes[box] = self._getconst(const) def make_virtual(self, virtualbox, known_class, fielddescrs, fieldboxes): vinfo = VirtualInfo(known_class, fielddescrs) @@ -109,15 +137,15 @@ self._make_virtual(virtualbox, vinfo, itemboxes) def _make_virtual(self, virtualbox, vinfo, fieldboxes): - assert self.liveboxes[virtualbox] == 0 - self.liveboxes[virtualbox] = len(self.virtuals) | VIRTUAL_FLAG + assert tagged_eq(self.liveboxes[virtualbox], UNASSIGNED) + self.liveboxes[virtualbox] = tag(len(self.virtuals), TAGVIRTUAL) self.virtuals.append(vinfo) self.vfieldboxes.append(fieldboxes) self._register_boxes(fieldboxes) def register_box(self, box): if isinstance(box, Box) and box not in self.liveboxes: - self.liveboxes[box] = 0 + self.liveboxes[box] = UNASSIGNED return True return False @@ -126,7 +154,9 @@ self.register_box(box) def is_virtual(self, virtualbox): - return self.liveboxes[virtualbox] >= VIRTUAL_FLAG + tagged = self.liveboxes[virtualbox] + _, tagbits = untag(tagged) + return tagbits == TAGVIRTUAL def _flatten_frame_info(self): storage = self.storage @@ -149,21 +179,21 @@ storage = self.storage liveboxes = [] for box in self.liveboxes.iterkeys(): - if self.liveboxes[box] == 0: - self.liveboxes[box] = len(liveboxes) + if tagged_eq(self.liveboxes[box], UNASSIGNED): + self.liveboxes[box] = tag(len(liveboxes), TAGBOX) liveboxes.append(box) - nums = storage.rd_nums = [0]*self.nnums + nums = storage.rd_nums = [rffi.r_short(0)]*self.nnums i = self.nnums-1 snapshot = self.storage.rd_snapshot while True: # at least one boxes = snapshot.boxes - nums[i] = -1 + nums[i] = NEXTFRAME i -= 1 for j in range(len(boxes)-1, -1, -1): box = boxes[j] if box in values: box = values[box].get_key_box() - nums[i] = self._getboxindex(box) + nums[i] = self._gettagged(box) i -= 1 snapshot = snapshot.prev if snapshot is None: @@ -174,7 +204,7 @@ for i in range(len(storage.rd_virtuals)): vinfo = storage.rd_virtuals[i] fieldboxes = self.vfieldboxes[i] - vinfo.fieldnums = [self._getboxindex(box) + vinfo.fieldnums = [self._gettagged(box) for box in fieldboxes] storage.rd_consts = self.consts[:] storage.rd_snapshot = None @@ -182,14 +212,14 @@ dump_storage(storage, liveboxes) return liveboxes - def _getboxindex(self, box): + def _gettagged(self, box): if isinstance(box, Const): - return self._getconstindex(box) + return self._getconst(box) else: return self.liveboxes[box] - def _getconstindex(self, const): - result = -2 - len(self.consts) + def _getconst(self, const): + result = tag(len(self.consts), TAGCONST) self.consts.append(const) return result @@ -289,19 +319,21 @@ while True: num = self.nums[self.i_boxes] self.i_boxes += 1 - if num == -1: + if tagged_eq(num, NEXTFRAME): break boxes.append(self._decode_box(num)) return boxes def _decode_box(self, num): - if num < 0: - return self.consts[-2 - num] - elif num & VIRTUAL_FLAG: + num, tag = untag(num) + if tag == TAGCONST: + return self.consts[num] + elif tag == TAGVIRTUAL: virtuals = self.virtuals assert virtuals is not None - return virtuals[num - VIRTUAL_FLAG] + return virtuals[num] else: + assert tag == TAGBOX return self.liveboxes[num] def has_more_frame_infos(self): Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Mon Oct 5 19:20:58 2009 @@ -10,7 +10,7 @@ from pypy.jit.metainterp.optimizeopt import optimize_loop_1 from pypy.jit.metainterp.optimizeutil import InvalidLoop from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt -from pypy.jit.metainterp import resume, executor, compile +from pypy.jit.metainterp import executor, compile, resume from pypy.jit.metainterp.resoperation import rop, opname, ResOperation from pypy.jit.metainterp.test.oparser import pure_parse @@ -25,6 +25,7 @@ def test_store_final_boxes_in_guard(): from pypy.jit.metainterp.compile import ResumeGuardDescr + from pypy.jit.metainterp.resume import tag, NEXTFRAME, TAGBOX b0 = BoxInt() b1 = BoxInt() opt = optimizeopt.Optimizer(None, None) @@ -40,10 +41,12 @@ # opt.store_final_boxes_in_guard(op) if op.fail_args == [b0, b1]: - assert fdescr.rd_nums == [0, -1, 1, -1] + assert fdescr.rd_nums == [tag(0, TAGBOX), NEXTFRAME, + tag(1, TAGBOX), NEXTFRAME] else: assert op.fail_args == [b1, b0] - assert fdescr.rd_nums == [1, -1, 0, -1] + assert fdescr.rd_nums == [tag(1, TAGBOX), NEXTFRAME, + tag(0, TAGBOX), NEXTFRAME] assert fdescr.rd_virtuals is None assert fdescr.rd_consts == [] assert fdescr.rd_frame_infos == fi Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Mon Oct 5 19:20:58 2009 @@ -1,5 +1,5 @@ import py -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.jit.metainterp.resume import * from pypy.jit.metainterp.history import BoxInt, BoxPtr, ConstInt, ConstAddr from pypy.jit.metainterp.history import ConstPtr @@ -9,15 +9,45 @@ class Storage: pass +def test_tag(): + assert tag(3, 1) == rffi.r_short(3<<2|1) + assert tag(-3, 2) == rffi.r_short(-3<<2|2) + assert tag((1<<13)-1, 3) == rffi.r_short(((1<<15)-1)|3) + assert tag(-1<<13, 3) == rffi.r_short((-1<<15)|3) + py.test.raises(ValueError, tag, 3, 5) + py.test.raises(ValueError, tag, 1<<13, 0) + py.test.raises(ValueError, tag, (1<<13)+1, 0) + py.test.raises(ValueError, tag, (-1<<13)-1, 0) + py.test.raises(ValueError, tag, (-1<<13)-5, 0) + +def test_untag(): + assert untag(tag(3, 1)) == (3, 1) + assert untag(tag(-3, 2)) == (-3, 2) + assert untag(tag((1<<13)-1, 3)) == ((1<<13)-1, 3) + assert untag(tag(-1<<13, 3)) == (-1<<13, 3) + +def test_tagged_eq(): + assert tagged_eq(NEXTFRAME, NEXTFRAME) + assert not tagged_eq(NEXTFRAME, UNASSIGNED) + def test_simple_read(): b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] storage = Storage() storage.rd_frame_infos = [] storage.rd_consts = [c1, c2, c3] - storage.rd_nums = [0, -2, 0, 1, -1, - -3, -4, -1, - 0, 1, 2, -1 + storage.rd_nums = [tag(0, TAGBOX), + tag(0, TAGCONST), + tag(0, TAGBOX), + tag(1, TAGBOX), + NEXTFRAME, + tag(1, TAGCONST), + tag(2, TAGCONST), + NEXTFRAME, + tag(0, TAGBOX), + tag(1, TAGBOX), + tag(2, TAGBOX), + NEXTFRAME ] storage.rd_virtuals = None b1s, b2s, b3s = [BoxInt(), BoxPtr(), BoxInt()] @@ -175,7 +205,8 @@ modifier = ResumeDataVirtualAdder(storage) modifier.walk_snapshots({}) - assert modifier.liveboxes == {b1: 0, b2: 0, b3: 0} + assert modifier.liveboxes == {b1: UNASSIGNED, b2: UNASSIGNED, + b3: UNASSIGNED} assert modifier.nnums == len(env)+1+len(env1)+1 b1_2 = BoxInt() @@ -191,7 +222,7 @@ modifier = ResumeDataVirtualAdder(storage) modifier.walk_snapshots({b1: val, b2: val}) - assert modifier.liveboxes == {b1_2: 0, b3: 0} + assert modifier.liveboxes == {b1_2: UNASSIGNED, b3: UNASSIGNED} assert modifier.nnums == len(env)+1+len(env1)+1 def test_flatten_frame_info(): Modified: pypy/trunk/pypy/rpython/rlist.py ============================================================================== --- pypy/trunk/pypy/rpython/rlist.py (original) +++ pypy/trunk/pypy/rpython/rlist.py Mon Oct 5 19:20:58 2009 @@ -5,11 +5,11 @@ from pypy.rpython.rmodel import Repr, IteratorRepr, IntegerRepr, inputconst from pypy.rpython.rstr import AbstractStringRepr, AbstractCharRepr from pypy.rpython.lltypesystem.lltype import typeOf, Ptr, Void, Signed, Bool -from pypy.rpython.lltypesystem.lltype import nullptr, Char, UniChar +from pypy.rpython.lltypesystem.lltype import nullptr, Char, UniChar, Number from pypy.rpython import robject from pypy.rlib.objectmodel import malloc_zero_filled from pypy.rlib.debug import ll_assert -from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib.rarithmetic import ovfcheck, widen from pypy.rpython.annlowlevel import ADTInterface ADTIFixedList = ADTInterface(None, { @@ -493,6 +493,8 @@ T = typeOf(item) if T is Char or T is UniChar: check = ord(item) + elif isinstance(T, Number): + check = widen(item) else: check = item if (not malloc_zero_filled) or check: # as long as malloc it is known to zero the allocated memory avoid zeroing twice Modified: pypy/trunk/pypy/rpython/test/test_rlist.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_rlist.py (original) +++ pypy/trunk/pypy/rpython/test/test_rlist.py Mon Oct 5 19:20:58 2009 @@ -1296,6 +1296,16 @@ return ''.join(l) py.test.raises(TyperError, self.interpret, f, [5]) + def test_r_short_list(self): + from pypy.rpython.lltypesystem.rffi import r_short + from pypy.rlib import rarithmetic + def f(i): + l = [r_short(0)] * 10 + l[i+1] = r_short(3) + return rarithmetic.widen(l[i]) + res = self.interpret(f, [3]) + assert res == 0 + class TestLLtype(BaseTestRlist, LLRtypeMixin): rlist = ll_rlist From cfbolz at codespeak.net Mon Oct 5 21:43:46 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 5 Oct 2009 21:43:46 +0200 (CEST) Subject: [pypy-svn] r68191 - pypy/trunk/pypy/interpreter Message-ID: <20091005194346.25A89168405@codespeak.net> Author: cfbolz Date: Mon Oct 5 21:43:44 2009 New Revision: 68191 Modified: pypy/trunk/pypy/interpreter/typedef.py Log: make Member immutable, to help the JIT with properties. Modified: pypy/trunk/pypy/interpreter/typedef.py ============================================================================== --- pypy/trunk/pypy/interpreter/typedef.py (original) +++ pypy/trunk/pypy/interpreter/typedef.py Mon Oct 5 21:43:44 2009 @@ -508,6 +508,7 @@ class Member(Wrappable): """For slots.""" + _immutable_ = True def __init__(self, index, name, w_cls): self.index = index self.name = name From cfbolz at codespeak.net Mon Oct 5 22:10:54 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 5 Oct 2009 22:10:54 +0200 (CEST) Subject: [pypy-svn] r68192 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091005201054.1CA0816801E@codespeak.net> Author: cfbolz Date: Mon Oct 5 22:10:52 2009 New Revision: 68192 Modified: pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/jit/metainterp/test/test_resume.py Log: (pedronis jumping around, cfbolz): use a new tag for not-too-large integers in the resume data to make stuff more compact. Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Mon Oct 5 22:10:52 2009 @@ -3,6 +3,7 @@ from pypy.jit.metainterp.resoperation import rop from pypy.rpython.lltypesystem import rffi from pypy.rlib import rarithmetic +from pypy.rlib.objectmodel import we_are_translated # Logic to encode the chain of frames and the state of the boxes at a # guard operation, and to decode it again. This is a bit advanced, @@ -80,8 +81,9 @@ return rarithmetic.widen(x) == rarithmetic.widen(y) TAGCONST = 0 -TAGBOX = 1 -TAGVIRTUAL = 2 +TAGINT = 1 +TAGBOX = 2 +TAGVIRTUAL = 3 NEXTFRAME = tag(-1, TAGVIRTUAL) UNASSIGNED = tag(-1, TAGBOX) @@ -219,6 +221,15 @@ return self.liveboxes[box] def _getconst(self, const): + if isinstance(const, ConstInt): + val = const.getint() + try: + if not we_are_translated() and not isinstance(val, int): + # unhappiness, probably a symbolic + raise ValueError + return tag(val, TAGINT) + except ValueError: + pass result = tag(len(self.consts), TAGCONST) self.consts.append(const) return result @@ -332,6 +343,8 @@ virtuals = self.virtuals assert virtuals is not None return virtuals[num] + elif tag == TAGINT: + return ConstInt(num) else: assert tag == TAGBOX return self.liveboxes[num] Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Mon Oct 5 22:10:52 2009 @@ -60,6 +60,34 @@ lst = reader.consume_boxes() assert lst == [b1s, b2s, b3s] +def test_simple_read_tagged_ints(): + b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] + storage = Storage() + storage.rd_frame_infos = [] + storage.rd_consts = [] + storage.rd_nums = [tag(0, TAGBOX), + tag(1, TAGINT), + tag(0, TAGBOX), + tag(1, TAGBOX), + NEXTFRAME, + tag(2, TAGINT), + tag(3, TAGINT), + NEXTFRAME, + tag(0, TAGBOX), + tag(1, TAGBOX), + tag(2, TAGBOX), + NEXTFRAME + ] + storage.rd_virtuals = None + b1s, b2s, b3s = [BoxInt(), BoxPtr(), BoxInt()] + assert b1s != b3s + reader = ResumeDataReader(storage, [b1s, b2s, b3s]) + lst = reader.consume_boxes() + assert lst == [b1s, ConstInt(1), b1s, b2s] + lst = reader.consume_boxes() + assert lst == [ConstInt(2), ConstInt(3)] + lst = reader.consume_boxes() + assert lst == [b1s, b2s, b3s] def test_frame_info(): storage = Storage() @@ -298,6 +326,25 @@ assert lst == [b1t, b2t, b3t] assert metainterp.trace == [] +def test_virtual_adder_int_constants(): + b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)] + storage = make_storage(b1s, b2s, b3s) + modifier = ResumeDataVirtualAdder(storage) + modifier.walk_snapshots({}) + liveboxes = modifier.finish({}) + assert storage.rd_snapshot is None + metainterp = MyMetaInterp(LLtypeMixin.cpu) + reader = ResumeDataReader(storage, [], metainterp) + lst = reader.consume_boxes() + assert lst == [ConstInt(sys.maxint), ConstInt(1), ConstInt(sys.maxint), + ConstInt(2**16)] + lst = reader.consume_boxes() + assert lst == [ConstInt(2), ConstInt(3)] + lst = reader.consume_boxes() + assert lst == [b1s, b2s, b3s] + assert metainterp.trace == [] + + def test_virtual_adder_no_op_renaming(): b1s, b2s, b3s = [BoxInt(1), BoxInt(2), BoxInt(3)] storage = make_storage(b1s, b2s, b3s) From cfbolz at codespeak.net Tue Oct 6 10:44:56 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 6 Oct 2009 10:44:56 +0200 (CEST) Subject: [pypy-svn] r68193 - pypy/trunk/pypy/rlib Message-ID: <20091006084456.928FD16844E@codespeak.net> Author: cfbolz Date: Tue Oct 6 10:44:55 2009 New Revision: 68193 Modified: pypy/trunk/pypy/rlib/rbigint.py Log: Stop trying to share 0L and 1L, seems to be more trouble than it's worth (it's the most likely cause for the JIT long failures). Modified: pypy/trunk/pypy/rlib/rbigint.py ============================================================================== --- pypy/trunk/pypy/rlib/rbigint.py (original) +++ pypy/trunk/pypy/rlib/rbigint.py Tue Oct 6 10:44:55 2009 @@ -49,8 +49,8 @@ class rbigint(object): """This is a reimplementation of longs using a list of digits.""" - def __init__(self, digits, sign=0): - if len(digits) == 0: + def __init__(self, digits=None, sign=0): + if digits is None or len(digits) == 0: digits = [0] self.digits = digits self.sign = sign @@ -73,7 +73,7 @@ sign = 1 ival = r_uint(intval) else: - return rbigint.ZERO + return rbigint() # Count the number of Python digits. # We used to pick 5 ("big enough for anything"), but that's a # waste of time and space given that 5*15 = 75 bits are rarely @@ -95,8 +95,8 @@ def frombool(b): if b: - return rbigint.ONE - return rbigint.ZERO + return rbigint([1], 1) + return rbigint() frombool = staticmethod(frombool) def fromlong(l): @@ -113,7 +113,7 @@ dval = -dval frac, expo = math.frexp(dval) # dval = frac*2**expo; 0.0 <= frac < 1.0 if expo <= 0: - return rbigint.ZERO + return rbigint() ndig = (expo-1) // SHIFT + 1 # Number of 'digits' in result v = rbigint([0] * ndig, 1) frac = math.ldexp(frac, (expo-1) % SHIFT + 1) @@ -350,7 +350,7 @@ div, mod = _divrem(v, w) if mod.sign * w.sign == -1: mod = mod.add(w) - div = div.sub(rbigint.ONE) + div = div.sub(rbigint([1], 1)) return div, mod def pow(a, b, c=None): @@ -382,7 +382,7 @@ # if modulus == 1: # return 0 if c._numdigits() == 1 and c.digits[0] == 1: - return rbigint.ZERO + return rbigint() # if base < 0: # base = base % modulus @@ -488,7 +488,7 @@ wordshift = int_other // SHIFT newsize = self._numdigits() - wordshift if newsize <= 0: - return rbigint.ZERO + return rbigint() loshift = int_other % SHIFT hishift = SHIFT - loshift @@ -558,8 +558,6 @@ def __repr__(self): return "" % (self.digits, self.sign, self.str()) -rbigint.ZERO = rbigint([0], 0) -rbigint.ONE = rbigint([1], 1) #_________________________________________________________________ From pedronis at codespeak.net Tue Oct 6 10:52:20 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 6 Oct 2009 10:52:20 +0200 (CEST) Subject: [pypy-svn] r68194 - pypy/trunk/pypy/translator/c/test Message-ID: <20091006085220.95911168458@codespeak.net> Author: pedronis Date: Tue Oct 6 10:52:20 2009 New Revision: 68194 Modified: pypy/trunk/pypy/translator/c/test/test_dlltool.py Log: make them xfail with a comment. The fact that they broke is a bit of a sign of too much abuse and fragility Modified: pypy/trunk/pypy/translator/c/test/test_dlltool.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_dlltool.py (original) +++ pypy/trunk/pypy/translator/c/test/test_dlltool.py Tue Oct 6 10:52:20 2009 @@ -4,7 +4,9 @@ import py class TestDLLTool(object): + @py.test.mark.xfail def test_basic(self): + # XXX abusing get_entry_point to get a so name makes no sense def f(x): return x @@ -17,6 +19,7 @@ assert dll.f(3) == 3 assert dll.b(10) == 12 + @py.test.mark.xfail def test_split_criteria(self): def f(x): return x From fijal at codespeak.net Tue Oct 6 10:57:03 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 6 Oct 2009 10:57:03 +0200 (CEST) Subject: [pypy-svn] r68195 - in pypy/trunk/pypy/translator/c: . test Message-ID: <20091006085703.0E870168458@codespeak.net> Author: fijal Date: Tue Oct 6 10:57:02 2009 New Revision: 68195 Modified: pypy/trunk/pypy/translator/c/dlltool.py pypy/trunk/pypy/translator/c/test/test_dlltool.py Log: Accept keyword arg and ignore it Modified: pypy/trunk/pypy/translator/c/dlltool.py ============================================================================== --- pypy/trunk/pypy/translator/c/dlltool.py (original) +++ pypy/trunk/pypy/translator/c/dlltool.py Tue Oct 6 10:57:02 2009 @@ -32,7 +32,7 @@ standalone=False, outputfilename=oname) - def get_entry_point(self): + def get_entry_point(self, isolated=False): return self.so_name class DLLDef(object): Modified: pypy/trunk/pypy/translator/c/test/test_dlltool.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_dlltool.py (original) +++ pypy/trunk/pypy/translator/c/test/test_dlltool.py Tue Oct 6 10:57:02 2009 @@ -4,7 +4,6 @@ import py class TestDLLTool(object): - @py.test.mark.xfail def test_basic(self): # XXX abusing get_entry_point to get a so name makes no sense def f(x): @@ -19,7 +18,6 @@ assert dll.f(3) == 3 assert dll.b(10) == 12 - @py.test.mark.xfail def test_split_criteria(self): def f(x): return x From arigo at codespeak.net Tue Oct 6 11:18:09 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 6 Oct 2009 11:18:09 +0200 (CEST) Subject: [pypy-svn] r68197 - pypy/trunk/pypy/jit/backend/x86 Message-ID: <20091006091809.BD54E168457@codespeak.net> Author: arigo Date: Tue Oct 6 11:18:08 2009 New Revision: 68197 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py Log: No point in generating MOV(reg, same_reg). Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Tue Oct 6 11:18:08 2009 @@ -218,7 +218,8 @@ else: self.mc.MOV(target, addr_add(imm(self.fail_box_int_addr), imm(i*WORD))) - self.mc.MOV(loc, target) + if target is not loc: + self.mc.MOV(loc, target) for i in range(len(floatlocs)): loc = floatlocs[i] if loc is None: From pedronis at codespeak.net Tue Oct 6 12:16:27 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 6 Oct 2009 12:16:27 +0200 (CEST) Subject: [pypy-svn] r68198 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091006101627.5390E16845C@codespeak.net> Author: pedronis Date: Tue Oct 6 12:16:26 2009 New Revision: 68198 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/jit/metainterp/simple_optimize.py pypy/trunk/pypy/jit/metainterp/test/test_resume.py Log: (cfbolz, pedronis) more sharing and interning of consts across the faildescrs of a loop Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Tue Oct 6 12:16:26 2009 @@ -365,6 +365,7 @@ self.values_to_clean = {} self.interned_refs = {} + self.resumedata_memo = resume.ResumeDataLoopMemo(cpu) def getinterned(self, box): constbox = self.get_constant_box(box) @@ -514,7 +515,7 @@ def store_final_boxes_in_guard(self, op): descr = op.descr assert isinstance(descr, compile.ResumeGuardDescr) - modifier = resume.ResumeDataVirtualAdder(descr) + modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo) modifier.walk_snapshots(self.values) newboxes = modifier.finish(self.values) descr.store_final_boxes(op, newboxes) Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Tue Oct 6 12:16:26 2009 @@ -1,5 +1,5 @@ import sys -from pypy.jit.metainterp.history import Box, Const, ConstInt +from pypy.jit.metainterp.history import Box, Const, ConstInt, INT, REF from pypy.jit.metainterp.resoperation import rop from pypy.rpython.lltypesystem import rffi from pypy.rlib import rarithmetic @@ -92,14 +92,54 @@ VIRTUAL_FLAG = int((sys.maxint+1) // 2) assert not (VIRTUAL_FLAG & (VIRTUAL_FLAG-1)) # a power of two +class ResumeDataLoopMemo(object): + + def __init__(self, cpu): + self.cpu = cpu + self.consts = [] + self.large_ints = {} + self.refs = {} + + def getconst(self, const): + if const.type == INT: + val = const.getint() + if not we_are_translated() and not isinstance(val, int): + # unhappiness, probably a symbolic + return self._newconst(const) + try: + return tag(val, TAGINT) + except ValueError: + pass + tagged = self.large_ints.get(val, UNASSIGNED) + if not tagged_eq(tagged, UNASSIGNED): + return tagged + tagged = self._newconst(const) + self.large_ints[val] = tagged + return tagged + elif const.type == REF: + val = const.getref_base() + val = self.cpu.ts.cast_ref_to_hashable(self.cpu, val) + tagged = self.refs.get(val, UNASSIGNED) + if not tagged_eq(tagged, UNASSIGNED): + return tagged + tagged = self._newconst(const) + self.refs[val] = tagged + return tagged + return self._newconst(const) + + def _newconst(self, const): + result = tag(len(self.consts), TAGCONST) + self.consts.append(const) + return result + _frame_info_placeholder = (None, 0, 0) class ResumeDataVirtualAdder(object): - def __init__(self, storage): + def __init__(self, storage, memo): self.storage = storage + self.memo = memo self.liveboxes = {} - self.consts = [] self.virtuals = [] self.vfieldboxes = [] @@ -124,7 +164,7 @@ def make_constant(self, box, const): # this part of the interface is not used so far by optimizeopt.py if tagged_eq(self.liveboxes[box], UNASSIGNED): - self.liveboxes[box] = self._getconst(const) + self.liveboxes[box] = self.memo.getconst(const) def make_virtual(self, virtualbox, known_class, fielddescrs, fieldboxes): vinfo = VirtualInfo(known_class, fielddescrs) @@ -208,7 +248,7 @@ fieldboxes = self.vfieldboxes[i] vinfo.fieldnums = [self._gettagged(box) for box in fieldboxes] - storage.rd_consts = self.consts[:] + storage.rd_consts = self.memo.consts storage.rd_snapshot = None if debug: dump_storage(storage, liveboxes) @@ -216,25 +256,10 @@ def _gettagged(self, box): if isinstance(box, Const): - return self._getconst(box) + return self.memo.getconst(box) else: return self.liveboxes[box] - def _getconst(self, const): - if isinstance(const, ConstInt): - val = const.getint() - try: - if not we_are_translated() and not isinstance(val, int): - # unhappiness, probably a symbolic - raise ValueError - return tag(val, TAGINT) - except ValueError: - pass - result = tag(len(self.consts), TAGCONST) - self.consts.append(const) - return result - - class AbstractVirtualInfo(object): def allocate(self, metainterp): raise NotImplementedError Modified: pypy/trunk/pypy/jit/metainterp/simple_optimize.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/simple_optimize.py (original) +++ pypy/trunk/pypy/jit/metainterp/simple_optimize.py Tue Oct 6 12:16:26 2009 @@ -7,7 +7,7 @@ EMPTY_VALUES = {} -def optimize_loop(options, old_loops, loop, cpu=None): +def optimize_loop(metainterp_sd, old_loops, loop, cpu=None): if old_loops: assert len(old_loops) == 1 return old_loops[0] @@ -16,12 +16,13 @@ # we need it since the backend can modify those lists, which make # get_guard_op in compile.py invalid # in fact, x86 modifies this list for moving GCs + memo = resume.ResumeDataLoopMemo(cpu) newoperations = [] for op in loop.operations: if op.is_guard(): descr = op.descr assert isinstance(descr, compile.ResumeGuardDescr) - modifier = resume.ResumeDataVirtualAdder(descr) + modifier = resume.ResumeDataVirtualAdder(descr, memo) modifier.walk_snapshots(EMPTY_VALUES) newboxes = modifier.finish(EMPTY_VALUES) descr.store_final_boxes(op, newboxes) @@ -29,6 +30,6 @@ loop.operations = newoperations return None -def optimize_bridge(options, old_loops, loop, cpu=None): - optimize_loop(options, [], loop, cpu) +def optimize_bridge(metainterp_sd, old_loops, loop, cpu=None): + optimize_loop(metainterp_sd, [], loop, cpu) return old_loops[0] Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Tue Oct 6 12:16:26 2009 @@ -2,7 +2,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.jit.metainterp.resume import * from pypy.jit.metainterp.history import BoxInt, BoxPtr, ConstInt, ConstAddr -from pypy.jit.metainterp.history import ConstPtr +from pypy.jit.metainterp.history import ConstPtr, ConstFloat from pypy.jit.metainterp.test.test_optimizefindnode import LLtypeMixin from pypy.jit.metainterp import executor @@ -230,7 +230,7 @@ storage = Storage() storage.rd_snapshot = snap1 - modifier = ResumeDataVirtualAdder(storage) + modifier = ResumeDataVirtualAdder(storage, None) modifier.walk_snapshots({}) assert modifier.liveboxes == {b1: UNASSIGNED, b2: UNASSIGNED, @@ -247,7 +247,7 @@ storage = Storage() storage.rd_snapshot = snap1 - modifier = ResumeDataVirtualAdder(storage) + modifier = ResumeDataVirtualAdder(storage, None) modifier.walk_snapshots({b1: val, b2: val}) assert modifier.liveboxes == {b1_2: UNASSIGNED, b3: UNASSIGNED} @@ -262,7 +262,7 @@ storage = Storage() storage.rd_frame_info_list = fi1 - modifier = ResumeDataVirtualAdder(storage) + modifier = ResumeDataVirtualAdder(storage, None) modifier._flatten_frame_info() assert storage.rd_frame_info_list is None @@ -270,6 +270,52 @@ ("JITCODE1", 3, 4)] +def test_ResumeDataLoopMemo_ints(): + memo = ResumeDataLoopMemo(None) + tagged = memo.getconst(ConstInt(44)) + assert untag(tagged) == (44, TAGINT) + tagged = memo.getconst(ConstInt(-3)) + assert untag(tagged) == (-3, TAGINT) + const = ConstInt(50000) + tagged = memo.getconst(const) + index, tagbits = untag(tagged) + assert tagbits == TAGCONST + assert memo.consts[index] is const + tagged = memo.getconst(ConstInt(50000)) + index2, tagbits = untag(tagged) + assert tagbits == TAGCONST + assert index2 == index + +demo55 = lltype.malloc(LLtypeMixin.NODE) +demo55o = lltype.cast_opaque_ptr(llmemory.GCREF, demo55) +demo66 = lltype.malloc(LLtypeMixin.NODE) +demo66o = lltype.cast_opaque_ptr(llmemory.GCREF, demo66) + +def test_ResumeDataLoopMemo_refs(): + cpu = LLtypeMixin.cpu + memo = ResumeDataLoopMemo(cpu) + const = cpu.ts.ConstRef(demo55o) + tagged = memo.getconst(const) + index, tagbits = untag(tagged) + assert tagbits == TAGCONST + assert memo.consts[index] is const + tagged = memo.getconst(cpu.ts.ConstRef(demo55o)) + index2, tagbits = untag(tagged) + assert tagbits == TAGCONST + assert index2 == index + tagged = memo.getconst(cpu.ts.ConstRef(demo66o)) + index3, tagbits = untag(tagged) + assert tagbits == TAGCONST + assert index3 != index + +def test_ResumeDataLoopMemo_other(): + memo = ResumeDataLoopMemo(None) + const = ConstFloat(-1.0) + tagged = memo.getconst(const) + index, tagbits = untag(tagged) + assert tagbits == TAGCONST + assert memo.consts[index] is const + class MyMetaInterp: def __init__(self, cpu): self.cpu = cpu @@ -282,9 +328,6 @@ descr)) return resbox -demo55 = lltype.malloc(LLtypeMixin.NODE) -demo55o = lltype.cast_opaque_ptr(llmemory.GCREF, demo55) - def _resume_remap(liveboxes, expected, *newvalues): newboxes = [] for box in liveboxes: @@ -306,7 +349,8 @@ def test_virtual_adder_no_op(): b1s, b2s, b3s = [BoxInt(1), BoxPtr(), BoxInt(3)] storage = make_storage(b1s, b2s, b3s) - modifier = ResumeDataVirtualAdder(storage) + memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + modifier = ResumeDataVirtualAdder(storage, memo) modifier.walk_snapshots({}) assert not modifier.is_virtual(b1s) assert not modifier.is_virtual(b2s) @@ -329,7 +373,8 @@ def test_virtual_adder_int_constants(): b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)] storage = make_storage(b1s, b2s, b3s) - modifier = ResumeDataVirtualAdder(storage) + memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + modifier = ResumeDataVirtualAdder(storage, memo) modifier.walk_snapshots({}) liveboxes = modifier.finish({}) assert storage.rd_snapshot is None @@ -345,10 +390,30 @@ assert metainterp.trace == [] +def test_virtual_adder_memo_const_sharing(): + b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)] + storage = make_storage(b1s, b2s, b3s) + memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + modifier = ResumeDataVirtualAdder(storage, memo) + modifier.walk_snapshots({}) + modifier.finish({}) + assert len(memo.consts) == 2 + assert storage.rd_consts is memo.consts + + b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**17), ConstInt(-65)] + storage2 = make_storage(b1s, b2s, b3s) + modifier2 = ResumeDataVirtualAdder(storage2, memo) + modifier2.walk_snapshots({}) + modifier2.finish({}) + assert len(memo.consts) == 3 + assert storage2.rd_consts is memo.consts + + def test_virtual_adder_no_op_renaming(): b1s, b2s, b3s = [BoxInt(1), BoxInt(2), BoxInt(3)] storage = make_storage(b1s, b2s, b3s) - modifier = ResumeDataVirtualAdder(storage) + memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + modifier = ResumeDataVirtualAdder(storage, memo) b1_2 = BoxInt() class FakeValue(object): @@ -383,7 +448,8 @@ BoxPtr(), BoxPtr()] c1s = ConstInt(111) storage = make_storage(b1s, b2s, b3s) - modifier = ResumeDataVirtualAdder(storage) + memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + modifier = ResumeDataVirtualAdder(storage, memo) modifier.walk_snapshots({}) modifier.make_virtual(b2s, ConstAddr(LLtypeMixin.node_vtable_adr, @@ -454,7 +520,8 @@ # the list of liveboxes b1s = ConstInt(111) storage = make_storage(b1s, b2s, b3s) - modifier = ResumeDataVirtualAdder(storage) + memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + modifier = ResumeDataVirtualAdder(storage, memo) modifier.walk_snapshots({}) if testnumber == 1: @@ -484,7 +551,8 @@ b1s, b2s, b3s, b4s = [BoxInt(1), BoxPtr(), BoxInt(3), BoxInt(4)] c1s = ConstInt(111) storage = make_storage(b1s, b2s, b3s) - modifier = ResumeDataVirtualAdder(storage) + memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + modifier = ResumeDataVirtualAdder(storage, memo) modifier.walk_snapshots({}) modifier.make_varray(b2s, LLtypeMixin.arraydescr, @@ -531,7 +599,8 @@ b1s, b2s, b3s, b4s = [BoxInt(1), BoxPtr(), BoxInt(3), BoxPtr()] c1s = ConstInt(111) storage = make_storage(b1s, b2s, b3s) - modifier = ResumeDataVirtualAdder(storage) + memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + modifier = ResumeDataVirtualAdder(storage, memo) modifier.walk_snapshots({}) modifier.make_vstruct(b2s, LLtypeMixin.ssize, From arigo at codespeak.net Tue Oct 6 12:31:07 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 6 Oct 2009 12:31:07 +0200 (CEST) Subject: [pypy-svn] r68199 - pypy/trunk/pypy/module/pypyjit Message-ID: <20091006103107.D38A816845C@codespeak.net> Author: arigo Date: Tue Oct 6 12:31:07 2009 New Revision: 68199 Modified: pypy/trunk/pypy/module/pypyjit/__init__.py Log: For now a comment, fix me later. Modified: pypy/trunk/pypy/module/pypyjit/__init__.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/__init__.py (original) +++ pypy/trunk/pypy/module/pypyjit/__init__.py Tue Oct 6 12:31:07 2009 @@ -14,5 +14,6 @@ # add the 'defaults' attribute from pypy.rlib.jit import PARAMETERS space = self.space + # XXX this is not really the default compiled into a pypy-c-jit XXX w_obj = space.wrap(PARAMETERS) space.setattr(space.wrap(self), space.wrap('defaults'), w_obj) From cfbolz at codespeak.net Tue Oct 6 13:28:16 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 6 Oct 2009 13:28:16 +0200 (CEST) Subject: [pypy-svn] r68200 - pypy/trunk/pypy/rlib Message-ID: <20091006112816.28550168455@codespeak.net> Author: cfbolz Date: Tue Oct 6 13:28:15 2009 New Revision: 68200 Modified: pypy/trunk/pypy/rlib/rbigint.py Log: make the digit list of longs non-resizable Modified: pypy/trunk/pypy/rlib/rbigint.py ============================================================================== --- pypy/trunk/pypy/rlib/rbigint.py (original) +++ pypy/trunk/pypy/rlib/rbigint.py Tue Oct 6 13:28:15 2009 @@ -1,5 +1,6 @@ from pypy.rlib.rarithmetic import LONG_BIT, intmask, r_uint, r_ulonglong from pypy.rlib.rarithmetic import ovfcheck, r_longlong, widen +from pypy.rlib.debug import make_sure_not_resized import math, sys @@ -52,6 +53,7 @@ def __init__(self, digits=None, sign=0): if digits is None or len(digits) == 0: digits = [0] + make_sure_not_resized(digits) self.digits = digits self.sign = sign @@ -585,7 +587,7 @@ digits.append(mask_digit(l)) l = l >> SHIFT if not l: - return digits + return digits[:] # to make it non-resizable digits_from_nonneg_long._annspecialcase_ = "specialize:argtype(0)" def digits_for_most_neg_long(l): @@ -601,7 +603,7 @@ l = -intmask(l) assert l >= 0 digits.append(l) - return digits + return digits[:] # to make it non-resizable digits_for_most_neg_long._annspecialcase_ = "specialize:argtype(0)" def args_from_rarith_int1(x): From pedronis at codespeak.net Tue Oct 6 15:28:42 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 6 Oct 2009 15:28:42 +0200 (CEST) Subject: [pypy-svn] r68203 - pypy/extradoc/planning Message-ID: <20091006132842.BA9A41683DD@codespeak.net> Author: pedronis Date: Tue Oct 6 15:28:41 2009 New Revision: 68203 Modified: pypy/extradoc/planning/jit.txt Log: (cfbolz, pedronis) some gardening and updates Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Tue Oct 6 15:28:41 2009 @@ -14,10 +14,7 @@ pypy-jit behaviour that explicitely check whether the loops make sense -- objectmodel.instantiate is currently not seen by the JIT. This makes all - instance allocation in the Python interpreter escape - -- store sinking also for get/setarrayitem +- store sinking also for get/setarrayitem WIP - lose less information across residual calls - the tragedy of the skipped tests @@ -33,24 +30,36 @@ - improve on predictability: delegate the caching, don't trace into signals ... but produce just a conditional call +- we should think about merging several consecutive guards, to make the + assembler smaller and to save resume data space + - update things in metainterp/doc + Python interpreter: -- make shared dicts support shadowtracking + +- goal: on average <=5 guards per original bytecode + - don't look for the global-lookup-cache when a function contains no LOAD_GLOBAL - it's a bit silly that it's possible to change the code objects of builtin functions - raising an exception tends to escape frames, due to the traceback capturing +- calls with keywords produce intractably huge traces + inlining discussion -------------------- +- need to not always inline, but clever about which functions not to inline +- at some point we need to merge the tails of loops, to avoid exponential + explosion - need to remember when we gave up - tracing aggressively will put pressure on the speed of tracing - what should we do about recursive calls? - connecting compiled loops accross a call? + things we know are missing --------------------------- @@ -58,9 +67,10 @@ - virtualizables are not finished: fix 'test_external_read_sometimes' -- float support +- save space on all the guard operations (see resume.py): + - share more between the guards of a loop + - think of ways to share information about virtuals -- save space on all the guard operations (see resume.py) backend: - speed of backend? From pedronis at codespeak.net Tue Oct 6 16:12:46 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 6 Oct 2009 16:12:46 +0200 (CEST) Subject: [pypy-svn] r68204 - in pypy/trunk/pypy: . jit/backend/llvm/test jit/backend/x86/test jit/metainterp jit/metainterp/test translator/backendopt translator/backendopt/test Message-ID: <20091006141246.1F0AC168429@codespeak.net> Author: pedronis Date: Tue Oct 6 16:12:45 2009 New Revision: 68204 Added: pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py - copied unchanged from r68203, pypy/branch/better-getfield-opts/pypy/translator/backendopt/test/test_writeanalyze.py pypy/trunk/pypy/translator/backendopt/writeanalyze.py - copied unchanged from r68203, pypy/branch/better-getfield-opts/pypy/translator/backendopt/writeanalyze.py Modified: pypy/trunk/pypy/ (props changed) pypy/trunk/pypy/jit/backend/llvm/test/conftest.py (props changed) pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py (props changed) pypy/trunk/pypy/jit/metainterp/logger.py (props changed) pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: (cfbolz, pedronis): merge the better-getfield-ops branch. Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Tue Oct 6 16:12:45 2009 @@ -41,9 +41,8 @@ class OptValue(object): - _attrs_ = ('box', 'known_class', 'level', '_fields') + _attrs_ = ('box', 'known_class', 'level') level = LEVEL_UNKNOWN - _fields = None def __init__(self, box): self.box = box @@ -169,6 +168,7 @@ class AbstractVirtualStructValue(AbstractVirtualValue): + _attrs_ = ('_fields', ) def __init__(self, optimizer, keybox, source_op=None): AbstractVirtualValue.__init__(self, optimizer, keybox, source_op) @@ -357,15 +357,9 @@ self.cpu = cpu self.loop = loop self.values = {} - # OptValues to clean when we see an operation with side-effects - # they are ordered by fielddescrs of the affected fields - # note: it is important that this is not a av_newdict2 dict! - # we want more precision to not clear unrelated fields, just because - # they are at the same offset (but in a different struct type) - self.values_to_clean = {} - self.interned_refs = {} self.resumedata_memo = resume.ResumeDataLoopMemo(cpu) + self.heap_op_optimizer = HeapOpOptimizer(self) def getinterned(self, box): constbox = self.get_constant_box(box) @@ -497,6 +491,7 @@ self.loop.operations = self.newoperations def emit_operation(self, op, must_clone=True): + self.heap_op_optimizer.emitting_operation(op) for i in range(len(op.args)): arg = op.args[i] if arg in self.values: @@ -520,21 +515,6 @@ newboxes = modifier.finish(self.values) descr.store_final_boxes(op, newboxes) - def clean_fields_of_values(self, descr=None): - if descr is None: - for descr, values in self.values_to_clean.iteritems(): - for value in values: - value._fields.clear() - self.values_to_clean = {} - else: - for value in self.values_to_clean.get(descr, []): - del value._fields[descr] - self.values_to_clean[descr] = [] - - def register_value_to_clean(self, value, descr): - self.values_to_clean.setdefault(descr, []).append(value) - - def optimize_default(self, op): if op.is_always_pure(): for arg in op.args: @@ -546,8 +526,6 @@ resbox = execute_nonspec(self.cpu, op.opnum, argboxes, op.descr) self.make_constant(op.result, resbox.constbox()) return - elif not op.has_no_side_effect() and not op.is_ovf(): - self.clean_fields_of_values() # otherwise, the operation remains self.emit_operation(op) @@ -666,18 +644,8 @@ assert fieldvalue is not None self.make_equal_to(op.result, fieldvalue) else: - # check if the field was read from another getfield_gc just before - if value._fields is None: - value._fields = av_newdict2() - elif op.descr in value._fields: - self.make_equal_to(op.result, value._fields[op.descr]) - return - # default case: produce the operation value.make_nonnull() - self.optimize_default(op) - # then remember the result of reading the field - value._fields[op.descr] = self.getvalue(op.result) - self.register_value_to_clean(value, op.descr) + self.heap_op_optimizer.optimize_GETFIELD_GC(op, value) # note: the following line does not mean that the two operations are # completely equivalent, because GETFIELD_GC_PURE is_always_pure(). @@ -689,15 +657,8 @@ value.setfield(op.descr, self.getvalue(op.args[1])) else: value.make_nonnull() - self.emit_operation(op) - # kill all fields with the same descr, as those could be affected - # by this setfield (via aliasing) - self.clean_fields_of_values(op.descr) - # remember the result of future reads of the field - if value._fields is None: - value._fields = av_newdict2() - value._fields[op.descr] = self.getvalue(op.args[1]) - self.register_value_to_clean(value, op.descr) + fieldvalue = self.getvalue(op.args[1]) + self.heap_op_optimizer.optimize_SETFIELD_GC(op, value, fieldvalue) def optimize_NEW_WITH_VTABLE(self, op): self.make_virtual(op.args[0], op.result, op) @@ -736,7 +697,7 @@ self.make_equal_to(op.result, itemvalue) return value.make_nonnull() - self.optimize_default(op) + self.heap_op_optimizer.optimize_GETARRAYITEM_GC(op, value) # note: the following line does not mean that the two operations are # completely equivalent, because GETARRAYITEM_GC_PURE is_always_pure(). @@ -750,9 +711,8 @@ value.setitem(indexbox.getint(), self.getvalue(op.args[2])) return value.make_nonnull() - # don't use optimize_default, because otherwise unrelated struct - # fields will be cleared - self.emit_operation(op) + fieldvalue = self.getvalue(op.args[2]) + self.heap_op_optimizer.optimize_SETARRAYITEM_GC(op, value, fieldvalue) def optimize_INSTANCEOF(self, op): value = self.getvalue(op.args[0]) @@ -766,9 +726,141 @@ self.emit_operation(op) def optimize_DEBUG_MERGE_POINT(self, op): - # special-case this operation to prevent e.g. the handling of - # 'values_to_clean' (the op cannot be marked as side-effect-free) - # otherwise it would be removed - self.newoperations.append(op) + self.emit_operation(op) optimize_ops = _findall(Optimizer, 'optimize_') + + +class CachedArrayItems(object): + def __init__(self): + self.fixed_index_items = {} + self.var_index_item = None + self.var_index_indexvalue = None + + +class HeapOpOptimizer(object): + def __init__(self, optimizer): + self.optimizer = optimizer + # cached OptValues for each field descr + # NOTE: it is important that this is not a av_newdict2 dict! + # we want more precision to prevent mixing up of unrelated fields, just + # because they are at the same offset (but in a different struct type) + self.cached_fields = {} + + # cached OptValues for each field descr + self.cached_arrayitems = {} + + def clean_caches(self): + self.cached_fields.clear() + self.cached_arrayitems.clear() + + def cache_field_value(self, descr, value, fieldvalue, write=False): + if write: + d = self.cached_fields[descr] = {} + else: + d = self.cached_fields.setdefault(descr, {}) + d[value] = fieldvalue + + def read_cached_field(self, descr, value): + d = self.cached_fields.get(descr, None) + if d is None: + return None + return d.get(value, None) + + def cache_arrayitem_value(self, descr, value, indexvalue, fieldvalue, write=False): + d = self.cached_arrayitems.get(descr, None) + if d is None: + d = self.cached_arrayitems[descr] = {} + cache = d.get(value, None) + if cache is None: + cache = d[value] = CachedArrayItems() + indexbox = self.optimizer.get_constant_box(indexvalue.box) + if indexbox is not None: + index = indexbox.getint() + if write: + for value, othercache in d.iteritems(): + # fixed index, clean the variable index cache, in case the + # index is the same + othercache.var_index_indexvalue = None + othercache.var_index_item = None + try: + del othercache.fixed_index_items[index] + except KeyError: + pass + cache.fixed_index_items[index] = fieldvalue + else: + if write: + for value, othercache in d.iteritems(): + # variable index, clear all caches for this descr + othercache.var_index_indexvalue = None + othercache.var_index_item = None + othercache.fixed_index_items.clear() + cache.var_index_indexvalue = indexvalue + cache.var_index_item = fieldvalue + + def read_cached_arrayitem(self, descr, value, indexvalue): + d = self.cached_arrayitems.get(descr, None) + if d is None: + return None + cache = d.get(value, None) + if cache is None: + return None + indexbox = self.optimizer.get_constant_box(indexvalue.box) + if indexbox is not None: + return cache.fixed_index_items.get(indexbox.getint(), None) + elif cache.var_index_indexvalue is indexvalue: + return cache.var_index_item + return None + + def emitting_operation(self, op): + if op.is_always_pure(): + return + if op.has_no_side_effect(): + return + if op.is_ovf(): + return + if op.is_guard(): + return + opnum = op.opnum + if (opnum == rop.SETFIELD_GC or + opnum == rop.SETARRAYITEM_GC or + opnum == rop.DEBUG_MERGE_POINT): + return + self.clean_caches() + + def optimize_GETFIELD_GC(self, op, value): + # check if the field was read from another getfield_gc just before + # or has been written to recently + fieldvalue = self.read_cached_field(op.descr, value) + if fieldvalue is not None: + self.optimizer.make_equal_to(op.result, fieldvalue) + return + # default case: produce the operation + value.make_nonnull() + self.optimizer.optimize_default(op) + # then remember the result of reading the field + fieldvalue = self.optimizer.getvalue(op.result) + self.cache_field_value(op.descr, value, fieldvalue) + + def optimize_SETFIELD_GC(self, op, value, fieldvalue): + self.optimizer.emit_operation(op) + # remember the result of future reads of the field + self.cache_field_value(op.descr, value, fieldvalue, write=True) + + def optimize_GETARRAYITEM_GC(self, op, value): + indexvalue = self.optimizer.getvalue(op.args[1]) + fieldvalue = self.read_cached_arrayitem(op.descr, value, indexvalue) + if fieldvalue is not None: + self.optimizer.make_equal_to(op.result, fieldvalue) + return + self.optimizer.optimize_default(op) + fieldvalue = self.optimizer.getvalue(op.result) + self.cache_arrayitem_value(op.descr, value, indexvalue, fieldvalue) + + def optimize_SETARRAYITEM_GC(self, op, value, fieldvalue): + self.optimizer.emit_operation(op) + indexvalue = self.optimizer.getvalue(op.args[1]) + self.cache_arrayitem_value(op.descr, value, indexvalue, fieldvalue, + write=True) + + Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Tue Oct 6 16:12:45 2009 @@ -1023,21 +1023,28 @@ def test_duplicate_getfield_1(self): ops = """ - [p1] + [p1, p2] i1 = getfield_gc(p1, descr=valuedescr) - i2 = getfield_gc(p1, descr=valuedescr) + i2 = getfield_gc(p2, descr=valuedescr) + i3 = getfield_gc(p1, descr=valuedescr) + i4 = getfield_gc(p2, descr=valuedescr) escape(i1) escape(i2) - jump(p1) + escape(i3) + escape(i4) + jump(p1, p2) """ expected = """ - [p1] + [p1, p2] i1 = getfield_gc(p1, descr=valuedescr) + i2 = getfield_gc(p2, descr=valuedescr) escape(i1) + escape(i2) escape(i1) - jump(p1) + escape(i2) + jump(p1, p2) """ - self.optimize_loop(ops, 'Not', expected) + self.optimize_loop(ops, 'Not, Not', expected) def test_getfield_after_setfield(self): ops = """ @@ -1206,6 +1213,138 @@ """ self.optimize_loop(ops, 'Not, Not', ops) + def test_duplicate_getarrayitem_1(self): + ops = """ + [p1] + p2 = getarrayitem_gc(p1, 0, descr=arraydescr2) + p3 = getarrayitem_gc(p1, 1, descr=arraydescr2) + p4 = getarrayitem_gc(p1, 0, descr=arraydescr2) + p5 = getarrayitem_gc(p1, 1, descr=arraydescr2) + escape(p2) + escape(p3) + escape(p4) + escape(p5) + jump(p1) + """ + expected = """ + [p1] + p2 = getarrayitem_gc(p1, 0, descr=arraydescr2) + p3 = getarrayitem_gc(p1, 1, descr=arraydescr2) + escape(p2) + escape(p3) + escape(p2) + escape(p3) + jump(p1) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_duplicate_getarrayitem_after_setarrayitem_1(self): + ops = """ + [p1, p2] + setarrayitem_gc(p1, 0, p2, descr=arraydescr2) + p3 = getarrayitem_gc(p1, 0, descr=arraydescr2) + escape(p3) + jump(p1, p3) + """ + expected = """ + [p1, p2] + setarrayitem_gc(p1, 0, p2, descr=arraydescr2) + escape(p2) + jump(p1, p2) + """ + self.optimize_loop(ops, 'Not, Not', expected) + + def test_duplicate_getarrayitem_after_setarrayitem_2(self): + ops = """ + [p1, p2, p3, i1] + setarrayitem_gc(p1, 0, p2, descr=arraydescr2) + setarrayitem_gc(p1, i1, p3, descr=arraydescr2) + p4 = getarrayitem_gc(p1, 0, descr=arraydescr2) + p5 = getarrayitem_gc(p1, i1, descr=arraydescr2) + escape(p4) + escape(p5) + jump(p1, p2, p3, i1) + """ + expected = """ + [p1, p2, p3, i1] + setarrayitem_gc(p1, 0, p2, descr=arraydescr2) + setarrayitem_gc(p1, i1, p3, descr=arraydescr2) + p4 = getarrayitem_gc(p1, 0, descr=arraydescr2) + escape(p4) + escape(p3) + jump(p1, p2, p3, i1) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', expected) + + def test_duplicate_getarrayitem_after_setarrayitem_3(self): + ops = """ + [p1, p2, p3, p4, i1] + setarrayitem_gc(p1, i1, p2, descr=arraydescr2) + setarrayitem_gc(p1, 0, p3, descr=arraydescr2) + setarrayitem_gc(p1, 1, p4, descr=arraydescr2) + p5 = getarrayitem_gc(p1, i1, descr=arraydescr2) + p6 = getarrayitem_gc(p1, 0, descr=arraydescr2) + p7 = getarrayitem_gc(p1, 1, descr=arraydescr2) + escape(p5) + escape(p6) + escape(p7) + jump(p1, p2, p3, p4, i1) + """ + expected = """ + [p1, p2, p3, p4, i1] + setarrayitem_gc(p1, i1, p2, descr=arraydescr2) + setarrayitem_gc(p1, 0, p3, descr=arraydescr2) + setarrayitem_gc(p1, 1, p4, descr=arraydescr2) + p5 = getarrayitem_gc(p1, i1, descr=arraydescr2) + escape(p5) + escape(p3) + escape(p4) + jump(p1, p2, p3, p4, i1) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not, Not', expected) + + def test_getarrayitem_pure_does_not_invalidate(self): + ops = """ + [p1, p2] + p3 = getarrayitem_gc(p1, 0, descr=arraydescr2) + i4 = getfield_gc_pure(ConstPtr(myptr), descr=valuedescr) + p5 = getarrayitem_gc(p1, 0, descr=arraydescr2) + escape(p3) + escape(i4) + escape(p5) + jump(p1, p2) + """ + expected = """ + [p1, p2] + p3 = getarrayitem_gc(p1, 0, descr=arraydescr2) + escape(p3) + escape(5) + escape(p3) + jump(p1, p2) + """ + self.optimize_loop(ops, 'Not, Not', expected) + + def test_duplicate_getarrayitem_after_setarrayitem_two_arrays(self): + ops = """ + [p1, p2, p3, p4, i1] + setarrayitem_gc(p1, 0, p3, descr=arraydescr2) + setarrayitem_gc(p2, 1, p4, descr=arraydescr2) + p5 = getarrayitem_gc(p1, 0, descr=arraydescr2) + p6 = getarrayitem_gc(p2, 1, descr=arraydescr2) + escape(p5) + escape(p6) + jump(p1, p2, p3, p4, i1) + """ + expected = """ + [p1, p2, p3, p4, i1] + setarrayitem_gc(p1, 0, p3, descr=arraydescr2) + setarrayitem_gc(p2, 1, p4, descr=arraydescr2) + escape(p3) + escape(p4) + jump(p1, p2, p3, p4, i1) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not, Not', expected) + def test_bug_1(self): ops = """ [i0, p1] From pedronis at codespeak.net Tue Oct 6 16:13:10 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 6 Oct 2009 16:13:10 +0200 (CEST) Subject: [pypy-svn] r68205 - pypy/branch/better-getfield-opts Message-ID: <20091006141310.BA56E16842D@codespeak.net> Author: pedronis Date: Tue Oct 6 16:13:10 2009 New Revision: 68205 Removed: pypy/branch/better-getfield-opts/ Log: (cfbolz, pedronis) kill merge branch From pedronis at codespeak.net Tue Oct 6 16:30:12 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 6 Oct 2009 16:30:12 +0200 (CEST) Subject: [pypy-svn] r68206 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091006143012.3F738168032@codespeak.net> Author: pedronis Date: Tue Oct 6 16:30:11 2009 New Revision: 68206 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: (cfbolz, pedronis): don't make a list of fielddescrs for virtual structs every time Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Tue Oct 6 16:30:11 2009 @@ -168,11 +168,12 @@ class AbstractVirtualStructValue(AbstractVirtualValue): - _attrs_ = ('_fields', ) + _attrs_ = ('_fields', '_cached_sorted_fields') def __init__(self, optimizer, keybox, source_op=None): AbstractVirtualValue.__init__(self, optimizer, keybox, source_op) self._fields = av_newdict2() + self._cached_sorted_fields = None def getfield(self, ofs, default): return self._fields.get(ofs, default) @@ -200,12 +201,22 @@ self._fields = None return self.box + def _get_field_descr_list(self): + _cached_sorted_fields = self._cached_sorted_fields + if (_cached_sorted_fields is not None and + len(self._fields) == len(_cached_sorted_fields)): + lst = self._cached_sorted_fields + else: + lst = self._fields.keys() + sort_descrs(lst) + self._cached_sorted_fields = lst + return lst + def get_args_for_fail(self, modifier): if self.box is None and not modifier.is_virtual(self.keybox): # modifier.is_virtual() checks for recursion: it is False unless # we have already seen the very same keybox - lst = self._fields.keys() - sort_descrs(lst) + lst = self._get_field_descr_list() fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst] self._make_virtual(modifier, lst, fieldboxes) for ofs in lst: Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Tue Oct 6 16:30:11 2009 @@ -50,6 +50,19 @@ assert fdescr.rd_virtuals is None assert fdescr.rd_consts == [] assert fdescr.rd_frame_infos == fi + +def test_sharing_field_lists_of_virtual(): + virt1 = optimizeopt.AbstractVirtualStructValue(None, None) + lst1 = virt1._get_field_descr_list() + assert lst1 == [] + lst2 = virt1._get_field_descr_list() + assert lst1 is lst2 + virt1.setfield(LLtypeMixin.valuedescr, optimizeopt.OptValue(None)) + lst1 = virt1._get_field_descr_list() + assert lst1 == [LLtypeMixin.valuedescr] + lst2 = virt1._get_field_descr_list() + assert lst1 is lst2 + # ____________________________________________________________ def equaloplists(oplist1, oplist2, strict_fail_args=True, remap={}): From pedronis at codespeak.net Tue Oct 6 16:36:19 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 6 Oct 2009 16:36:19 +0200 (CEST) Subject: [pypy-svn] r68207 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091006143619.CBC14168032@codespeak.net> Author: pedronis Date: Tue Oct 6 16:36:18 2009 New Revision: 68207 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py Log: add a comment Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Tue Oct 6 16:36:18 2009 @@ -202,6 +202,7 @@ return self.box def _get_field_descr_list(self): + # this shares only per instance and not per type, but better than nothing _cached_sorted_fields = self._cached_sorted_fields if (_cached_sorted_fields is not None and len(self._fields) == len(_cached_sorted_fields)): From pedronis at codespeak.net Tue Oct 6 17:21:07 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 6 Oct 2009 17:21:07 +0200 (CEST) Subject: [pypy-svn] r68210 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091006152107.A9226168422@codespeak.net> Author: pedronis Date: Tue Oct 6 17:21:07 2009 New Revision: 68210 Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/jit/metainterp/test/test_resume.py Log: (pedronis, cfbolz): refactor resuming in preparation for further changes Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Tue Oct 6 17:21:07 2009 @@ -1678,15 +1678,11 @@ def rebuild_state_after_failure(self, resumedescr, newboxes): vinfo = self.staticdata.virtualizable_info - resumereader = resume.ResumeDataReader(resumedescr, newboxes, self) self.framestack = [] - while resumereader.has_more_frame_infos(): - jitcode, pc, exception_target = resumereader.consume_frame_info() - env = resumereader.consume_boxes() - f = self.newframe(jitcode) - f.setup_resume_at_op(pc, exception_target, env) - if vinfo is not None: - self.virtualizable_boxes = resumereader.consume_boxes() + expect_virtualizable = vinfo is not None + virtualizable_boxes = resume.rebuild_from_resumedata(self, newboxes, resumedescr, expect_virtualizable) + if expect_virtualizable: + self.virtualizable_boxes = virtualizable_boxes # just jumped away from assembler (case 4 in the comment in # virtualizable.py) into tracing (case 2); check that vable_rti # is and stays NULL. Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Tue Oct 6 17:21:07 2009 @@ -10,7 +10,7 @@ # because it needs to support optimize.py which encodes virtuals with # arbitrary cycles. -# XXX I guess that building the data so that it is as compact as possible +# XXX building the data so that it is as compact as possible # on the 'storage' object would be a big win. debug = False @@ -331,6 +331,19 @@ self.fieldnums) +def rebuild_from_resumedata(metainterp, newboxes, resumedescr, expects_virtualizables): + resumereader = ResumeDataReader(resumedescr, newboxes, metainterp) + while resumereader.has_more_frame_infos(): + jitcode, pc, exception_target = resumereader.consume_frame_info() + env = resumereader.consume_boxes() + f = metainterp.newframe(jitcode) + f.setup_resume_at_op(pc, exception_target, env) + if expects_virtualizables: + virtualizable_boxes = resumereader.consume_boxes() + return virtualizable_boxes + return None + + class ResumeDataReader(object): i_frame_infos = 0 i_boxes = 0 Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Tue Oct 6 17:21:07 2009 @@ -119,6 +119,14 @@ self.exception_target = exc_target self.env = list(boxes) + def setup_resume_at_op(self, pc, exception_target, env): + self.__init__(self.jitcode, pc, exception_target, *env) + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + def __ne__(self, other): + return self.__dict__ != other.__dict__ + def test_Snapshot_create(): l = ['b0', 'b1'] snap = Snapshot(None, l) @@ -216,6 +224,69 @@ assert snapshot.prev is fs[2].parent_resumedata_snapshot assert snapshot.boxes == fs[2].env +def test_rebuild_from_resumedata(): + class FakeMetaInterp(object): + def __init__(self): + self.framestack = [] + def newframe(self, jitcode): + frame = FakeFrame(jitcode, -1, -1) + self.framestack.append(frame) + return frame + b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] + c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] + storage = Storage() + fs = [FakeFrame("code0", 0, -1, b1, c1, b2), + FakeFrame("code1", 3, 7, b3, c2, b1), + FakeFrame("code2", 9, -1, c3, b2)] + capture_resumedata(fs, None, storage) + memo = ResumeDataLoopMemo(None) + modifier = ResumeDataVirtualAdder(storage, memo) + modifier.walk_snapshots({}) + liveboxes = modifier.finish({}) + metainterp = FakeMetaInterp() + + b1t, b2t, b3t = [BoxInt(), BoxPtr(), BoxInt()] + newboxes = _resume_remap(liveboxes, [b1, b2, b3], b1t, b2t, b3t) + + result = rebuild_from_resumedata(metainterp, newboxes, storage, + False) + assert result is None + fs2 = [FakeFrame("code0", 0, -1, b1t, c1, b2t), + FakeFrame("code1", 3, 7, b3t, c2, b1t), + FakeFrame("code2", 9, -1, c3, b2t)] + assert metainterp.framestack == fs2 + +def test_rebuild_from_resumedata_with_virtualizable(): + class FakeMetaInterp(object): + def __init__(self): + self.framestack = [] + def newframe(self, jitcode): + frame = FakeFrame(jitcode, -1, -1) + self.framestack.append(frame) + return frame + b1, b2, b3, b4 = [BoxInt(), BoxPtr(), BoxInt(), BoxPtr()] + c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] + storage = Storage() + fs = [FakeFrame("code0", 0, -1, b1, c1, b2), + FakeFrame("code1", 3, 7, b3, c2, b1), + FakeFrame("code2", 9, -1, c3, b2)] + capture_resumedata(fs, [b4], storage) + memo = ResumeDataLoopMemo(None) + modifier = ResumeDataVirtualAdder(storage, memo) + modifier.walk_snapshots({}) + liveboxes = modifier.finish({}) + metainterp = FakeMetaInterp() + + b1t, b2t, b3t, b4t = [BoxInt(), BoxPtr(), BoxInt(), BoxPtr()] + newboxes = _resume_remap(liveboxes, [b1, b2, b3, b4], b1t, b2t, b3t, b4t) + + result = rebuild_from_resumedata(metainterp, newboxes, storage, + True) + assert result == [b4t] + fs2 = [FakeFrame("code0", 0, -1, b1t, c1, b2t), + FakeFrame("code1", 3, 7, b3t, c2, b1t), + FakeFrame("code2", 9, -1, c3, b2t)] + assert metainterp.framestack == fs2 # ____________________________________________________________ def test_walk_snapshots(): From pedronis at codespeak.net Tue Oct 6 17:56:20 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 6 Oct 2009 17:56:20 +0200 (CEST) Subject: [pypy-svn] r68211 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091006155620.BD100168426@codespeak.net> Author: pedronis Date: Tue Oct 6 17:56:20 2009 New Revision: 68211 Modified: pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_resume.py Log: (cfbolz, pedronis): keep the frame infos as chained lists. fix dump_storage Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Tue Oct 6 17:56:20 2009 @@ -217,7 +217,6 @@ storage.rd_frame_infos = frame_infos def finish(self, values): - self._flatten_frame_info() storage = self.storage liveboxes = [] for box in self.liveboxes.iterkeys(): @@ -225,18 +224,18 @@ self.liveboxes[box] = tag(len(liveboxes), TAGBOX) liveboxes.append(box) nums = storage.rd_nums = [rffi.r_short(0)]*self.nnums - i = self.nnums-1 + i = 0 snapshot = self.storage.rd_snapshot while True: # at least one boxes = snapshot.boxes - nums[i] = NEXTFRAME - i -= 1 - for j in range(len(boxes)-1, -1, -1): + for j in range(len(boxes)): box = boxes[j] if box in values: box = values[box].get_key_box() nums[i] = self._gettagged(box) - i -= 1 + i += 1 + nums[i] = NEXTFRAME + i += 1 snapshot = snapshot.prev if snapshot is None: break @@ -292,7 +291,7 @@ return 'VirtualInfo("%s", %s, %s)' % ( self.known_class, ['"%s"' % (fd,) for fd in self.fielddescrs], - self.fieldnums) + [untag(i) for i in self.fieldnums]) class VStructInfo(AbstractVirtualStructInfo): def __init__(self, typedescr, fielddescrs): @@ -306,7 +305,7 @@ return 'VStructInfo("%s", %s, %s)' % ( self.typedescr, ['"%s"' % (fd,) for fd in self.fielddescrs], - self.fieldnums) + [untag(i) for i in self.fieldnums]) class VArrayInfo(AbstractVirtualInfo): def __init__(self, arraydescr): @@ -327,21 +326,26 @@ box, ConstInt(i), itembox) def repr_rpython(self): - return 'VArrayInfo("%s", %s)' % (self.arraydescr, - self.fieldnums) + return 'VArrayInfo("%s", %s)' % ( + self.arraydescr, + [untag(i) for i in self.fieldnums]) -def rebuild_from_resumedata(metainterp, newboxes, resumedescr, expects_virtualizables): - resumereader = ResumeDataReader(resumedescr, newboxes, metainterp) - while resumereader.has_more_frame_infos(): - jitcode, pc, exception_target = resumereader.consume_frame_info() - env = resumereader.consume_boxes() - f = metainterp.newframe(jitcode) - f.setup_resume_at_op(pc, exception_target, env) +def rebuild_from_resumedata(metainterp, newboxes, storage, expects_virtualizables): + resumereader = ResumeDataReader(storage, newboxes, metainterp) + virtualizable_boxes = None if expects_virtualizables: virtualizable_boxes = resumereader.consume_boxes() - return virtualizable_boxes - return None + frameinfo = storage.rd_frame_info_list + while True: + env = resumereader.consume_boxes() + f = metainterp.newframe(frameinfo.jitcode) + f.setup_resume_at_op(frameinfo.pc, frameinfo.exception_target, env) + frameinfo = frameinfo.prev + if frameinfo is None: + break + metainterp.framestack.reverse() + return virtualizable_boxes class ResumeDataReader(object): @@ -350,7 +354,6 @@ virtuals = None def __init__(self, storage, liveboxes, metainterp=None): - self.frame_infos = storage.rd_frame_infos self.nums = storage.rd_nums self.consts = storage.rd_consts self.liveboxes = liveboxes @@ -387,14 +390,6 @@ assert tag == TAGBOX return self.liveboxes[num] - def has_more_frame_infos(self): - return self.i_frame_infos < len(self.frame_infos) - - def consume_frame_info(self): - frame_info = self.frame_infos[self.i_frame_infos] - self.i_frame_infos += 1 - return frame_info - # ____________________________________________________________ def dump_storage(storage, liveboxes): @@ -403,9 +398,14 @@ from pypy.rlib import objectmodel fd = os.open('log.storage', os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0666) os.write(fd, 'Log(%d, [\n' % objectmodel.compute_unique_id(storage)) - for frame_info in storage.rd_frame_infos: - os.write(fd, '\t("%s", %d, %d),\n' % frame_info) - os.write(fd, '\t],\n\t%s,\n' % (storage.rd_nums,)) + frameinfo = storage.rd_frame_info_list + while True: + os.write(fd, '\t("%s", %d, %d),\n' % ( + frameinfo.jitcode, frameinfo.pc, frameinfo.exception_target)) + frameinfo = frameinfo.prev + if frameinfo is None: + break + os.write(fd, '\t],\n\t%s,\n' % ([untag(i) for i in storage.rd_nums],)) os.write(fd, '\t[\n') for const in storage.rd_consts: os.write(fd, '\t"%s",\n' % (const.repr_rpython(),)) Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Tue Oct 6 17:56:20 2009 @@ -32,7 +32,6 @@ fdescr = ResumeGuardDescr(None) op = ResOperation(rop.GUARD_TRUE, [], None, descr=fdescr) # setup rd data - fi = [("code0", 1, 2), ("code1", 3, -1)] fi0 = resume.FrameInfo(None, FakeFrame("code0", 1, 2)) fdescr.rd_frame_info_list = resume.FrameInfo(fi0, FakeFrame("code1", 3, -1)) @@ -41,15 +40,14 @@ # opt.store_final_boxes_in_guard(op) if op.fail_args == [b0, b1]: - assert fdescr.rd_nums == [tag(0, TAGBOX), NEXTFRAME, - tag(1, TAGBOX), NEXTFRAME] - else: - assert op.fail_args == [b1, b0] assert fdescr.rd_nums == [tag(1, TAGBOX), NEXTFRAME, tag(0, TAGBOX), NEXTFRAME] + else: + assert op.fail_args == [b1, b0] + assert fdescr.rd_nums == [tag(0, TAGBOX), NEXTFRAME, + tag(1, TAGBOX), NEXTFRAME] assert fdescr.rd_virtuals is None assert fdescr.rd_consts == [] - assert fdescr.rd_frame_infos == fi def test_sharing_field_lists_of_virtual(): virt1 = optimizeopt.AbstractVirtualStructValue(None, None) Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Tue Oct 6 17:56:20 2009 @@ -89,23 +89,7 @@ lst = reader.consume_boxes() assert lst == [b1s, b2s, b3s] -def test_frame_info(): - storage = Storage() - storage.rd_frame_infos = [(1, 2, 5), (3, 4, 7)] - storage.rd_consts = [] - storage.rd_nums = [] - storage.rd_virtuals = None - # - reader = ResumeDataReader(storage, []) - assert reader.has_more_frame_infos() - fi = reader.consume_frame_info() - assert fi == (1, 2, 5) - assert reader.has_more_frame_infos() - fi = reader.consume_frame_info() - assert fi == (3, 4, 7) - assert not reader.has_more_frame_infos() - -# ____________________________________________________________ + # ____________________________________________________________ @@ -434,11 +418,11 @@ metainterp = MyMetaInterp(LLtypeMixin.cpu) reader = ResumeDataReader(storage, newboxes, metainterp) lst = reader.consume_boxes() - assert lst == [b1t, ConstInt(1), b1t, b2t] + assert lst == [b1t, b2t, b3t] lst = reader.consume_boxes() assert lst == [ConstInt(2), ConstInt(3)] lst = reader.consume_boxes() - assert lst == [b1t, b2t, b3t] + assert lst == [b1t, ConstInt(1), b1t, b2t] assert metainterp.trace == [] def test_virtual_adder_int_constants(): @@ -452,12 +436,12 @@ metainterp = MyMetaInterp(LLtypeMixin.cpu) reader = ResumeDataReader(storage, [], metainterp) lst = reader.consume_boxes() - assert lst == [ConstInt(sys.maxint), ConstInt(1), ConstInt(sys.maxint), - ConstInt(2**16)] + assert lst == [b1s, b2s, b3s] lst = reader.consume_boxes() assert lst == [ConstInt(2), ConstInt(3)] lst = reader.consume_boxes() - assert lst == [b1s, b2s, b3s] + assert lst == [ConstInt(sys.maxint), ConstInt(1), ConstInt(sys.maxint), + ConstInt(2**16)] assert metainterp.trace == [] @@ -507,11 +491,11 @@ metainterp = MyMetaInterp(LLtypeMixin.cpu) reader = ResumeDataReader(storage, newboxes, metainterp) lst = reader.consume_boxes() - assert lst == [b1t, ConstInt(1), b1t, b1t] + assert lst == [b1t, b1t, b3t] lst = reader.consume_boxes() assert lst == [ConstInt(2), ConstInt(3)] lst = reader.consume_boxes() - assert lst == [b1t, b1t, b3t] + assert lst == [b1t, ConstInt(1), b1t, b1t] assert metainterp.trace == [] def test_virtual_adder_make_virtual(): @@ -551,12 +535,19 @@ # metainterp = MyMetaInterp(LLtypeMixin.cpu) reader = ResumeDataReader(storage, newboxes, metainterp) + trace = metainterp.trace[:] + del metainterp.trace[:] + lst = reader.consume_boxes() + b2t = lst[1] + assert lst == [b1t, b2t, b3t] + lst = reader.consume_boxes() + assert lst == [ConstInt(2), ConstInt(3)] lst = reader.consume_boxes() - b2t = lst[-1] + assert metainterp.trace == [] assert lst == [b1t, ConstInt(1), b1t, b2t] b4tx = b2t.value._obj.container._as_ptr().next b4tx = lltype.cast_opaque_ptr(llmemory.GCREF, b4tx) - assert metainterp.trace == [ + assert trace == [ (rop.NEW_WITH_VTABLE, [LLtypeMixin.node_vtable_adr], b2t.value, None), (rop.NEW_WITH_VTABLE, [LLtypeMixin.node_vtable_adr2], b4tx, None), (rop.SETFIELD_GC, [b2t.value, b4tx], None, LLtypeMixin.nextdescr), @@ -565,13 +556,6 @@ (rop.SETFIELD_GC, [b4tx, b3t.value], None, LLtypeMixin.valuedescr), (rop.SETFIELD_GC, [b4tx, b5t.value], None, LLtypeMixin.otherdescr), ] - del metainterp.trace[:] - lst = reader.consume_boxes() - assert lst == [ConstInt(2), ConstInt(3)] - assert metainterp.trace == [] - lst = reader.consume_boxes() - assert lst == [b1t, b2t, b3t] - assert metainterp.trace == [] # ptr = b2t.value._obj.container._as_ptr() assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.NODE) @@ -610,11 +594,11 @@ reader = ResumeDataReader(storage, newboxes, metainterp) lst = reader.consume_boxes() c1t = ConstInt(111) - assert lst == [c1t, ConstInt(1), c1t, b2t] + assert lst == [c1t, b2t, b3t] lst = reader.consume_boxes() assert lst == [ConstInt(2), ConstInt(3)] lst = reader.consume_boxes() - assert lst == [c1t, b2t, b3t] + assert lst == [c1t, ConstInt(1), c1t, b2t] assert metainterp.trace == [] @@ -643,20 +627,21 @@ # metainterp = MyMetaInterp(LLtypeMixin.cpu) reader = ResumeDataReader(storage, newboxes, metainterp) + trace = metainterp.trace[:] + del metainterp.trace[:] lst = reader.consume_boxes() - b2t = lst[-1] - assert lst == [b1t, ConstInt(1), b1t, b2t] - assert metainterp.trace == [ + b2t = lst[1] + assert lst == [b1t, b2t, b3t] + assert trace == [ (rop.NEW_ARRAY, [2], b2t.value, LLtypeMixin.arraydescr), (rop.SETARRAYITEM_GC, [b2t.value,0,44], None, LLtypeMixin.arraydescr), (rop.SETARRAYITEM_GC, [b2t.value,1,111], None, LLtypeMixin.arraydescr), ] - del metainterp.trace[:] lst = reader.consume_boxes() assert lst == [ConstInt(2), ConstInt(3)] assert metainterp.trace == [] lst = reader.consume_boxes() - assert lst == [b1t, b2t, b3t] + assert lst == [b1t, ConstInt(1), b1t, b2t] assert metainterp.trace == [] # ptr = b2t.value._obj.container._as_ptr() @@ -694,9 +679,11 @@ metainterp = MyMetaInterp(LLtypeMixin.cpu) reader = ResumeDataReader(storage, newboxes, metainterp) lst = reader.consume_boxes() - b2t = lst[-1] - assert lst == [b1t, ConstInt(1), b1t, b2t] - assert metainterp.trace == [ + trace = metainterp.trace[:] + del metainterp.trace[:] + b2t = lst[1] + assert lst == [b1t, b2t, b3t] + assert trace == [ (rop.NEW, [], b2t.value, LLtypeMixin.ssize), (rop.SETFIELD_GC, [b2t.value, 111], None, LLtypeMixin.adescr), (rop.SETFIELD_GC, [b2t.value, NULL], None, LLtypeMixin.bdescr), @@ -706,7 +693,7 @@ assert lst == [ConstInt(2), ConstInt(3)] assert metainterp.trace == [] lst = reader.consume_boxes() - assert lst == [b1t, b2t, b3t] + assert lst == [b1t, ConstInt(1), b1t, b2t] assert metainterp.trace == [] # ptr = b2t.value._obj.container._as_ptr() From pedronis at codespeak.net Tue Oct 6 18:31:27 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 6 Oct 2009 18:31:27 +0200 (CEST) Subject: [pypy-svn] r68212 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20091006163127.F3CCD16842D@codespeak.net> Author: pedronis Date: Tue Oct 6 18:31:27 2009 New Revision: 68212 Added: pypy/trunk/pypy/jit/metainterp/test/test_ztranslation.py (contents, props changed) Modified: pypy/trunk/pypy/jit/metainterp/test/test_zrpy_basic.py Log: (pedronis, cfbolz): add a test that tries to translate with a jit that uses a number of features. Thus we can make test_zrpy_basic also a slow test. Modified: pypy/trunk/pypy/jit/metainterp/test/test_zrpy_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_zrpy_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_zrpy_basic.py Tue Oct 6 18:31:27 2009 @@ -31,7 +31,8 @@ assert res == 490 def test_loop_2(self): - # this test runs even without the --slow option, to see at least one + if not option.run_slow_tests: + py.test.skip("use --slow to execute this long-running test") jitdriver = JitDriver(greens = [], reds = ['i', 'total']) def f(i): total = 0 Added: pypy/trunk/pypy/jit/metainterp/test/test_ztranslation.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/jit/metainterp/test/test_ztranslation.py Tue Oct 6 18:31:27 2009 @@ -0,0 +1,61 @@ +import py +from pypy.jit.metainterp.warmspot import rpython_ll_meta_interp, ll_meta_interp +from pypy.jit.metainterp.test import test_basic +from pypy.jit.backend.llgraph import runner +from pypy.rlib.jit import JitDriver, OPTIMIZER_FULL, unroll_parameters, PARAMETERS +from pypy.jit.conftest import option +from pypy.jit.metainterp.jitprof import Profiler + +class TranslationTest: + + CPUClass = None + type_system = None + + def test_stuff_translates(self): + # this is a basic test that tries to hit a number of features and their + # translation: + # - jitting of loops and bridges + # - virtualizables + # - set_param interface + # - profiler + # - full optimizer + + class Frame(object): + _virtualizable2_ = ['i'] + + def __init__(self, i): + self.i = i + + jitdriver = JitDriver(greens = [], reds = ['frame', 'total'], + virtualizables = ['frame']) + def f(i): + for param in unroll_parameters: + defl = PARAMETERS[param] + jitdriver.set_param(param, defl) + jitdriver.set_param("threshold", 3) + jitdriver.set_param("trace_eagerness", 2) + total = 0 + frame = Frame(i) + while frame.i > 3: + jitdriver.can_enter_jit(frame=frame, total=total) + jitdriver.jit_merge_point(frame=frame, total=total) + total += frame.i + if frame.i >= 20: + frame.i -= 2 + frame.i -= 1 + return total * 10 + res = ll_meta_interp(f, [40], CPUClass=self.CPUClass, + type_system=self.type_system) + assert res == f(40) + from pypy.jit.metainterp import optimize + res = rpython_ll_meta_interp(f, [40], loops=2, CPUClass=self.CPUClass, + type_system=self.type_system, + optimizer=OPTIMIZER_FULL, + ProfilerClass=Profiler) + assert res == f(40) + + +class TestTranslationLLtype(TranslationTest): + + CPUClass = runner.LLtypeCPU + type_system = 'lltype' From pedronis at codespeak.net Tue Oct 6 18:38:02 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 6 Oct 2009 18:38:02 +0200 (CEST) Subject: [pypy-svn] r68214 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20091006163802.E4D71168432@codespeak.net> Author: pedronis Date: Tue Oct 6 18:38:02 2009 New Revision: 68214 Removed: pypy/trunk/pypy/jit/metainterp/test/test_zrpy_recursive.py pypy/trunk/pypy/jit/metainterp/test/test_zrpy_send.py pypy/trunk/pypy/jit/metainterp/test/test_zrpy_set_param.py pypy/trunk/pypy/jit/metainterp/test/test_zrpy_slist.py pypy/trunk/pypy/jit/metainterp/test/test_zrpy_virtualizable.py Log: (pedronis, cfbolz): kill tests that nobody ever runs ever From cfbolz at codespeak.net Tue Oct 6 18:39:38 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 6 Oct 2009 18:39:38 +0200 (CEST) Subject: [pypy-svn] r68215 - pypy/trunk/pypy/jit/tl Message-ID: <20091006163938.CC50A168432@codespeak.net> Author: cfbolz Date: Tue Oct 6 18:39:38 2009 New Revision: 68215 Modified: pypy/trunk/pypy/jit/tl/pypyjit.py Log: small hack to fix pypyjit.py Modified: pypy/trunk/pypy/jit/tl/pypyjit.py ============================================================================== --- pypy/trunk/pypy/jit/tl/pypyjit.py (original) +++ pypy/trunk/pypy/jit/tl/pypyjit.py Tue Oct 6 18:39:38 2009 @@ -31,6 +31,7 @@ config = get_pypy_config(translating=True) config.translation.backendopt.inline_threshold = 0 +config.translation.gc = 'boehm' config.objspace.nofaking = True config.objspace.compiler = "ast" config.translating = True From afa at codespeak.net Tue Oct 6 19:23:59 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 6 Oct 2009 19:23:59 +0200 (CEST) Subject: [pypy-svn] r68216 - pypy/trunk/pypy/module/posix Message-ID: <20091006172359.10C91168422@codespeak.net> Author: afa Date: Tue Oct 6 19:23:59 2009 New Revision: 68216 Modified: pypy/trunk/pypy/module/posix/interp_posix.py Log: Correctly wrap the exception raised by os.getsid() Modified: pypy/trunk/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/trunk/pypy/module/posix/interp_posix.py (original) +++ pypy/trunk/pypy/module/posix/interp_posix.py Tue Oct 6 19:23:59 2009 @@ -781,7 +781,11 @@ Call the system call getsid(). """ - return space.wrap(os.getsid(pid)) + try: + sid = os.getsid(pid) + except OSError, e: + raise wrap_oserror(space, e) + return space.wrap(sid) getsid.unwrap_spec = [ObjSpace, int] def setsid(space): From afa at codespeak.net Tue Oct 6 19:37:41 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 6 Oct 2009 19:37:41 +0200 (CEST) Subject: [pypy-svn] r68217 - pypy/trunk/pypy/module/posix/test Message-ID: <20091006173741.F33E0168425@codespeak.net> Author: afa Date: Tue Oct 6 19:37:41 2009 New Revision: 68217 Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py Log: A test for the previous fix Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/trunk/pypy/module/posix/test/test_posix2.py (original) +++ pypy/trunk/pypy/module/posix/test/test_posix2.py Tue Oct 6 19:37:41 2009 @@ -47,6 +47,8 @@ cls.w_geteuid = space.wrap(os.geteuid()) if hasattr(os, 'getgid'): cls.w_getgid = space.wrap(os.getgid()) + if hasattr(os, 'getsid'): + cls.w_getsid0 = space.wrap(os.getsid(0)) if hasattr(os, 'sysconf'): sysconf_name = os.sysconf_names.keys()[0] cls.w_sysconf_name = space.wrap(sysconf_name) @@ -367,6 +369,12 @@ os = self.posix raises(OSError, os.setgid, -100000) + if hasattr(os, 'getsid'): + def test_os_getsid(self): + os = self.posix + assert os.getsid(0) == self.getsid0 + raises(OSError, os.getsid, -100000) + if hasattr(os, 'sysconf'): def test_os_sysconf(self): os = self.posix From cfbolz at codespeak.net Wed Oct 7 00:06:46 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 7 Oct 2009 00:06:46 +0200 (CEST) Subject: [pypy-svn] r68219 - pypy/trunk/pypy/objspace/std Message-ID: <20091006220646.97A1616842A@codespeak.net> Author: cfbolz Date: Wed Oct 7 00:06:44 2009 New Revision: 68219 Modified: pypy/trunk/pypy/objspace/std/typeobject.py Log: when instantiating something, promote it early to get rid of some silly guards Modified: pypy/trunk/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/typeobject.py (original) +++ pypy/trunk/pypy/objspace/std/typeobject.py Wed Oct 7 00:06:44 2009 @@ -587,6 +587,7 @@ # ____________________________________________________________ def call__Type(space, w_type, __args__): + w_type = hint(w_type, promote=True) # special case for type(x) if space.is_w(w_type, space.w_type): try: From cfbolz at codespeak.net Wed Oct 7 00:31:25 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 7 Oct 2009 00:31:25 +0200 (CEST) Subject: [pypy-svn] r68220 - pypy/trunk/pypy/objspace/std Message-ID: <20091006223125.5467016842D@codespeak.net> Author: cfbolz Date: Wed Oct 7 00:31:24 2009 New Revision: 68220 Modified: pypy/trunk/pypy/objspace/std/typeobject.py Log: This proves the essential silliness of itself. Modified: pypy/trunk/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/typeobject.py (original) +++ pypy/trunk/pypy/objspace/std/typeobject.py Wed Oct 7 00:31:24 2009 @@ -644,9 +644,9 @@ else: version_tag1 = w_type1.version_tag version_tag2 = w_type2.version_tag + version_tag1 = hint(version_tag1, promote=True) + version_tag2 = hint(version_tag2, promote=True) if version_tag1 is not None and version_tag2 is not None: - version_tag1 = hint(version_tag1, promote=True) - version_tag2 = hint(version_tag2, promote=True) res = _pure_issubtype(w_type1, w_type2, version_tag1, version_tag2) return space.newbool(res) res = _issubtype(w_type1, w_type2) From benjamin at codespeak.net Wed Oct 7 01:03:02 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 7 Oct 2009 01:03:02 +0200 (CEST) Subject: [pypy-svn] r68221 - pypy/trunk/pypy/jit/backend/x86 Message-ID: <20091006230302.D809516842D@codespeak.net> Author: benjamin Date: Wed Oct 7 01:03:02 2009 New Revision: 68221 Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py Log: put assignment after function def which it aliases Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Wed Oct 7 01:03:02 2009 @@ -765,6 +765,7 @@ result_loc = self.force_allocate_reg(op.result) self.Perform(op, [base_loc, ofs_loc, size_loc], result_loc) + consider_getfield_raw = consider_getfield_gc consider_getfield_gc_pure = consider_getfield_gc def consider_getarrayitem_gc(self, op, ignored): @@ -775,7 +776,6 @@ result_loc = self.force_allocate_reg(op.result) self.Perform(op, [base_loc, ofs_loc, imm(scale), imm(ofs)], result_loc) - consider_getfield_raw = consider_getfield_gc consider_getarrayitem_gc_pure = consider_getarrayitem_gc From benjamin at codespeak.net Wed Oct 7 02:20:57 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 7 Oct 2009 02:20:57 +0200 (CEST) Subject: [pypy-svn] r68222 - in pypy/trunk/pypy/jit/backend: test x86 Message-ID: <20091007002057.93A9616842F@codespeak.net> Author: benjamin Date: Wed Oct 7 02:20:56 2009 New Revision: 68222 Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/regalloc.py Log: (benjamin, fijal advising) generate code for GETFIELD_RAW_PURE Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Wed Oct 7 02:20:56 2009 @@ -980,6 +980,20 @@ assert s.x == chr(190) assert s.y == chr(150) + def test_field_raw_pure(self): + # This is really testing the same thing as test_field_basic but can't + # hurt... + S = lltype.Struct('S', ('x', lltype.Signed)) + s = lltype.malloc(S, flavor='raw') + s_box = BoxInt(rffi.cast(lltype.Signed, s)) + for get_op, set_op in ((rop.GETFIELD_RAW, rop.SETFIELD_RAW), + (rop.GETFIELD_RAW_PURE, rop.SETFIELD_RAW)): + fd = self.cpu.fielddescrof(S, 'x') + self.execute_operation(set_op, [s_box, BoxInt(32)], 'void', + descr=fd) + res = self.execute_operation(get_op, [s_box], 'int', descr=fd) + assert res.getint() == 32 + def test_new_with_vtable(self): cpu = self.cpu t_box, T_box = self.alloc_instance(self.T) Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Wed Oct 7 02:20:56 2009 @@ -558,6 +558,8 @@ else: raise NotImplementedError("getfield size = %d" % size) + genop_getfield_raw = genop_getfield_gc + genop_getfield_raw_pure = genop_getfield_gc genop_getfield_gc_pure = genop_getfield_gc def genop_getarrayitem_gc(self, op, arglocs, resloc): @@ -578,7 +580,6 @@ print "[asmgen]setarrayitem unsupported size: %d" % scale.value raise NotImplementedError() - genop_getfield_raw = genop_getfield_gc genop_getarrayitem_gc_pure = genop_getarrayitem_gc def genop_discard_setfield_gc(self, op, arglocs): Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Wed Oct 7 02:20:56 2009 @@ -766,6 +766,7 @@ self.Perform(op, [base_loc, ofs_loc, size_loc], result_loc) consider_getfield_raw = consider_getfield_gc + consider_getfield_raw_pure = consider_getfield_gc consider_getfield_gc_pure = consider_getfield_gc def consider_getarrayitem_gc(self, op, ignored): From arigo at codespeak.net Wed Oct 7 11:08:25 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Oct 2009 11:08:25 +0200 (CEST) Subject: [pypy-svn] r68224 - pypy/trunk/pypy/translator/c/gcc Message-ID: <20091007090825.68ABA168404@codespeak.net> Author: arigo Date: Wed Oct 7 11:08:24 2009 New Revision: 68224 Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Log: Remove the incomplete .lbl.s file in case of an exception. Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Wed Oct 7 11:08:24 2009 @@ -1211,9 +1211,14 @@ assert fn.endswith('.s') lblfn = fn[:-2] + '.lbl.s' g = open(lblfn, 'w') - tracker.process(f, g, filename=fn) - f.close() + try: + tracker.process(f, g, filename=fn) + except: + g.close() + os.unlink(lblfn) + raise g.close() + f.close() if output_raw_table: tracker.dump_raw_table(sys.stdout) tracker.clear() From pedronis at codespeak.net Wed Oct 7 11:29:28 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 7 Oct 2009 11:29:28 +0200 (CEST) Subject: [pypy-svn] r68225 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091007092928.60E32168405@codespeak.net> Author: pedronis Date: Wed Oct 7 11:29:28 2009 New Revision: 68225 Modified: pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/jit/metainterp/test/test_resume.py Log: (cfbolz, pedronis) this is not used anymore Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Wed Oct 7 11:29:28 2009 @@ -200,22 +200,6 @@ _, tagbits = untag(tagged) return tagbits == TAGVIRTUAL - def _flatten_frame_info(self): - storage = self.storage - frame_info_list = storage.rd_frame_info_list - storage.rd_frame_info_list = None - j = frame_info_list.level - frame_infos = [_frame_info_placeholder]*j - j -= 1 - while True: # at least one - frame_infos[j] = (frame_info_list.jitcode, frame_info_list.pc, - frame_info_list.exception_target) - frame_info_list = frame_info_list.prev - if frame_info_list is None: - break - j -= 1 - storage.rd_frame_infos = frame_infos - def finish(self, values): storage = self.storage liveboxes = [] Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Wed Oct 7 11:29:28 2009 @@ -308,23 +308,6 @@ assert modifier.liveboxes == {b1_2: UNASSIGNED, b3: UNASSIGNED} assert modifier.nnums == len(env)+1+len(env1)+1 -def test_flatten_frame_info(): - frame = FakeFrame("JITCODE", 1, 2) - fi = FrameInfo(None, frame) - frame1 = FakeFrame("JITCODE1", 3, 4) - fi1 = FrameInfo(fi, frame1) - - storage = Storage() - storage.rd_frame_info_list = fi1 - - modifier = ResumeDataVirtualAdder(storage, None) - modifier._flatten_frame_info() - assert storage.rd_frame_info_list is None - - assert storage.rd_frame_infos == [("JITCODE", 1, 2), - ("JITCODE1", 3, 4)] - - def test_ResumeDataLoopMemo_ints(): memo = ResumeDataLoopMemo(None) tagged = memo.getconst(ConstInt(44)) From fijal at codespeak.net Wed Oct 7 11:37:07 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Oct 2009 11:37:07 +0200 (CEST) Subject: [pypy-svn] r68226 - pypy/trunk/pypy/jit/backend/test Message-ID: <20091007093707.AC28316840C@codespeak.net> Author: fijal Date: Wed Oct 7 11:37:07 2009 New Revision: 68226 Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py Log: Fix the test Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Wed Oct 7 11:37:07 2009 @@ -985,7 +985,8 @@ # hurt... S = lltype.Struct('S', ('x', lltype.Signed)) s = lltype.malloc(S, flavor='raw') - s_box = BoxInt(rffi.cast(lltype.Signed, s)) + sa = llmemory.cast_ptr_to_adr(s) + s_box = BoxInt(self.cpu.cast_adr_to_int(sa)) for get_op, set_op in ((rop.GETFIELD_RAW, rop.SETFIELD_RAW), (rop.GETFIELD_RAW_PURE, rop.SETFIELD_RAW)): fd = self.cpu.fielddescrof(S, 'x') From fijal at codespeak.net Wed Oct 7 11:38:52 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Oct 2009 11:38:52 +0200 (CEST) Subject: [pypy-svn] r68227 - pypy/trunk/pypy/jit/backend/llgraph Message-ID: <20091007093852.BB9AB16840C@codespeak.net> Author: fijal Date: Wed Oct 7 11:38:51 2009 New Revision: 68227 Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py Log: missing (rare) op Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/llimpl.py Wed Oct 7 11:38:51 2009 @@ -679,6 +679,8 @@ else: raise NotImplementedError + op_getfield_raw_pure = op_getfield_raw + def op_new(self, size): return do_new(size.ofs) From arigo at codespeak.net Wed Oct 7 12:03:54 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Oct 2009 12:03:54 +0200 (CEST) Subject: [pypy-svn] r68228 - pypy/trunk/pypy/translator/c/gcc Message-ID: <20091007100354.09EAB168407@codespeak.net> Author: arigo Date: Wed Oct 7 12:03:54 2009 New Revision: 68228 Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Log: Support for gcc-4.4, which uses occasionally function names that can be any label like "T.14426". Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Wed Oct 7 12:03:54 2009 @@ -2,8 +2,9 @@ import re, sys, os, random -r_functionstart_elf = re.compile(r"\t.type\s+(\w+),\s*[@]function\s*$") -r_functionend_elf = re.compile(r"\t.size\s+(\w+),\s*[.]-(\w+)\s*$") +LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' +r_functionstart_elf = re.compile(r"\t.type\s+"+LABEL+",\s*[@]function\s*$") +r_functionend_elf = re.compile(r"\t.size\s+"+LABEL+",\s*[.]-"+LABEL+"\s*$") # darwin r_textstart = re.compile(r"\t.text\s*$") @@ -25,9 +26,8 @@ OFFSET_LABELS = 2**30 # inside functions -LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' r_label = re.compile(LABEL+"[:]\s*$") -r_globl = re.compile(r"\t[.]globl\t(\w+)\s*$") +r_globl = re.compile(r"\t[.]globl\t"+LABEL+"\s*$") r_globllabel = re.compile(LABEL+r"=[.][+]%d\s*$"%OFFSET_LABELS) r_insn = re.compile(r"\t([a-z]\w*)\s") r_jump = re.compile(r"\tj\w+\s+"+LABEL+"\s*$") From afa at codespeak.net Wed Oct 7 13:27:54 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 7 Oct 2009 13:27:54 +0200 (CEST) Subject: [pypy-svn] r68229 - pypy/trunk/pypy/translator/jvm Message-ID: <20091007112754.2414C168409@codespeak.net> Author: afa Date: Wed Oct 7 13:27:54 2009 New Revision: 68229 Modified: pypy/trunk/pypy/translator/jvm/metavm.py pypy/trunk/pypy/translator/jvm/opcodes.py Log: Fix --backend=jvm translation I'm not sure about the SignedLongLong->UnsignedLongLong cast, though. Modified: pypy/trunk/pypy/translator/jvm/metavm.py ============================================================================== --- pypy/trunk/pypy/translator/jvm/metavm.py (original) +++ pypy/trunk/pypy/translator/jvm/metavm.py Wed Oct 7 13:27:54 2009 @@ -95,6 +95,7 @@ (ootype.UnsignedLongLong, ootype.Unsigned): jvm.L2I, (ootype.UnsignedLongLong, ootype.Signed): jvm.L2I, (ootype.UnsignedLongLong, ootype.SignedLongLong): None, + (ootype.SignedLongLong, ootype.UnsignedLongLong): None, } class _CastPrimitive(MicroInstruction): Modified: pypy/trunk/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/trunk/pypy/translator/jvm/opcodes.py (original) +++ pypy/trunk/pypy/translator/jvm/opcodes.py Wed Oct 7 13:27:54 2009 @@ -96,6 +96,7 @@ 'gc__collect': jvm.SYSTEMGC, 'gc_set_max_heap_size': Ignore, 'resume_point': Ignore, + 'jit_marker': Ignore, 'promote_virtualizable': Ignore, 'debug_assert': [], # TODO: implement? From antocuni at codespeak.net Wed Oct 7 14:25:38 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 7 Oct 2009 14:25:38 +0200 (CEST) Subject: [pypy-svn] r68230 - in pypy/trunk/pypy: rpython/test translator/cli translator/cli/src translator/cli/test translator/jvm/test translator/oosupport Message-ID: <20091007122538.80D9F168409@codespeak.net> Author: antocuni Date: Wed Oct 7 14:25:37 2009 New Revision: 68230 Modified: pypy/trunk/pypy/rpython/test/test_rbuiltin.py pypy/trunk/pypy/translator/cli/cts.py pypy/trunk/pypy/translator/cli/ilgenerator.py pypy/trunk/pypy/translator/cli/metavm.py pypy/trunk/pypy/translator/cli/src/pypylib.cs pypy/trunk/pypy/translator/cli/test/runtest.py pypy/trunk/pypy/translator/jvm/test/test_builtin.py pypy/trunk/pypy/translator/jvm/test/test_list.py pypy/trunk/pypy/translator/oosupport/constant.py Log: - move test_cast_primitive to the base class, so it's tested also for ootype - teach the cli backend about the SHORT type, making test_cast_primitive and test_r_short_list passing - skip those two tests in the jvm backend Modified: pypy/trunk/pypy/rpython/test/test_rbuiltin.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_rbuiltin.py (original) +++ pypy/trunk/pypy/rpython/test/test_rbuiltin.py Wed Oct 7 14:25:37 2009 @@ -462,32 +462,6 @@ assert x1 == intmask(x0) assert x3 == intmask(x2) -class TestLLtype(BaseTestRbuiltin, LLRtypeMixin): - - def test_isinstance_obj(self): - _1 = lltype.pyobjectptr(1) - def f(x): - return isinstance(x, int) - res = self.interpret(f, [_1], someobjects=True) - assert res is True - _1_0 = lltype.pyobjectptr(1.0) - res = self.interpret(f, [_1_0], someobjects=True) - assert res is False - - def test_hasattr(self): - class A(object): - def __init__(self): - self.x = 42 - def f(i): - a = A() - if i==0: return int(hasattr(A, '__init__')) - if i==1: return int(hasattr(A, 'y')) - if i==2: return int(hasattr(42, 'x')) - for x, y in zip(range(3), (1, 0, 0)): - res = self.interpret(f, [x], someobjects=True) - assert res._obj.value == y - # hmm, would like to test against PyObj, is this the wrong place/way? - def test_cast_primitive(self): from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy def llf(u): @@ -516,6 +490,36 @@ return lltype.cast_primitive(rffi.SHORT, v) res = self.interpret(llf, [123], policy=LowLevelAnnotatorPolicy()) assert res == 123 + def llf(v): + return lltype.cast_primitive(lltype.Signed, v) + res = self.interpret(llf, [rffi.r_short(123)], policy=LowLevelAnnotatorPolicy()) + assert res == 123 + +class TestLLtype(BaseTestRbuiltin, LLRtypeMixin): + + def test_isinstance_obj(self): + _1 = lltype.pyobjectptr(1) + def f(x): + return isinstance(x, int) + res = self.interpret(f, [_1], someobjects=True) + assert res is True + _1_0 = lltype.pyobjectptr(1.0) + res = self.interpret(f, [_1_0], someobjects=True) + assert res is False + + def test_hasattr(self): + class A(object): + def __init__(self): + self.x = 42 + def f(i): + a = A() + if i==0: return int(hasattr(A, '__init__')) + if i==1: return int(hasattr(A, 'y')) + if i==2: return int(hasattr(42, 'x')) + for x, y in zip(range(3), (1, 0, 0)): + res = self.interpret(f, [x], someobjects=True) + assert res._obj.value == y + # hmm, would like to test against PyObj, is this the wrong place/way? def test_cast(self): def llfn(v): Modified: pypy/trunk/pypy/translator/cli/cts.py ============================================================================== --- pypy/trunk/pypy/translator/cli/cts.py (original) +++ pypy/trunk/pypy/translator/cli/cts.py Wed Oct 7 14:25:37 2009 @@ -6,6 +6,7 @@ from py.builtin import set from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import rffi from pypy.rpython.ootypesystem import ootype from pypy.translator.cli.option import getoption from pypy.translator.cli import oopspec @@ -107,6 +108,7 @@ T = CliPrimitiveType class types: void = T('void') + int16 = T('int16') int32 = T('int32') uint32 = T('unsigned int32') int64 = T('int64') @@ -134,6 +136,7 @@ _lltype_to_cts = { ootype.Void: types.void, + rffi.SHORT: types.int16, ootype.Signed: types.int32, ootype.Unsigned: types.uint32, ootype.SignedLongLong: types.int64, Modified: pypy/trunk/pypy/translator/cli/ilgenerator.py ============================================================================== --- pypy/trunk/pypy/translator/cli/ilgenerator.py (original) +++ pypy/trunk/pypy/translator/cli/ilgenerator.py Wed Oct 7 14:25:37 2009 @@ -1,5 +1,6 @@ from pypy.rpython.lltypesystem.lltype import Signed, Unsigned, Void, Bool, Float from pypy.rpython.lltypesystem.lltype import SignedLongLong, UnsignedLongLong +from pypy.rpython.lltypesystem import rffi from pypy.rlib.objectmodel import CDefinedIntSymbolic from pypy.rlib.rarithmetic import isnan, isinf from pypy.rpython.ootypesystem import ootype @@ -406,7 +407,7 @@ ilasm.opcode('ldc.r8', repr(value)) elif isinstance(value, CDefinedIntSymbolic): ilasm.opcode('ldc.i4', DEFINED_INT_SYMBOLICS[value.expr]) - elif TYPE in (ootype.Signed, ootype.Unsigned): + elif TYPE in (ootype.Signed, ootype.Unsigned, rffi.SHORT): ilasm.opcode('ldc.i4', str(value)) elif TYPE in (ootype.SignedLongLong, ootype.UnsignedLongLong): ilasm.opcode('ldc.i8', str(value)) Modified: pypy/trunk/pypy/translator/cli/metavm.py ============================================================================== --- pypy/trunk/pypy/translator/cli/metavm.py (original) +++ pypy/trunk/pypy/translator/cli/metavm.py Wed Oct 7 14:25:37 2009 @@ -1,5 +1,6 @@ from pypy.translator.cli import oopspec from pypy.rpython.ootypesystem import ootype +from pypy.rpython.lltypesystem import rffi from pypy.translator.oosupport.metavm import Generator, InstructionList, MicroInstruction,\ PushAllArgs, StoreResult, GetField, SetField, DownCast from pypy.translator.oosupport.metavm import _Call as _OOCall @@ -274,6 +275,7 @@ ootype.Bool: 'i1', ootype.Char: 'i2', ootype.UniChar: 'i2', + rffi.SHORT: 'i2', ootype.Signed: 'i4', ootype.SignedLongLong: 'i8', ootype.Unsigned: 'u4', Modified: pypy/trunk/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/trunk/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/trunk/pypy/translator/cli/src/pypylib.cs Wed Oct 7 14:25:37 2009 @@ -18,6 +18,7 @@ public static string ToPython(uint x) { return x.ToString(); } public static string ToPython(long x) { return x.ToString(); } public static string ToPython(ulong x) { return x.ToString(); } + public static string ToPython(short x) { return x.ToString(); } // XXX: it does not support strings containing "'". public static string ToPython(string x) { if (x == null) Modified: pypy/trunk/pypy/translator/cli/test/runtest.py ============================================================================== --- pypy/trunk/pypy/translator/cli/test/runtest.py (original) +++ pypy/trunk/pypy/translator/cli/test/runtest.py Wed Oct 7 14:25:37 2009 @@ -126,6 +126,7 @@ def __convert_method(self, arg_type): _conv = { + CTS.types.int16: 'ToInt16', CTS.types.int32: 'ToInt32', CTS.types.uint32: 'ToUInt32', CTS.types.int64: 'ToInt64', @@ -285,7 +286,8 @@ self._ann = None self._cli_func = None - def _compile(self, fn, args, ann=None, backendopt=True, auto_raise_exc=False, exctrans=False): + def _compile(self, fn, args, ann=None, backendopt=True, + auto_raise_exc=False, exctrans=False, policy=None): if ann is None: ann = [get_annotation(x) for x in args] if self._func is fn and self._ann == ann: @@ -293,7 +295,7 @@ else: self._cli_func = compile_function(fn, ann, backendopt=backendopt, auto_raise_exc=auto_raise_exc, - exctrans=exctrans) + exctrans=exctrans, annotatorpolicy=policy) self._func = fn self._ann = ann return self._cli_func @@ -314,7 +316,7 @@ backendopt = getattr(self, 'backendopt', True) # enable it by default return backendopt - def interpret(self, fn, args, annotation=None, backendopt=None, exctrans=False): + def interpret(self, fn, args, annotation=None, backendopt=None, exctrans=False, policy=None): backendopt = self._get_backendopt(backendopt) f = self._compile(fn, args, annotation, backendopt=backendopt, exctrans=exctrans) res = f(*args) Modified: pypy/trunk/pypy/translator/jvm/test/test_builtin.py ============================================================================== --- pypy/trunk/pypy/translator/jvm/test/test_builtin.py (original) +++ pypy/trunk/pypy/translator/jvm/test/test_builtin.py Wed Oct 7 14:25:37 2009 @@ -34,6 +34,10 @@ 'http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6539705') BaseTestBuiltin.test_os_access(self) + def test_cast_primitive(self): + py.test.skip('fixme!') + + class TestJvmTime(JvmTest, BaseTestTime): pass Modified: pypy/trunk/pypy/translator/jvm/test/test_list.py ============================================================================== --- pypy/trunk/pypy/translator/jvm/test/test_list.py (original) +++ pypy/trunk/pypy/translator/jvm/test/test_list.py Wed Oct 7 14:25:37 2009 @@ -9,6 +9,9 @@ def test_getitem_exc(self): py.test.skip('fixme!') + def test_r_short_list(self): + py.test.skip('fixme!') + def test_zeroed_list(self): def fn(): lst = [0] * 16 Modified: pypy/trunk/pypy/translator/oosupport/constant.py ============================================================================== --- pypy/trunk/pypy/translator/oosupport/constant.py (original) +++ pypy/trunk/pypy/translator/oosupport/constant.py Wed Oct 7 14:25:37 2009 @@ -26,6 +26,7 @@ """ from pypy.rpython.ootypesystem import ootype +from pypy.rpython.lltypesystem import rffi import operator MAX_CONST_PER_STEP = 50 @@ -33,7 +34,7 @@ PRIMITIVE_TYPES = set([ootype.Void, ootype.Bool, ootype.Char, ootype.UniChar, ootype.Float, ootype.Signed, ootype.Unsigned, ootype.String, ootype.Unicode, ootype.SignedLongLong, - ootype.UnsignedLongLong]) + ootype.UnsignedLongLong, rffi.SHORT]) def is_primitive(TYPE): return TYPE in PRIMITIVE_TYPES From pedronis at codespeak.net Wed Oct 7 15:10:39 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 7 Oct 2009 15:10:39 +0200 (CEST) Subject: [pypy-svn] r68231 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20091007131039.76EA9168407@codespeak.net> Author: pedronis Date: Wed Oct 7 15:10:37 2009 New Revision: 68231 Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py Log: (cfbolz, pedronis) integration tests in preparation of deeper changes to increase the sharing of resume data Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Wed Oct 7 15:10:37 2009 @@ -110,6 +110,9 @@ return self.__dict__ == other.__dict__ def __ne__(self, other): return self.__dict__ != other.__dict__ + def __repr__(self): + return "" % (self.jitcode, self.pc, + self.exception_target, self.env) def test_Snapshot_create(): l = ['b0', 'b1'] @@ -209,13 +212,6 @@ assert snapshot.boxes == fs[2].env def test_rebuild_from_resumedata(): - class FakeMetaInterp(object): - def __init__(self): - self.framestack = [] - def newframe(self, jitcode): - frame = FakeFrame(jitcode, -1, -1) - self.framestack.append(frame) - return frame b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] storage = Storage() @@ -227,7 +223,7 @@ modifier = ResumeDataVirtualAdder(storage, memo) modifier.walk_snapshots({}) liveboxes = modifier.finish({}) - metainterp = FakeMetaInterp() + metainterp = MyMetaInterp(None) b1t, b2t, b3t = [BoxInt(), BoxPtr(), BoxInt()] newboxes = _resume_remap(liveboxes, [b1, b2, b3], b1t, b2t, b3t) @@ -241,13 +237,6 @@ assert metainterp.framestack == fs2 def test_rebuild_from_resumedata_with_virtualizable(): - class FakeMetaInterp(object): - def __init__(self): - self.framestack = [] - def newframe(self, jitcode): - frame = FakeFrame(jitcode, -1, -1) - self.framestack.append(frame) - return frame b1, b2, b3, b4 = [BoxInt(), BoxPtr(), BoxInt(), BoxPtr()] c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] storage = Storage() @@ -259,7 +248,7 @@ modifier = ResumeDataVirtualAdder(storage, memo) modifier.walk_snapshots({}) liveboxes = modifier.finish({}) - metainterp = FakeMetaInterp() + metainterp = MyMetaInterp(None) b1t, b2t, b3t, b4t = [BoxInt(), BoxPtr(), BoxInt(), BoxPtr()] newboxes = _resume_remap(liveboxes, [b1, b2, b3, b4], b1t, b2t, b3t, b4t) @@ -271,6 +260,116 @@ FakeFrame("code1", 3, 7, b3t, c2, b1t), FakeFrame("code2", 9, -1, c3, b2t)] assert metainterp.framestack == fs2 + +def test_rebuild_from_resumedata_two_guards(): + b1, b2, b3, b4 = [BoxInt(), BoxPtr(), BoxInt(), BoxInt()] + c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] + storage = Storage() + fs = [FakeFrame("code0", 0, -1, b1, c1, b2), + FakeFrame("code1", 3, 7, b3, c2, b1), + FakeFrame("code2", 9, -1, c3, b2)] + capture_resumedata(fs, None, storage) + storage2 = Storage() + fs = fs[:-1] + [FakeFrame("code2", 10, -1, c3, b2, b4)] + capture_resumedata(fs, None, storage2) + + memo = ResumeDataLoopMemo(None) + modifier = ResumeDataVirtualAdder(storage, memo) + modifier.walk_snapshots({}) + liveboxes = modifier.finish({}) + + modifier = ResumeDataVirtualAdder(storage2, memo) + modifier.walk_snapshots({}) + liveboxes2 = modifier.finish({}) + + metainterp = MyMetaInterp(None) + + b1t, b2t, b3t, b4t = [BoxInt(), BoxPtr(), BoxInt(), BoxInt()] + newboxes = _resume_remap(liveboxes, [b1, b2, b3], b1t, b2t, b3t) + + result = rebuild_from_resumedata(metainterp, newboxes, storage, + False) + assert result is None + fs2 = [FakeFrame("code0", 0, -1, b1t, c1, b2t), + FakeFrame("code1", 3, 7, b3t, c2, b1t), + FakeFrame("code2", 9, -1, c3, b2t)] + assert metainterp.framestack == fs2 + + newboxes = _resume_remap(liveboxes2, [b1, b2, b3, b4], b1t, b2t, b3t, b4t) + + metainterp.framestack = [] + result = rebuild_from_resumedata(metainterp, newboxes, storage2, + False) + assert result is None + fs2 = fs2[:-1] + [FakeFrame("code2", 10, -1, c3, b2t, b4t)] + assert metainterp.framestack == fs2 + + +def test_rebuild_from_resumedata_two_guards_w_virtuals(): + + b1, b2, b3, b4, b5 = [BoxInt(), BoxPtr(), BoxInt(), BoxInt(), BoxInt()] + c1, c2, c3, c4 = [ConstInt(1), ConstInt(2), ConstInt(3), + LLtypeMixin.nodebox.constbox()] + class FakeValue(object): + + def register_value(self, modifier): + if modifier.register_box(b2): + modifier.make_virtual(b2, ConstAddr(LLtypeMixin.node_vtable_adr, + LLtypeMixin.cpu), + [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr], + [c4, fieldbox]) + + def get_key_box(self): + return b2 + + values = {b2: FakeValue()} + + storage = Storage() + fs = [FakeFrame("code0", 0, -1, b1, c1, b2), + FakeFrame("code1", 3, 7, b3, c2, b1), + FakeFrame("code2", 9, -1, c3, b2)] + capture_resumedata(fs, None, storage) + storage2 = Storage() + fs = fs[:-1] + [FakeFrame("code2", 10, -1, c3, b2, b4)] + capture_resumedata(fs, None, storage2) + + memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + fieldbox = b5 + modifier = ResumeDataVirtualAdder(storage, memo) + modifier.walk_snapshots(values) + liveboxes = modifier.finish(values) + + fieldbox = b4 + modifier = ResumeDataVirtualAdder(storage2, memo) + modifier.walk_snapshots(values) + liveboxes2 = modifier.finish(values) + + metainterp = MyMetaInterp(LLtypeMixin.cpu) + + b1t, b3t, b4t, b5t = [BoxInt(), BoxInt(), BoxInt(), BoxInt()] + newboxes = _resume_remap(liveboxes, [b1, b3, b5], b1t, b3t, b5t) + + result = rebuild_from_resumedata(metainterp, newboxes, storage, + False) + + b2t = metainterp.resboxes[0] + fs2 = [FakeFrame("code0", 0, -1, b1t, c1, b2t), + FakeFrame("code1", 3, 7, b3t, c2, b1t), + FakeFrame("code2", 9, -1, c3, b2t)] + assert metainterp.framestack == fs2 + + newboxes = _resume_remap(liveboxes2, [b1, b3, b4], b1t, b3t, b4t) + + metainterp = MyMetaInterp(LLtypeMixin.cpu) + result = rebuild_from_resumedata(metainterp, newboxes, storage2, + False) + b2t = metainterp.resboxes[0] + fs2 = [FakeFrame("code0", 0, -1, b1t, c1, b2t), + FakeFrame("code1", 3, 7, b3t, c2, b1t), + FakeFrame("code2", 10, -1, c3, b2t, b4t)] + assert metainterp.framestack == fs2 + + # ____________________________________________________________ def test_walk_snapshots(): @@ -358,12 +457,21 @@ def __init__(self, cpu): self.cpu = cpu self.trace = [] + self.framestack = [] + self.resboxes = [] + + def newframe(self, jitcode): + frame = FakeFrame(jitcode, -1, -1) + self.framestack.append(frame) + return frame + def execute_and_record(self, opnum, descr, *argboxes): resbox = executor.execute(self.cpu, opnum, descr, *argboxes) self.trace.append((opnum, [box.value for box in argboxes], resbox and resbox.value, descr)) + self.resboxes.append(resbox) return resbox def _resume_remap(liveboxes, expected, *newvalues): From arigo at codespeak.net Wed Oct 7 15:27:21 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Oct 2009 15:27:21 +0200 (CEST) Subject: [pypy-svn] r68232 - pypy/branch/gc-compress Message-ID: <20091007132721.7EAD616841D@codespeak.net> Author: arigo Date: Wed Oct 7 15:27:20 2009 New Revision: 68232 Added: pypy/branch/gc-compress/ - copied from r68231, pypy/trunk/ Log: A branch in which to play with compressing the header of objects that have a vtable. The idea is to somehow drop the 'typeptr' as a separate field and instead use the type number stored in the header word added by the GC. From fijal at codespeak.net Wed Oct 7 15:49:01 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 7 Oct 2009 15:49:01 +0200 (CEST) Subject: [pypy-svn] r68233 - pypy/branch/remove-rewrite-assembler Message-ID: <20091007134901.DF2C116840C@codespeak.net> Author: fijal Date: Wed Oct 7 15:49:01 2009 New Revision: 68233 Added: pypy/branch/remove-rewrite-assembler/ - copied from r68232, pypy/trunk/ Log: A branch to try teaching gc how to deal with ConstPtrs encoded in the assembler From arigo at codespeak.net Wed Oct 7 16:14:40 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Oct 2009 16:14:40 +0200 (CEST) Subject: [pypy-svn] r68234 - in pypy/branch/gc-compress/pypy/rpython/lltypesystem: . test Message-ID: <20091007141440.9E0F616840D@codespeak.net> Author: arigo Date: Wed Oct 7 16:14:39 2009 New Revision: 68234 Added: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py (contents, props changed) pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llgroup.py (contents, props changed) Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py Log: Add the notion of 'group', tentatively. Added: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py ============================================================================== --- (empty file) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py Wed Oct 7 16:14:39 2009 @@ -0,0 +1,57 @@ +import weakref +from pypy.rpython.lltypesystem import lltype, llmemory, rffi + + +class GroupType(lltype.ContainerType): + """A 'group' that stores static structs together in memory. + The point is that they can be referenced by a GroupMemberOffset + which only takes 2 bytes (a USHORT), so the total size of a group + is limited to 18 or 19 bits (= the 16 bits in a USHORT, plus 2 or + 3 bits at the end that are zero and so don't need to be stored). + """ + +Group = GroupType() + + +class group(lltype._container): + _TYPE = Group + + def __init__(self, name): + self.name = name + self.members = [] + + def add_member(self, structptr): + TYPE = lltype.typeOf(structptr) + assert isinstance(TYPE.TO, lltype.Struct) + assert TYPE.TO._gckind == 'raw' + struct = structptr._as_obj() + assert struct not in _membership,"cannot be a member of several groups" + self.members.append(struct) + _membership[struct] = self + +def member_of_group(structptr): + return _membership.get(structptr._as_obj(), None) + +_membership = weakref.WeakKeyDictionary() + + +class GroupMemberOffset(llmemory.Symbolic): + """The offset of a struct inside a group, stored compactly in a USHORT. + Can only be used by the lloperation 'get_group_member'. + """ + def annotation(self): + from pypy.annotation import model + return model.SomeInteger(knowntype=rffi.r_ushort) + + def lltype(self): + return rffi.USHORT + + def __init__(self, grp, member): + assert lltype.typeOf(grp) == Group + assert member._as_obj() in grp.members + self.grpptr = grp._as_ptr() + self.member = member._as_ptr() + + def _get_group_member(self, grpptr): + assert grpptr == self.grpptr, "get_group_member: wrong group!" + return self.member Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py Wed Oct 7 16:14:39 2009 @@ -410,6 +410,7 @@ 'cast_adr_to_ptr': LLOp(canfold=True), 'cast_adr_to_int': LLOp(sideeffects=False), 'cast_int_to_adr': LLOp(canfold=True), # not implemented in llinterp + 'get_group_member': LLOp(canfold=True), # __________ used by the JIT ________ Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py Wed Oct 7 16:14:39 2009 @@ -388,6 +388,13 @@ def op_promote_virtualizable(object, fieldname, flags): pass # XXX should do something +def op_get_group_member(TYPE, grpptr, memberoffset): + from pypy.rpython.lltypesystem import llgroup + assert isinstance(memberoffset, llgroup.GroupMemberOffset) + member = memberoffset._get_group_member(grpptr) + return lltype.cast_pointer(TYPE, member) +op_get_group_member.need_result_type = True + # ____________________________________________________________ def get_op_impl(opname): Added: pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llgroup.py ============================================================================== --- (empty file) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llgroup.py Wed Oct 7 16:14:39 2009 @@ -0,0 +1,71 @@ +from pypy.rpython.lltypesystem.llgroup import * +from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rpython.test.test_llinterp import interpret + + +class TestLLGroup(object): + + def _freeze_(self): + return True + + def build(self): + grp = group('testing') + S1 = lltype.Struct('S1', ('x', lltype.Signed)) + S2 = lltype.Struct('S2', ('y', lltype.Signed), ('z', lltype.Signed)) + p1a = lltype.malloc(S1, immortal=True, zero=True) + p1b = lltype.malloc(S1, immortal=True, zero=True) + p2a = lltype.malloc(S2, immortal=True, zero=True) + p2b = lltype.malloc(S2, immortal=True, zero=True) + p1a.x = 123 + p1b.x = 456 + p2a.y = 789 + p2b.z = -12 + grp.add_member(p1a) + grp.add_member(p2a) + grp.add_member(p2b) + grp.add_member(p1b) + self.g1a = GroupMemberOffset(grp, p1a) + self.g1b = GroupMemberOffset(grp, p1b) + self.g2a = GroupMemberOffset(grp, p2a) + self.g2b = GroupMemberOffset(grp, p2b) + self.p1a = p1a + self.p1b = p1b + self.p2a = p2a + self.p2b = p2b + self.S1 = S1 + self.S2 = S2 + self.grp = grp + self.grpptr = grp._as_ptr() + + def test_simple(self): + self.build() + grpptr = self.grpptr + S1 = self.S1 + S2 = self.S2 + Ptr = lltype.Ptr + assert llop.get_group_member(Ptr(S1), grpptr, self.g1a) == self.p1a + assert llop.get_group_member(Ptr(S1), grpptr, self.g1b) == self.p1b + assert llop.get_group_member(Ptr(S2), grpptr, self.g2a) == self.p2a + assert llop.get_group_member(Ptr(S2), grpptr, self.g2b) == self.p2b + + def test_member_of_group(self): + self.build() + assert member_of_group(self.p1a) == self.grp + assert member_of_group(self.p2b) == self.grp + pnew = lltype.malloc(self.S2, immortal=True) + assert member_of_group(pnew) is None + + def test_rpython(self): + self.build() + grpptr = self.grpptr + def f(): + p = llop.get_group_member(lltype.Ptr(self.S1), grpptr, self.g1a) + assert p == self.p1a + p = llop.get_group_member(lltype.Ptr(self.S1), grpptr, self.g1b) + assert p == self.p1b + p = llop.get_group_member(lltype.Ptr(self.S2), grpptr, self.g2a) + assert p == self.p2a + p = llop.get_group_member(lltype.Ptr(self.S2), grpptr, self.g2b) + assert p == self.p2b + return 3 + assert interpret(f, []) == 3 From arigo at codespeak.net Wed Oct 7 16:28:30 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Oct 2009 16:28:30 +0200 (CEST) Subject: [pypy-svn] r68235 - in pypy/branch/gc-compress/pypy/rpython/lltypesystem: . test Message-ID: <20091007142830.92B0B16840D@codespeak.net> Author: arigo Date: Wed Oct 7 16:28:30 2009 New Revision: 68235 Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llgroup.py Log: An ad-hoc extension. Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py Wed Oct 7 16:28:30 2009 @@ -26,6 +26,7 @@ assert TYPE.TO._gckind == 'raw' struct = structptr._as_obj() assert struct not in _membership,"cannot be a member of several groups" + assert struct._parentstructure() is None self.members.append(struct) _membership[struct] = self @@ -48,10 +49,20 @@ def __init__(self, grp, member): assert lltype.typeOf(grp) == Group - assert member._as_obj() in grp.members self.grpptr = grp._as_ptr() self.member = member._as_ptr() + self.index = grp.members.index(member._as_obj()) def _get_group_member(self, grpptr): assert grpptr == self.grpptr, "get_group_member: wrong group!" return self.member + + def _get_next_group_member(self, grpptr, skipoffset): + # ad-hoc: returns a pointer to the group member that follows this one, + # given information in 'skipoffset' about how much to skip -- which + # is the size of the current member. + assert grpptr == self.grpptr, "get_next_group_member: wrong group!" + assert isinstance(skipoffset, llmemory.ItemOffset) + assert skipoffset.TYPE == lltype.typeOf(self.member).TO + assert skipoffset.repeat == 1 + return self.grpptr._as_obj().members[self.index + 1]._as_ptr() Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py Wed Oct 7 16:28:30 2009 @@ -411,6 +411,7 @@ 'cast_adr_to_int': LLOp(sideeffects=False), 'cast_int_to_adr': LLOp(canfold=True), # not implemented in llinterp 'get_group_member': LLOp(canfold=True), + 'get_next_group_member':LLOp(canfold=True), # __________ used by the JIT ________ Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py Wed Oct 7 16:28:30 2009 @@ -395,6 +395,13 @@ return lltype.cast_pointer(TYPE, member) op_get_group_member.need_result_type = True +def op_get_next_group_member(TYPE, grpptr, memberoffset, skipoffset): + from pypy.rpython.lltypesystem import llgroup + assert isinstance(memberoffset, llgroup.GroupMemberOffset) + member = memberoffset._get_next_group_member(grpptr, skipoffset) + return lltype.cast_pointer(TYPE, member) +op_get_next_group_member.need_result_type = True + # ____________________________________________________________ def get_op_impl(opname): Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llgroup.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llgroup.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llgroup.py Wed Oct 7 16:28:30 2009 @@ -55,6 +55,24 @@ pnew = lltype.malloc(self.S2, immortal=True) assert member_of_group(pnew) is None + def test_next_group_member(self): + self.build() + grpptr = self.grpptr + S1 = self.S1 + S2 = self.S2 + Ptr = lltype.Ptr + p = llop.get_next_group_member(Ptr(S2), grpptr, + self.g1a, llmemory.sizeof(S1)) + assert p == self.p2a + # + p = llop.get_next_group_member(Ptr(S2), grpptr, + self.g2a, llmemory.sizeof(S2)) + assert p == self.p2b + # + p = llop.get_next_group_member(Ptr(S1), grpptr, + self.g2b, llmemory.sizeof(S2)) + assert p == self.p1b + def test_rpython(self): self.build() grpptr = self.grpptr From arigo at codespeak.net Wed Oct 7 16:31:25 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 7 Oct 2009 16:31:25 +0200 (CEST) Subject: [pypy-svn] r68236 - pypy/branch/gc-compress/pypy/rpython/lltypesystem Message-ID: <20091007143125.A195B16840D@codespeak.net> Author: arigo Date: Wed Oct 7 16:31:25 2009 New Revision: 68236 Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py Log: Oups. It seems to make more sense that way. Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py Wed Oct 7 16:31:25 2009 @@ -33,7 +33,7 @@ def member_of_group(structptr): return _membership.get(structptr._as_obj(), None) -_membership = weakref.WeakKeyDictionary() +_membership = weakref.WeakValueDictionary() class GroupMemberOffset(llmemory.Symbolic): From afa at codespeak.net Wed Oct 7 17:58:58 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 7 Oct 2009 17:58:58 +0200 (CEST) Subject: [pypy-svn] r68237 - in pypy/trunk/pypy: rlib rpython/module Message-ID: <20091007155858.E00CB16840D@codespeak.net> Author: afa Date: Wed Oct 7 17:58:55 2009 New Revision: 68237 Modified: pypy/trunk/pypy/rlib/rwin32.py pypy/trunk/pypy/rpython/module/ll_os.py Log: Fix --backend=cli translation on Windows, by providing a dumb message for WindowsError exceptions. Also fix a memory leak. There ought to be a better way for backends to override functions. Modified: pypy/trunk/pypy/rlib/rwin32.py ============================================================================== --- pypy/trunk/pypy/rlib/rwin32.py (original) +++ pypy/trunk/pypy/rlib/rwin32.py Wed Oct 7 17:58:55 2009 @@ -5,7 +5,8 @@ from pypy.rpython.tool import rffi_platform from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.lltypesystem import lltype, rffi -import os +from pypy.rlib.rarithmetic import intmask +import os, sys # This module can be imported on any platform, # but most symbols are not usable... @@ -95,24 +96,38 @@ # A bit like strerror... def FormatError(code): + return llimpl_FormatError(code) + + def llimpl_FormatError(code): "Return a message corresponding to the given Windows error code." buf = lltype.malloc(rffi.VOIDPP.TO, 1, flavor='raw') - msglen = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM, - None, - code, - DEFAULT_LANGUAGE, - rffi.cast(rffi.VOIDP, buf), - 0, None) - - # FormatMessage always appends a \n. - msglen -= 1 - - result = ''.join([buf[0][i] for i in range(msglen)]) - LocalFree(buf[0]) + try: + msglen = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM, + None, + code, + DEFAULT_LANGUAGE, + rffi.cast(rffi.VOIDP, buf), + 0, None) + + if msglen <= 2 or msglen > sys.maxint: + return fake_FormatError(code) + + # FormatMessage always appends \r\n. + buflen = intmask(msglen - 2) + assert buflen > 0 + + result = rffi.charpsize2str(buf[0], buflen) + LocalFree(buf[0]) + finally: + lltype.free(buf, flavor='raw') + return result + def fake_FormatError(code): + return 'Windows Error %d' % (code,) + def lastWindowsError(context="Windows Error"): code = GetLastError() return WindowsError(code, context) Modified: pypy/trunk/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/trunk/pypy/rpython/module/ll_os.py (original) +++ pypy/trunk/pypy/rpython/module/ll_os.py Wed Oct 7 17:58:55 2009 @@ -1507,3 +1507,20 @@ def getcontroller(self): from pypy.rpython.module.ll_os_environ import OsEnvironController return OsEnvironController() + +# ____________________________________________________________ +# Support for the WindowsError exception + +if sys.platform == 'win32': + from pypy.rlib import rwin32 + + class RegisterFormatError(BaseLazyRegistering): + def __init__(self): + pass + + @registering(rwin32.FormatError) + def register_rwin32_FormatError(self): + return extdef([int], str, + "rwin32_FormatError", + llimpl=rwin32.llimpl_FormatError, + ooimpl=rwin32.fake_FormatError) From afa at codespeak.net Wed Oct 7 18:05:22 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 7 Oct 2009 18:05:22 +0200 (CEST) Subject: [pypy-svn] r68238 - pypy/branch/asmgcc-mingw32-2 Message-ID: <20091007160522.2D27F168010@codespeak.net> Author: afa Date: Wed Oct 7 18:05:21 2009 New Revision: 68238 Added: pypy/branch/asmgcc-mingw32-2/ - copied from r68237, pypy/trunk/ Log: A short-lived branch to test my changes to trackgcroot for mingw32 support From afa at codespeak.net Wed Oct 7 18:10:47 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 7 Oct 2009 18:10:47 +0200 (CEST) Subject: [pypy-svn] r68239 - pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc Message-ID: <20091007161047.521B7168010@codespeak.net> Author: afa Date: Wed Oct 7 18:10:46 2009 New Revision: 68239 Modified: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py Log: Apply changes. Mostly the same as the ones I reverted some time ago, just be more careful to not disturb other platforms. Modified: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py Wed Oct 7 18:10:46 2009 @@ -219,7 +219,22 @@ if functionlines: yield in_function, functionlines - _find_functions_mingw32 = _find_functions_darwin + def _find_functions_mingw32(self, iterlines): + functionlines = [] + in_text = False + in_function = False + for n, line in enumerate(iterlines): + if r_textstart.match(line): + in_text = True + elif r_sectionstart.match(line): + in_text = False + elif in_text and r_functionstart_darwin.match(line): + yield in_function, functionlines + functionlines = [] + in_function = True + functionlines.append(line) + if functionlines: + yield in_function, functionlines def process(self, iterlines, newfile, entrypoint='main', filename='?'): if self.format in ('darwin', 'mingw32'): @@ -271,7 +286,7 @@ funcname = '_'+match.group(1) else: assert False, "unknown format: %s" % format - + self.funcname = funcname self.lines = lines self.uses_frame_pointer = False @@ -650,14 +665,19 @@ # only for leal -12(%ebp), %esp in function epilogues source = match.group(1) match = r_localvar_ebp.match(source) - if not match: - framesize = None # strange instruction - else: + if match: if not self.uses_frame_pointer: raise UnrecognizedOperation('epilogue without prologue') ofs_from_ebp = int(match.group(1) or '0') assert ofs_from_ebp <= 0 framesize = 4 - ofs_from_ebp + else: + match = r_localvar_esp.match(source) + # leal 12(%esp), %esp + if match: + return InsnStackAdjust(int(match.group(1))) + + framesize = None # strange instruction return InsnEpilogue(framesize) else: return self.binary_insn(line) @@ -717,12 +737,30 @@ return InsnRet() def visit_jmp(self, line): + tablelabel = None match = r_jmp_switch.match(line) if match: # this is a jmp *Label(%index), used for table-based switches. # Assume that the table is just a list of lines looking like # .long LABEL or .long 0, ending in a .text or .section .text.hot. tablelabel = match.group(1) + elif r_unaryinsn_star.match(line): + # maybe a jmp similar to the above, but stored in a + # registry: + # movl L9341(%eax), %eax + # jmp *%eax + operand = r_unaryinsn_star.match(line).group(1)[1:] + prev_line = self.lines[self.currentlineno-1] + match = r_insn.match(prev_line) + binaryinsn = r_binaryinsn.match(prev_line) + if (match and binaryinsn and + match.group(1) == 'movl' and binaryinsn.group(2) == operand + and '(' in binaryinsn.group(1)): + tablelabel = binaryinsn.group(1).split('(')[0] + if tablelabel not in self.labels: + # Probably an indirect tail-call. + tablelabel = None + if tablelabel: tablelin = self.labels[tablelabel].lineno + 1 while not r_jmptable_end.match(self.lines[tablelin]): match = r_jmptable_item.match(self.lines[tablelin]) From cfbolz at codespeak.net Wed Oct 7 22:27:50 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 7 Oct 2009 22:27:50 +0200 (CEST) Subject: [pypy-svn] r68240 - pypy/trunk/pypy/translator/microbench Message-ID: <20091007202750.B9C23168007@codespeak.net> Author: cfbolz Date: Wed Oct 7 22:27:48 2009 New Revision: 68240 Added: pypy/trunk/pypy/translator/microbench/test_call.py (contents, props changed) Modified: pypy/trunk/pypy/translator/microbench/test_count1.py Log: a few more cute benchmarks Added: pypy/trunk/pypy/translator/microbench/test_call.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/translator/microbench/test_call.py Wed Oct 7 22:27:48 2009 @@ -0,0 +1,79 @@ +N = int(2**19 - 1) + +def f1(x): + return x + 1 + +def test_call_1(): + c = 0 + n = N + while c < n: + c = f1(c) + +def f2(x, y): + return x + y + +def test_call_2(): + c = 0 + n = N + while c < n: + c = f2(c, 1) + + +def f3(x, y, z): + return x + y * z + +def test_call_3(): + c = 0 + n = N + while c < n: + c = f3(c, 1, 1) + + +def f4(w, x, y, z): + return x + y * z / w + +def test_call_4(): + c = 0 + n = N + while c < n: + c = f4(1, c, 1, 1) + +# __________________________________________ + + +def d4(x, y=1, z=1, w=1): + return x + y * z / w + +def test_call_default_1(): + c = 0 + n = N + while c < n: + c = d4(c) + +def test_call_default_2(): + c = 0 + n = N + while c < n: + c = d4(c, 1) + + +# __________________________________________ + + +def test_call_keyword_1(): + c = 0 + n = N + while c < n: + c = d4(c, z=1) + +def test_call_keyword_2(): + c = 0 + n = N + while c < n: + c = d4(c, z=1, w=1) + +def test_call_keyword_3(): + c = 0 + n = N + while c < n: + c = d4(c, z=1, w=1, y=1) Modified: pypy/trunk/pypy/translator/microbench/test_count1.py ============================================================================== --- pypy/trunk/pypy/translator/microbench/test_count1.py (original) +++ pypy/trunk/pypy/translator/microbench/test_count1.py Wed Oct 7 22:27:48 2009 @@ -190,7 +190,16 @@ while c < n: x = x + 1 c += 1 - + +def test_count_increment_in_global(): + global inc + c = 0 + x = 0 + inc = 1 + n = N + while c < n: + x = x + inc + c += inc def test_count_in_global2(): global y From cfbolz at codespeak.net Wed Oct 7 22:47:47 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 7 Oct 2009 22:47:47 +0200 (CEST) Subject: [pypy-svn] r68241 - pypy/trunk/pypy/translator/microbench Message-ID: <20091007204747.2D068168045@codespeak.net> Author: cfbolz Date: Wed Oct 7 22:47:46 2009 New Revision: 68241 Modified: pypy/trunk/pypy/translator/microbench/microbench.py Log: print just the time if one argument is given Modified: pypy/trunk/pypy/translator/microbench/microbench.py ============================================================================== --- pypy/trunk/pypy/translator/microbench/microbench.py (original) +++ pypy/trunk/pypy/translator/microbench/microbench.py Wed Oct 7 22:47:46 2009 @@ -59,8 +59,10 @@ if args[0].startswith('-F'): fmt = "%" + args[0][2:] args.pop(0) + inner = True else: fmt = "%.2f" + inner = False if '-k' in args: i = args.index('-k') @@ -92,9 +94,15 @@ for k, v in benchdata.iteritems(): result.append( (v / benchdata_ref[k], k) ) result.sort() - for r in result: - slowdown, testcase = r + for slowdown, testcase in result: print '%5.2fx slower on %s' % (slowdown, testcase) + if not inner and n == 0: + result = [] + for k, v in benchdata.iteritems(): + result.append((v, k)) + result.sort() + for v, testcase in result: + print 'took %5.5fs for %s' % (v, testcase) if not executables: run(test_cases, fmt) From cfbolz at codespeak.net Wed Oct 7 23:16:37 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 7 Oct 2009 23:16:37 +0200 (CEST) Subject: [pypy-svn] r68242 - pypy/trunk/pypy/interpreter Message-ID: <20091007211637.5AFDF16802F@codespeak.net> Author: cfbolz Date: Wed Oct 7 23:16:36 2009 New Revision: 68242 Modified: pypy/trunk/pypy/interpreter/nestedscope.py Log: kill some beautiful code from revision 1263 Modified: pypy/trunk/pypy/interpreter/nestedscope.py ============================================================================== --- pypy/trunk/pypy/interpreter/nestedscope.py (original) +++ pypy/trunk/pypy/interpreter/nestedscope.py Wed Oct 7 23:16:36 2009 @@ -195,11 +195,7 @@ def STORE_DEREF(f, varindex, *ignored): # nested scopes: access a variable through its cell object w_newvalue = f.popvalue() - #try: cell = f.cells[varindex] - #except IndexError: - # import pdb; pdb.set_trace() - # raise cell.set(w_newvalue) def MAKE_CLOSURE(f, numdefaults, *ignored): From fijal at codespeak.net Thu Oct 8 07:54:27 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Oct 2009 07:54:27 +0200 (CEST) Subject: [pypy-svn] r68243 - pypy/trunk/pypy/rpython/tool/test Message-ID: <20091008055427.95B55168049@codespeak.net> Author: fijal Date: Thu Oct 8 07:54:26 2009 New Revision: 68243 Removed: pypy/trunk/pypy/rpython/tool/test/test_c.py Log: I don't think this test is testing anything reasonable these days. Feel free to revert it back if it is. From arigo at codespeak.net Thu Oct 8 11:26:00 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Oct 2009 11:26:00 +0200 (CEST) Subject: [pypy-svn] r68244 - in pypy/branch/gc-compress/pypy/rpython/lltypesystem: . test Message-ID: <20091008092600.73B33498451@codespeak.net> Author: arigo Date: Thu Oct 8 11:25:58 2009 New Revision: 68244 Added: pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/__init__.py (contents, props changed) Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llgroup.py Log: Improve the test and write it in Yet Another Style (argh) to be reused directly by the C backend. Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py Thu Oct 8 11:25:58 2009 @@ -9,6 +9,7 @@ is limited to 18 or 19 bits (= the 16 bits in a USHORT, plus 2 or 3 bits at the end that are zero and so don't need to be stored). """ + _gckind = 'raw' Group = GroupType() Added: pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/__init__.py ============================================================================== Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llgroup.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llgroup.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llgroup.py Thu Oct 8 11:25:58 2009 @@ -55,35 +55,46 @@ pnew = lltype.malloc(self.S2, immortal=True) assert member_of_group(pnew) is None - def test_next_group_member(self): - self.build() - grpptr = self.grpptr - S1 = self.S1 - S2 = self.S2 - Ptr = lltype.Ptr - p = llop.get_next_group_member(Ptr(S2), grpptr, - self.g1a, llmemory.sizeof(S1)) - assert p == self.p2a - # - p = llop.get_next_group_member(Ptr(S2), grpptr, - self.g2a, llmemory.sizeof(S2)) - assert p == self.p2b - # - p = llop.get_next_group_member(Ptr(S1), grpptr, - self.g2b, llmemory.sizeof(S2)) - assert p == self.p1b + def test_interpreted(self): + f = build_test() + res = f() + assert res == 42 def test_rpython(self): - self.build() - grpptr = self.grpptr - def f(): - p = llop.get_group_member(lltype.Ptr(self.S1), grpptr, self.g1a) - assert p == self.p1a - p = llop.get_group_member(lltype.Ptr(self.S1), grpptr, self.g1b) - assert p == self.p1b - p = llop.get_group_member(lltype.Ptr(self.S2), grpptr, self.g2a) - assert p == self.p2a - p = llop.get_group_member(lltype.Ptr(self.S2), grpptr, self.g2b) - assert p == self.p2b - return 3 - assert interpret(f, []) == 3 + f = build_test() + res = interpret(f, []) + assert res == 42 + + +def build_test(): + test = TestLLGroup() + test.build() + grpptr = test.grpptr + g1x = [test.g1a, test.g1b] + def f(): + p = llop.get_group_member(lltype.Ptr(test.S1), grpptr, test.g1a) + assert p == test.p1a + p = llop.get_group_member(lltype.Ptr(test.S1), grpptr, test.g1b) + assert p == test.p1b + p = llop.get_group_member(lltype.Ptr(test.S2), grpptr, test.g2a) + assert p == test.p2a + p = llop.get_group_member(lltype.Ptr(test.S2), grpptr, test.g2b) + assert p == test.p2b + # + p = llop.get_next_group_member(lltype.Ptr(test.S2), grpptr, + test.g1a, llmemory.sizeof(test.S1)) + assert p == test.p2a + p = llop.get_next_group_member(lltype.Ptr(test.S2), grpptr, + test.g2a, llmemory.sizeof(test.S2)) + assert p == test.p2b + p = llop.get_next_group_member(lltype.Ptr(test.S1), grpptr, + test.g2b, llmemory.sizeof(test.S2)) + assert p == test.p1b + # + expected = [123, 456] + for i in range(2): + p = llop.get_group_member(lltype.Ptr(test.S1), grpptr, g1x[i]) + assert p.x == expected[i] + # + return 42 + return f From arigo at codespeak.net Thu Oct 8 12:01:27 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Oct 2009 12:01:27 +0200 (CEST) Subject: [pypy-svn] r68245 - in pypy/branch/gc-compress/pypy/translator/c: . src test Message-ID: <20091008100127.8E71749844D@codespeak.net> Author: arigo Date: Thu Oct 8 12:01:27 2009 New Revision: 68245 Added: pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h (contents, props changed) Modified: pypy/branch/gc-compress/pypy/translator/c/database.py pypy/branch/gc-compress/pypy/translator/c/node.py pypy/branch/gc-compress/pypy/translator/c/primitive.py pypy/branch/gc-compress/pypy/translator/c/src/g_include.h pypy/branch/gc-compress/pypy/translator/c/test/test_lltyped.py Log: Support for llgroups in the C backend. Modified: pypy/branch/gc-compress/pypy/translator/c/database.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/database.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/database.py Thu Oct 8 12:01:27 2009 @@ -5,6 +5,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem.llmemory import WeakRef, _WeakRefType, GCREF from pypy.rpython.lltypesystem.rffi import CConstant +from pypy.rpython.lltypesystem import llgroup from pypy.tool.sourcetools import valid_identifier from pypy.translator.c.primitive import PrimitiveName, PrimitiveType from pypy.translator.c.node import StructDefNode, ArrayDefNode @@ -141,6 +142,8 @@ #raise Exception("don't know about opaque type %r" % (T,)) return 'struct %s @' % ( valid_identifier('pypy_opaque_' + T.tag),) + elif isinstance(T, llgroup.GroupType): + return 'XXX-dont-use-me @' else: raise Exception("don't know about type %r" % (T,)) Modified: pypy/branch/gc-compress/pypy/translator/c/node.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/node.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/node.py Thu Oct 8 12:01:27 2009 @@ -3,7 +3,7 @@ GcStruct, GcArray, RttiStruct, ContainerType, \ parentlink, Ptr, PyObject, Void, OpaqueType, Float, \ RuntimeTypeInfo, getRuntimeTypeInfo, Char, _subarray -from pypy.rpython.lltypesystem import llmemory +from pypy.rpython.lltypesystem import llmemory, llgroup from pypy.translator.c.funcgen import FunctionCodeGenerator from pypy.translator.c.external import CExternalFunctionCodeGenerator from pypy.translator.c.support import USESLOTS # set to False if necessary while refactoring @@ -464,11 +464,15 @@ return hasattr(self.T, "_hints") and self.T._hints.get('thread_local') def forward_declaration(self): + if llgroup.member_of_group(self.obj): + return yield '%s;' % ( forward_cdecl(self.implementationtypename, self.name, self.db.standalone, self.is_thread_local())) def implementation(self): + if llgroup.member_of_group(self.obj): + return [] lines = list(self.initializationexpr()) lines[0] = '%s = %s' % ( cdecl(self.implementationtypename, self.name, self.is_thread_local()), @@ -898,6 +902,58 @@ #obj._converted_weakref = container # hack for genllvm :-/ return db.getcontainernode(container, _dont_write_c_code=False) +class GroupNode(ContainerNode): + nodekind = 'group' + count_members = None + + def __init__(self, *args): + ContainerNode.__init__(self, *args) + self.implementationtypename = 'struct group_%s_s @' % self.name + + def basename(self): + return self.obj.name + + def enum_dependencies(self): + for member in self.obj.members: + yield member._as_ptr() + + def _fix_members(self): + if self.count_members is None: + self.count_members = len(self.obj.members) + else: + # make sure no new member showed up, because it's too late + assert len(self.obj.members) == self.count_members + + def forward_declaration(self): + self._fix_members() + ctype = ['%s {' % cdecl(self.implementationtypename, '')] + for i, member in enumerate(self.obj.members): + structtypename = self.db.gettype(typeOf(member)) + ctype.append('\t%s;' % cdecl(structtypename, 'member%d' % i)) + ctype.append('} @') + ctype = '\n'.join(ctype) + yield '%s;' % ( + forward_cdecl(ctype, self.name, self.db.standalone, + self.is_thread_local())) + for i, member in enumerate(self.obj.members): + structnode = self.db.getcontainernode(member) + yield '#define %s %s.member%d' % (structnode.name, + self.name, i) + + def initializationexpr(self): + self._fix_members() + lines = ['{'] + lasti = len(self.obj.members) - 1 + for i, member in enumerate(self.obj.members): + structnode = self.db.getcontainernode(member) + lines1 = list(structnode.initializationexpr()) + lines1[0] += '\t/* member%d: %s */' % (i, structnode.name) + if i != lasti: + lines1[-1] += ',' + lines.extend(lines1) + lines.append('}') + return lines + ContainerNodeFactory = { Struct: StructNode, @@ -909,4 +965,5 @@ OpaqueType: opaquenode_factory, PyObjectType: PyObjectNode, llmemory._WeakRefType: weakrefnode_factory, + llgroup.GroupType: GroupNode, } Modified: pypy/branch/gc-compress/pypy/translator/c/primitive.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/primitive.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/primitive.py Thu Oct 8 12:01:27 2009 @@ -3,7 +3,7 @@ from pypy.rlib.objectmodel import CDefinedIntSymbolic from pypy.rlib.rarithmetic import r_longlong, isinf, isnan from pypy.rpython.lltypesystem.lltype import * -from pypy.rpython.lltypesystem import rffi +from pypy.rpython.lltypesystem import rffi, llgroup from pypy.rpython.lltypesystem.llmemory import Address, \ AddressOffset, ItemOffset, ArrayItemsOffset, FieldOffset, \ CompositeOffset, ArrayLengthOffset, \ @@ -136,6 +136,19 @@ else: return 'NULL' +def name_ushort(value, db): + if isinstance(value, Symbolic): + if isinstance(value, llgroup.GroupMemberOffset): + groupnode = db.getcontainernode(value.grpptr._as_obj()) + structnode = db.getcontainernode(value.member._as_obj()) + return 'GROUP_MEMBER_OFFSET(%s, %s)' % ( + groupnode.name, + structnode.name, + ) + else: + raise Exception("unimplemented symbolic %r" % value) + return str(value) + # On 64 bit machines, SignedLongLong and Signed are the same, so the # order matters, because we want the Signed implementation. PrimitiveName = { @@ -151,6 +164,7 @@ Void: name_void, Address: name_address, GCREF: name_gcref, + rffi.USHORT: name_ushort, } PrimitiveType = { @@ -166,6 +180,7 @@ Void: 'void @', Address: 'void* @', GCREF: 'void* @', + rffi.USHORT: 'unsigned short @', } def define_c_primitive(ll_type, c_name): @@ -181,7 +196,7 @@ for ll_type, c_name in [(rffi.SIGNEDCHAR, 'signed char'), (rffi.UCHAR, 'unsigned char'), (rffi.SHORT, 'short'), - (rffi.USHORT, 'unsigned short'), + #(rffi.USHORT, 'unsigned short'), (rffi.INT, 'int'), (rffi.UINT, 'unsigned int'), (rffi.LONG, 'long'), Modified: pypy/branch/gc-compress/pypy/translator/c/src/g_include.h ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/src/g_include.h (original) +++ pypy/branch/gc-compress/pypy/translator/c/src/g_include.h Thu Oct 8 12:01:27 2009 @@ -34,6 +34,7 @@ #ifndef AVR #include "src/unichar.h" #endif +#include "src/llgroup.h" #include "src/instrument.h" Added: pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h ============================================================================== --- (empty file) +++ pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h Thu Oct 8 12:01:27 2009 @@ -0,0 +1,10 @@ + + +#define GROUP_MEMBER_OFFSET(group, membername) \ + ((unsigned short)((((char*)&membername) - ((char*)&group)) / sizeof(long))) + +#define OP_GET_GROUP_MEMBER(groupptr, compactoffset, r) \ + r = ((char*)groupptr) + ((long)compactoffset)*sizeof(long) + +#define OP_GET_NEXT_GROUP_MEMBER(groupptr, compactoffset, skipoffset, r) \ + r = ((char*)groupptr) + ((long)compactoffset)*sizeof(long) + skipoffset Modified: pypy/branch/gc-compress/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/test/test_lltyped.py Thu Oct 8 12:01:27 2009 @@ -701,5 +701,9 @@ fn = self.getcompiled(llf) fn() - - + def test_llgroup(self): + from pypy.rpython.lltypesystem.test import test_llgroup + f = test_llgroup.build_test() + fn = self.getcompiled(f) + res = fn() + assert res == 42 From antocuni at codespeak.net Thu Oct 8 15:46:27 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 8 Oct 2009 15:46:27 +0200 (CEST) Subject: [pypy-svn] r68246 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091008134627.79E83498454@codespeak.net> Author: antocuni Date: Thu Oct 8 15:46:26 2009 New Revision: 68246 Modified: pypy/trunk/pypy/jit/metainterp/resume.py Log: don't store null keys into the dictionary, as it doesn't work on cli Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Thu Oct 8 15:46:26 2009 @@ -99,6 +99,20 @@ self.consts = [] self.large_ints = {} self.refs = {} + self.nullref = UNASSIGNED + + # we cannot store null keys into dictionaries when translating to CLI, so + # we special case them + def getref(self, key): + if not key: + return self.nullref + return self.refs.get(key, UNASSIGNED) + + def setref(self, val, tagged): + if not val: + self.nullref = tagged + else: + self.refs[val] = tagged def getconst(self, const): if const.type == INT: @@ -119,11 +133,11 @@ elif const.type == REF: val = const.getref_base() val = self.cpu.ts.cast_ref_to_hashable(self.cpu, val) - tagged = self.refs.get(val, UNASSIGNED) + tagged = self.getref(val) if not tagged_eq(tagged, UNASSIGNED): return tagged tagged = self._newconst(const) - self.refs[val] = tagged + self.setref(val, tagged) return tagged return self._newconst(const) From arigo at codespeak.net Thu Oct 8 15:48:22 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Oct 2009 15:48:22 +0200 (CEST) Subject: [pypy-svn] r68247 - in pypy/branch/gc-compress/pypy: rpython/lltypesystem rpython/lltypesystem/test translator/c translator/c/src translator/c/test Message-ID: <20091008134822.57D22498454@codespeak.net> Author: arigo Date: Thu Oct 8 15:48:20 2009 New Revision: 68247 Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llgroup.py pypy/branch/gc-compress/pypy/translator/c/node.py pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h pypy/branch/gc-compress/pypy/translator/c/test/test_lltyped.py Log: Detect at compile-time when the size of a group is greater than the maximum permitted, i.e. 2**16 * sizeof(long). Uses a really obscure C hack. Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py Thu Oct 8 15:48:20 2009 @@ -28,8 +28,10 @@ struct = structptr._as_obj() assert struct not in _membership,"cannot be a member of several groups" assert struct._parentstructure() is None + index = len(self.members) self.members.append(struct) _membership[struct] = self + return GroupMemberOffset(self, index) def member_of_group(structptr): return _membership.get(structptr._as_obj(), None) @@ -48,11 +50,11 @@ def lltype(self): return rffi.USHORT - def __init__(self, grp, member): + def __init__(self, grp, memberindex): assert lltype.typeOf(grp) == Group self.grpptr = grp._as_ptr() - self.member = member._as_ptr() - self.index = grp.members.index(member._as_obj()) + self.index = memberindex + self.member = grp.members[memberindex]._as_ptr() def _get_group_member(self, grpptr): assert grpptr == self.grpptr, "get_group_member: wrong group!" Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llgroup.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llgroup.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llgroup.py Thu Oct 8 15:48:20 2009 @@ -20,14 +20,10 @@ p1b.x = 456 p2a.y = 789 p2b.z = -12 - grp.add_member(p1a) - grp.add_member(p2a) - grp.add_member(p2b) - grp.add_member(p1b) - self.g1a = GroupMemberOffset(grp, p1a) - self.g1b = GroupMemberOffset(grp, p1b) - self.g2a = GroupMemberOffset(grp, p2a) - self.g2b = GroupMemberOffset(grp, p2b) + self.g1a = grp.add_member(p1a) + self.g2a = grp.add_member(p2a) + self.g2b = grp.add_member(p2b) + self.g1b = grp.add_member(p1b) self.p1a = p1a self.p1b = p1b self.p2a = p2a Modified: pypy/branch/gc-compress/pypy/translator/c/node.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/node.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/node.py Thu Oct 8 15:48:20 2009 @@ -926,6 +926,7 @@ def forward_declaration(self): self._fix_members() + yield '' ctype = ['%s {' % cdecl(self.implementationtypename, '')] for i, member in enumerate(self.obj.members): structtypename = self.db.gettype(typeOf(member)) @@ -935,10 +936,13 @@ yield '%s;' % ( forward_cdecl(ctype, self.name, self.db.standalone, self.is_thread_local())) + yield '#include "src/llgroup.h"' + yield 'PYPY_GROUP_CHECK_SIZE(%s);' % self.name for i, member in enumerate(self.obj.members): structnode = self.db.getcontainernode(member) yield '#define %s %s.member%d' % (structnode.name, self.name, i) + yield '' def initializationexpr(self): self._fix_members() Modified: pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h (original) +++ pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h Thu Oct 8 15:48:20 2009 @@ -1,3 +1,5 @@ +#ifndef _PYPY_LL_GROUP_H_ +#define _PYPY_LL_GROUP_H_ #define GROUP_MEMBER_OFFSET(group, membername) \ @@ -8,3 +10,12 @@ #define OP_GET_NEXT_GROUP_MEMBER(groupptr, compactoffset, skipoffset, r) \ r = ((char*)groupptr) + ((long)compactoffset)*sizeof(long) + skipoffset + +/* A macro to crash at compile-time if sizeof(group) is too large. + Uses a hack that I've found on some random forum. Haaaaaaaaaackish. */ +#define PYPY_GROUP_CHECK_SIZE(groupname) \ + typedef char group_##groupname##_is_too_large[2*(sizeof(groupname) \ + <= 65536 * sizeof(long))-1] + + +#endif /* _PYPY_LL_GROUP_H_ */ Modified: pypy/branch/gc-compress/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/test/test_lltyped.py Thu Oct 8 15:48:20 2009 @@ -707,3 +707,37 @@ fn = self.getcompiled(f) res = fn() assert res == 42 + + def test_llgroup_size_limit(self): + yield self._test_size_limit, True + yield self._test_size_limit, False + + def _test_size_limit(self, toobig): + from pypy.rpython.lltypesystem import llgroup + from pypy.rpython.lltypesystem.lloperation import llop + from pypy.translator.platform import CompilationError + grp = llgroup.group("big") + S1 = Struct('S1', ('x', Signed), ('y', Signed), + ('z', Signed), ('u', Signed), + ('x2', Signed), ('y2', Signed), + ('z2', Signed), ('u2', Signed), + ('x3', Signed), ('y3', Signed), + ('z3', Signed), ('u3', Signed), + ('x4', Signed), ('y4', Signed), + ('z4', Signed), ('u4', Signed)) + goffsets = [] + for i in range(4096 + toobig): + goffsets.append(grp.add_member(malloc(S1, immortal=True))) + grpptr = grp._as_ptr() + def f(n): + p = llop.get_group_member(Ptr(S1), grpptr, goffsets[n]) + q = llop.get_group_member(Ptr(S1), grpptr, goffsets[0]) + p.x = 5 + q.x = 666 + return p.x + if toobig: + py.test.raises(CompilationError, self.getcompiled, f, [int]) + else: + fn = self.getcompiled(f, [int]) + res = fn(-1) + assert res == 5 From arigo at codespeak.net Thu Oct 8 16:22:33 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Oct 2009 16:22:33 +0200 (CEST) Subject: [pypy-svn] r68248 - pypy/branch/gc-compress/pypy/rpython/memory/test Message-ID: <20091008142233.AFF67168030@codespeak.net> Author: arigo Date: Thu Oct 8 16:22:33 2009 New Revision: 68248 Modified: pypy/branch/gc-compress/pypy/rpython/memory/test/test_gctypelayout.py Log: Before changing this logic, add a test that these simple queries are constant-folded away if the typeid is constant. Modified: pypy/branch/gc-compress/pypy/rpython/memory/test/test_gctypelayout.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/test/test_gctypelayout.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/test/test_gctypelayout.py Thu Oct 8 16:22:33 2009 @@ -1,6 +1,8 @@ from pypy.rpython.memory.gctypelayout import TypeLayoutBuilder, GCData from pypy.rpython.memory.gctypelayout import offsets_to_gc_pointers from pypy.rpython.lltypesystem import lltype +from pypy.rpython.test.test_llinterp import get_interpreter +from pypy.objspace.flow.model import Constant def getname(T): try: @@ -40,3 +42,29 @@ lst1 = gcdata.q_varsize_offsets_to_gcpointers_in_var_part(tid1) lst2 = gcdata.q_offsets_to_gc_pointers(tid2) assert len(lst1) == len(lst2) + +def test_constfold(): + layoutbuilder = TypeLayoutBuilder() + tid1 = layoutbuilder.get_type_id(GC_A) + tid2 = layoutbuilder.get_type_id(GC_S3) + class MockGC: + def set_query_functions(self, is_varsize, + has_gcptr_in_varsize, + is_gcarrayofgcptr, + *rest): + self.is_varsize = is_varsize + self.has_gcptr_in_varsize = has_gcptr_in_varsize + self.is_gcarrayofgcptr = is_gcarrayofgcptr + gc = MockGC() + layoutbuilder.initialize_gc_query_function(gc) + # + def f(): + return (1 * gc.is_varsize(tid1) + + 10 * gc.has_gcptr_in_varsize(tid1) + + 100 * gc.is_gcarrayofgcptr(tid1) + + 1000 * gc.is_varsize(tid2) + + 10000 * gc.has_gcptr_in_varsize(tid2) + + 100000 * gc.is_gcarrayofgcptr(tid2)) + interp, graph = get_interpreter(f, [], backendopt=True) + assert interp.eval_graph(graph, []) == 11001 + assert graph.startblock.exits[0].args == [Constant(11001, lltype.Signed)] From arigo at codespeak.net Thu Oct 8 16:59:57 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Oct 2009 16:59:57 +0200 (CEST) Subject: [pypy-svn] r68249 - pypy/branch/gc-compress/pypy/rpython/memory Message-ID: <20091008145957.6856A16803D@codespeak.net> Author: arigo Date: Thu Oct 8 16:59:54 2009 New Revision: 68249 Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py Log: Stop trying to be clever about the 3 lower bits of the typeid. Instead just use a bit field in the type table. Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py Thu Oct 8 16:59:54 2009 @@ -17,6 +17,7 @@ # structure describing the layout of a typeid TYPE_INFO = lltype.Struct("type_info", + ("infobits", lltype.Signed), # combination of the T_xxx consts ("finalizer", FINALIZERTYPE), ("fixedsize", lltype.Signed), ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), @@ -24,7 +25,6 @@ ("ofstovar", lltype.Signed), ("ofstolength", lltype.Signed), ("varofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), - ("weakptrofs", lltype.Signed), ) TYPE_INFO_TABLE = lltype.Array(TYPE_INFO) @@ -36,16 +36,18 @@ def q_is_varsize(self, typeid): ll_assert(typeid > 0, "invalid type_id") - return (typeid & T_IS_FIXSIZE) == 0 + infobits = self.type_info_table[typeid].infobits + return (infobits & T_IS_VARSIZE) != 0 def q_has_gcptr_in_varsize(self, typeid): ll_assert(typeid > 0, "invalid type_id") - return (typeid & (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE)) == 0 + infobits = self.type_info_table[typeid].infobits + return (infobits & T_HAS_GCPTR_IN_VARSIZE) != 0 def q_is_gcarrayofgcptr(self, typeid): ll_assert(typeid > 0, "invalid type_id") - return (typeid & - (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE|T_NOT_SIMPLE_GCARRAY)) == 0 + infobits = self.type_info_table[typeid].infobits + return (infobits & T_IS_GCARRAY_OF_GCPTR) != 0 def q_finalizer(self, typeid): ll_assert(typeid > 0, "invalid type_id") @@ -77,7 +79,10 @@ def q_weakpointer_offset(self, typeid): ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].weakptrofs + infobits = self.type_info_table[typeid].infobits + if infobits & T_IS_WEAKREF: + return weakptr_offset + return -1 def set_query_functions(self, gc): gc.set_query_functions( @@ -93,58 +98,20 @@ self.q_varsize_offsets_to_gcpointers_in_var_part, self.q_weakpointer_offset) -# For the q_xxx functions that return flags, we use bit patterns -# in the typeid instead of entries in the type_info_table. The -# following flag combinations are used (the idea being that it's -# very fast on CPUs to check if all flags in a set are all zero): - -# * if T_IS_FIXSIZE is set, the gc object is not var-sized -# * if T_IS_FIXSIZE and T_NO_GCPTR_IN_VARSIZE are both cleared, -# there are gc ptrs in the var-sized part -# * if T_IS_FIXSIZE, T_NO_GCPTR_IN_VARSIZE and T_NOT_SIMPLE_GCARRAY -# are all cleared, the shape is just like GcArray(gcptr) - -T_IS_FIXSIZE = 0x4 -T_NO_GCPTR_IN_VARSIZE = 0x2 -T_NOT_SIMPLE_GCARRAY = 0x1 - -def get_typeid_bitmask(TYPE): - """Return the bits that we would like to be set or cleared in the type_id - corresponding to TYPE. This returns (mask, expected_value), where - the condition is that 'type_id & mask == expected_value'. - """ - if not TYPE._is_varsize(): - return (T_IS_FIXSIZE, T_IS_FIXSIZE) # not var-sized - - if (isinstance(TYPE, lltype.GcArray) - and isinstance(TYPE.OF, lltype.Ptr) - and TYPE.OF.TO._gckind == 'gc'): - # a simple GcArray(gcptr) - return (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE|T_NOT_SIMPLE_GCARRAY, 0) - if isinstance(TYPE, lltype.Struct): - ARRAY = TYPE._flds[TYPE._arrayfld] - else: - ARRAY = TYPE - assert isinstance(ARRAY, lltype.Array) - if ARRAY.OF != lltype.Void and len(offsets_to_gc_pointers(ARRAY.OF)) > 0: - # var-sized, with gc pointers in the variable part - return (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE|T_NOT_SIMPLE_GCARRAY, - T_NOT_SIMPLE_GCARRAY) - else: - # var-sized, but no gc pointer in the variable part - return (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE, T_NO_GCPTR_IN_VARSIZE) +T_IS_VARSIZE = 0x01 +T_HAS_GCPTR_IN_VARSIZE = 0x02 +T_IS_GCARRAY_OF_GCPTR = 0x04 +T_IS_WEAKREF = 0x08 def encode_type_shape(builder, info, TYPE): """Encode the shape of the TYPE into the TYPE_INFO structure 'info'.""" offsets = offsets_to_gc_pointers(TYPE) + infobits = 0 info.ofstoptrs = builder.offsets2table(offsets, TYPE) info.finalizer = builder.make_finalizer_funcptr_for_type(TYPE) - info.weakptrofs = weakpointer_offset(TYPE) if not TYPE._is_varsize(): - #info.isvarsize = False - #info.gcptrinvarsize = False info.fixedsize = llarena.round_up_for_allocation( llmemory.sizeof(TYPE)) info.ofstolength = -1 @@ -153,7 +120,7 @@ # varsize ones, the GC must anyway compute the size at run-time # and round up that result. else: - #info.isvarsize = True + infobits |= T_IS_VARSIZE info.fixedsize = llmemory.sizeof(TYPE, 0) if isinstance(TYPE, lltype.Struct): ARRAY = TYPE._flds[TYPE._arrayfld] @@ -161,7 +128,11 @@ info.ofstolength = ofs1 + llmemory.ArrayLengthOffset(ARRAY) info.ofstovar = ofs1 + llmemory.itemoffsetof(ARRAY, 0) else: + assert isinstance(TYPE, lltype.GcArray) ARRAY = TYPE + if (isinstance(ARRAY.OF, lltype.Ptr) + and ARRAY.OF.TO._gckind == 'gc'): + infobits |= T_IS_GCARRAY_OF_GCPTR info.ofstolength = llmemory.ArrayLengthOffset(ARRAY) info.ofstovar = llmemory.itemoffsetof(TYPE, 0) assert isinstance(ARRAY, lltype.Array) @@ -169,12 +140,13 @@ offsets = offsets_to_gc_pointers(ARRAY.OF) else: offsets = () + if len(offsets) > 0: + infobits |= T_HAS_GCPTR_IN_VARSIZE info.varofstoptrs = builder.offsets2table(offsets, ARRAY.OF) info.varitemsize = llmemory.sizeof(ARRAY.OF) - #info.gcptrinvarsize = len(offsets) > 0 - #info.gcarrayofgcptr = (isinstance(TYPE, lltype.GcArray) - # and isinstance(TYPE.OF, lltype.Ptr) - # and TYPE.OF.TO._gckind == 'gc') + if TYPE == WEAKREF: + infobits |= T_IS_WEAKREF + info.infobits = infobits # ____________________________________________________________ @@ -199,7 +171,6 @@ self.all_prebuilt_gc = [] self.finalizer_funcptrs = {} self.offsettable_cache = {} - self.next_typeid_cache = {} def get_type_id(self, TYPE): try: @@ -211,27 +182,16 @@ # It goes into a list for now, which will be turned into a # TYPE_INFO_TABLE in flatten_table() by the gc transformer. - # pick the next type_id with the correct bits set or cleared - mask, expected = get_typeid_bitmask(TYPE) - type_id = self.next_typeid_cache.get((mask, expected), 1) - while True: - if type_id == len(self.type_info_list): - self.type_info_list.append(None) - if (self.type_info_list[type_id] is None and - (type_id & mask) == expected): - break # can use this type_id - else: - type_id += 1 # continue searching - self.next_typeid_cache[mask, expected] = type_id + 1 - assert type_id & 0xffff == type_id # make sure it fits into 2 bytes - # build the TYPE_INFO structure info = lltype.malloc(GCData.TYPE_INFO, immortal=True, zero=True) if self.can_encode_type_shape: encode_type_shape(self, info, TYPE) else: self._pending_type_shapes.append((info, TYPE)) - self.type_info_list[type_id] = info + # store it + type_id = len(self.type_info_list) + assert type_id & 0xffff == type_id # make sure it fits into 2 bytes + self.type_info_list.append(info) self.id_of_type[TYPE] = type_id return type_id @@ -268,7 +228,7 @@ fieldnames = GCData.TYPE_INFO._names for tableentry, newcontent in zip(table, self.type_info_list): if newcontent is None: # empty entry - tableentry.weakptrofs = -1 + tableentry.infobits = 0 tableentry.ofstolength = -1 else: for name in fieldnames: @@ -349,11 +309,6 @@ offsets.append(0) return offsets -def weakpointer_offset(TYPE): - if TYPE == WEAKREF: - return llmemory.offsetof(WEAKREF, "weakptr") - return -1 - def gc_pointers_inside(v, adr, mutable_only=False): t = lltype.typeOf(v) if isinstance(t, lltype.Struct): @@ -409,6 +364,7 @@ sizeof_weakref= llmemory.sizeof(WEAKREF) empty_weakref = lltype.malloc(WEAKREF, immortal=True) empty_weakref.weakptr = llmemory.NULL +weakptr_offset = llmemory.offsetof(WEAKREF, "weakptr") def ll_weakref_deref(wref): wref = llmemory.cast_weakrefptr_to_ptr(WEAKREFPTR, wref) From pedronis at codespeak.net Thu Oct 8 17:54:30 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 8 Oct 2009 17:54:30 +0200 (CEST) Subject: [pypy-svn] r68250 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091008155430.C4B7D498458@codespeak.net> Author: pedronis Date: Thu Oct 8 17:54:29 2009 New Revision: 68250 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/jit/metainterp/simple_optimize.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_resume.py pypy/trunk/pypy/jit/metainterp/test/test_virtual.py pypy/trunk/pypy/jit/metainterp/typesystem.py Log: (cfbolz, pedronis) code to share numberings among guards up to forcing. puh. Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Thu Oct 8 17:54:29 2009 @@ -56,11 +56,6 @@ def get_key_box(self): return self.box - def register_value(self, modifier): - box = self.get_key_box() # may be a Const, too - if modifier.register_box(box): - self.get_args_for_fail(modifier) - def get_args_for_fail(self, modifier): pass @@ -144,9 +139,7 @@ CONST_1 = ConstInt(1) CVAL_ZERO = ConstantValue(CONST_0) CVAL_ZERO_FLOAT = ConstantValue(ConstFloat(0.0)) -llhelper.CONST_NULL = ConstPtr(ConstPtr.value) llhelper.CVAL_NULLREF = ConstantValue(llhelper.CONST_NULL) -oohelper.CONST_NULL = ConstObj(ConstObj.value) oohelper.CVAL_NULLREF = ConstantValue(oohelper.CONST_NULL) @@ -166,6 +159,12 @@ return self.keybox return self.box + def force_box(self): + if self.box is None: + self.optimizer.forget_numberings(self.keybox) + self._really_force() + return self.box + class AbstractVirtualStructValue(AbstractVirtualValue): _attrs_ = ('_fields', '_cached_sorted_fields') @@ -182,24 +181,22 @@ assert isinstance(fieldvalue, OptValue) self._fields[ofs] = fieldvalue - def force_box(self): - if self.box is None: - assert self.source_op is not None - newoperations = self.optimizer.newoperations - newoperations.append(self.source_op) - self.box = box = self.source_op.result - # - iteritems = self._fields.iteritems() - if not we_are_translated(): #random order is fine, except for tests - iteritems = list(iteritems) - iteritems.sort(key = lambda (x,y): x.sort_key()) - for ofs, value in iteritems: - subbox = value.force_box() - op = ResOperation(rop.SETFIELD_GC, [box, subbox], None, - descr=ofs) - newoperations.append(op) - self._fields = None - return self.box + def _really_force(self): + assert self.source_op is not None + newoperations = self.optimizer.newoperations + newoperations.append(self.source_op) + self.box = box = self.source_op.result + # + iteritems = self._fields.iteritems() + if not we_are_translated(): #random order is fine, except for tests + iteritems = list(iteritems) + iteritems.sort(key = lambda (x,y): x.sort_key()) + for ofs, value in iteritems: + subbox = value.force_box() + op = ResOperation(rop.SETFIELD_GC, [box, subbox], None, + descr=ofs) + newoperations.append(op) + self._fields = None def _get_field_descr_list(self): # this shares only per instance and not per type, but better than nothing @@ -214,8 +211,9 @@ return lst def get_args_for_fail(self, modifier): - if self.box is None and not modifier.is_virtual(self.keybox): - # modifier.is_virtual() checks for recursion: it is False unless + if self.box is None and not modifier.already_seen_virtual(self.keybox): + # modifier.already_seen_virtual() + # checks for recursion: it is False unless # we have already seen the very same keybox lst = self._get_field_descr_list() fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst] @@ -272,25 +270,24 @@ assert isinstance(itemvalue, OptValue) self._items[index] = itemvalue - def force_box(self): - if self.box is None: - assert self.source_op is not None - newoperations = self.optimizer.newoperations - newoperations.append(self.source_op) - self.box = box = self.source_op.result - for index in range(len(self._items)): - subvalue = self._items[index] - if subvalue is not None: - subbox = subvalue.force_box() - op = ResOperation(rop.SETARRAYITEM_GC, - [box, ConstInt(index), subbox], None, - descr=self.arraydescr) - newoperations.append(op) - return self.box + def _really_force(self): + assert self.source_op is not None + newoperations = self.optimizer.newoperations + newoperations.append(self.source_op) + self.box = box = self.source_op.result + for index in range(len(self._items)): + subvalue = self._items[index] + if subvalue is not None: + subbox = subvalue.force_box() + op = ResOperation(rop.SETARRAYITEM_GC, + [box, ConstInt(index), subbox], None, + descr=self.arraydescr) + newoperations.append(op) def get_args_for_fail(self, modifier): - if self.box is None and not modifier.is_virtual(self.keybox): - # modifier.is_virtual() checks for recursion: it is False unless + if self.box is None and not modifier.already_seen_virtual(self.keybox): + # modifier.already_seen_virtual() + # checks for recursion: it is False unless # we have already seen the very same keybox itemboxes = [] const = self.optimizer.new_const_item(self.arraydescr) @@ -373,6 +370,9 @@ self.resumedata_memo = resume.ResumeDataLoopMemo(cpu) self.heap_op_optimizer = HeapOpOptimizer(self) + def forget_numberings(self, virtualbox): + self.resumedata_memo.forget_numberings(virtualbox) + def getinterned(self, box): constbox = self.get_constant_box(box) if constbox is None: @@ -523,7 +523,6 @@ descr = op.descr assert isinstance(descr, compile.ResumeGuardDescr) modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo) - modifier.walk_snapshots(self.values) newboxes = modifier.finish(self.values) descr.store_final_boxes(op, newboxes) Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Thu Oct 8 17:54:29 2009 @@ -8,10 +8,7 @@ # Logic to encode the chain of frames and the state of the boxes at a # guard operation, and to decode it again. This is a bit advanced, # because it needs to support optimize.py which encodes virtuals with -# arbitrary cycles. - -# XXX building the data so that it is as compact as possible -# on the 'storage' object would be a big win. +# arbitrary cycles and also to compress the information debug = False @@ -36,7 +33,7 @@ self.pc = frame.pc self.exception_target = frame.exception_target -def _ensure_parent_resumedata(framestack, n): +def _ensure_parent_resumedata(framestack, n): target = framestack[n] if n == 0 or target.parent_resumedata_frame_info_list is not None: return @@ -61,6 +58,13 @@ snapshot = Snapshot(snapshot, virtualizable_boxes[:]) # xxx for now storage.rd_snapshot = snapshot +class Numbering(object): + __slots__ = ('prev', 'nums') + + def __init__(self, prev, nums): + self.prev = prev + self.nums = nums + TAGMASK = 3 def tag(value, tagbits): @@ -85,10 +89,9 @@ TAGBOX = 2 TAGVIRTUAL = 3 -NEXTFRAME = tag(-1, TAGVIRTUAL) UNASSIGNED = tag(-1, TAGBOX) +NULLREF = tag(-1, TAGCONST) - VIRTUAL_FLAG = int((sys.maxint+1) // 2) assert not (VIRTUAL_FLAG & (VIRTUAL_FLAG-1)) # a power of two @@ -99,20 +102,7 @@ self.consts = [] self.large_ints = {} self.refs = {} - self.nullref = UNASSIGNED - - # we cannot store null keys into dictionaries when translating to CLI, so - # we special case them - def getref(self, key): - if not key: - return self.nullref - return self.refs.get(key, UNASSIGNED) - - def setref(self, val, tagged): - if not val: - self.nullref = tagged - else: - self.refs[val] = tagged + self.numberings = {} def getconst(self, const): if const.type == INT: @@ -132,20 +122,62 @@ return tagged elif const.type == REF: val = const.getref_base() + if not val: + return NULLREF val = self.cpu.ts.cast_ref_to_hashable(self.cpu, val) - tagged = self.getref(val) + tagged = self.refs.get(val, UNASSIGNED) if not tagged_eq(tagged, UNASSIGNED): return tagged tagged = self._newconst(const) - self.setref(val, tagged) - return tagged + self.refs[val] = tagged + return tagged return self._newconst(const) def _newconst(self, const): result = tag(len(self.consts), TAGCONST) self.consts.append(const) - return result - + return result + + def number(self, values, snapshot): + if snapshot is None: + return None, {}, 0 + if snapshot in self.numberings: + numb, liveboxes, v = self.numberings[snapshot] + return numb, liveboxes.copy(), v + + numb1, liveboxes, v = self.number(values, snapshot.prev) + n = len(liveboxes)-v + boxes = snapshot.boxes + length = len(boxes) + nums = [UNASSIGNED] * length + for i in range(length): + box = boxes[i] + value = values.get(box, None) + if value is not None: + box = value.get_key_box() + + if isinstance(box, Const): + tagged = self.getconst(box) + elif box in liveboxes: + tagged = liveboxes[box] + else: + if value is not None and value.is_virtual(): + tagged = tag(v, TAGVIRTUAL) + v += 1 + else: + tagged = tag(n, TAGBOX) + n += 1 + liveboxes[box] = tagged + nums[i] = tagged + numb = Numbering(numb1, nums) + self.numberings[snapshot] = numb, liveboxes, v + return numb, liveboxes.copy(), v + + def forget_numberings(self, virtualbox): + # XXX ideally clear only the affected numberings + self.numberings.clear() + + _frame_info_placeholder = (None, 0, 0) class ResumeDataVirtualAdder(object): @@ -153,32 +185,8 @@ def __init__(self, storage, memo): self.storage = storage self.memo = memo - self.liveboxes = {} - self.virtuals = [] - self.vfieldboxes = [] - - def walk_snapshots(self, values): - nnums = 0 - snapshot = self.storage.rd_snapshot - assert snapshot - while True: # at least one - boxes = snapshot.boxes - nnums += len(boxes)+1 - for box in boxes: - if box in values: - value = values[box] - value.register_value(self) - else: - self.register_box(box) - snapshot = snapshot.prev - if snapshot is None: - break - self.nnums = nnums - - def make_constant(self, box, const): - # this part of the interface is not used so far by optimizeopt.py - if tagged_eq(self.liveboxes[box], UNASSIGNED): - self.liveboxes[box] = self.memo.getconst(const) + #self.virtuals = [] + #self.vfieldboxes = [] def make_virtual(self, virtualbox, known_class, fielddescrs, fieldboxes): vinfo = VirtualInfo(known_class, fielddescrs) @@ -193,50 +201,80 @@ self._make_virtual(virtualbox, vinfo, itemboxes) def _make_virtual(self, virtualbox, vinfo, fieldboxes): - assert tagged_eq(self.liveboxes[virtualbox], UNASSIGNED) - self.liveboxes[virtualbox] = tag(len(self.virtuals), TAGVIRTUAL) - self.virtuals.append(vinfo) - self.vfieldboxes.append(fieldboxes) + if virtualbox in self.liveboxes_from_env: + tagged = self.liveboxes_from_env[virtualbox] + i, _ = untag(tagged) + assert self.virtuals[i] is None + self.virtuals[i] = vinfo + self.vfieldboxes[i] = fieldboxes + else: + tagged = tag(len(self.virtuals), TAGVIRTUAL) + self.virtuals.append(vinfo) + self.vfieldboxes.append(fieldboxes) + self.liveboxes[virtualbox] = tagged self._register_boxes(fieldboxes) def register_box(self, box): - if isinstance(box, Box) and box not in self.liveboxes: + if (isinstance(box, Box) and box not in self.liveboxes_from_env + and box not in self.liveboxes): self.liveboxes[box] = UNASSIGNED return True return False - + def _register_boxes(self, boxes): for box in boxes: self.register_box(box) - def is_virtual(self, virtualbox): - tagged = self.liveboxes[virtualbox] + def already_seen_virtual(self, virtualbox): + if virtualbox not in self.liveboxes: + assert virtualbox in self.liveboxes_from_env + assert untag(self.liveboxes_from_env[virtualbox])[1] == TAGVIRTUAL + return False + tagged = self.liveboxes[virtualbox] _, tagbits = untag(tagged) return tagbits == TAGVIRTUAL def finish(self, values): + # compute the numbering storage = self.storage - liveboxes = [] - for box in self.liveboxes.iterkeys(): - if tagged_eq(self.liveboxes[box], UNASSIGNED): + numb, liveboxes_from_env, v = self.memo.number(values, storage.rd_snapshot) + self.liveboxes_from_env = liveboxes_from_env + self.liveboxes = {} + storage.rd_numb = numb + storage.rd_snapshot = None + + # collect liveboxes and virtuals + n = len(liveboxes_from_env) - v + liveboxes = [None]*n + self.virtuals = [None]*v + self.vfieldboxes = [None]*v + for box, tagged in liveboxes_from_env.iteritems(): + i, tagbits = untag(tagged) + if tagbits == TAGBOX: + liveboxes[i] = box + else: + assert tagbits == TAGVIRTUAL + value = values[box] + value.get_args_for_fail(self) + + self._number_virtuals(liveboxes) + + storage.rd_consts = self.memo.consts + if debug: + dump_storage(storage, liveboxes) + return liveboxes[:] + + def _number_virtuals(self, liveboxes): + for box, tagged in self.liveboxes.iteritems(): + i, tagbits = untag(tagged) + if tagbits == TAGBOX: + assert tagged_eq(tagged, UNASSIGNED) self.liveboxes[box] = tag(len(liveboxes), TAGBOX) liveboxes.append(box) - nums = storage.rd_nums = [rffi.r_short(0)]*self.nnums - i = 0 - snapshot = self.storage.rd_snapshot - while True: # at least one - boxes = snapshot.boxes - for j in range(len(boxes)): - box = boxes[j] - if box in values: - box = values[box].get_key_box() - nums[i] = self._gettagged(box) - i += 1 - nums[i] = NEXTFRAME - i += 1 - snapshot = snapshot.prev - if snapshot is None: - break + else: + assert tagbits == TAGVIRTUAL + + storage = self.storage storage.rd_virtuals = None if len(self.virtuals) > 0: storage.rd_virtuals = self.virtuals[:] @@ -245,16 +283,13 @@ fieldboxes = self.vfieldboxes[i] vinfo.fieldnums = [self._gettagged(box) for box in fieldboxes] - storage.rd_consts = self.memo.consts - storage.rd_snapshot = None - if debug: - dump_storage(storage, liveboxes) - return liveboxes def _gettagged(self, box): if isinstance(box, Const): return self.memo.getconst(box) else: + if box in self.liveboxes_from_env: + return self.liveboxes_from_env[box] return self.liveboxes[box] class AbstractVirtualInfo(object): @@ -347,14 +382,13 @@ class ResumeDataReader(object): - i_frame_infos = 0 - i_boxes = 0 virtuals = None def __init__(self, storage, liveboxes, metainterp=None): - self.nums = storage.rd_nums + self.cur_numb = storage.rd_numb self.consts = storage.rd_consts self.liveboxes = liveboxes + self.cpu = metainterp.cpu self._prepare_virtuals(metainterp, storage.rd_virtuals) def _prepare_virtuals(self, metainterp, virtuals): @@ -365,18 +399,21 @@ vinfo.setfields(metainterp, self.virtuals[i], self._decode_box) def consume_boxes(self): - boxes = [] - while True: - num = self.nums[self.i_boxes] - self.i_boxes += 1 - if tagged_eq(num, NEXTFRAME): - break - boxes.append(self._decode_box(num)) + numb = self.cur_numb + assert numb is not None + nums = numb.nums + n = len(nums) + boxes = [None] * n + for i in range(n): + boxes[i] = self._decode_box(nums[i]) + self.cur_numb = numb.prev return boxes - def _decode_box(self, num): - num, tag = untag(num) + def _decode_box(self, tagged): + num, tag = untag(tagged) if tag == TAGCONST: + if tagged_eq(tagged, NULLREF): + return self.cpu.ts.CONST_NULL return self.consts[num] elif tag == TAGVIRTUAL: virtuals = self.virtuals @@ -403,8 +440,14 @@ frameinfo = frameinfo.prev if frameinfo is None: break - os.write(fd, '\t],\n\t%s,\n' % ([untag(i) for i in storage.rd_nums],)) - os.write(fd, '\t[\n') + os.write(fd, '\t],\n\t[\n') + numb = storage.rd_numb + while True: + os.write(fd, '\t\t%s,\n' % ([untag(i) for i in numb.nums],)) + numb = numb.prev + if numb is None: + break + os.write(fd, '\t], [\n') for const in storage.rd_consts: os.write(fd, '\t"%s",\n' % (const.repr_rpython(),)) os.write(fd, '\t], [\n') Modified: pypy/trunk/pypy/jit/metainterp/simple_optimize.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/simple_optimize.py (original) +++ pypy/trunk/pypy/jit/metainterp/simple_optimize.py Thu Oct 8 17:54:29 2009 @@ -23,7 +23,6 @@ descr = op.descr assert isinstance(descr, compile.ResumeGuardDescr) modifier = resume.ResumeDataVirtualAdder(descr, memo) - modifier.walk_snapshots(EMPTY_VALUES) newboxes = modifier.finish(EMPTY_VALUES) descr.store_final_boxes(op, newboxes) newoperations.append(op) Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Thu Oct 8 17:54:29 2009 @@ -25,7 +25,7 @@ def test_store_final_boxes_in_guard(): from pypy.jit.metainterp.compile import ResumeGuardDescr - from pypy.jit.metainterp.resume import tag, NEXTFRAME, TAGBOX + from pypy.jit.metainterp.resume import tag, TAGBOX b0 = BoxInt() b1 = BoxInt() opt = optimizeopt.Optimizer(None, None) @@ -40,12 +40,12 @@ # opt.store_final_boxes_in_guard(op) if op.fail_args == [b0, b1]: - assert fdescr.rd_nums == [tag(1, TAGBOX), NEXTFRAME, - tag(0, TAGBOX), NEXTFRAME] + assert fdescr.rd_numb.nums == [tag(1, TAGBOX)] + assert fdescr.rd_numb.prev.nums == [tag(0, TAGBOX)] else: assert op.fail_args == [b1, b0] - assert fdescr.rd_nums == [tag(0, TAGBOX), NEXTFRAME, - tag(1, TAGBOX), NEXTFRAME] + assert fdescr.rd_numb.nums == [tag(0, TAGBOX)] + assert fdescr.rd_numb.prev.nums == [tag(1, TAGBOX)] assert fdescr.rd_virtuals is None assert fdescr.rd_consts == [] Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Thu Oct 8 17:54:29 2009 @@ -1,5 +1,6 @@ import py from pypy.rpython.lltypesystem import lltype, llmemory, rffi +from pypy.jit.metainterp.optimizeopt import VirtualValue, OptValue from pypy.jit.metainterp.resume import * from pypy.jit.metainterp.history import BoxInt, BoxPtr, ConstInt, ConstAddr from pypy.jit.metainterp.history import ConstPtr, ConstFloat @@ -16,9 +17,9 @@ assert tag(-1<<13, 3) == rffi.r_short((-1<<15)|3) py.test.raises(ValueError, tag, 3, 5) py.test.raises(ValueError, tag, 1<<13, 0) - py.test.raises(ValueError, tag, (1<<13)+1, 0) + py.test.raises(ValueError, tag, (1<<13)+1, 0) py.test.raises(ValueError, tag, (-1<<13)-1, 0) - py.test.raises(ValueError, tag, (-1<<13)-5, 0) + py.test.raises(ValueError, tag, (-1<<13)-5, 0) def test_untag(): assert untag(tag(3, 1)) == (3, 1) @@ -27,34 +28,54 @@ assert untag(tag(-1<<13, 3)) == (-1<<13, 3) def test_tagged_eq(): - assert tagged_eq(NEXTFRAME, NEXTFRAME) - assert not tagged_eq(NEXTFRAME, UNASSIGNED) + assert tagged_eq(UNASSIGNED, UNASSIGNED) + assert not tagged_eq(tag(1, TAGBOX), UNASSIGNED) + +class MyMetaInterp: + def __init__(self, cpu=None): + if cpu is None: + cpu = LLtypeMixin.cpu + self.cpu = cpu + self.trace = [] + self.framestack = [] + self.resboxes = [] + + def newframe(self, jitcode): + frame = FakeFrame(jitcode, -1, -1) + self.framestack.append(frame) + return frame + + def execute_and_record(self, opnum, descr, *argboxes): + resbox = executor.execute(self.cpu, opnum, descr, *argboxes) + self.trace.append((opnum, + list(argboxes), + resbox, + descr)) + if resbox is not None: + self.resboxes.append(resbox) + return resbox + def test_simple_read(): b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] storage = Storage() - storage.rd_frame_infos = [] storage.rd_consts = [c1, c2, c3] - storage.rd_nums = [tag(0, TAGBOX), - tag(0, TAGCONST), - tag(0, TAGBOX), - tag(1, TAGBOX), - NEXTFRAME, - tag(1, TAGCONST), - tag(2, TAGCONST), - NEXTFRAME, - tag(0, TAGBOX), - tag(1, TAGBOX), - tag(2, TAGBOX), - NEXTFRAME - ] + numb = Numbering(None, [tag(0, TAGBOX), tag(1, TAGBOX), tag(2, TAGBOX)]) + numb = Numbering(numb, [tag(1, TAGCONST), tag(2, TAGCONST)]) + numb = Numbering(numb, [tag(0, TAGBOX), + tag(0, TAGCONST), + NULLREF, + tag(0, TAGBOX), + tag(1, TAGBOX)]) + storage.rd_numb = numb storage.rd_virtuals = None + b1s, b2s, b3s = [BoxInt(), BoxPtr(), BoxInt()] assert b1s != b3s - reader = ResumeDataReader(storage, [b1s, b2s, b3s]) + reader = ResumeDataReader(storage, [b1s, b2s, b3s], MyMetaInterp()) lst = reader.consume_boxes() - assert lst == [b1s, ConstInt(1), b1s, b2s] + assert lst == [b1s, ConstInt(1), LLtypeMixin.cpu.ts.CONST_NULL, b1s, b2s] lst = reader.consume_boxes() assert lst == [ConstInt(2), ConstInt(3)] lst = reader.consume_boxes() @@ -63,25 +84,18 @@ def test_simple_read_tagged_ints(): b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] storage = Storage() - storage.rd_frame_infos = [] storage.rd_consts = [] - storage.rd_nums = [tag(0, TAGBOX), - tag(1, TAGINT), - tag(0, TAGBOX), - tag(1, TAGBOX), - NEXTFRAME, - tag(2, TAGINT), - tag(3, TAGINT), - NEXTFRAME, - tag(0, TAGBOX), - tag(1, TAGBOX), - tag(2, TAGBOX), - NEXTFRAME - ] + numb = Numbering(None, [tag(0, TAGBOX), tag(1, TAGBOX), tag(2, TAGBOX)]) + numb = Numbering(numb, [tag(2, TAGINT), tag(3, TAGINT)]) + numb = Numbering(numb, [tag(0, TAGBOX), + tag(1, TAGINT), + tag(0, TAGBOX), + tag(1, TAGBOX)]) + storage.rd_numb = numb storage.rd_virtuals = None b1s, b2s, b3s = [BoxInt(), BoxPtr(), BoxInt()] assert b1s != b3s - reader = ResumeDataReader(storage, [b1s, b2s, b3s]) + reader = ResumeDataReader(storage, [b1s, b2s, b3s], MyMetaInterp()) lst = reader.consume_boxes() assert lst == [b1s, ConstInt(1), b1s, b2s] lst = reader.consume_boxes() @@ -89,7 +103,7 @@ lst = reader.consume_boxes() assert lst == [b1s, b2s, b3s] - # ____________________________________________________________ +# ____________________________________________________________ @@ -127,7 +141,7 @@ def test_FrameInfo_create(): jitcode = "JITCODE" - frame = FakeFrame(jitcode, 1, 2) + frame = FakeFrame(jitcode, 1, 2) fi = FrameInfo(None, frame) assert fi.prev is None assert fi.jitcode is jitcode @@ -136,7 +150,7 @@ assert fi.level == 1 jitcode1 = "JITCODE1" - frame1 = FakeFrame(jitcode, 3, 4) + frame1 = FakeFrame(jitcode, 3, 4) fi1 = FrameInfo(fi, frame1) assert fi1.prev is fi assert fi1.jitcode is jitcode @@ -144,9 +158,20 @@ assert fi1.exception_target == 4 assert fi1.level == 2 +def test_Numbering_create(): + l = [1, 2] + numb = Numbering(None, l) + assert numb.prev is None + assert numb.nums is l + + l1 = ['b3'] + numb1 = Numbering(numb, l1) + assert numb1.prev is numb + assert numb1.nums is l1 + def test_capture_resumedata(): b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] - c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] + c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] fs = [FakeFrame("code0", 0, -1, b1, c1, b2)] storage = Storage() @@ -159,7 +184,7 @@ assert storage.rd_frame_info_list.jitcode == 'code0' assert storage.rd_snapshot.prev is None assert storage.rd_snapshot.boxes == fs[0].env - assert storage.rd_snapshot.boxes is not fs[0].env + assert storage.rd_snapshot.boxes is not fs[0].env storage = Storage() fs = [FakeFrame("code0", 0, -1, b1, c1, b2), @@ -171,7 +196,7 @@ assert frame_info_list.prev is fs[2].parent_resumedata_frame_info_list assert frame_info_list.jitcode == 'code2' assert frame_info_list.pc == 9 - + snapshot = storage.rd_snapshot assert snapshot.prev is fs[2].parent_resumedata_snapshot assert snapshot.boxes == fs[2].env @@ -221,9 +246,8 @@ capture_resumedata(fs, None, storage) memo = ResumeDataLoopMemo(None) modifier = ResumeDataVirtualAdder(storage, memo) - modifier.walk_snapshots({}) liveboxes = modifier.finish({}) - metainterp = MyMetaInterp(None) + metainterp = MyMetaInterp() b1t, b2t, b3t = [BoxInt(), BoxPtr(), BoxInt()] newboxes = _resume_remap(liveboxes, [b1, b2, b3], b1t, b2t, b3t) @@ -246,9 +270,8 @@ capture_resumedata(fs, [b4], storage) memo = ResumeDataLoopMemo(None) modifier = ResumeDataVirtualAdder(storage, memo) - modifier.walk_snapshots({}) liveboxes = modifier.finish({}) - metainterp = MyMetaInterp(None) + metainterp = MyMetaInterp() b1t, b2t, b3t, b4t = [BoxInt(), BoxPtr(), BoxInt(), BoxPtr()] newboxes = _resume_remap(liveboxes, [b1, b2, b3, b4], b1t, b2t, b3t, b4t) @@ -275,14 +298,12 @@ memo = ResumeDataLoopMemo(None) modifier = ResumeDataVirtualAdder(storage, memo) - modifier.walk_snapshots({}) liveboxes = modifier.finish({}) modifier = ResumeDataVirtualAdder(storage2, memo) - modifier.walk_snapshots({}) liveboxes2 = modifier.finish({}) - metainterp = MyMetaInterp(None) + metainterp = MyMetaInterp() b1t, b2t, b3t, b4t = [BoxInt(), BoxPtr(), BoxInt(), BoxInt()] newboxes = _resume_remap(liveboxes, [b1, b2, b3], b1t, b2t, b3t) @@ -305,25 +326,20 @@ assert metainterp.framestack == fs2 +def virtual_value(keybox, value, next): + vv = VirtualValue(None, ConstAddr(LLtypeMixin.node_vtable_adr, + LLtypeMixin.cpu), keybox) + if not isinstance(next, OptValue): + next = OptValue(next) + vv.setfield(LLtypeMixin.nextdescr, next) + vv.setfield(LLtypeMixin.valuedescr, OptValue(value)) + return vv + def test_rebuild_from_resumedata_two_guards_w_virtuals(): b1, b2, b3, b4, b5 = [BoxInt(), BoxPtr(), BoxInt(), BoxInt(), BoxInt()] c1, c2, c3, c4 = [ConstInt(1), ConstInt(2), ConstInt(3), LLtypeMixin.nodebox.constbox()] - class FakeValue(object): - - def register_value(self, modifier): - if modifier.register_box(b2): - modifier.make_virtual(b2, ConstAddr(LLtypeMixin.node_vtable_adr, - LLtypeMixin.cpu), - [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr], - [c4, fieldbox]) - - def get_key_box(self): - return b2 - - values = {b2: FakeValue()} - storage = Storage() fs = [FakeFrame("code0", 0, -1, b1, c1, b2), FakeFrame("code1", 3, 7, b3, c2, b1), @@ -334,17 +350,27 @@ capture_resumedata(fs, None, storage2) memo = ResumeDataLoopMemo(LLtypeMixin.cpu) - fieldbox = b5 + values = {b2: virtual_value(b2, b5, c4)} modifier = ResumeDataVirtualAdder(storage, memo) - modifier.walk_snapshots(values) liveboxes = modifier.finish(values) - - fieldbox = b4 + assert len(storage.rd_virtuals) == 1 + assert storage.rd_virtuals[0].fieldnums == [tag(len(liveboxes)-1, TAGBOX), + tag(0, TAGCONST)] + + b6 = BoxPtr() + v6 = virtual_value(b6, c2, None) + v6.setfield(LLtypeMixin.nextdescr, v6) + values = {b2: virtual_value(b2, b4, v6), b6: v6} modifier = ResumeDataVirtualAdder(storage2, memo) - modifier.walk_snapshots(values) liveboxes2 = modifier.finish(values) + assert len(storage2.rd_virtuals) == 2 + assert storage2.rd_virtuals[0].fieldnums == [tag(len(liveboxes2)-1, TAGBOX), + tag(1, TAGVIRTUAL)] + assert storage2.rd_virtuals[1].fieldnums == [tag(2, TAGINT), + tag(1, TAGVIRTUAL)] - metainterp = MyMetaInterp(LLtypeMixin.cpu) + # now on to resuming + metainterp = MyMetaInterp() b1t, b3t, b4t, b5t = [BoxInt(), BoxInt(), BoxInt(), BoxInt()] newboxes = _resume_remap(liveboxes, [b1, b3, b5], b1t, b3t, b5t) @@ -360,52 +386,39 @@ newboxes = _resume_remap(liveboxes2, [b1, b3, b4], b1t, b3t, b4t) - metainterp = MyMetaInterp(LLtypeMixin.cpu) + metainterp = MyMetaInterp() result = rebuild_from_resumedata(metainterp, newboxes, storage2, False) b2t = metainterp.resboxes[0] + assert len(metainterp.resboxes) == 2 fs2 = [FakeFrame("code0", 0, -1, b1t, c1, b2t), FakeFrame("code1", 3, 7, b3t, c2, b1t), FakeFrame("code2", 10, -1, c3, b2t, b4t)] assert metainterp.framestack == fs2 - - -# ____________________________________________________________ - -def test_walk_snapshots(): - b1, b2, b3 = [BoxInt(), BoxInt(), BoxInt()] - c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] - - env = [b1, c1, b2, b1, c2] - snap = Snapshot(None, env) - env1 = [c3, b3, b1, c1] - snap1 = Snapshot(snap, env1) +def test_resumedata_top_recursive_virtuals(): + b1, b2, b3 = [BoxPtr(), BoxPtr(), BoxInt()] storage = Storage() - storage.rd_snapshot = snap1 - - modifier = ResumeDataVirtualAdder(storage, None) - modifier.walk_snapshots({}) - - assert modifier.liveboxes == {b1: UNASSIGNED, b2: UNASSIGNED, - b3: UNASSIGNED} - assert modifier.nnums == len(env)+1+len(env1)+1 - - b1_2 = BoxInt() - class FakeValue(object): - - def register_value(self, modifier): - modifier.register_box(b1_2) - val = FakeValue() + fs = [FakeFrame("code0", 0, -1, b1, b2)] + capture_resumedata(fs, None, storage) + + memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + v1 = virtual_value(b1, b3, None) + v2 = virtual_value(b2, b3, v1) + v1.setfield(LLtypeMixin.nextdescr, v2) + values = {b1: v1, b2: v2} + modifier = ResumeDataVirtualAdder(storage, memo) + liveboxes = modifier.finish(values) + assert liveboxes == [b3] + assert len(storage.rd_virtuals) == 2 + assert storage.rd_virtuals[0].fieldnums == [tag(0, TAGBOX), + tag(1, TAGVIRTUAL)] + assert storage.rd_virtuals[1].fieldnums == [tag(0, TAGBOX), + tag(0, TAGVIRTUAL)] - storage = Storage() - storage.rd_snapshot = snap1 - modifier = ResumeDataVirtualAdder(storage, None) - modifier.walk_snapshots({b1: val, b2: val}) +# ____________________________________________________________ - assert modifier.liveboxes == {b1_2: UNASSIGNED, b3: UNASSIGNED} - assert modifier.nnums == len(env)+1+len(env1)+1 def test_ResumeDataLoopMemo_ints(): memo = ResumeDataLoopMemo(None) @@ -444,6 +457,8 @@ index3, tagbits = untag(tagged) assert tagbits == TAGCONST assert index3 != index + tagged = memo.getconst(cpu.ts.CONST_NULL) + assert tagged == NULLREF def test_ResumeDataLoopMemo_other(): memo = ResumeDataLoopMemo(None) @@ -453,26 +468,114 @@ assert tagbits == TAGCONST assert memo.consts[index] is const -class MyMetaInterp: - def __init__(self, cpu): - self.cpu = cpu - self.trace = [] - self.framestack = [] - self.resboxes = [] +def test_ResumeDataLoopMemo_number(): + b1, b2, b3, b4, b5 = [BoxInt(), BoxInt(), BoxInt(), BoxPtr(), BoxPtr()] + c1, c2, c3, c4 = [ConstInt(1), ConstInt(2), ConstInt(3), ConstInt(4)] - def newframe(self, jitcode): - frame = FakeFrame(jitcode, -1, -1) - self.framestack.append(frame) - return frame + env = [b1, c1, b2, b1, c2] + snap = Snapshot(None, env) + env1 = [c3, b3, b1, c1] + snap1 = Snapshot(snap, env1) + env2 = [c3, b3, b1, c3] + snap2 = Snapshot(snap, env2) - def execute_and_record(self, opnum, descr, *argboxes): - resbox = executor.execute(self.cpu, opnum, descr, *argboxes) - self.trace.append((opnum, - [box.value for box in argboxes], - resbox and resbox.value, - descr)) - self.resboxes.append(resbox) - return resbox + memo = ResumeDataLoopMemo(None) + + numb, liveboxes, v = memo.number({}, snap1) + assert v == 0 + + assert liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), + b3: tag(2, TAGBOX)} + assert numb.nums == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), + tag(1, TAGINT)] + assert numb.prev.nums == [tag(0, TAGBOX), tag(1, TAGINT), tag(1, TAGBOX), + tag(0, TAGBOX), tag(2, TAGINT)] + assert numb.prev.prev is None + + numb2, liveboxes2, v = memo.number({}, snap2) + assert v == 0 + + assert liveboxes2 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), + b3: tag(2, TAGBOX)} + assert liveboxes2 is not liveboxes + assert numb2.nums == [tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), + tag(3, TAGINT)] + assert numb2.prev is numb.prev + + env3 = [c3, b3, b1, c3] + snap3 = Snapshot(snap, env3) + + class FakeValue(object): + def __init__(self, virt, box): + self.virt = virt + self.valuebox = box + + def get_key_box(self): + return self.valuebox + + def is_virtual(self): + return self.virt + + # renamed + numb3, liveboxes3, v = memo.number({b3: FakeValue(False, c4)}, snap3) + assert v == 0 + + assert liveboxes3 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX)} + assert numb3.nums == [tag(3, TAGINT), tag(4, TAGINT), tag(0, TAGBOX), + tag(3, TAGINT)] + assert numb3.prev is numb.prev + + # virtual + env4 = [c3, b4, b1, c3] + snap4 = Snapshot(snap, env4) + + numb4, liveboxes4, v = memo.number({b4: FakeValue(True, b4)}, snap4) + assert v == 1 + + assert liveboxes4 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), + b4: tag(0, TAGVIRTUAL)} + assert numb4.nums == [tag(3, TAGINT), tag(0, TAGVIRTUAL), tag(0, TAGBOX), + tag(3, TAGINT)] + assert numb4.prev is numb.prev + + env5 = [b1, b4, b5] + snap5 = Snapshot(snap4, env5) + + numb5, liveboxes5, v = memo.number({b4: FakeValue(True, b4), + b5: FakeValue(True, b5)}, snap5) + assert v == 2 + + assert liveboxes5 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), + b4: tag(0, TAGVIRTUAL), b5: tag(1, TAGVIRTUAL)} + assert numb5.nums == [tag(0, TAGBOX), tag(0, TAGVIRTUAL), + tag(1, TAGVIRTUAL)] + assert numb5.prev is numb4 + + +def test__make_virtual(): + b1, b2 = BoxInt(), BoxInt() + vbox = BoxPtr() + modifier = ResumeDataVirtualAdder(None, None) + modifier.liveboxes_from_env = {} + modifier.liveboxes = {} + modifier.virtuals = [] + modifier.vfieldboxes = [] + modifier.make_virtual(vbox, None, ['a', 'b'], [b1, b2]) + assert modifier.liveboxes == {vbox: tag(0, TAGVIRTUAL), b1: UNASSIGNED, + b2: UNASSIGNED} + assert len(modifier.virtuals) == 1 + assert modifier.vfieldboxes == [[b1, b2]] + + modifier = ResumeDataVirtualAdder(None, None) + modifier.liveboxes_from_env = {vbox: tag(0, TAGVIRTUAL)} + modifier.liveboxes = {} + modifier.virtuals = [None] + modifier.vfieldboxes = [None] + modifier.make_virtual(vbox, None, ['a', 'b', 'c'], [b1, b2, vbox]) + assert modifier.liveboxes == {b1: UNASSIGNED, b2: UNASSIGNED, + vbox: tag(0, TAGVIRTUAL)} + assert len(modifier.virtuals) == 1 + assert modifier.vfieldboxes == [[b1, b2, vbox]] def _resume_remap(liveboxes, expected, *newvalues): newboxes = [] @@ -488,43 +591,16 @@ snapshot = Snapshot(snapshot, [ConstInt(2), ConstInt(3)]) snapshot = Snapshot(snapshot, [b1, b2, b3]) storage.rd_snapshot = snapshot - # just to placate _flatten_frame_info - storage.rd_frame_info_list = FrameInfo(None, FakeFrame("", 0, -1)) return storage -def test_virtual_adder_no_op(): - b1s, b2s, b3s = [BoxInt(1), BoxPtr(), BoxInt(3)] - storage = make_storage(b1s, b2s, b3s) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) - modifier = ResumeDataVirtualAdder(storage, memo) - modifier.walk_snapshots({}) - assert not modifier.is_virtual(b1s) - assert not modifier.is_virtual(b2s) - assert not modifier.is_virtual(b3s) - # done - liveboxes = modifier.finish({}) - assert storage.rd_snapshot is None - b1t, b2t, b3t = [BoxInt(11), BoxPtr(demo55o), BoxInt(33)] - newboxes = _resume_remap(liveboxes, [b1s, b2s, b3s], b1t, b2t, b3t) - metainterp = MyMetaInterp(LLtypeMixin.cpu) - reader = ResumeDataReader(storage, newboxes, metainterp) - lst = reader.consume_boxes() - assert lst == [b1t, b2t, b3t] - lst = reader.consume_boxes() - assert lst == [ConstInt(2), ConstInt(3)] - lst = reader.consume_boxes() - assert lst == [b1t, ConstInt(1), b1t, b2t] - assert metainterp.trace == [] - def test_virtual_adder_int_constants(): b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)] storage = make_storage(b1s, b2s, b3s) memo = ResumeDataLoopMemo(LLtypeMixin.cpu) modifier = ResumeDataVirtualAdder(storage, memo) - modifier.walk_snapshots({}) liveboxes = modifier.finish({}) assert storage.rd_snapshot is None - metainterp = MyMetaInterp(LLtypeMixin.cpu) + metainterp = MyMetaInterp() reader = ResumeDataReader(storage, [], metainterp) lst = reader.consume_boxes() assert lst == [b1s, b2s, b3s] @@ -541,7 +617,6 @@ storage = make_storage(b1s, b2s, b3s) memo = ResumeDataLoopMemo(LLtypeMixin.cpu) modifier = ResumeDataVirtualAdder(storage, memo) - modifier.walk_snapshots({}) modifier.finish({}) assert len(memo.consts) == 2 assert storage.rd_consts is memo.consts @@ -549,7 +624,6 @@ b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**17), ConstInt(-65)] storage2 = make_storage(b1s, b2s, b3s) modifier2 = ResumeDataVirtualAdder(storage2, memo) - modifier2.walk_snapshots({}) modifier2.finish({}) assert len(memo.consts) == 3 assert storage2.rd_consts is memo.consts @@ -563,23 +637,19 @@ b1_2 = BoxInt() class FakeValue(object): - def register_value(self, modifier): - modifier.register_box(b1_2) + def is_virtual(self): + return False def get_key_box(self): return b1_2 val = FakeValue() values = {b1s: val, b2s: val} - modifier.walk_snapshots(values) - assert not modifier.is_virtual(b1_2) - assert not modifier.is_virtual(b3s) - # done liveboxes = modifier.finish(values) assert storage.rd_snapshot is None b1t, b3t = [BoxInt(11), BoxInt(33)] newboxes = _resume_remap(liveboxes, [b1_2, b3s], b1t, b3t) - metainterp = MyMetaInterp(LLtypeMixin.cpu) + metainterp = MyMetaInterp() reader = ResumeDataReader(storage, newboxes, metainterp) lst = reader.consume_boxes() assert lst == [b1t, b1t, b3t] @@ -589,14 +659,38 @@ assert lst == [b1t, ConstInt(1), b1t, b1t] assert metainterp.trace == [] + +def test_virtual_adder_make_constant(): + b1s, b2s, b3s = [BoxInt(1), BoxPtr(), BoxInt(3)] + b1s = ConstInt(111) + storage = make_storage(b1s, b2s, b3s) + memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + modifier = ResumeDataVirtualAdder(storage, memo) + liveboxes = modifier.finish({}) + b2t, b3t = [BoxPtr(demo55o), BoxInt(33)] + newboxes = _resume_remap(liveboxes, [b2s, b3s], b2t, b3t) + metainterp = MyMetaInterp() + reader = ResumeDataReader(storage, newboxes, metainterp) + lst = reader.consume_boxes() + c1t = ConstInt(111) + assert lst == [c1t, b2t, b3t] + lst = reader.consume_boxes() + assert lst == [ConstInt(2), ConstInt(3)] + lst = reader.consume_boxes() + assert lst == [c1t, ConstInt(1), c1t, b2t] + assert metainterp.trace == [] + + def test_virtual_adder_make_virtual(): - b1s, b2s, b3s, b4s, b5s = [BoxInt(1), BoxPtr(), BoxInt(3), - BoxPtr(), BoxPtr()] + b2s, b3s, b4s, b5s = [BoxPtr(), BoxInt(3), BoxPtr(), BoxPtr()] c1s = ConstInt(111) - storage = make_storage(b1s, b2s, b3s) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + storage = Storage() + memo = ResumeDataLoopMemo(LLtypeMixin.cpu) modifier = ResumeDataVirtualAdder(storage, memo) - modifier.walk_snapshots({}) + modifier.liveboxes_from_env = {} + modifier.liveboxes = {} + modifier.virtuals = [] + modifier.vfieldboxes = [] modifier.make_virtual(b2s, ConstAddr(LLtypeMixin.node_vtable_adr, LLtypeMixin.cpu), @@ -608,46 +702,41 @@ [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr, LLtypeMixin.otherdescr], [b2s, b3s, b5s]) # new fields - assert not modifier.is_virtual(b1s) - assert modifier.is_virtual(b2s) - assert not modifier.is_virtual(b3s) - assert modifier.is_virtual(b4s) - assert not modifier.is_virtual(b5s) - # done - liveboxes = modifier.finish({}) - b1t, b3t, b5t = [BoxInt(11), BoxInt(33), BoxPtr(demo55o)] - newboxes = _resume_remap(liveboxes, [b1s, - #b2s -- virtual - b3s, - #b4s -- virtual - #b2s -- again, shared - #b3s -- again, shared - b5s], b1t, b3t, b5t) - # - metainterp = MyMetaInterp(LLtypeMixin.cpu) + + liveboxes = [] + modifier._number_virtuals(liveboxes) + storage.rd_consts = memo.consts[:] + storage.rd_numb = None + # resume + b3t, b5t = [BoxInt(33), BoxPtr(demo55o)] + newboxes = _resume_remap(liveboxes, [#b2s -- virtual + b3s, + #b4s -- virtual + #b2s -- again, shared + #b3s -- again, shared + b5s], b3t, b5t) + + metainterp = MyMetaInterp() reader = ResumeDataReader(storage, newboxes, metainterp) - trace = metainterp.trace[:] - del metainterp.trace[:] - lst = reader.consume_boxes() - b2t = lst[1] - assert lst == [b1t, b2t, b3t] - lst = reader.consume_boxes() - assert lst == [ConstInt(2), ConstInt(3)] - lst = reader.consume_boxes() - assert metainterp.trace == [] - assert lst == [b1t, ConstInt(1), b1t, b2t] - b4tx = b2t.value._obj.container._as_ptr().next - b4tx = lltype.cast_opaque_ptr(llmemory.GCREF, b4tx) - assert trace == [ - (rop.NEW_WITH_VTABLE, [LLtypeMixin.node_vtable_adr], b2t.value, None), - (rop.NEW_WITH_VTABLE, [LLtypeMixin.node_vtable_adr2], b4tx, None), - (rop.SETFIELD_GC, [b2t.value, b4tx], None, LLtypeMixin.nextdescr), - (rop.SETFIELD_GC, [b2t.value, c1s.value],None, LLtypeMixin.valuedescr), - (rop.SETFIELD_GC, [b4tx, b2t.value], None, LLtypeMixin.nextdescr), - (rop.SETFIELD_GC, [b4tx, b3t.value], None, LLtypeMixin.valuedescr), - (rop.SETFIELD_GC, [b4tx, b5t.value], None, LLtypeMixin.otherdescr), + assert len(reader.virtuals) == 2 + b2t = reader._decode_box(tag(0, TAGVIRTUAL)) + b4t = reader._decode_box(tag(1, TAGVIRTUAL)) + trace = metainterp.trace + expected = [ + (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr, + LLtypeMixin.cpu)], + b2t, None), + (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr2, + LLtypeMixin.cpu)], + b4t, None), + (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.nextdescr), + (rop.SETFIELD_GC, [b2t, c1s], None, LLtypeMixin.valuedescr), + (rop.SETFIELD_GC, [b4t, b2t], None, LLtypeMixin.nextdescr), + (rop.SETFIELD_GC, [b4t, b3t], None, LLtypeMixin.valuedescr), + (rop.SETFIELD_GC, [b4t, b5t], None, LLtypeMixin.otherdescr), ] - # + for x, y in zip(expected, trace): + assert x == y ptr = b2t.value._obj.container._as_ptr() assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.NODE) assert ptr.value == 111 @@ -657,83 +746,43 @@ assert ptr2.parent.value == 33 assert ptr2.parent.next == ptr - -def test_virtual_adder_make_constant(): - for testnumber in [0, 1]: - b1s, b2s, b3s = [BoxInt(1), BoxPtr(), BoxInt(3)] - if testnumber == 0: - # I. making a constant by directly specifying a constant in - # the list of liveboxes - b1s = ConstInt(111) - storage = make_storage(b1s, b2s, b3s) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) - modifier = ResumeDataVirtualAdder(storage, memo) - modifier.walk_snapshots({}) - - if testnumber == 1: - # II. making a constant with make_constant() - modifier.make_constant(b1s, ConstInt(111)) - assert not modifier.is_virtual(b1s) - - assert not modifier.is_virtual(b2s) - assert not modifier.is_virtual(b3s) - # done - liveboxes = modifier.finish({}) - b2t, b3t = [BoxPtr(demo55o), BoxInt(33)] - newboxes = _resume_remap(liveboxes, [b2s, b3s], b2t, b3t) - metainterp = MyMetaInterp(LLtypeMixin.cpu) - reader = ResumeDataReader(storage, newboxes, metainterp) - lst = reader.consume_boxes() - c1t = ConstInt(111) - assert lst == [c1t, b2t, b3t] - lst = reader.consume_boxes() - assert lst == [ConstInt(2), ConstInt(3)] - lst = reader.consume_boxes() - assert lst == [c1t, ConstInt(1), c1t, b2t] - assert metainterp.trace == [] - - def test_virtual_adder_make_varray(): - b1s, b2s, b3s, b4s = [BoxInt(1), BoxPtr(), BoxInt(3), BoxInt(4)] + b2s, b4s = [BoxPtr(), BoxInt(4)] c1s = ConstInt(111) - storage = make_storage(b1s, b2s, b3s) + storage = Storage() memo = ResumeDataLoopMemo(LLtypeMixin.cpu) modifier = ResumeDataVirtualAdder(storage, memo) - modifier.walk_snapshots({}) + modifier.liveboxes_from_env = {} + modifier.liveboxes = {} + modifier.virtuals = [] + modifier.vfieldboxes = [] modifier.make_varray(b2s, LLtypeMixin.arraydescr, [b4s, c1s]) # new fields - assert not modifier.is_virtual(b1s) - assert modifier.is_virtual(b2s) - assert not modifier.is_virtual(b3s) - assert not modifier.is_virtual(b4s) - # done - liveboxes = modifier.finish({}) + liveboxes = [] + modifier._number_virtuals(liveboxes) + storage.rd_consts = memo.consts[:] + storage.rd_numb = None + # resume b1t, b3t, b4t = [BoxInt(11), BoxInt(33), BoxInt(44)] - newboxes = _resume_remap(liveboxes, [b1s, - #b2s -- virtual - b3s, - b4s], - b1t, b3t, b4t) - # - metainterp = MyMetaInterp(LLtypeMixin.cpu) + newboxes = _resume_remap(liveboxes, [#b2s -- virtual + b4s], + b4t) + # resume + metainterp = MyMetaInterp() reader = ResumeDataReader(storage, newboxes, metainterp) - trace = metainterp.trace[:] - del metainterp.trace[:] - lst = reader.consume_boxes() - b2t = lst[1] - assert lst == [b1t, b2t, b3t] - assert trace == [ - (rop.NEW_ARRAY, [2], b2t.value, LLtypeMixin.arraydescr), - (rop.SETARRAYITEM_GC, [b2t.value,0,44], None, LLtypeMixin.arraydescr), - (rop.SETARRAYITEM_GC, [b2t.value,1,111], None, LLtypeMixin.arraydescr), + assert len(reader.virtuals) == 1 + b2t = reader._decode_box(tag(0, TAGVIRTUAL)) + trace = metainterp.trace + expected = [ + (rop.NEW_ARRAY, [ConstInt(2)], b2t, LLtypeMixin.arraydescr), + (rop.SETARRAYITEM_GC, [b2t,ConstInt(0), b4t],None, + LLtypeMixin.arraydescr), + (rop.SETARRAYITEM_GC, [b2t,ConstInt(1), c1s], None, + LLtypeMixin.arraydescr), ] - lst = reader.consume_boxes() - assert lst == [ConstInt(2), ConstInt(3)] - assert metainterp.trace == [] - lst = reader.consume_boxes() - assert lst == [b1t, ConstInt(1), b1t, b2t] - assert metainterp.trace == [] + for x, y in zip(expected, trace): + assert x == y # ptr = b2t.value._obj.container._as_ptr() assert lltype.typeOf(ptr) == lltype.Ptr(lltype.GcArray(lltype.Signed)) @@ -743,49 +792,41 @@ def test_virtual_adder_make_vstruct(): - b1s, b2s, b3s, b4s = [BoxInt(1), BoxPtr(), BoxInt(3), BoxPtr()] + b2s, b4s = [BoxPtr(), BoxPtr()] c1s = ConstInt(111) - storage = make_storage(b1s, b2s, b3s) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + storage = Storage() + memo = ResumeDataLoopMemo(LLtypeMixin.cpu) modifier = ResumeDataVirtualAdder(storage, memo) - modifier.walk_snapshots({}) + modifier.liveboxes_from_env = {} + modifier.liveboxes = {} + modifier.virtuals = [] + modifier.vfieldboxes = [] modifier.make_vstruct(b2s, LLtypeMixin.ssize, [LLtypeMixin.adescr, LLtypeMixin.bdescr], [c1s, b4s]) # new fields - assert not modifier.is_virtual(b1s) - assert modifier.is_virtual(b2s) - assert not modifier.is_virtual(b3s) - assert not modifier.is_virtual(b4s) - # done - liveboxes = modifier.finish({}) - b1t, b3t, b4t = [BoxInt(11), BoxInt(33), BoxPtr()] - newboxes = _resume_remap(liveboxes, [b1s, - #b2s -- virtual - b3s, - b4s], - b1t, b3t, b4t) + liveboxes = [] + modifier._number_virtuals(liveboxes) + storage.rd_consts = memo.consts[:] + storage.rd_numb = None + b4t = BoxPtr() + newboxes = _resume_remap(liveboxes, [#b2s -- virtual + b4s], b4t) # NULL = ConstPtr.value - metainterp = MyMetaInterp(LLtypeMixin.cpu) + metainterp = MyMetaInterp() reader = ResumeDataReader(storage, newboxes, metainterp) - lst = reader.consume_boxes() - trace = metainterp.trace[:] - del metainterp.trace[:] - b2t = lst[1] - assert lst == [b1t, b2t, b3t] - assert trace == [ - (rop.NEW, [], b2t.value, LLtypeMixin.ssize), - (rop.SETFIELD_GC, [b2t.value, 111], None, LLtypeMixin.adescr), - (rop.SETFIELD_GC, [b2t.value, NULL], None, LLtypeMixin.bdescr), + assert len(reader.virtuals) == 1 + b2t = reader._decode_box(tag(0, TAGVIRTUAL)) + + trace = metainterp.trace + expected = [ + (rop.NEW, [], b2t, LLtypeMixin.ssize), + (rop.SETFIELD_GC, [b2t, c1s], None, LLtypeMixin.adescr), + (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.bdescr), ] - del metainterp.trace[:] - lst = reader.consume_boxes() - assert lst == [ConstInt(2), ConstInt(3)] - assert metainterp.trace == [] - lst = reader.consume_boxes() - assert lst == [b1t, ConstInt(1), b1t, b2t] - assert metainterp.trace == [] + for x, y in zip(expected, trace): + assert x == y # ptr = b2t.value._obj.container._as_ptr() assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.S) Modified: pypy/trunk/pypy/jit/metainterp/test/test_virtual.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_virtual.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_virtual.py Thu Oct 8 17:54:29 2009 @@ -302,6 +302,31 @@ # ENTER - compile the leaving path self.check_enter_count(4) + def test_guards_around_forcing(self): + class A(object): + def __init__(self, x): + self.x = x + mydriver = JitDriver(reds = ['n'], greens = []) + global_a = A(0) + + def g(a): + n = a.x + if n < 10: + n += 1 + global_a.forced = a + if n < 20: + assert global_a.forced is a + + def f(n): + while n > 0: + mydriver.can_enter_jit(n=n) + mydriver.jit_merge_point(n=n) + a = A(n) + g(a) + n -= 1 + return 0 + self.meta_interp(f, [50]) + # ____________________________________________________________ # Run 1: all the tests instantiate a real RPython class Modified: pypy/trunk/pypy/jit/metainterp/typesystem.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/typesystem.py (original) +++ pypy/trunk/pypy/jit/metainterp/typesystem.py Thu Oct 8 17:54:29 2009 @@ -46,6 +46,7 @@ ConstRef = history.ConstPtr ConstAddr = history.ConstAddr loops_done_with_this_frame_ref = None # patched by compile.py + CONST_NULL = history.ConstPtr(history.ConstPtr.value) CVAL_NULLREF = None # patched by optimizeopt.py def get_VABLERTI(self): @@ -146,6 +147,7 @@ ConstRef = history.ConstObj ConstAddr = history.ConstObj loops_done_with_this_frame_ref = None # patched by compile.py + CONST_NULL = history.ConstObj(history.ConstObj.value) CVAL_NULLREF = None # patched by optimizeopt.py def get_VABLERTI(self): From arigo at codespeak.net Thu Oct 8 18:01:04 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Oct 2009 18:01:04 +0200 (CEST) Subject: [pypy-svn] r68251 - pypy/branch/gc-compress/pypy/rpython/memory Message-ID: <20091008160104.F2B9E498458@codespeak.net> Author: arigo Date: Thu Oct 8 18:01:03 2009 New Revision: 68251 Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py Log: Change this code to put all type info into an llgroup instead of an array. While I'm at it, already use the fact that the group members can be of varying size, by making the entries for non-varsized types twice as small. Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py Thu Oct 8 18:01:03 2009 @@ -1,4 +1,6 @@ -from pypy.rpython.lltypesystem import lltype, llmemory, llarena +from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup +from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import ll_assert @@ -21,65 +23,71 @@ ("finalizer", FINALIZERTYPE), ("fixedsize", lltype.Signed), ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), + ) + VARSIZE_TYPE_INFO = lltype.Struct("varsize_type_info", + ("header", TYPE_INFO), ("varitemsize", lltype.Signed), ("ofstovar", lltype.Signed), ("ofstolength", lltype.Signed), ("varofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), ) - TYPE_INFO_TABLE = lltype.Array(TYPE_INFO) + TYPE_INFO_PTR = lltype.Ptr(TYPE_INFO) + VARSIZE_TYPE_INFO_PTR = lltype.Ptr(VARSIZE_TYPE_INFO) - def __init__(self, type_info_table): - self.type_info_table = type_info_table - # 'type_info_table' is a list of TYPE_INFO structures when - # running with gcwrapper, or a real TYPE_INFO_TABLE after - # the gctransformer. + def __init__(self, type_info_group): + assert isinstance(type_info_group, llgroup.group) + self.type_info_group = type_info_group + self.type_info_group_ptr = type_info_group._as_ptr() + + def get(self, typeid): + if we_are_translated(): + ll_assert(typeid, "invalid type_id") + return llop.get_group_member(GCData.TYPE_INFO_PTR, + self.type_info_group_ptr, + typeid) + + def get_varsize(self, typeid): + if we_are_translated(): + ll_assert(typeid, "invalid type_id") + return llop.get_group_member(GCData.VARSIZE_TYPE_INFO_PTR, + self.type_info_group_ptr, + typeid) def q_is_varsize(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - infobits = self.type_info_table[typeid].infobits + infobits = self.get(typeid).infobits return (infobits & T_IS_VARSIZE) != 0 def q_has_gcptr_in_varsize(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - infobits = self.type_info_table[typeid].infobits + infobits = self.get(typeid).infobits return (infobits & T_HAS_GCPTR_IN_VARSIZE) != 0 def q_is_gcarrayofgcptr(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - infobits = self.type_info_table[typeid].infobits + infobits = self.get(typeid).infobits return (infobits & T_IS_GCARRAY_OF_GCPTR) != 0 def q_finalizer(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].finalizer + return self.get(typeid).finalizer def q_offsets_to_gc_pointers(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].ofstoptrs + return self.get(typeid).ofstoptrs def q_fixed_size(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].fixedsize + return self.get(typeid).fixedsize def q_varsize_item_sizes(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].varitemsize + return self.get_varsize(typeid).varitemsize def q_varsize_offset_to_variable_part(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].ofstovar + return self.get_varsize(typeid).ofstovar def q_varsize_offset_to_length(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].ofstolength + return self.get_varsize(typeid).ofstolength def q_varsize_offsets_to_gcpointers_in_var_part(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].varofstoptrs + return self.get_varsize(typeid).varofstoptrs def q_weakpointer_offset(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - infobits = self.type_info_table[typeid].infobits + infobits = self.get(typeid).infobits if infobits & T_IS_WEAKREF: return weakptr_offset return -1 @@ -114,27 +122,27 @@ if not TYPE._is_varsize(): info.fixedsize = llarena.round_up_for_allocation( llmemory.sizeof(TYPE)) - info.ofstolength = -1 # note about round_up_for_allocation(): in the 'info' table # we put a rounded-up size only for fixed-size objects. For # varsize ones, the GC must anyway compute the size at run-time # and round up that result. else: infobits |= T_IS_VARSIZE + varinfo = lltype.cast_pointer(GCData.VARSIZE_TYPE_INFO_PTR, info) info.fixedsize = llmemory.sizeof(TYPE, 0) if isinstance(TYPE, lltype.Struct): ARRAY = TYPE._flds[TYPE._arrayfld] ofs1 = llmemory.offsetof(TYPE, TYPE._arrayfld) - info.ofstolength = ofs1 + llmemory.ArrayLengthOffset(ARRAY) - info.ofstovar = ofs1 + llmemory.itemoffsetof(ARRAY, 0) + varinfo.ofstolength = ofs1 + llmemory.ArrayLengthOffset(ARRAY) + varinfo.ofstovar = ofs1 + llmemory.itemoffsetof(ARRAY, 0) else: assert isinstance(TYPE, lltype.GcArray) ARRAY = TYPE if (isinstance(ARRAY.OF, lltype.Ptr) and ARRAY.OF.TO._gckind == 'gc'): infobits |= T_IS_GCARRAY_OF_GCPTR - info.ofstolength = llmemory.ArrayLengthOffset(ARRAY) - info.ofstovar = llmemory.itemoffsetof(TYPE, 0) + varinfo.ofstolength = llmemory.ArrayLengthOffset(ARRAY) + varinfo.ofstovar = llmemory.itemoffsetof(TYPE, 0) assert isinstance(ARRAY, lltype.Array) if ARRAY.OF != lltype.Void: offsets = offsets_to_gc_pointers(ARRAY.OF) @@ -142,8 +150,8 @@ offsets = () if len(offsets) > 0: infobits |= T_HAS_GCPTR_IN_VARSIZE - info.varofstoptrs = builder.offsets2table(offsets, ARRAY.OF) - info.varitemsize = llmemory.sizeof(ARRAY.OF) + varinfo.varofstoptrs = builder.offsets2table(offsets, ARRAY.OF) + varinfo.varitemsize = llmemory.sizeof(ARRAY.OF) if TYPE == WEAKREF: infobits |= T_IS_WEAKREF info.infobits = infobits @@ -156,7 +164,7 @@ can_encode_type_shape = True # set to False initially by the JIT def __init__(self): - self.type_info_list = [None] # don't use typeid 0, helps debugging + self.make_type_info_group() self.id_of_type = {} # {LLTYPE: type_id} self.seen_roots = {} # the following are lists of addresses of gc pointers living inside the @@ -172,6 +180,13 @@ self.finalizer_funcptrs = {} self.offsettable_cache = {} + def make_type_info_group(self): + self.type_info_group = llgroup.group("typeinfo") + # don't use typeid 0, may help debugging + DUMMY = lltype.Struct("dummy", ('x', lltype.Signed)) + dummy = lltype.malloc(DUMMY, immortal=True, zero=True) + self.type_info_group.add_member(dummy) + def get_type_id(self, TYPE): try: return self.id_of_type[TYPE] @@ -183,15 +198,20 @@ # TYPE_INFO_TABLE in flatten_table() by the gc transformer. # build the TYPE_INFO structure - info = lltype.malloc(GCData.TYPE_INFO, immortal=True, zero=True) + if not TYPE._is_varsize(): + fullinfo = lltype.malloc(GCData.TYPE_INFO, + immortal=True, zero=True) + info = fullinfo + else: + fullinfo = lltype.malloc(GCData.VARSIZE_TYPE_INFO, + immortal=True, zero=True) + info = fullinfo.header if self.can_encode_type_shape: encode_type_shape(self, info, TYPE) else: self._pending_type_shapes.append((info, TYPE)) # store it - type_id = len(self.type_info_list) - assert type_id & 0xffff == type_id # make sure it fits into 2 bytes - self.type_info_list.append(info) + type_id = self.type_info_group.add_member(fullinfo) self.id_of_type[TYPE] = type_id return type_id @@ -247,7 +267,7 @@ return lltype.nullptr(GCData.ADDRESS_VOID_FUNC) def initialize_gc_query_function(self, gc): - return GCData(self.type_info_list).set_query_functions(gc) + return GCData(self.type_info_group).set_query_functions(gc) def consider_constant(self, TYPE, value, gc): if value is not lltype.top_container(value): From arigo at codespeak.net Thu Oct 8 18:04:15 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Oct 2009 18:04:15 +0200 (CEST) Subject: [pypy-svn] r68252 - pypy/branch/gc-compress/pypy/rpython/memory/gc Message-ID: <20091008160415.4F287498458@codespeak.net> Author: arigo Date: Thu Oct 8 18:04:13 2009 New Revision: 68252 Modified: pypy/branch/gc-compress/pypy/rpython/memory/gc/semispace.py Log: Fix the semispace. Modified: pypy/branch/gc-compress/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gc/semispace.py Thu Oct 8 18:04:13 2009 @@ -4,7 +4,7 @@ from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE from pypy.rpython.memory.support import get_address_stack, get_address_deque from pypy.rpython.memory.support import AddressDict -from pypy.rpython.lltypesystem import lltype, llmemory, llarena +from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi from pypy.rlib.objectmodel import free_non_gc_object from pypy.rlib.debug import ll_assert from pypy.rpython.lltypesystem.lloperation import llop @@ -13,8 +13,7 @@ import sys, os, time -TYPEID_MASK = 0xffff -first_gcflag = 1 << 16 +first_gcflag = 1 GCFLAG_FORWARDED = first_gcflag # GCFLAG_EXTERNAL is set on objects not living in the semispace: # either immortal objects or (for HybridGC) externally raw_malloc'ed @@ -23,6 +22,21 @@ memoryError = MemoryError() +# Handlers for the adt methods getflags() etc. on the HDR. +# These are mostly just workarounds for the limited support +# for 16-bits integer, providing the necessary casts. +def _hdr_getflags(hdr): + return lltype.cast_primitive(lltype.Signed, hdr.flags16) +def _hdr_setflags(hdr, flags): + hdr.flags16 = lltype.cast_primitive(rffi.USHORT, flags) +def _hdr_addflag(hdr, flag): + flags = lltype.cast_primitive(lltype.Signed, hdr.flags16) + hdr.flags16 = lltype.cast_primitive(rffi.USHORT, flags | flag) +def _hdr_delflag(hdr, flag): + flags = lltype.cast_primitive(lltype.Signed, hdr.flags16) + hdr.flags16 = lltype.cast_primitive(rffi.USHORT, flags & ~flag) + + class SemiSpaceGC(MovingGCBase): _alloc_flavor_ = "raw" inline_simple_malloc = True @@ -32,7 +46,14 @@ total_collection_time = 0.0 total_collection_count = 0 - HDR = lltype.Struct('header', ('tid', lltype.Signed)) + HDR = lltype.Struct('header', ('typeid16', rffi.USHORT), + ('flags16', rffi.USHORT), + adtmeths = { + 'getflags': _hdr_getflags, + 'setflags': _hdr_setflags, + 'addflag': _hdr_addflag, + 'delflag': _hdr_delflag, + }) FORWARDSTUB = lltype.GcStruct('forwarding_stub', ('forw', llmemory.Address)) FORWARDSTUBPTR = lltype.Ptr(FORWARDSTUB) @@ -350,12 +371,12 @@ return self.is_forwarded(obj) def is_forwarded(self, obj): - return self.header(obj).tid & GCFLAG_FORWARDED != 0 + return self.header(obj).getflags() & GCFLAG_FORWARDED != 0 # note: all prebuilt objects also have this flag set def get_forwarding_address(self, obj): - tid = self.header(obj).tid - if tid & GCFLAG_EXTERNAL: + flags = self.header(obj).getflags() + if flags & GCFLAG_EXTERNAL: self.visit_external_object(obj) return obj # external or prebuilt objects are "forwarded" # to themselves @@ -373,35 +394,38 @@ # writes after translation to C. size_gc_header = self.size_gc_header() stubsize = llmemory.sizeof(self.FORWARDSTUB) - tid = self.header(obj).tid - ll_assert(tid & GCFLAG_EXTERNAL == 0, "unexpected GCFLAG_EXTERNAL") - ll_assert(tid & GCFLAG_FORWARDED == 0, "unexpected GCFLAG_FORWARDED") + flags = self.header(obj).getflags() + ll_assert(flags & GCFLAG_EXTERNAL == 0, "unexpected GCFLAG_EXTERNAL") + ll_assert(flags & GCFLAG_FORWARDED == 0, "unexpected GCFLAG_FORWARDED") # replace the object at 'obj' with a FORWARDSTUB. hdraddr = obj - size_gc_header llarena.arena_reset(hdraddr, size_gc_header + objsize, False) llarena.arena_reserve(hdraddr, size_gc_header + stubsize) hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(self.HDR)) - hdr.tid = tid | GCFLAG_FORWARDED + hdr.addflag(GCFLAG_FORWARDED) stub = llmemory.cast_adr_to_ptr(obj, self.FORWARDSTUBPTR) stub.forw = newobj def get_type_id(self, addr): - tid = self.header(addr).tid - ll_assert(tid & (GCFLAG_FORWARDED|GCFLAG_EXTERNAL) != GCFLAG_FORWARDED, + hdr = self.header(addr) + flg = hdr.getflags() + ll_assert(flg & (GCFLAG_FORWARDED|GCFLAG_EXTERNAL) != GCFLAG_FORWARDED, "get_type_id on forwarded obj") # Non-prebuilt forwarded objects are overwritten with a FORWARDSTUB. # Although calling get_type_id() on a forwarded object works by itself, # we catch it as an error because it's likely that what is then # done with the typeid is bogus. - return tid & TYPEID_MASK + return hdr.typeid16 def init_gc_object(self, addr, typeid, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.tid = typeid | flags + hdr.typeid16 = typeid + hdr.setflags(flags) def init_gc_object_immortal(self, addr, typeid, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.tid = typeid | flags | GCFLAG_EXTERNAL | GCFLAG_FORWARDED + hdr.typeid16 = typeid + hdr.setflags(flags | GCFLAG_EXTERNAL | GCFLAG_FORWARDED) # immortal objects always have GCFLAG_FORWARDED set; # see get_forwarding_address(). @@ -466,13 +490,13 @@ if self.surviving(obj): newobj = self.get_forwarding_address(obj) hdr = self.header(newobj) - if hdr.tid & GCFLAG_FINALIZATION_ORDERING: + if hdr.getflags() & GCFLAG_FINALIZATION_ORDERING: return 2 else: return 3 else: hdr = self.header(obj) - if hdr.tid & GCFLAG_FINALIZATION_ORDERING: + if hdr.getflags() & GCFLAG_FINALIZATION_ORDERING: return 1 else: return 0 @@ -481,7 +505,7 @@ ll_assert(self._finalization_state(obj) == 0, "unexpected finalization state != 0") hdr = self.header(obj) - hdr.tid |= GCFLAG_FINALIZATION_ORDERING + hdr.addflag(GCFLAG_FINALIZATION_ORDERING) def _recursively_bump_finalization_state_from_2_to_3(self, obj): ll_assert(self._finalization_state(obj) == 2, @@ -493,8 +517,10 @@ while pending.non_empty(): y = pending.pop() hdr = self.header(y) - if hdr.tid & GCFLAG_FINALIZATION_ORDERING: # state 2 ? - hdr.tid &= ~GCFLAG_FINALIZATION_ORDERING # change to state 3 + flags = hdr.getflags() + if flags & GCFLAG_FINALIZATION_ORDERING: # state 2 ? + flags &= ~GCFLAG_FINALIZATION_ORDERING # change to state 3 + hdr.setflags(flags) self.trace(y, self._append_if_nonnull, pending) def _recursively_bump_finalization_state_from_1_to_2(self, obj, scan): @@ -540,21 +566,22 @@ self.run_finalizers = new_run_finalizer def _is_external(self, obj): - return (self.header(obj).tid & GCFLAG_EXTERNAL) != 0 + return (self.header(obj).getflags() & GCFLAG_EXTERNAL) != 0 def debug_check_object(self, obj): """Check the invariants about 'obj' that should be true between collections.""" - tid = self.header(obj).tid - if tid & GCFLAG_EXTERNAL: - ll_assert(tid & GCFLAG_FORWARDED, "bug: external+!forwarded") + flags = self.header(obj).getflags() + if flags & GCFLAG_EXTERNAL: + ll_assert(flags & GCFLAG_FORWARDED, "bug: external+!forwarded") ll_assert(not (self.tospace <= obj < self.free), "external flag but object inside the semispaces") else: - ll_assert(not (tid & GCFLAG_FORWARDED), "bug: !external+forwarded") + ll_assert(not (flags & GCFLAG_FORWARDED), + "bug: !external+forwarded") ll_assert(self.tospace <= obj < self.free, "!external flag but object outside the semispaces") - ll_assert(not (tid & GCFLAG_FINALIZATION_ORDERING), + ll_assert(not (flags & GCFLAG_FINALIZATION_ORDERING), "unexpected GCFLAG_FINALIZATION_ORDERING") def debug_check_can_copy(self, obj): From afa at codespeak.net Thu Oct 8 18:09:25 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 8 Oct 2009 18:09:25 +0200 (CEST) Subject: [pypy-svn] r68253 - pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/test Message-ID: <20091008160925.DADBF498458@codespeak.net> Author: afa Date: Thu Oct 8 18:09:25 2009 New Revision: 68253 Modified: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/test/test_asmgcroot.py Log: Fix these tests on Windows; now that all translated functions are processed in setup_class, skipping one test is not enough to avoid the error. Modified: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/test/test_asmgcroot.py Thu Oct 8 18:09:25 2009 @@ -156,3 +156,6 @@ if sys.platform == 'win32': def test_callback_with_collect(self): py.test.skip("No libffi yet with mingw32") + + def define_callback_with_collect(cls): + return lambda: 0 From arigo at codespeak.net Thu Oct 8 18:40:41 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Oct 2009 18:40:41 +0200 (CEST) Subject: [pypy-svn] r68254 - in pypy/branch/gc-compress/pypy: rpython/lltypesystem rpython/memory rpython/memory/gc rpython/memory/gctransform rpython/memory/test translator/c/src Message-ID: <20091008164041.C034B498458@codespeak.net> Author: arigo Date: Thu Oct 8 18:40:40 2009 New Revision: 68254 Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py pypy/branch/gc-compress/pypy/rpython/memory/gc/semispace.py pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py pypy/branch/gc-compress/pypy/rpython/memory/test/test_gctypelayout.py pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h Log: Progress: now semispace can again be translated. Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py Thu Oct 8 18:40:40 2009 @@ -412,6 +412,7 @@ 'cast_int_to_adr': LLOp(canfold=True), # not implemented in llinterp 'get_group_member': LLOp(canfold=True), 'get_next_group_member':LLOp(canfold=True), + 'is_group_member_zero': LLOp(canfold=True), # __________ used by the JIT ________ Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py Thu Oct 8 18:40:40 2009 @@ -402,6 +402,14 @@ return lltype.cast_pointer(TYPE, member) op_get_next_group_member.need_result_type = True +def op_is_group_member_zero(memberoffset): + from pypy.rpython.lltypesystem import llgroup + if isinstance(memberoffset, llgroup.GroupMemberOffset): + return memberoffset.index == 0 + else: + assert isinstance(memberoffset, int) + return memberoffset == 0 + # ____________________________________________________________ def get_op_impl(opname): Modified: pypy/branch/gc-compress/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gc/semispace.py Thu Oct 8 18:40:40 2009 @@ -85,7 +85,7 @@ # This class only defines the malloc_{fixed,var}size_clear() methods # because the spaces are filled with zeroes in advance. - def malloc_fixedsize_clear(self, typeid, size, can_collect, + def malloc_fixedsize_clear(self, typeid16, size, can_collect, has_finalizer=False, contains_weakptr=False): size_gc_header = self.gcheaderbuilder.size_gc_header totalsize = size_gc_header + size @@ -95,7 +95,7 @@ raise memoryError result = self.obtain_free_space(totalsize) llarena.arena_reserve(result, totalsize) - self.init_gc_object(result, typeid) + self.init_gc_object(result, typeid16) self.free = result + totalsize if has_finalizer: self.objects_with_finalizers.append(result + size_gc_header) @@ -103,7 +103,7 @@ self.objects_with_weakrefs.append(result + size_gc_header) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) - def malloc_varsize_clear(self, typeid, length, size, itemsize, + def malloc_varsize_clear(self, typeid16, length, size, itemsize, offset_to_length, can_collect): size_gc_header = self.gcheaderbuilder.size_gc_header nonvarsize = size_gc_header + size @@ -118,7 +118,7 @@ raise memoryError result = self.obtain_free_space(totalsize) llarena.arena_reserve(result, totalsize) - self.init_gc_object(result, typeid) + self.init_gc_object(result, typeid16) (result + size_gc_header + offset_to_length).signed[0] = length self.free = result + llarena.round_up_for_allocation(totalsize) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) @@ -417,14 +417,14 @@ # done with the typeid is bogus. return hdr.typeid16 - def init_gc_object(self, addr, typeid, flags=0): + def init_gc_object(self, addr, typeid16, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.typeid16 = typeid + hdr.typeid16 = typeid16 hdr.setflags(flags) - def init_gc_object_immortal(self, addr, typeid, flags=0): + def init_gc_object_immortal(self, addr, typeid16, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.typeid16 = typeid + hdr.typeid16 = typeid16 hdr.setflags(flags | GCFLAG_EXTERNAL | GCFLAG_FORWARDED) # immortal objects always have GCFLAG_FORWARDED set; # see get_forwarding_address(). Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py Thu Oct 8 18:40:40 2009 @@ -1,7 +1,7 @@ from pypy.rpython.memory.gctransform.transform import GCTransformer from pypy.rpython.memory.gctransform.support import find_gc_ptrs_in_type, \ get_rtti, ll_call_destructor, type_contains_pyobjs, var_ispyobj -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython import rmodel from pypy.rpython.memory import gctypelayout from pypy.rpython.memory.gc import marksweep @@ -22,6 +22,9 @@ import sys, types +TYPE_ID = rffi.USHORT + + class CollectAnalyzer(graphanalyze.BoolGraphAnalyzer): def analyze_direct_call(self, graph, seen=None): @@ -131,11 +134,9 @@ self.layoutbuilder.transformer = self self.get_type_id = self.layoutbuilder.get_type_id - # set up dummy a table, to be overwritten with the real one in finish() - type_info_table = lltype._ptr( - lltype.Ptr(gctypelayout.GCData.TYPE_INFO_TABLE), - "delayed!type_info_table", solid=True) - gcdata = gctypelayout.GCData(type_info_table) + # set up GCData with the llgroup from the layoutbuilder, which + # will grow as more TYPE_INFO members are added to it + gcdata = gctypelayout.GCData(self.layoutbuilder.type_info_group) # initialize the following two fields with a random non-NULL address, # to make the annotator happy. The fields are patched in finish() @@ -163,6 +164,8 @@ gcdata.gc.setup() bk = self.translator.annotator.bookkeeper + r_typeid16 = rffi.platform.numbertype_to_rclass[TYPE_ID] + s_typeid16 = annmodel.SomeInteger(knowntype=r_typeid16) # the point of this little dance is to not annotate # self.gcdata.static_root_xyz as constants. XXX is it still needed?? @@ -212,7 +215,7 @@ malloc_fixedsize_clear_meth = GCClass.malloc_fixedsize_clear.im_func self.malloc_fixedsize_clear_ptr = getfn( malloc_fixedsize_clear_meth, - [s_gc, annmodel.SomeInteger(nonneg=True), + [s_gc, s_typeid16, annmodel.SomeInteger(nonneg=True), annmodel.SomeBool(), annmodel.SomeBool(), annmodel.SomeBool()], s_gcref, @@ -221,7 +224,7 @@ malloc_fixedsize_meth = GCClass.malloc_fixedsize.im_func self.malloc_fixedsize_ptr = getfn( malloc_fixedsize_meth, - [s_gc, annmodel.SomeInteger(nonneg=True), + [s_gc, s_typeid16, annmodel.SomeInteger(nonneg=True), annmodel.SomeBool(), annmodel.SomeBool(), annmodel.SomeBool()], s_gcref, @@ -235,7 +238,8 @@ ## + [annmodel.SomeBool(), annmodel.SomeBool()], s_gcref) self.malloc_varsize_clear_ptr = getfn( GCClass.malloc_varsize_clear.im_func, - [s_gc] + [annmodel.SomeInteger(nonneg=True) for i in range(5)] + [s_gc, s_typeid16] + + [annmodel.SomeInteger(nonneg=True) for i in range(4)] + [annmodel.SomeBool()], s_gcref) self.collect_ptr = getfn(GCClass.collect.im_func, [s_gc, annmodel.SomeInteger()], annmodel.s_None) @@ -268,7 +272,7 @@ s_True = annmodel.SomeBool(); s_True .const = True self.malloc_fast_ptr = getfn( malloc_fast, - [s_gc, annmodel.SomeInteger(nonneg=True), + [s_gc, s_typeid16, annmodel.SomeInteger(nonneg=True), s_True, s_False, s_False], s_gcref, @@ -288,7 +292,7 @@ s_True = annmodel.SomeBool(); s_True .const = True self.malloc_varsize_clear_fast_ptr = getfn( malloc_varsize_clear_fast, - [s_gc, annmodel.SomeInteger(nonneg=True), + [s_gc, s_typeid16, annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True), @@ -304,7 +308,7 @@ "malloc_varsize_nonmovable") self.malloc_varsize_nonmovable_ptr = getfn( malloc_nonmovable, - [s_gc, annmodel.SomeInteger(nonneg=True), + [s_gc, s_typeid16, annmodel.SomeInteger(nonneg=True)], s_gcref) else: self.malloc_varsize_nonmovable_ptr = None @@ -315,7 +319,7 @@ "malloc_varsize_resizable") self.malloc_varsize_resizable_ptr = getfn( malloc_resizable, - [s_gc, annmodel.SomeInteger(nonneg=True), + [s_gc, s_typeid16, annmodel.SomeInteger(nonneg=True)], s_gcref) else: self.malloc_varsize_resizable_ptr = None @@ -421,21 +425,14 @@ return [getattr(hdr, fldname) for fldname in HDR._names] def finish_tables(self): - table = self.layoutbuilder.flatten_table() - log.info("assigned %s typeids" % (len(table), )) + group = self.layoutbuilder.close_table() + log.info("assigned %s typeids" % (len(group.members), )) log.info("added %s push/pop stack root instructions" % ( self.num_pushs, )) if self.write_barrier_ptr: log.info("inserted %s write barrier calls" % ( self.write_barrier_calls, )) - # replace the type_info_table pointer in gcdata -- at this point, - # the database is in principle complete, so it has already seen - # the delayed pointer. We need to force it to consider the new - # array now. - - self.gcdata.type_info_table._become(table) - # XXX because we call inputconst already in replace_malloc, we can't # modify the instance, we have to modify the 'rtyped instance' # instead. horrors. is there a better way? @@ -467,12 +464,14 @@ """write out the list of type ids together with some info""" from pypy.tool.udir import udir # XXX not ideal since it is not per compilation, but per run + # XXX argh argh, this only gives the member index but not the + # real typeid, which is a complete mess to obtain now... + all_ids = self.layoutbuilder.id_of_type.items() + all_ids = [(typeinfo.index, TYPE) for (TYPE, typeinfo) in all_ids] + all_ids = dict(all_ids) f = udir.join("typeids.txt").open("w") - all = [(typeid, TYPE) - for TYPE, typeid in self.layoutbuilder.id_of_type.iteritems()] - all.sort() - for typeid, TYPE in all: - f.write("%s %s\n" % (typeid, TYPE)) + for index in range(len(self.layoutbuilder.type_info_group.members)): + f.write("member%-4d %s\n" % (index, all_ids.get(index, '?'))) f.close() def transform_graph(self, graph): @@ -502,8 +501,8 @@ assert PTRTYPE.TO == TYPE type_id = self.get_type_id(TYPE) - c_type_id = rmodel.inputconst(lltype.Signed, type_id) - info = self.layoutbuilder.type_info_list[type_id] + c_type_id = rmodel.inputconst(TYPE_ID, type_id) + info = self.layoutbuilder.get_info(type_id) c_size = rmodel.inputconst(lltype.Signed, info.fixedsize) has_finalizer = bool(self.finalizer_funcptr_for_type(TYPE)) c_has_finalizer = rmodel.inputconst(lltype.Bool, has_finalizer) @@ -523,9 +522,12 @@ c_has_finalizer, rmodel.inputconst(lltype.Bool, False)] else: assert not c_has_finalizer.value + info_varsize = self.layoutbuilder.get_info_varsize(type_id) v_length = op.args[-1] - c_ofstolength = rmodel.inputconst(lltype.Signed, info.ofstolength) - c_varitemsize = rmodel.inputconst(lltype.Signed, info.varitemsize) + c_ofstolength = rmodel.inputconst(lltype.Signed, + info_varsize.ofstolength) + c_varitemsize = rmodel.inputconst(lltype.Signed, + info_varsize.varitemsize) if flags.get('resizable') and self.malloc_varsize_resizable_ptr: assert c_can_collect.value malloc_ptr = self.malloc_varsize_resizable_ptr @@ -656,8 +658,8 @@ type_id = self.get_type_id(WEAKREF) - c_type_id = rmodel.inputconst(lltype.Signed, type_id) - info = self.layoutbuilder.type_info_list[type_id] + c_type_id = rmodel.inputconst(TYPE_ID, type_id) + info = self.layoutbuilder.get_info(type_id) c_size = rmodel.inputconst(lltype.Signed, info.fixedsize) malloc_ptr = self.malloc_fixedsize_ptr c_has_finalizer = rmodel.inputconst(lltype.Bool, False) Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py Thu Oct 8 18:40:40 2009 @@ -40,15 +40,15 @@ self.type_info_group_ptr = type_info_group._as_ptr() def get(self, typeid): - if we_are_translated(): - ll_assert(typeid, "invalid type_id") + ll_assert(not llop.is_group_member_zero(lltype.Bool, typeid), + "invalid type_id") return llop.get_group_member(GCData.TYPE_INFO_PTR, self.type_info_group_ptr, typeid) def get_varsize(self, typeid): - if we_are_translated(): - ll_assert(typeid, "invalid type_id") + ll_assert(not llop.is_group_member_zero(lltype.Bool, typeid), + "invalid type_id") return llop.get_group_member(GCData.VARSIZE_TYPE_INFO_PTR, self.type_info_group_ptr, typeid) @@ -194,9 +194,6 @@ assert self.can_add_new_types assert isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)) # Record the new type_id description as a TYPE_INFO structure. - # It goes into a list for now, which will be turned into a - # TYPE_INFO_TABLE in flatten_table() by the gc transformer. - # build the TYPE_INFO structure if not TYPE._is_varsize(): fullinfo = lltype.malloc(GCData.TYPE_INFO, @@ -215,6 +212,16 @@ self.id_of_type[TYPE] = type_id return type_id + def get_info(self, type_id): + return llop.get_group_member(GCData.TYPE_INFO_PTR, + self.type_info_group._as_ptr(), + type_id) + + def get_info_varsize(self, type_id): + return llop.get_group_member(GCData.VARSIZE_TYPE_INFO_PTR, + self.type_info_group._as_ptr(), + type_id) + def encode_type_shapes_now(self): if not self.can_encode_type_shape: self.can_encode_type_shape = True @@ -240,20 +247,11 @@ self.offsettable_cache[TYPE] = cachedarray return cachedarray - def flatten_table(self): + def close_table(self): + # make sure we no longer add members to the type_info_group. self.can_add_new_types = False self.offsettable_cache = None - table = lltype.malloc(GCData.TYPE_INFO_TABLE, len(self.type_info_list), - immortal=True) - fieldnames = GCData.TYPE_INFO._names - for tableentry, newcontent in zip(table, self.type_info_list): - if newcontent is None: # empty entry - tableentry.infobits = 0 - tableentry.ofstolength = -1 - else: - for name in fieldnames: - setattr(tableentry, name, getattr(newcontent, name)) - return table + return self.type_info_group def finalizer_funcptr_for_type(self, TYPE): if TYPE in self.finalizer_funcptrs: Modified: pypy/branch/gc-compress/pypy/rpython/memory/test/test_gctypelayout.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/test/test_gctypelayout.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/test/test_gctypelayout.py Thu Oct 8 18:40:40 2009 @@ -38,7 +38,7 @@ for T1, T2 in [(GC_A, GC_S), (GC_A2, GC_S2), (GC_S3, GC_S2)]: tid1 = layoutbuilder.get_type_id(T1) tid2 = layoutbuilder.get_type_id(T2) - gcdata = GCData(layoutbuilder.type_info_list) + gcdata = GCData(layoutbuilder.type_info_group) lst1 = gcdata.q_varsize_offsets_to_gcpointers_in_var_part(tid1) lst2 = gcdata.q_offsets_to_gc_pointers(tid2) assert len(lst1) == len(lst2) Modified: pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h (original) +++ pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h Thu Oct 8 18:40:40 2009 @@ -11,6 +11,9 @@ #define OP_GET_NEXT_GROUP_MEMBER(groupptr, compactoffset, skipoffset, r) \ r = ((char*)groupptr) + ((long)compactoffset)*sizeof(long) + skipoffset +#define OP_IS_GROUP_MEMBER_ZERO(compactoffset, r) \ + r = (compactoffset == 0) + /* A macro to crash at compile-time if sizeof(group) is too large. Uses a hack that I've found on some random forum. Haaaaaaaaaackish. */ #define PYPY_GROUP_CHECK_SIZE(groupname) \ From arigo at codespeak.net Thu Oct 8 19:03:09 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Oct 2009 19:03:09 +0200 (CEST) Subject: [pypy-svn] r68255 - pypy/branch/gc-compress/pypy/rpython/memory/gc Message-ID: <20091008170309.A933E498457@codespeak.net> Author: arigo Date: Thu Oct 8 19:03:09 2009 New Revision: 68255 Modified: pypy/branch/gc-compress/pypy/rpython/memory/gc/generation.py pypy/branch/gc-compress/pypy/rpython/memory/gc/hybrid.py pypy/branch/gc-compress/pypy/rpython/memory/gc/marksweep.py Log: Fix these three GCs. Modified: pypy/branch/gc-compress/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gc/generation.py Thu Oct 8 19:03:09 2009 @@ -244,7 +244,7 @@ newobj = SemiSpaceGC.make_a_copy(self, obj, objsize) # During a full collect, all copied objects might implicitly come # from the nursery. In case they do, we must add this flag: - self.header(newobj).tid |= GCFLAG_NO_YOUNG_PTRS + self.header(newobj).addflag(GCFLAG_NO_YOUNG_PTRS) return newobj # history: this was missing and caused an object to become old but without the # flag set. Such an object is bogus in the sense that the write_barrier doesn't @@ -263,7 +263,7 @@ while oldlist.non_empty(): obj = oldlist.pop() hdr = self.header(obj) - hdr.tid |= GCFLAG_NO_YOUNG_PTRS + hdr.addflag(GCFLAG_NO_YOUNG_PTRS) def weakrefs_grow_older(self): while self.young_objects_with_weakrefs.non_empty(): @@ -293,7 +293,7 @@ self.last_generation_root_objects = self.AddressStack() while stack.non_empty(): obj = stack.pop() - self.header(obj).tid |= GCFLAG_NO_HEAP_PTRS + self.header(obj).addflag(GCFLAG_NO_HEAP_PTRS) # ^^^ the flag we just added will be removed immediately if # the object still contains pointers to younger objects self.trace(obj, self._trace_external_obj, obj) @@ -360,7 +360,7 @@ count += 1 obj = oldlist.pop() hdr = self.header(obj) - hdr.tid |= GCFLAG_NO_YOUNG_PTRS + hdr.addflag(GCFLAG_NO_YOUNG_PTRS) self.trace_and_drag_out_of_nursery(obj) if self.config.gcconfig.debugprint: llop.debug_print(lltype.Void, "collect_oldrefs_to_nursery", count) @@ -423,10 +423,10 @@ # for the JIT: a minimal description of the write_barrier() method # (the JIT assumes it is of the shape # "if newvalue.int0 & JIT_WB_IF_FLAG: remember_young_pointer()") - JIT_WB_IF_FLAG = GCFLAG_NO_YOUNG_PTRS + JIT_WB_IF_FLAG = "XXXX" # GCFLAG_NO_YOUNG_PTRS def write_barrier(self, newvalue, addr_struct): - if self.header(addr_struct).tid & GCFLAG_NO_YOUNG_PTRS: + if self.header(addr_struct).getflags() & GCFLAG_NO_YOUNG_PTRS: self.remember_young_pointer(addr_struct, newvalue) def _setup_wb(self): @@ -444,7 +444,7 @@ "nursery object with GCFLAG_NO_YOUNG_PTRS") if self.is_in_nursery(addr): self.old_objects_pointing_to_young.append(addr_struct) - self.header(addr_struct).tid &= ~GCFLAG_NO_YOUNG_PTRS + self.header(addr_struct).delflag(GCFLAG_NO_YOUNG_PTRS) elif addr == NULL: return self.write_into_last_generation_obj(addr_struct, addr) @@ -453,23 +453,25 @@ def assume_young_pointers(self, addr_struct): objhdr = self.header(addr_struct) - if objhdr.tid & GCFLAG_NO_YOUNG_PTRS: + flags = objhdr.getflags() + if flags & GCFLAG_NO_YOUNG_PTRS: self.old_objects_pointing_to_young.append(addr_struct) - objhdr.tid &= ~GCFLAG_NO_YOUNG_PTRS - if objhdr.tid & GCFLAG_NO_HEAP_PTRS: - objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS + objhdr.delflag(GCFLAG_NO_YOUNG_PTRS) + if flags & GCFLAG_NO_HEAP_PTRS: + objhdr.delflag(GCFLAG_NO_HEAP_PTRS) self.last_generation_root_objects.append(addr_struct) def write_into_last_generation_obj(self, addr_struct, addr): objhdr = self.header(addr_struct) - if objhdr.tid & GCFLAG_NO_HEAP_PTRS: + flags = objhdr.getflags() + if flags & GCFLAG_NO_HEAP_PTRS: if not self.is_last_generation(addr): - objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS + objhdr.delflag(GCFLAG_NO_HEAP_PTRS) self.last_generation_root_objects.append(addr_struct) def is_last_generation(self, obj): # overridden by HybridGC - return (self.header(obj).tid & GCFLAG_EXTERNAL) != 0 + return (self.header(obj).getflags() & GCFLAG_EXTERNAL) != 0 def _compute_id(self, obj): if self.is_in_nursery(obj): @@ -500,15 +502,15 @@ """Check the invariants about 'obj' that should be true between collections.""" SemiSpaceGC.debug_check_object(self, obj) - tid = self.header(obj).tid - if tid & GCFLAG_NO_YOUNG_PTRS: + flags = self.header(obj).getflags() + if flags & GCFLAG_NO_YOUNG_PTRS: ll_assert(not self.is_in_nursery(obj), "nursery object with GCFLAG_NO_YOUNG_PTRS") self.trace(obj, self._debug_no_nursery_pointer, None) elif not self.is_in_nursery(obj): ll_assert(self._d_oopty.contains(obj), "missing from old_objects_pointing_to_young") - if tid & GCFLAG_NO_HEAP_PTRS: + if flags & GCFLAG_NO_HEAP_PTRS: ll_assert(self.is_last_generation(obj), "GCFLAG_NO_HEAP_PTRS on non-3rd-generation object") self.trace(obj, self._debug_no_gen1or2_pointer, None) @@ -537,10 +539,10 @@ self._debug_check_flag_2, None) def _debug_check_flag_1(self, obj, ignored): - ll_assert(not (self.header(obj).tid & GCFLAG_NO_YOUNG_PTRS), + ll_assert(not (self.header(obj).getflags() & GCFLAG_NO_YOUNG_PTRS), "unexpected GCFLAG_NO_YOUNG_PTRS") def _debug_check_flag_2(self, obj, ignored): - ll_assert(not (self.header(obj).tid & GCFLAG_NO_HEAP_PTRS), + ll_assert(not (self.header(obj).getflags() & GCFLAG_NO_HEAP_PTRS), "unexpected GCFLAG_NO_HEAP_PTRS") def debug_check_can_copy(self, obj): Modified: pypy/branch/gc-compress/pypy/rpython/memory/gc/hybrid.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gc/hybrid.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gc/hybrid.py Thu Oct 8 19:03:09 2009 @@ -34,10 +34,10 @@ # * last_generation_root_objects: gen3 objs that point to gen1or2 objs # # How to tell the objects apart: -# * external: tid & GCFLAG_EXTERNAL +# * external: flags & GCFLAG_EXTERNAL # * gen1: is_in_nursery(obj) -# * gen3: (tid & (GCFLAG_EXTERNAL|GCFLAG_AGE_MASK)) == -# (GCFLAG_EXTERNAL|GCFLAG_AGE_MAX) +# * gen3: (flags & (GCFLAG_EXTERNAL|GCFLAG_AGE_MASK)) == +# (GCFLAG_EXTERNAL|GCFLAG_AGE_MAX) # # Some invariants: # * gen3 are either GCFLAG_NO_HEAP_PTRS or in 'last_generation_root_objects' @@ -222,7 +222,6 @@ def realloc(self, ptr, newlength, fixedsize, itemsize, lengthofs, grow): size_gc_header = self.size_gc_header() addr = llmemory.cast_ptr_to_adr(ptr) - tid = self.get_type_id(addr) nonvarsize = size_gc_header + fixedsize try: varsize = ovfcheck(itemsize * newlength) @@ -253,8 +252,7 @@ return llmemory.cast_adr_to_ptr(result + size_gc_header, llmemory.GCREF) def can_move(self, addr): - tid = self.header(addr).tid - return not (tid & GCFLAG_EXTERNAL) + return not (self.header(addr).getflags() & GCFLAG_EXTERNAL) def malloc_varsize_collecting_nursery(self, totalsize): result = self.collect_nursery() @@ -343,9 +341,9 @@ self._nonmoving_copy_size = 0 def _set_gcflag_unvisited(self, obj, ignored): - ll_assert(not (self.header(obj).tid & GCFLAG_UNVISITED), + ll_assert(not (self.header(obj).getflags() & GCFLAG_UNVISITED), "bogus GCFLAG_UNVISITED on gen3 obj") - self.header(obj).tid |= GCFLAG_UNVISITED + self.header(obj).addflag(GCFLAG_UNVISITED) def collect_roots(self): if not self.is_collecting_gen3(): @@ -364,18 +362,20 @@ # ones with GCFLAG_FORWARDED set and GCFLAG_UNVISITED not set. # This is equivalent to self.is_forwarded() for all objects except # the ones obtained by raw_malloc. - flags = self.header(obj).tid & (GCFLAG_FORWARDED|GCFLAG_UNVISITED) - return flags == GCFLAG_FORWARDED + flags = self.header(obj).getflags() + return ((flags & (GCFLAG_FORWARDED|GCFLAG_UNVISITED)) == + GCFLAG_FORWARDED) def is_last_generation(self, obj): - return ((self.header(obj).tid & (GCFLAG_EXTERNAL|GCFLAG_AGE_MASK)) == + flags = self.header(obj).getflags() + return ((flags & (GCFLAG_EXTERNAL|GCFLAG_AGE_MASK)) == (GCFLAG_EXTERNAL|GCFLAG_AGE_MAX)) def visit_external_object(self, obj): hdr = self.header(obj) - if hdr.tid & GCFLAG_UNVISITED: + if hdr.getflags() & GCFLAG_UNVISITED: # This is a not-visited-yet raw_malloced object. - hdr.tid -= GCFLAG_UNVISITED + hdr.delflag(GCFLAG_UNVISITED) self.rawmalloced_objects_to_trace.append(obj) def make_a_copy(self, obj, objsize): @@ -384,26 +384,26 @@ # If they don't, we count how many times they are copied and when # some threshold is reached we make the copy a non-movable "external" # object. The threshold is MAX_SEMISPACE_AGE. - tid = self.header(obj).tid + flags = self.header(obj).getflags() # XXX the following logic is not doing exactly what is explained # above: any object without GCFLAG_NO_YOUNG_PTRS has its age not # incremented. This is accidental: it means that objects that # are very often modified to point to young objects don't reach # the 3rd generation. For now I'll leave it this way because # I'm not sure that it's a bad thing. - if not (tid & GCFLAG_NO_YOUNG_PTRS): - tid |= GCFLAG_NO_YOUNG_PTRS # object comes from the nursery - elif (tid & GCFLAG_AGE_MASK) < GCFLAG_AGE_MAX: - tid += GCFLAG_AGE_ONE + if not (flags & GCFLAG_NO_YOUNG_PTRS): + flags |= GCFLAG_NO_YOUNG_PTRS # object comes from the nursery + elif (flags & GCFLAG_AGE_MASK) < GCFLAG_AGE_MAX: + flags += GCFLAG_AGE_ONE else: newobj = self.make_a_nonmoving_copy(obj, objsize) if newobj: return newobj - tid &= ~GCFLAG_AGE_MASK + flags &= ~GCFLAG_AGE_MASK # skip GenerationGC.make_a_copy() as we already did the right # thing about GCFLAG_NO_YOUNG_PTRS newobj = SemiSpaceGC.make_a_copy(self, obj, objsize) - self.header(newobj).tid = tid + self.header(newobj).setflags(flags) return newobj def make_a_nonmoving_copy(self, obj, objsize): @@ -420,7 +420,7 @@ llmemory.raw_memcopy(obj - self.size_gc_header(), newaddr, totalsize) newobj = newaddr + self.size_gc_header() hdr = self.header(newobj) - hdr.tid |= self.GCFLAGS_FOR_NEW_EXTERNAL_OBJECTS + hdr.addflag(self.GCFLAGS_FOR_NEW_EXTERNAL_OBJECTS) # GCFLAG_UNVISITED is not set # GCFLAG_NO_HEAP_PTRS is not set either, conservatively. It may be # set by the next collection's collect_last_generation_roots(). @@ -485,7 +485,7 @@ newgen3roots = self.AddressStack() while gen3roots.non_empty(): obj = gen3roots.pop() - if not (self.header(obj).tid & GCFLAG_UNVISITED): + if not (self.header(obj).getflags() & GCFLAG_UNVISITED): newgen3roots.append(obj) gen3roots.delete() self.last_generation_root_objects = newgen3roots @@ -500,8 +500,8 @@ alive_count = alive_size = dead_count = dead_size = 0 while objects.non_empty(): obj = objects.pop() - tid = self.header(obj).tid - if tid & GCFLAG_UNVISITED: + flags = self.header(obj).getflags() + if flags & GCFLAG_UNVISITED: if self.config.gcconfig.debugprint: dead_count+=1 dead_size+=raw_malloc_usage(self.get_size(obj)) @@ -514,24 +514,24 @@ if generation == 3: surviving_objects.append(obj) elif generation == 2: - ll_assert((tid & GCFLAG_AGE_MASK) < GCFLAG_AGE_MAX, + ll_assert((flags & GCFLAG_AGE_MASK) < GCFLAG_AGE_MAX, "wrong age for generation 2 object") - tid += GCFLAG_AGE_ONE - if (tid & GCFLAG_AGE_MASK) == GCFLAG_AGE_MAX: + flags += GCFLAG_AGE_ONE + if (flags & GCFLAG_AGE_MASK) == GCFLAG_AGE_MAX: # the object becomes part of generation 3 self.gen3_rawmalloced_objects.append(obj) # GCFLAG_NO_HEAP_PTRS not set yet, conservatively self.last_generation_root_objects.append(obj) else: # the object stays in generation 2 - tid |= GCFLAG_UNVISITED + flags |= GCFLAG_UNVISITED surviving_objects.append(obj) - self.header(obj).tid = tid + self.header(obj).setflags(flags) elif generation == -2: # the object stays in generation -2 - tid |= GCFLAG_UNVISITED + flags |= GCFLAG_UNVISITED surviving_objects.append(obj) - self.header(obj).tid = tid + self.header(obj).setflags(flags) objects.delete() if generation == 2: self.gen2_rawmalloced_objects = surviving_objects @@ -575,8 +575,8 @@ """Check the invariants about 'obj' that should be true between collections.""" GenerationGC.debug_check_object(self, obj) - tid = self.header(obj).tid - if tid & GCFLAG_UNVISITED: + flags = self.header(obj).getflags() + if flags & GCFLAG_UNVISITED: ll_assert(self._d_gen2ro.contains(obj), "GCFLAG_UNVISITED on non-gen2 object") @@ -589,20 +589,20 @@ self.gen3_rawmalloced_objects.foreach(self._debug_check_gen3, None) def _debug_check_gen2(self, obj, ignored): - tid = self.header(obj).tid - ll_assert(bool(tid & GCFLAG_EXTERNAL), + flags = self.header(obj).getflags() + ll_assert(bool(flags & GCFLAG_EXTERNAL), "gen2: missing GCFLAG_EXTERNAL") - ll_assert(bool(tid & GCFLAG_UNVISITED), + ll_assert(bool(flags & GCFLAG_UNVISITED), "gen2: missing GCFLAG_UNVISITED") - ll_assert((tid & GCFLAG_AGE_MASK) < GCFLAG_AGE_MAX, + ll_assert((flags & GCFLAG_AGE_MASK) < GCFLAG_AGE_MAX, "gen2: age field too large") def _debug_check_gen3(self, obj, ignored): - tid = self.header(obj).tid - ll_assert(bool(tid & GCFLAG_EXTERNAL), + flags = self.header(obj).getflags() + ll_assert(bool(flags & GCFLAG_EXTERNAL), "gen3: missing GCFLAG_EXTERNAL") - ll_assert(not (tid & GCFLAG_UNVISITED), + ll_assert(not (flags & GCFLAG_UNVISITED), "gen3: unexpected GCFLAG_UNVISITED") - ll_assert((tid & GCFLAG_AGE_MASK) == GCFLAG_AGE_MAX, + ll_assert((flags & GCFLAG_AGE_MASK) == GCFLAG_AGE_MAX, "gen3: wrong age field") def can_malloc_nonmovable(self): Modified: pypy/branch/gc-compress/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gc/marksweep.py Thu Oct 8 19:03:09 2009 @@ -4,7 +4,7 @@ from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE from pypy.rpython.memory.support import get_address_stack from pypy.rpython.memory.gcheader import GCHeaderBuilder -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rlib.objectmodel import free_non_gc_object from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck @@ -25,7 +25,9 @@ HDRPTR = lltype.Ptr(HDR) # need to maintain a linked list of malloced objects, since we used the # systems allocator and can't walk the heap - HDR.become(lltype.Struct('header', ('typeid', lltype.Signed), + HDR.become(lltype.Struct('header', ('typeid16', rffi.USHORT), + ('mark', lltype.Bool), + ('curpool_flag', lltype.Bool), ('next', HDRPTR))) POOL = lltype.GcStruct('gc_pool') @@ -75,14 +77,14 @@ if self.bytes_malloced > self.bytes_malloced_threshold: self.collect() - def write_malloc_statistics(self, typeid, size, result, varsize): + def write_malloc_statistics(self, typeid16, size, result, varsize): pass - def write_free_statistics(self, typeid, result): + def write_free_statistics(self, typeid16, result): pass - def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, - contains_weakptr=False): + def malloc_fixedsize(self, typeid16, size, can_collect, + has_finalizer=False, contains_weakptr=False): if can_collect: self.maybe_collect() size_gc_header = self.gcheaderbuilder.size_gc_header @@ -97,7 +99,9 @@ if not result: raise memoryError hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) - hdr.typeid = typeid << 1 + hdr.typeid16 = typeid16 + hdr.mark = False + hdr.curpool_flag = False if has_finalizer: hdr.next = self.malloced_objects_with_finalizer self.malloced_objects_with_finalizer = hdr @@ -109,13 +113,13 @@ self.malloced_objects = hdr self.bytes_malloced = bytes_malloced result += size_gc_header - #llop.debug_print(lltype.Void, 'malloc typeid', typeid, + #llop.debug_print(lltype.Void, 'malloc typeid', typeid16, # '->', llmemory.cast_adr_to_int(result)) - self.write_malloc_statistics(typeid, tot_size, result, False) + self.write_malloc_statistics(typeid16, tot_size, result, False) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) malloc_fixedsize._dont_inline_ = True - def malloc_fixedsize_clear(self, typeid, size, can_collect, + def malloc_fixedsize_clear(self, typeid16, size, can_collect, has_finalizer=False, contains_weakptr=False): if can_collect: self.maybe_collect() @@ -132,7 +136,9 @@ raise memoryError raw_memclear(result, tot_size) hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) - hdr.typeid = typeid << 1 + hdr.typeid16 = typeid16 + hdr.mark = False + hdr.curpool_flag = False if has_finalizer: hdr.next = self.malloced_objects_with_finalizer self.malloced_objects_with_finalizer = hdr @@ -144,14 +150,14 @@ self.malloced_objects = hdr self.bytes_malloced = bytes_malloced result += size_gc_header - #llop.debug_print(lltype.Void, 'malloc typeid', typeid, + #llop.debug_print(lltype.Void, 'malloc typeid', typeid16, # '->', llmemory.cast_adr_to_int(result)) - self.write_malloc_statistics(typeid, tot_size, result, False) + self.write_malloc_statistics(typeid16, tot_size, result, False) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) malloc_fixedsize_clear._dont_inline_ = True - def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, - can_collect): + def malloc_varsize(self, typeid16, length, size, itemsize, + offset_to_length, can_collect): if can_collect: self.maybe_collect() size_gc_header = self.gcheaderbuilder.size_gc_header @@ -169,20 +175,22 @@ raise memoryError (result + size_gc_header + offset_to_length).signed[0] = length hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) - hdr.typeid = typeid << 1 + hdr.typeid16 = typeid16 + hdr.mark = False + hdr.curpool_flag = False hdr.next = self.malloced_objects self.malloced_objects = hdr self.bytes_malloced = bytes_malloced result += size_gc_header #llop.debug_print(lltype.Void, 'malloc_varsize length', length, - # 'typeid', typeid, + # 'typeid', typeid16, # '->', llmemory.cast_adr_to_int(result)) - self.write_malloc_statistics(typeid, tot_size, result, True) + self.write_malloc_statistics(typeid16, tot_size, result, True) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) malloc_varsize._dont_inline_ = True - def malloc_varsize_clear(self, typeid, length, size, itemsize, + def malloc_varsize_clear(self, typeid16, length, size, itemsize, offset_to_length, can_collect): if can_collect: self.maybe_collect() @@ -202,16 +210,18 @@ raw_memclear(result, tot_size) (result + size_gc_header + offset_to_length).signed[0] = length hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) - hdr.typeid = typeid << 1 + hdr.typeid16 = typeid16 + hdr.mark = False + hdr.curpool_flag = False hdr.next = self.malloced_objects self.malloced_objects = hdr self.bytes_malloced = bytes_malloced result += size_gc_header #llop.debug_print(lltype.Void, 'malloc_varsize length', length, - # 'typeid', typeid, + # 'typeid', typeid16, # '->', llmemory.cast_adr_to_int(result)) - self.write_malloc_statistics(typeid, tot_size, result, True) + self.write_malloc_statistics(typeid16, tot_size, result, True) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) malloc_varsize_clear._dont_inline_ = True @@ -251,10 +261,10 @@ hdr = self.malloced_objects_with_finalizer while hdr: next = hdr.next - typeid = hdr.typeid >> 1 + typeid = hdr.typeid16 gc_info = llmemory.cast_ptr_to_adr(hdr) obj = gc_info + size_gc_header - if not hdr.typeid & 1: + if not hdr.mark: self.add_reachable_to_stack(obj, objects) addr = llmemory.cast_ptr_to_adr(hdr) size = self.fixed_size(typeid) @@ -271,31 +281,30 @@ curr = objects.pop() gc_info = curr - size_gc_header hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - if hdr.typeid & 1: + if hdr.mark: continue self.add_reachable_to_stack(curr, objects) - hdr.typeid = hdr.typeid | 1 + hdr.mark = True objects.delete() # also mark self.curpool if self.curpool: gc_info = llmemory.cast_ptr_to_adr(self.curpool) - size_gc_header hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - hdr.typeid = hdr.typeid | 1 + hdr.mark = True # go through the list of objects containing weak pointers # and kill the links if they go to dead objects # if the object itself is not marked, free it hdr = self.objects_with_weak_pointers surviving = lltype.nullptr(self.HDR) while hdr: - typeid = hdr.typeid >> 1 + typeid = hdr.typeid16 next = hdr.next addr = llmemory.cast_ptr_to_adr(hdr) size = self.fixed_size(typeid) estimate = raw_malloc_usage(size_gc_header + size) - if hdr.typeid & 1: - typeid = hdr.typeid >> 1 + if hdr.mark: offset = self.weakpointer_offset(typeid) - hdr.typeid = hdr.typeid & (~1) + hdr.mark = False gc_info = llmemory.cast_ptr_to_adr(hdr) weakref_obj = gc_info + size_gc_header pointing_to = (weakref_obj + offset).address[0] @@ -306,7 +315,7 @@ # pointed to object will die # XXX what to do if the object has a finalizer which resurrects # the object? - if not hdr_pointing_to.typeid & 1: + if not hdr_pointing_to.mark: (weakref_obj + offset).address[0] = NULL hdr.next = surviving surviving = hdr @@ -331,7 +340,7 @@ ppnext += llmemory.offsetof(self.POOLNODE, 'linkedlist') hdr = poolnode.linkedlist while hdr: #sweep - typeid = hdr.typeid >> 1 + typeid = hdr.typeid16 next = hdr.next addr = llmemory.cast_ptr_to_adr(hdr) size = self.fixed_size(typeid) @@ -339,8 +348,8 @@ length = (addr + size_gc_header + self.varsize_offset_to_length(typeid)).signed[0] size += self.varsize_item_sizes(typeid) * length estimate = raw_malloc_usage(size_gc_header + size) - if hdr.typeid & 1: - hdr.typeid = hdr.typeid & (~1) + if hdr.mark: + hdr.mark = False ppnext.address[0] = addr ppnext = llmemory.cast_ptr_to_adr(hdr) ppnext += llmemory.offsetof(self.HDR, 'next') @@ -423,17 +432,17 @@ last = lltype.nullptr(self.HDR) while hdr: next = hdr.next - if hdr.typeid & 1: + if hdr.mark: hdr.next = lltype.nullptr(self.HDR) if not self.malloced_objects_with_finalizer: self.malloced_objects_with_finalizer = hdr else: last.next = hdr - hdr.typeid = hdr.typeid & (~1) + hdr.mark = False last = hdr else: obj = llmemory.cast_ptr_to_adr(hdr) + size_gc_header - finalizer = self.getfinalizer(hdr.typeid >> 1) + finalizer = self.getfinalizer(hdr.typeid16) # make malloced_objects_with_finalizer consistent # for the sake of a possible collection caused by finalizer if not self.malloced_objects_with_finalizer: @@ -473,7 +482,7 @@ size_gc_header = self.gcheaderbuilder.size_gc_header gc_info = gcobjectaddr - size_gc_header hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - hdr.typeid = hdr.typeid & (~1) + hdr.mark = False STAT_HEAP_USAGE = 0 STAT_BYTES_MALLOCED = 1 @@ -483,7 +492,7 @@ size_gc_header = self.gcheaderbuilder.size_gc_header gc_info = obj - size_gc_header hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - return hdr.typeid >> 1 + return hdr.typeid16 def add_reachable_to_stack(self, obj, objects): self.trace(obj, self._add_reachable, objects) @@ -504,13 +513,17 @@ def init_gc_object(self, addr, typeid): hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) - hdr.typeid = typeid << 1 + hdr.typeid16 = typeid + hdr.mark = False + hdr.curpool_flag = False def init_gc_object_immortal(self, addr, typeid, flags=0): # prebuilt gc structures always have the mark bit set # ignore flags hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) - hdr.typeid = (typeid << 1) | 1 + hdr.typeid16 = typeid + hdr.mark = True + hdr.curpool_flag = False # experimental support for thread cloning def x_swap_pool(self, newpool): @@ -566,7 +579,6 @@ # in the specified pool. A new pool is built to contain the # copies, and the 'gcobjectptr' and 'pool' fields of clonedata # are adjusted to refer to the result. - CURPOOL_FLAG = sys.maxint // 2 + 1 # install a new pool into which all the mallocs go curpool = self.x_swap_pool(lltype.nullptr(X_POOL)) @@ -583,7 +595,7 @@ hdr = hdr.next # skip the POOL object itself while hdr: next = hdr.next - hdr.typeid |= CURPOOL_FLAG # mark all objects from malloced_list + hdr.curpool_flag = True # mark all objects from malloced_list hdr.next = lltype.nullptr(self.HDR) # abused to point to the copy oldobjects.append(llmemory.cast_ptr_to_adr(hdr)) hdr = next @@ -600,12 +612,11 @@ continue # pointer is NULL oldhdr = llmemory.cast_adr_to_ptr(oldobj_addr - size_gc_header, self.HDRPTR) - typeid = oldhdr.typeid - if not (typeid & CURPOOL_FLAG): + if not oldhdr.curpool_flag: continue # ignore objects that were not in the malloced_list newhdr = oldhdr.next # abused to point to the copy if not newhdr: - typeid = (typeid & ~CURPOOL_FLAG) >> 1 + typeid = oldhdr.typeid16 size = self.fixed_size(typeid) # XXX! collect() at the beginning if the free heap is low if self.is_varsize(typeid): @@ -631,11 +642,15 @@ newhdr_addr = newobj_addr - size_gc_header newhdr = llmemory.cast_adr_to_ptr(newhdr_addr, self.HDRPTR) - saved_id = newhdr.typeid # XXX hack needed for genc + saved_id = newhdr.typeid16 # XXX hack needed for genc + saved_flg1 = newhdr.mark + saved_flg2 = newhdr.curpool_flag saved_next = newhdr.next # where size_gc_header == 0 raw_memcopy(oldobj_addr, newobj_addr, size) - newhdr.typeid = saved_id - newhdr.next = saved_next + newhdr.typeid16 = saved_id + newhdr.mark = saved_flg1 + newhdr.curpool_flag = saved_flg2 + newhdr.next = saved_next offsets = self.offsets_to_gc_pointers(typeid) i = 0 @@ -669,7 +684,7 @@ next = lltype.nullptr(self.HDR) while oldobjects.non_empty(): hdr = llmemory.cast_adr_to_ptr(oldobjects.pop(), self.HDRPTR) - hdr.typeid &= ~CURPOOL_FLAG # reset the flag + hdr.curpool_flag = False # reset the flag hdr.next = next next = hdr oldobjects.delete() From fijal at codespeak.net Thu Oct 8 19:36:15 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Oct 2009 19:36:15 +0200 (CEST) Subject: [pypy-svn] r68256 - pypy/branch/remove-rewrite-assembler Message-ID: <20091008173615.169B0498454@codespeak.net> Author: fijal Date: Thu Oct 8 19:36:14 2009 New Revision: 68256 Removed: pypy/branch/remove-rewrite-assembler/ Log: I don't think this branch is going anywhere, kill From fijal at codespeak.net Thu Oct 8 19:37:05 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 8 Oct 2009 19:37:05 +0200 (CEST) Subject: [pypy-svn] r68257 - pypy/branch/inline-fastpath-malloc Message-ID: <20091008173705.F3201498454@codespeak.net> Author: fijal Date: Thu Oct 8 19:37:05 2009 New Revision: 68257 Added: pypy/branch/inline-fastpath-malloc/ - copied from r68256, pypy/trunk/ Log: Create a branch for inlining fast path of malloc in JIT From pedronis at codespeak.net Thu Oct 8 20:47:21 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 8 Oct 2009 20:47:21 +0200 (CEST) Subject: [pypy-svn] r68258 - in pypy/trunk/pypy/jit: backend/x86/test metainterp Message-ID: <20091008184721.86D8D498453@codespeak.net> Author: pedronis Date: Thu Oct 8 20:47:20 2009 New Revision: 68258 Modified: pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py pypy/trunk/pypy/jit/metainterp/warmspot.py Log: (cfbolz, pedronis) - reorganize test_zrpy_gc in convoluted way such that the jit can be compiled only twice , this speeds it up a lot - make sure we always see the full optimizer, avoids lots of annotation issues when only the simple one is specified Modified: pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py Thu Oct 8 20:47:20 2009 @@ -4,8 +4,10 @@ however, is the correct handling of GC, i.e. if objects are freed as soon as possible (at least in a simple case). """ + import weakref, random import py +from pypy.annotation import policy as annpolicy from pypy.rlib import rgc from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.lltypesystem.lloperation import llop @@ -17,6 +19,9 @@ stack_pos = X86StackManager.stack_pos class X(object): + def __init__(self, x=0): + self.x = x + next = None class CheckError(Exception): @@ -26,22 +31,27 @@ if not flag: raise CheckError - -def get_test(main): +def get_g(main): main._dont_inline_ = True - - def g(n): + def g(num, n): x = X() x.foo = 2 main(n, x) x.foo = 5 return weakref.ref(x) g._dont_inline_ = True + return g + + +def get_entry(g): def entrypoint(args): + num = 0 + if len(args) == 2: + num = int(args[1]) r_list = [] for i in range(20): - r = g(2000) + r = g(num, 2000) r_list.append(r) rgc.collect() rgc.collect(); rgc.collect() @@ -55,7 +65,7 @@ return entrypoint -def compile_and_run(f, gc, CPUClass=CPU386, **kwds): +def compile(f, gc, **kwds): from pypy.annotation.listdef import s_list_of_strings from pypy.translator.translator import TranslationContext from pypy.jit.metainterp.warmspot import apply_jit @@ -66,17 +76,26 @@ t.config.translation.gcconfig.debugprint = True for name, value in kwds.items(): setattr(t.config.translation, name, value) - t.buildannotator().build_types(f, [s_list_of_strings]) + ann = t.buildannotator(policy=annpolicy.StrictAnnotatorPolicy()) + ann.build_types(f, [s_list_of_strings]) t.buildrtyper().specialize() if kwds['jit']: - apply_jit(t, CPUClass=CPUClass, optimizer=OPTIMIZER_SIMPLE) + apply_jit(t, optimizer=OPTIMIZER_SIMPLE) cbuilder = genc.CStandaloneBuilder(t, f, t.config) cbuilder.generate_source() cbuilder.compile() + return cbuilder + +def run(cbuilder, args=''): # - data = cbuilder.cmdexec('') + data = cbuilder.cmdexec(args) return data.strip() +def compile_and_run(f, gc, **kwds): + cbuilder = compile(f, gc, **kwds) + return run(cbuilder) + + def test_compile_boehm(): myjitdriver = JitDriver(greens = [], reds = ['n', 'x']) @@ -87,51 +106,100 @@ y = X() y.foo = x.foo n -= y.foo - res = compile_and_run(get_test(main), "boehm", jit=True) + res = compile_and_run(get_entry(get_g(main)), "boehm", jit=True) assert int(res) >= 16 -def test_compile_hybrid_1(): - # a moving GC. Supports malloc_varsize_nonmovable. Simple test, works - # without write_barriers and root stack enumeration. - myjitdriver = JitDriver(greens = [], reds = ['n', 'x']) - def main(n, x): - while n > 0: - myjitdriver.can_enter_jit(n=n, x=x) - myjitdriver.jit_merge_point(n=n, x=x) +# ______________________________________________________________________ + +class TestCompileHybrid(object): + def setup_class(cls): + funcs = [] + name_to_func = {} + for fullname in dir(cls): + if not fullname.startswith('define'): + continue + definefunc = getattr(cls, fullname) + _, name = fullname.split('_', 1) + beforefunc, loopfunc, afterfunc = definefunc.im_func(cls) + if beforefunc is None: + def beforefunc(n, x): + return n, x, None, None, None, None, None, None, None, None, None, '' + if afterfunc is None: + def afterfunc(n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s): + pass + beforefunc.func_name = 'before_'+name + loopfunc.func_name = 'loop_'+name + afterfunc.func_name = 'after_'+name + funcs.append((beforefunc, loopfunc, afterfunc)) + assert name not in name_to_func + name_to_func[name] = len(name_to_func) + def allfuncs(num, n): + x = X() + x.foo = 2 + main_allfuncs(num, n, x) + x.foo = 5 + return weakref.ref(x) + def main_allfuncs(num, n, x): + n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s = funcs[num][0](n, x) + while n > 0: + myjitdriver.can_enter_jit(num=num, n=n, x=x, x0=x0, x1=x1, + x2=x2, x3=x3, x4=x4, x5=x5, x6=x6, x7=x7, l=l, s=s) + myjitdriver.jit_merge_point(num=num, n=n, x=x, x0=x0, x1=x1, + x2=x2, x3=x3, x4=x4, x5=x5, x6=x6, x7=x7, l=l, s=s) + + n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s = funcs[num][1]( + n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s) + funcs[num][2](n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s) + myjitdriver = JitDriver(greens = ['num'], + reds = ['n', 'x', 'x0', 'x1', 'x2', 'x3', 'x4', + 'x5', 'x6', 'x7', 'l', 's']) + cls.main_allfuncs = staticmethod(main_allfuncs) + cls.name_to_func = name_to_func + cls.cbuilder = compile(get_entry(allfuncs), "hybrid", gcrootfinder="asmgcc", jit=True) + + def run(self, name): + num = self.name_to_func[name] + res = self.cbuilder.cmdexec(str(num)) + assert int(res) == 20 + + def run_orig(self, name, n, x): + num = self.name_to_func[name] + self.main_allfuncs(num, n, x) + + def define_compile_hybrid_1(cls): + # a moving GC. Supports malloc_varsize_nonmovable. Simple test, works + # without write_barriers and root stack enumeration. + def f(n, x, *args): y = X() y.foo = x.foo n -= y.foo - res = compile_and_run(get_test(main), "hybrid", gcrootfinder="asmgcc", - jit=True) - assert int(res) == 20 - -def test_compile_hybrid_2(): - # More complex test, requires root stack enumeration but - # not write_barriers. - myjitdriver = JitDriver(greens = [], reds = ['n', 'x']) - def main(n, x): - while n > 0: - myjitdriver.can_enter_jit(n=n, x=x) - myjitdriver.jit_merge_point(n=n, x=x) + return (n, x) + args + return None, f, None + + def test_compile_hybrid_1(self): + self.run('compile_hybrid_1') + + def define_compile_hybrid_2(cls): + # More complex test, requires root stack enumeration but + # not write_barriers. + def f(n, x, *args): prev = x - for j in range(101): # main() runs 20'000 times, thus allocates + for j in range(101): # f() runs 20'000 times, thus allocates y = X() # a total of 2'020'000 objects y.foo = prev.foo prev = y n -= prev.foo - res = compile_and_run(get_test(main), "hybrid", gcrootfinder="asmgcc", - jit=True) - assert int(res) == 20 + return (n, x) + args + return None, f, None -def test_compile_hybrid_3(): - # Third version of the test. Really requires write_barriers. - myjitdriver = JitDriver(greens = [], reds = ['n', 'x']) - def main(n, x): - while n > 0: - myjitdriver.can_enter_jit(n=n, x=x) - myjitdriver.jit_merge_point(n=n, x=x) + def test_compile_hybrid_2(self): + self.run('compile_hybrid_2') + + def define_compile_hybrid_3(cls): + # Third version of the test. Really requires write_barriers. + def f(n, x, *args): x.next = None - for j in range(101): # main() runs 20'000 times, thus allocates + for j in range(101): # f() runs 20'000 times, thus allocates y = X() # a total of 2'020'000 objects y.foo = j+1 y.next = x.next @@ -145,41 +213,40 @@ check(not y.next) check(total == 101*102/2) n -= x.foo - x_test = X() - x_test.foo = 5 - main(6, x_test) # check that it does not raise CheckError - res = compile_and_run(get_test(main), "hybrid", gcrootfinder="asmgcc", - jit=True) - assert int(res) == 20 - -def test_compile_hybrid_3_extra(): - # Extra version of the test, with tons of live vars around the residual - # call that all contain a GC pointer. - myjitdriver = JitDriver(greens = [], reds = ['n', 'x0', 'x1', 'x2', 'x3', - 'x4', 'x5', 'x6', 'x7']) - def residual(n=26): - x = X() - x.next = X() - x.next.foo = n - return x - residual._look_inside_me_ = False - # - def main(n, x): - residual(5) - x0 = residual() - x1 = residual() - x2 = residual() - x3 = residual() - x4 = residual() - x5 = residual() - x6 = residual() - x7 = residual() - n *= 19 - while n > 0: - myjitdriver.can_enter_jit(n=n, x0=x0, x1=x1, x2=x2, x3=x3, - x4=x4, x5=x5, x6=x6, x7=x7) - myjitdriver.jit_merge_point(n=n, x0=x0, x1=x1, x2=x2, x3=x3, - x4=x4, x5=x5, x6=x6, x7=x7) + return (n, x) + args + return None, f, None + + + + def test_compile_hybrid_3(self): + x_test = X() + x_test.foo = 5 + self.run_orig('compile_hybrid_3', 6, x_test) # check that it does not raise CheckError + self.run('compile_hybrid_3') + + def define_compile_hybrid_3_extra(cls): + # Extra version of the test, with tons of live vars around the residual + # call that all contain a GC pointer. + def residual(n=26): + x = X() + x.next = X() + x.next.foo = n + return x + residual._look_inside_me_ = False + # + def before(n, x): + residual(5) + x0 = residual() + x1 = residual() + x2 = residual() + x3 = residual() + x4 = residual() + x5 = residual() + x6 = residual() + x7 = residual() + n *= 19 + return n, None, x0, x1, x2, x3, x4, x5, x6, x7, None, None + def f(n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s): x8 = residual() x9 = residual() check(x0.next.foo == 26) @@ -194,84 +261,54 @@ check(x9.next.foo == 26) x0, x1, x2, x3, x4, x5, x6, x7 = x7, x4, x6, x5, x3, x2, x9, x8 n -= 1 - main(6, None) # check that it does not raise AssertionError - res = compile_and_run(get_test(main), "hybrid", gcrootfinder="asmgcc", - jit=True) - assert int(res) == 20 - -def test_compile_hybrid_4(): - # Fourth version of the test, with __del__. - from pypy.rlib.debug import debug_print - class Counter: - cnt = 0 - counter = Counter() - class Z: - def __del__(self): - counter.cnt -= 1 - myjitdriver = JitDriver(greens = [], reds = ['n', 'x']) - def main(n, x): - debug_print('counter.cnt =', counter.cnt) - check(counter.cnt < 5) - counter.cnt = n // x.foo - while n > 0: - myjitdriver.can_enter_jit(n=n, x=x) - myjitdriver.jit_merge_point(n=n, x=x) + return n, None, x0, x1, x2, x3, x4, x5, x6, x7, None, None + return before, f, None + + def test_compile_hybrid_3_extra(self): + self.run_orig('compile_hybrid_3_extra', 6, None) # check that it does not raise CheckError + self.run('compile_hybrid_3_extra') + + def define_compile_hybrid_4(cls): + # Fourth version of the test, with __del__. + from pypy.rlib.debug import debug_print + class Counter: + cnt = 0 + counter = Counter() + class Z: + def __del__(self): + counter.cnt -= 1 + def before(n, x): + debug_print('counter.cnt =', counter.cnt) + check(counter.cnt < 5) + counter.cnt = n // x.foo + return n, x, None, None, None, None, None, None, None, None, None, None + def f(n, x, *args): Z() n -= x.foo - res = compile_and_run(get_test(main), "hybrid", gcrootfinder="asmgcc", - jit=True) - assert int(res) == 20 - -def test_compile_hybrid_5(): - # Test string manipulation. - myjitdriver = JitDriver(greens = [], reds = ['n', 'x', 's']) - def main(n, x): - s = '' - while n > 0: - myjitdriver.can_enter_jit(n=n, x=x, s=s) - myjitdriver.jit_merge_point(n=n, x=x, s=s) + return (n, x) + args + return before, f, None + + def test_compile_hybrid_4(self): + self.run('compile_hybrid_4') + + def define_compile_hybrid_5(cls): + # Test string manipulation. + def f(n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s): n -= x.foo s += str(n) - check(len(s) == 1*5 + 2*45 + 3*450 + 4*500) - res = compile_and_run(get_test(main), "hybrid", gcrootfinder="asmgcc", - jit=True) - assert int(res) == 20 - -def test_compile_hybrid_6(): - # Array manipulation (i.e. fixed-sized list). - myjitdriver = JitDriver(greens = [], reds = ['n', 'x', 'l']) - def main(n, x): - l = [] - while n > 0: - myjitdriver.can_enter_jit(n=n, x=x, l=l) - myjitdriver.jit_merge_point(n=n, x=x, l=l) - if n < 200: - l = [n, n, n] - if n < 100: - check(len(l) == 3) - check(l[0] == n) - check(l[1] == n) - check(l[2] == n) - n -= x.foo - check(len(l) == 3) - check(l[0] == 2) - check(l[1] == 2) - check(l[2] == 2) - res = compile_and_run(get_test(main), "hybrid", gcrootfinder="asmgcc", - jit=True) - assert int(res) == 20 - -def test_compile_hybrid_7(): - # Array of pointers (test the write barrier for setarrayitem_gc) - class X: - def __init__(self, x): - self.x = x - myjitdriver = JitDriver(greens = [], reds = ['n', 'x', 'l']) - def main(n, x): - l = [X(123)] - while n > 0: - myjitdriver.can_enter_jit(n=n, x=x, l=l) - myjitdriver.jit_merge_point(n=n, x=x, l=l) + return n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s + def after(n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s): + check(len(s) == 1*5 + 2*45 + 3*450 + 4*500) + return None, f, after + + def test_compile_hybrid_5(self): + self.run('compile_hybrid_5') + + def define_compile_hybrid_7(cls): + # Array of pointers (test the write barrier for setarrayitem_gc) + def before(n, x): + return n, x, None, None, None, None, None, None, None, None, [X(123)], None + def f(n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s): if n < 1900: check(l[0].x == 123) l = [None] * 16 @@ -310,31 +347,26 @@ check(l[14].x == n+130) check(l[15].x == n+140) n -= x.foo - check(len(l) == 16) - check(l[0].x == 123) - check(l[1].x == 2) - check(l[2].x == 12) - check(l[3].x == 22) - check(l[4].x == 32) - check(l[5].x == 42) - check(l[6].x == 52) - check(l[7].x == 62) - check(l[8].x == 72) - check(l[9].x == 82) - check(l[10].x == 92) - check(l[11].x == 102) - check(l[12].x == 112) - check(l[13].x == 122) - check(l[14].x == 132) - check(l[15].x == 142) - - class CPU386CollectOnLeave(CPU386): - - def execute_operations(self, loop, verbose=False): - op = CPU386.execute_operations(self, loop, verbose) - rgc.collect(0) - return op - - res = compile_and_run(get_test(main), "hybrid", gcrootfinder="asmgcc", - CPUClass=CPU386CollectOnLeave, jit=True) - assert int(res) == 20 + return n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s + def after(n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s): + check(len(l) == 16) + check(l[0].x == 123) + check(l[1].x == 2) + check(l[2].x == 12) + check(l[3].x == 22) + check(l[4].x == 32) + check(l[5].x == 42) + check(l[6].x == 52) + check(l[7].x == 62) + check(l[8].x == 72) + check(l[9].x == 82) + check(l[10].x == 92) + check(l[11].x == 102) + check(l[12].x == 112) + check(l[13].x == 122) + check(l[14].x == 132) + check(l[15].x == 142) + return before, f, after + + def test_compile_hybrid_7(self): + self.run('compile_hybrid_7') Modified: pypy/trunk/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmspot.py Thu Oct 8 20:47:20 2009 @@ -751,6 +751,13 @@ set_future_value._annspecialcase_ = 'specialize:ll_and_arg(2)' class WarmEnterState: + # make sure we always see the saner optimizer from an annotation + # point of view, otherwise we get lots of blocked ops + from pypy.jit.metainterp import optimize as _optimize + optimize_loop = staticmethod(_optimize.optimize_loop) + optimize_bridge = staticmethod(_optimize.optimize_bridge) + del _optimize + def __init__(self): # initialize the state with the default values of the # parameters specified in rlib/jit.py From arigo at codespeak.net Thu Oct 8 22:59:35 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Oct 2009 22:59:35 +0200 (CEST) Subject: [pypy-svn] r68259 - in pypy/branch/gc-compress/pypy/rpython/lltypesystem: . test Message-ID: <20091008205935.17943168022@codespeak.net> Author: arigo Date: Thu Oct 8 22:59:31 2009 New Revision: 68259 Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llarena.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/llmemory.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/lltype.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llarena.py Log: Kill the llarena feature of replacing an object with a stub. Instead, add the feature of swapping the header of an object for a completely different header. Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/llarena.py Thu Oct 8 22:59:31 2009 @@ -103,6 +103,48 @@ Arena.object_arena_location[container] = self, offset Arena.old_object_arena_location[container] = self, offset + def get_header_ofs(self, obj, oldheadersize): + headers = [] + for offset, ptr in self.objectptrs.items(): + size = self.objectsizes[offset] + if offset >= obj: # object is at or after 'obj' + continue + if offset + size < obj: # object is before the header + continue + headers.append(offset) + assert len(headers) >= 1, "header not found" + assert len(headers) == 1, "uh? multiple headers?" + offset = headers[0] + size = self.objectsizes[offset] + assert offset + size == obj + self.objectsizes[obj], "not a header?" + assert obj - offset == llmemory.raw_malloc_usage(oldheadersize) + return offset + + def swap_header(self, startobj, oldheadersize, newheadersize): + prev_end = [0] + for offset, ptr in self.objectptrs.items(): + size = self.objectsizes[offset] + if offset + size < startobj: + prev_end.append((offset + size)) + prev_end = max(prev_end) + new_bytes = llmemory.raw_malloc_usage(newheadersize) + assert prev_end <= startobj - new_bytes, "new header too big" + old_offset = self.get_header_ofs(startobj, oldheadersize) + assert isinstance(newheadersize, llmemory.GCHeaderOffset) + obj = self.objectptrs[startobj] + oldheaderptr = self.objectptrs.pop(old_offset) + oldheadersize.gcheaderbuilder.detach_header(obj, oldheaderptr) + newheader = newheadersize.gcheaderbuilder.new_header(obj) + newheader = llmemory.cast_ptr_to_adr(newheader) + # + del self.objectsizes[old_offset] + self.setobject(newheader, startobj - new_bytes, + new_bytes + self.objectsizes[startobj]) + oldheaderobj = oldheaderptr._as_obj() + del Arena.object_arena_location[oldheaderobj] + oldheaderobj._free() + + class fakearenaaddress(llmemory.fakeaddress): def __init__(self, arena, offset): @@ -309,6 +351,13 @@ """ return Arena(ptr.arena.nbytes, False).getaddr(0) +def arena_swap_header(obj, oldheadersize, newheadersize): + """Free the old header attached to 'obj', and attach one + of the size 'newheadersize' instead.""" + arena_addr = _getfakearenaaddress(obj) + arena_addr.arena.swap_header(arena_addr.offset, oldheadersize, + newheadersize) + # ____________________________________________________________ # # Translation support: the functions above turn into the code below. Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/llmemory.py Thu Oct 8 22:59:31 2009 @@ -376,9 +376,7 @@ # NOTE: the 'ptr' in the addresses must be normalized. # Use cast_ptr_to_adr() instead of directly fakeaddress() if unsure. def __init__(self, ptr): - if ptr is not None and ptr._obj0 is None: - ptr = None # null ptr => None - self.ptr = ptr + self.ptr = ptr or None # null ptr => None def __repr__(self): if self.ptr is None: @@ -417,8 +415,8 @@ def __eq__(self, other): if isinstance(other, fakeaddress): try: - obj1 = self._fixup().ptr - obj2 = other._fixup().ptr + obj1 = self.ptr + obj2 = other.ptr if obj1 is not None: obj1 = obj1._obj if obj2 is not None: obj2 = obj2._obj return obj1 == obj2 @@ -457,9 +455,8 @@ return self.ptr def _cast_to_ptr(self, EXPECTED_TYPE): - addr = self._fixup() - if addr: - return cast_any_ptr(EXPECTED_TYPE, addr.ptr) + if self: + return cast_any_ptr(EXPECTED_TYPE, self.ptr) else: return lltype.nullptr(EXPECTED_TYPE.TO) @@ -469,14 +466,6 @@ else: return 0 - def _fixup(self): - if self.ptr is not None and self.ptr._was_freed(): - # hack to support llarena.test_replace_object_with_stub() - from pypy.rpython.lltypesystem import llarena - return llarena._getfakearenaaddress(self) - else: - return self - # ____________________________________________________________ class NullAddressError(Exception): Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/lltype.py Thu Oct 8 22:59:31 2009 @@ -1194,10 +1194,6 @@ from pypy.rpython.lltypesystem import llmemory if isinstance(self._T, FuncType): return llmemory.fakeaddress(self) - elif self._was_freed(): - # hack to support llarena.test_replace_object_with_stub() - from pypy.rpython.lltypesystem import llarena - return llarena._oldobj_to_address(self._getobj(check=False)) elif isinstance(self._obj, _subarray): return llmemory.fakeaddress(self) ## # return an address built as an offset in the whole array Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llarena.py Thu Oct 8 22:59:31 2009 @@ -5,6 +5,7 @@ from pypy.rpython.lltypesystem.llarena import arena_reserve, arena_free from pypy.rpython.lltypesystem.llarena import round_up_for_allocation from pypy.rpython.lltypesystem.llarena import ArenaError, arena_new_view +from pypy.rpython.lltypesystem.llarena import arena_swap_header def test_arena(): S = lltype.Struct('S', ('x',lltype.Signed)) @@ -217,13 +218,15 @@ assert llmemory.cast_adr_to_int(a) == llmemory.cast_adr_to_int(a1) assert llmemory.cast_adr_to_int(a+1) == llmemory.cast_adr_to_int(a1) + 1 -def test_replace_object_with_stub(): +def test_replace_object_header(): from pypy.rpython.memory.gcheader import GCHeaderBuilder HDR = lltype.Struct('HDR', ('x', lltype.Signed)) + STUB = lltype.Struct('STUB', ('t', lltype.Char)) S = lltype.GcStruct('S', ('y', lltype.Signed), ('z', lltype.Signed)) - STUB = lltype.GcStruct('STUB', ('t', lltype.Char)) gcheaderbuilder = GCHeaderBuilder(HDR) size_gc_header = gcheaderbuilder.size_gc_header + gcstubbuilder = GCHeaderBuilder(STUB) + size_stub = gcstubbuilder.size_gc_header ssize = llmemory.raw_malloc_usage(llmemory.sizeof(S)) a = arena_malloc(13*ssize, True) @@ -231,31 +234,28 @@ arena_reserve(hdraddr, size_gc_header + llmemory.sizeof(S)) hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR)) hdr.x = 42 - obj = llmemory.cast_adr_to_ptr(hdraddr + size_gc_header, lltype.Ptr(S)) + objaddr = hdraddr + size_gc_header + obj = llmemory.cast_adr_to_ptr(objaddr, lltype.Ptr(S)) obj.y = -5 obj.z = -6 - hdraddr = llmemory.cast_ptr_to_adr(obj) - size_gc_header - arena_reset(hdraddr, size_gc_header + llmemory.sizeof(S), False) - arena_reserve(hdraddr, size_gc_header + llmemory.sizeof(STUB)) - - # check that it possible to reach the newly reserved HDR+STUB - # via the header of the old 'obj' pointer, both via the existing - # 'hdraddr': - hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR)) - hdr.x = 46 - stub = llmemory.cast_adr_to_ptr(hdraddr + size_gc_header, lltype.Ptr(STUB)) - stub.t = '!' - - # and via a (now-invalid) pointer to the old 'obj': (this is needed - # because during a garbage collection there are still pointers to - # the old 'obj' around to be fixed) - hdraddr = llmemory.cast_ptr_to_adr(obj) - size_gc_header - hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR)) - assert hdr.x == 46 - stub = llmemory.cast_adr_to_ptr(hdraddr + size_gc_header, - lltype.Ptr(STUB)) - assert stub.t == '!' + arena_swap_header(objaddr, size_gc_header, size_stub) + assert obj.y == -5 + assert obj.z == -6 + stubaddr = objaddr - size_stub + stub = llmemory.cast_adr_to_ptr(stubaddr, lltype.Ptr(STUB)) + stub.t = 'A' + py.test.raises(RuntimeError, "hdr.x") + py.test.raises(KeyError, "objaddr - size_gc_header") + + arena_swap_header(objaddr, size_stub, size_gc_header) + hdr2addr = objaddr - size_gc_header + hdr2 = llmemory.cast_adr_to_ptr(hdr2addr, lltype.Ptr(HDR)) + hdr2.x = 42 + assert obj.y == -5 + assert obj.z == -6 + py.test.raises(RuntimeError, "stub.t") + py.test.raises(KeyError, "objaddr - size_stub") def test_llinterpreted(): From arigo at codespeak.net Thu Oct 8 23:01:14 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 8 Oct 2009 23:01:14 +0200 (CEST) Subject: [pypy-svn] r68260 - pypy/branch/gc-compress/pypy/rpython/memory Message-ID: <20091008210114.B4BD116802C@codespeak.net> Author: arigo Date: Thu Oct 8 23:01:12 2009 New Revision: 68260 Modified: pypy/branch/gc-compress/pypy/rpython/memory/gcheader.py Log: Sorry, this goes with the previous checkin. Modified: pypy/branch/gc-compress/pypy/rpython/memory/gcheader.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gcheader.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gcheader.py Thu Oct 8 23:01:12 2009 @@ -16,17 +16,17 @@ def header_of_object(self, gcptr): # XXX hackhackhack - gcptr = gcptr._as_obj(check=False) + gcptr = gcptr._as_obj() if isinstance(gcptr, llmemory._gctransformed_wref): - return self.obj2header[gcptr._ptr._as_obj(check=False)] + return self.obj2header[gcptr._ptr._as_obj()] return self.obj2header[gcptr] def object_from_header(headerptr): - return header2obj[headerptr._as_obj(check=False)] + return header2obj[headerptr._as_obj()] object_from_header = staticmethod(object_from_header) def get_header(self, gcptr): - return self.obj2header.get(gcptr._as_obj(check=False), None) + return self.obj2header.get(gcptr._as_obj(), None) def attach_header(self, gcptr, headerptr): gcobj = gcptr._as_obj() @@ -38,6 +38,12 @@ self.obj2header[gcobj] = headerptr header2obj[headerptr._obj] = gcptr._as_ptr() + def detach_header(self, gcptr, headerptr): + gcobj = gcptr._as_obj() + assert self.obj2header[gcobj] == headerptr + del self.obj2header[gcobj] + del header2obj[headerptr._obj] + def new_header(self, gcptr): headerptr = lltype.malloc(self.HDR, immortal=True) self.attach_header(gcptr, headerptr) From afa at codespeak.net Thu Oct 8 23:41:59 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 8 Oct 2009 23:41:59 +0200 (CEST) Subject: [pypy-svn] r68261 - pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc Message-ID: <20091008214159.CB8E2168079@codespeak.net> Author: afa Date: Thu Oct 8 23:41:58 2009 New Revision: 68261 Modified: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py Log: Compute insn.previous_insns as we go then we'll be able to walk backward in visit_jump() Modified: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py Thu Oct 8 23:41:58 2009 @@ -301,7 +301,6 @@ try: if not self.list_call_insns(): return [] - self.findprevinsns() self.findframesize() self.fixlocalvars() self.trackgcroots() @@ -363,6 +362,18 @@ assert label not in self.labels, "duplicate label" self.labels[label] = Label(label, lineno) + def append_instruction(self, insn): + # Add the instruction to the list, and link it to the previous one. + previnsn = self.insns[-1] + self.insns.append(insn) + + try: + lst = insn.previous_insns + except AttributeError: + lst = insn.previous_insns = [] + if not isinstance(previnsn, InsnStop): + lst.append(previnsn) + def parse_instructions(self): self.insns = [InsnFunctionStart()] ignore_insns = False @@ -390,10 +401,13 @@ match = r_label.match(line) if match: insn = self.labels[match.group(1)] + if isinstance(insn, list): - self.insns.extend(insn) + for i in insn: + self.append_instruction(i) else: - self.insns.append(insn) + self.append_instruction(insn) + del self.currentlineno def find_missing_visit_method(self, opname): @@ -407,20 +421,6 @@ setattr(FunctionGcRootTracker, 'visit_' + opname, visit_nop) return self.visit_nop - def findprevinsns(self): - # builds the previous_insns of each Insn. For Labels, all jumps - # to them are already registered; all that is left to do is to - # make each Insn point to the Insn just before it. - for i in range(len(self.insns)-1): - previnsn = self.insns[i] - nextinsn = self.insns[i+1] - try: - lst = nextinsn.previous_insns - except AttributeError: - lst = nextinsn.previous_insns = [] - if not isinstance(previnsn, InsnStop): - lst.append(previnsn) - def list_call_insns(self): return [insn for insn in self.insns if isinstance(insn, InsnCall)] From cfbolz at codespeak.net Thu Oct 8 23:48:05 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 8 Oct 2009 23:48:05 +0200 (CEST) Subject: [pypy-svn] r68262 - in pypy/trunk/pypy/module/unicodedata: . test Message-ID: <20091008214805.8EB12168078@codespeak.net> Author: cfbolz Date: Thu Oct 8 23:48:04 2009 New Revision: 68262 Modified: pypy/trunk/pypy/module/unicodedata/interp_ucd.py pypy/trunk/pypy/module/unicodedata/test/test_unicodedata.py Log: issue452 resolved apply (a variant of) haypo's patch. Thanks. Modified: pypy/trunk/pypy/module/unicodedata/interp_ucd.py ============================================================================== --- pypy/trunk/pypy/module/unicodedata/interp_ucd.py (original) +++ pypy/trunk/pypy/module/unicodedata/interp_ucd.py Thu Oct 8 23:48:04 2009 @@ -147,7 +147,7 @@ def normalize(self, space, form, w_unistr): if not space.is_true(space.isinstance(w_unistr, space.w_unicode)): - raise TypeError, 'argument 2 must be unicode' + raise OperationError(space.w_TypeError, space.wrap('argument 2 must be unicode')) if form == 'NFC': composed = True decomposition = self._canon_decomposition Modified: pypy/trunk/pypy/module/unicodedata/test/test_unicodedata.py ============================================================================== --- pypy/trunk/pypy/module/unicodedata/test/test_unicodedata.py (original) +++ pypy/trunk/pypy/module/unicodedata/test/test_unicodedata.py Thu Oct 8 23:48:04 2009 @@ -78,6 +78,10 @@ if sys.maxunicode == 65535: raises(KeyError, unicodedata.lookup, "GOTHIC LETTER FAIHU") + def test_normalize(self): + import unicodedata + raises(TypeError, unicodedata.normalize, 'x') + class TestUnicodeData(object): def setup_class(cls): import random, unicodedata @@ -169,3 +173,5 @@ raises(KeyError, unicodedb_4_1_0.lookup, 'BENZENE RING WITH CIRCLE') raises(KeyError, unicodedb_3_2_0.name, 9187) raises(KeyError, unicodedb_4_1_0.name, 9187) + + From arigo at codespeak.net Fri Oct 9 10:05:00 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Oct 2009 10:05:00 +0200 (CEST) Subject: [pypy-svn] r68263 - in pypy/branch/gc-compress/pypy/rpython: lltypesystem lltypesystem/test memory Message-ID: <20091009080500.5FDF9168077@codespeak.net> Author: arigo Date: Fri Oct 9 10:04:58 2009 New Revision: 68263 Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llarena.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/llmemory.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/lltype.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llarena.py pypy/branch/gc-compress/pypy/rpython/memory/gcheader.py Log: Revert r68259 and r68260. Does not seem to work... Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/llarena.py Fri Oct 9 10:04:58 2009 @@ -103,48 +103,6 @@ Arena.object_arena_location[container] = self, offset Arena.old_object_arena_location[container] = self, offset - def get_header_ofs(self, obj, oldheadersize): - headers = [] - for offset, ptr in self.objectptrs.items(): - size = self.objectsizes[offset] - if offset >= obj: # object is at or after 'obj' - continue - if offset + size < obj: # object is before the header - continue - headers.append(offset) - assert len(headers) >= 1, "header not found" - assert len(headers) == 1, "uh? multiple headers?" - offset = headers[0] - size = self.objectsizes[offset] - assert offset + size == obj + self.objectsizes[obj], "not a header?" - assert obj - offset == llmemory.raw_malloc_usage(oldheadersize) - return offset - - def swap_header(self, startobj, oldheadersize, newheadersize): - prev_end = [0] - for offset, ptr in self.objectptrs.items(): - size = self.objectsizes[offset] - if offset + size < startobj: - prev_end.append((offset + size)) - prev_end = max(prev_end) - new_bytes = llmemory.raw_malloc_usage(newheadersize) - assert prev_end <= startobj - new_bytes, "new header too big" - old_offset = self.get_header_ofs(startobj, oldheadersize) - assert isinstance(newheadersize, llmemory.GCHeaderOffset) - obj = self.objectptrs[startobj] - oldheaderptr = self.objectptrs.pop(old_offset) - oldheadersize.gcheaderbuilder.detach_header(obj, oldheaderptr) - newheader = newheadersize.gcheaderbuilder.new_header(obj) - newheader = llmemory.cast_ptr_to_adr(newheader) - # - del self.objectsizes[old_offset] - self.setobject(newheader, startobj - new_bytes, - new_bytes + self.objectsizes[startobj]) - oldheaderobj = oldheaderptr._as_obj() - del Arena.object_arena_location[oldheaderobj] - oldheaderobj._free() - - class fakearenaaddress(llmemory.fakeaddress): def __init__(self, arena, offset): @@ -351,13 +309,6 @@ """ return Arena(ptr.arena.nbytes, False).getaddr(0) -def arena_swap_header(obj, oldheadersize, newheadersize): - """Free the old header attached to 'obj', and attach one - of the size 'newheadersize' instead.""" - arena_addr = _getfakearenaaddress(obj) - arena_addr.arena.swap_header(arena_addr.offset, oldheadersize, - newheadersize) - # ____________________________________________________________ # # Translation support: the functions above turn into the code below. Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/llmemory.py Fri Oct 9 10:04:58 2009 @@ -376,7 +376,9 @@ # NOTE: the 'ptr' in the addresses must be normalized. # Use cast_ptr_to_adr() instead of directly fakeaddress() if unsure. def __init__(self, ptr): - self.ptr = ptr or None # null ptr => None + if ptr is not None and ptr._obj0 is None: + ptr = None # null ptr => None + self.ptr = ptr def __repr__(self): if self.ptr is None: @@ -415,8 +417,8 @@ def __eq__(self, other): if isinstance(other, fakeaddress): try: - obj1 = self.ptr - obj2 = other.ptr + obj1 = self._fixup().ptr + obj2 = other._fixup().ptr if obj1 is not None: obj1 = obj1._obj if obj2 is not None: obj2 = obj2._obj return obj1 == obj2 @@ -455,8 +457,9 @@ return self.ptr def _cast_to_ptr(self, EXPECTED_TYPE): - if self: - return cast_any_ptr(EXPECTED_TYPE, self.ptr) + addr = self._fixup() + if addr: + return cast_any_ptr(EXPECTED_TYPE, addr.ptr) else: return lltype.nullptr(EXPECTED_TYPE.TO) @@ -466,6 +469,14 @@ else: return 0 + def _fixup(self): + if self.ptr is not None and self.ptr._was_freed(): + # hack to support llarena.test_replace_object_with_stub() + from pypy.rpython.lltypesystem import llarena + return llarena._getfakearenaaddress(self) + else: + return self + # ____________________________________________________________ class NullAddressError(Exception): Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/lltype.py Fri Oct 9 10:04:58 2009 @@ -1194,6 +1194,10 @@ from pypy.rpython.lltypesystem import llmemory if isinstance(self._T, FuncType): return llmemory.fakeaddress(self) + elif self._was_freed(): + # hack to support llarena.test_replace_object_with_stub() + from pypy.rpython.lltypesystem import llarena + return llarena._oldobj_to_address(self._getobj(check=False)) elif isinstance(self._obj, _subarray): return llmemory.fakeaddress(self) ## # return an address built as an offset in the whole array Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llarena.py Fri Oct 9 10:04:58 2009 @@ -5,7 +5,6 @@ from pypy.rpython.lltypesystem.llarena import arena_reserve, arena_free from pypy.rpython.lltypesystem.llarena import round_up_for_allocation from pypy.rpython.lltypesystem.llarena import ArenaError, arena_new_view -from pypy.rpython.lltypesystem.llarena import arena_swap_header def test_arena(): S = lltype.Struct('S', ('x',lltype.Signed)) @@ -218,15 +217,13 @@ assert llmemory.cast_adr_to_int(a) == llmemory.cast_adr_to_int(a1) assert llmemory.cast_adr_to_int(a+1) == llmemory.cast_adr_to_int(a1) + 1 -def test_replace_object_header(): +def test_replace_object_with_stub(): from pypy.rpython.memory.gcheader import GCHeaderBuilder HDR = lltype.Struct('HDR', ('x', lltype.Signed)) - STUB = lltype.Struct('STUB', ('t', lltype.Char)) S = lltype.GcStruct('S', ('y', lltype.Signed), ('z', lltype.Signed)) + STUB = lltype.GcStruct('STUB', ('t', lltype.Char)) gcheaderbuilder = GCHeaderBuilder(HDR) size_gc_header = gcheaderbuilder.size_gc_header - gcstubbuilder = GCHeaderBuilder(STUB) - size_stub = gcstubbuilder.size_gc_header ssize = llmemory.raw_malloc_usage(llmemory.sizeof(S)) a = arena_malloc(13*ssize, True) @@ -234,28 +231,31 @@ arena_reserve(hdraddr, size_gc_header + llmemory.sizeof(S)) hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR)) hdr.x = 42 - objaddr = hdraddr + size_gc_header - obj = llmemory.cast_adr_to_ptr(objaddr, lltype.Ptr(S)) + obj = llmemory.cast_adr_to_ptr(hdraddr + size_gc_header, lltype.Ptr(S)) obj.y = -5 obj.z = -6 - arena_swap_header(objaddr, size_gc_header, size_stub) - assert obj.y == -5 - assert obj.z == -6 - stubaddr = objaddr - size_stub - stub = llmemory.cast_adr_to_ptr(stubaddr, lltype.Ptr(STUB)) - stub.t = 'A' - py.test.raises(RuntimeError, "hdr.x") - py.test.raises(KeyError, "objaddr - size_gc_header") - - arena_swap_header(objaddr, size_stub, size_gc_header) - hdr2addr = objaddr - size_gc_header - hdr2 = llmemory.cast_adr_to_ptr(hdr2addr, lltype.Ptr(HDR)) - hdr2.x = 42 - assert obj.y == -5 - assert obj.z == -6 - py.test.raises(RuntimeError, "stub.t") - py.test.raises(KeyError, "objaddr - size_stub") + hdraddr = llmemory.cast_ptr_to_adr(obj) - size_gc_header + arena_reset(hdraddr, size_gc_header + llmemory.sizeof(S), False) + arena_reserve(hdraddr, size_gc_header + llmemory.sizeof(STUB)) + + # check that it possible to reach the newly reserved HDR+STUB + # via the header of the old 'obj' pointer, both via the existing + # 'hdraddr': + hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR)) + hdr.x = 46 + stub = llmemory.cast_adr_to_ptr(hdraddr + size_gc_header, lltype.Ptr(STUB)) + stub.t = '!' + + # and via a (now-invalid) pointer to the old 'obj': (this is needed + # because during a garbage collection there are still pointers to + # the old 'obj' around to be fixed) + hdraddr = llmemory.cast_ptr_to_adr(obj) - size_gc_header + hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR)) + assert hdr.x == 46 + stub = llmemory.cast_adr_to_ptr(hdraddr + size_gc_header, + lltype.Ptr(STUB)) + assert stub.t == '!' def test_llinterpreted(): Modified: pypy/branch/gc-compress/pypy/rpython/memory/gcheader.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gcheader.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gcheader.py Fri Oct 9 10:04:58 2009 @@ -16,17 +16,17 @@ def header_of_object(self, gcptr): # XXX hackhackhack - gcptr = gcptr._as_obj() + gcptr = gcptr._as_obj(check=False) if isinstance(gcptr, llmemory._gctransformed_wref): - return self.obj2header[gcptr._ptr._as_obj()] + return self.obj2header[gcptr._ptr._as_obj(check=False)] return self.obj2header[gcptr] def object_from_header(headerptr): - return header2obj[headerptr._as_obj()] + return header2obj[headerptr._as_obj(check=False)] object_from_header = staticmethod(object_from_header) def get_header(self, gcptr): - return self.obj2header.get(gcptr._as_obj(), None) + return self.obj2header.get(gcptr._as_obj(check=False), None) def attach_header(self, gcptr, headerptr): gcobj = gcptr._as_obj() @@ -38,12 +38,6 @@ self.obj2header[gcobj] = headerptr header2obj[headerptr._obj] = gcptr._as_ptr() - def detach_header(self, gcptr, headerptr): - gcobj = gcptr._as_obj() - assert self.obj2header[gcobj] == headerptr - del self.obj2header[gcobj] - del header2obj[headerptr._obj] - def new_header(self, gcptr): headerptr = lltype.malloc(self.HDR, immortal=True) self.attach_header(gcptr, headerptr) From pedronis at codespeak.net Fri Oct 9 11:26:40 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 9 Oct 2009 11:26:40 +0200 (CEST) Subject: [pypy-svn] r68264 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091009092640.52B4E49844D@codespeak.net> Author: pedronis Date: Fri Oct 9 11:26:38 2009 New Revision: 68264 Modified: pypy/trunk/pypy/jit/metainterp/compile.py pypy/trunk/pypy/jit/metainterp/jitprof.py pypy/trunk/pypy/jit/metainterp/optimize.py pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/simple_optimize.py pypy/trunk/pypy/jit/metainterp/test/test_jitprof.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: (cfbolz, pedronis) - moure counts, optimized ops/guards and forcings, and aborts - pass the metainterp staticdata to the optimizers - sanitize Profiler.print_stats Modified: pypy/trunk/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/compile.py (original) +++ pypy/trunk/pypy/jit/metainterp/compile.py Fri Oct 9 11:26:38 2009 @@ -59,7 +59,7 @@ metainterp_sd = metainterp.staticdata try: old_loop_token = metainterp_sd.state.optimize_loop( - metainterp_sd, old_loop_tokens, loop, metainterp.cpu) + metainterp_sd, old_loop_tokens, loop) except InvalidLoop: return None if old_loop_token is not None: @@ -249,8 +249,7 @@ try: target_loop_token = metainterp_sd.state.optimize_bridge(metainterp_sd, old_loop_tokens, - new_loop, - metainterp.cpu) + new_loop) except InvalidLoop: assert 0, "InvalidLoop in optimize_bridge?" return None Modified: pypy/trunk/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/jitprof.py (original) +++ pypy/trunk/pypy/jit/metainterp/jitprof.py Fri Oct 9 11:26:38 2009 @@ -2,6 +2,7 @@ """ A small helper module for profiling JIT """ +import os import time from pypy.rlib.debug import debug_print @@ -14,6 +15,11 @@ RECORDED_OPS BLACKHOLED_OPS GUARDS +OPT_OPS +OPT_GUARDS +OPT_FORCINGS +ABORT_TOO_LONG +ABORT_BRIDGE """ def _setup(): @@ -143,25 +149,36 @@ cnt = self.counters tim = self.times calls = self.calls - lines = ("Tracing: \t%d\t%f\n" % (cnt[TRACING], tim[TRACING]) + - "Backend: \t%d\t%f\n" % (cnt[BACKEND], tim[BACKEND]) + - "Running asm:\t%d\t%f\n" % (cnt[RUNNING], tim[RUNNING]) + - "Blackhole: \t%d\t%f\n" % (cnt[BLACKHOLE], tim[BLACKHOLE]) + - "TOTAL: \t\t%f\n" % (self.tk - self.starttime) + - "ops: \t%d\n" % cnt[OPS] + - " calls: \t%d\n" % calls[0][0] + - " pure calls: \t%d\n" % calls[0][1] + - "recorded ops: \t%d\n" % cnt[RECORDED_OPS] + - " calls: \t%d\n" % calls[1][0] + - " pure calls: \t%d\n" % calls[1][1] + - "guards: \t%d\n" % cnt[GUARDS] + - "blackholed ops:\t%d\n" % cnt[BLACKHOLED_OPS] + - " calls: \t%d\n" % calls[2][0] + - " pure calls: \t%d\n" % calls[2][1] - ) - import os - os.write(2, lines) - + self._print_line_time("Tracing", cnt[TRACING], tim[TRACING]) + self._print_line_time("Backend", cnt[BACKEND], tim[BACKEND]) + self._print_line_time("Running asm", cnt[RUNNING], tim[RUNNING]) + self._print_line_time("Blackhole", cnt[BLACKHOLE], tim[BLACKHOLE]) + line = "TOTAL: \t\t%f\n" % (self.tk - self.starttime, ) + os.write(2, line) + self._print_intline("ops", cnt[OPS]) + self._print_intline(" calls", calls[0][0]) + self._print_intline(" pure calls", calls[0][1]) + self._print_intline("recorded ops", cnt[RECORDED_OPS]) + self._print_intline(" calls", calls[1][0]) + self._print_intline(" pure calls", calls[1][1]) + self._print_intline("guards", cnt[GUARDS]) + self._print_intline("blackholed ops", calls[2][0]) + self._print_intline(" pure calls", calls[2][1]) + self._print_intline("opt ops", cnt[OPT_OPS]) + self._print_intline("opt guards", cnt[OPT_GUARDS]) + self._print_intline("forcings", cnt[OPT_FORCINGS]) + self._print_intline("trace too long", cnt[ABORT_BRIDGE]) + + def _print_line_time(self, string, i, tim): + final = "%s:%s\t%d\t%f\n" % (string, " " * max(0, 13-len(string)), i, tim) + os.write(2, final) + + def _print_intline(self, string, i): + final = string + ':' + " " * max(0, 16-len(string)) + final += '\t' + str(i) + '\n' + os.write(2, final) + + class BrokenProfilerData(Exception): pass Modified: pypy/trunk/pypy/jit/metainterp/optimize.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimize.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimize.py Fri Oct 9 11:26:38 2009 @@ -4,14 +4,15 @@ from pypy.jit.metainterp.optimizeopt import optimize_loop_1 from pypy.jit.metainterp.specnode import equals_specnodes -def optimize_loop(metainterp_sd, old_loop_tokens, loop, cpu): +def optimize_loop(metainterp_sd, old_loop_tokens, loop): + cpu = metainterp_sd.cpu metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations) finder = PerfectSpecializationFinder(cpu) finder.find_nodes_loop(loop) for old_loop_token in old_loop_tokens: if equals_specnodes(old_loop_token.specnodes, loop.token.specnodes): return old_loop_token - optimize_loop_1(cpu, loop) + optimize_loop_1(metainterp_sd, loop) return None # ____________________________________________________________ @@ -19,14 +20,15 @@ from pypy.jit.metainterp.optimizefindnode import BridgeSpecializationFinder from pypy.jit.metainterp.optimizeopt import optimize_bridge_1 -def optimize_bridge(metainterp_sd, old_loop_tokens, bridge, cpu): +def optimize_bridge(metainterp_sd, old_loop_tokens, bridge): + cpu = metainterp_sd.cpu metainterp_sd.logger_noopt.log_loop(bridge.inputargs, bridge.operations) finder = BridgeSpecializationFinder(cpu) finder.find_nodes_bridge(bridge) for old_loop_token in old_loop_tokens: if finder.bridge_matches(old_loop_token.specnodes): bridge.operations[-1].descr = old_loop_token # patch jump target - optimize_bridge_1(cpu, bridge) + optimize_bridge_1(metainterp_sd, bridge) return old_loop_token return None Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Fri Oct 9 11:26:38 2009 @@ -2,6 +2,7 @@ ConstFloat from pypy.jit.metainterp.history import Const, ConstInt, ConstPtr, ConstObj, REF from pypy.jit.metainterp.resoperation import rop, ResOperation +from pypy.jit.metainterp.jitprof import OPT_OPS, OPT_GUARDS, OPT_FORCINGS from pypy.jit.metainterp.executor import execute_nonspec from pypy.jit.metainterp.specnode import SpecNode, NotSpecNode, ConstantSpecNode from pypy.jit.metainterp.specnode import AbstractVirtualStructSpecNode @@ -15,21 +16,21 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.rpython.lltypesystem import lltype -def optimize_loop_1(cpu, loop): +def optimize_loop_1(metainterp_sd, loop): """Optimize loop.operations to make it match the input of loop.specnodes and to remove internal overheadish operations. Note that loop.specnodes must be applicable to the loop; you will probably get an AssertionError if not. """ - optimizer = Optimizer(cpu, loop) + optimizer = Optimizer(metainterp_sd, loop) optimizer.setup_virtuals_and_constants() optimizer.propagate_forward() -def optimize_bridge_1(cpu, bridge): +def optimize_bridge_1(metainterp_sd, bridge): """The same, but for a bridge. The only difference is that we don't expect 'specnodes' on the bridge. """ - optimizer = Optimizer(cpu, bridge) + optimizer = Optimizer(metainterp_sd, bridge) optimizer.propagate_forward() # ____________________________________________________________ @@ -362,15 +363,17 @@ class Optimizer(object): - def __init__(self, cpu, loop): - self.cpu = cpu + def __init__(self, metainterp_sd, loop): + self.metainterp_sd = metainterp_sd + self.cpu = metainterp_sd.cpu self.loop = loop self.values = {} self.interned_refs = {} - self.resumedata_memo = resume.ResumeDataLoopMemo(cpu) + self.resumedata_memo = resume.ResumeDataLoopMemo(self.cpu) self.heap_op_optimizer = HeapOpOptimizer(self) def forget_numberings(self, virtualbox): + self.metainterp_sd.profiler.count(OPT_FORCINGS) self.resumedata_memo.forget_numberings(virtualbox) def getinterned(self, box): @@ -513,7 +516,9 @@ op = op.clone() must_clone = False op.args[i] = box + self.metainterp_sd.profiler.count(OPT_OPS) if op.is_guard(): + self.metainterp_sd.profiler.count(OPT_GUARDS) self.store_final_boxes_in_guard(op) elif op.can_raise(): self.exception_might_have_happened = True Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Fri Oct 9 11:26:38 2009 @@ -12,6 +12,7 @@ from pypy.jit.metainterp.logger import Logger from pypy.jit.metainterp.jitprof import BLACKHOLED_OPS, EmptyProfiler from pypy.jit.metainterp.jitprof import GUARDS, RECORDED_OPS +from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG, ABORT_BRIDGE from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import specialize from pypy.rlib.jit import DEBUG_OFF, DEBUG_PROFILE, DEBUG_STEPS, DEBUG_DETAILED @@ -803,6 +804,7 @@ try: self.metainterp.reached_can_enter_jit(self.env) except GiveUp: + self.metainterp.staticdata.profiler.count(ABORT_BRIDGE) self.metainterp.switch_to_blackhole() if self.metainterp.is_blackholing(): self.blackhole_reached_merge_point(self.env) @@ -1315,6 +1317,7 @@ if not self.is_blackholing(): warmrunnerstate = self.staticdata.state if len(self.history.operations) > warmrunnerstate.trace_limit: + self.staticdata.profiler.count(ABORT_TOO_LONG) self.switch_to_blackhole() def _interpret(self): Modified: pypy/trunk/pypy/jit/metainterp/simple_optimize.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/simple_optimize.py (original) +++ pypy/trunk/pypy/jit/metainterp/simple_optimize.py Fri Oct 9 11:26:38 2009 @@ -7,7 +7,7 @@ EMPTY_VALUES = {} -def optimize_loop(metainterp_sd, old_loops, loop, cpu=None): +def optimize_loop(metainterp_sd, old_loops, loop): if old_loops: assert len(old_loops) == 1 return old_loops[0] @@ -16,7 +16,7 @@ # we need it since the backend can modify those lists, which make # get_guard_op in compile.py invalid # in fact, x86 modifies this list for moving GCs - memo = resume.ResumeDataLoopMemo(cpu) + memo = resume.ResumeDataLoopMemo(metainterp_sd.cpu) newoperations = [] for op in loop.operations: if op.is_guard(): @@ -29,6 +29,6 @@ loop.operations = newoperations return None -def optimize_bridge(metainterp_sd, old_loops, loop, cpu=None): - optimize_loop(metainterp_sd, [], loop, cpu) +def optimize_bridge(metainterp_sd, old_loops, loop): + optimize_loop(metainterp_sd, [], loop) return old_loops[0] Modified: pypy/trunk/pypy/jit/metainterp/test/test_jitprof.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_jitprof.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_jitprof.py Fri Oct 9 11:26:38 2009 @@ -55,7 +55,7 @@ ] assert profiler.events == expected assert profiler.times == [2, 1, 1, 1] - assert profiler.counters == [1, 1, 1, 1, 4, 3, 1, 1] + assert profiler.counters == [1, 1, 1, 1, 4, 3, 1, 1, 7, 1, 0, 0, 0] def test_simple_loop_with_call(self): @dont_look_inside Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Fri Oct 9 11:26:38 2009 @@ -10,6 +10,7 @@ from pypy.jit.metainterp.optimizeopt import optimize_loop_1 from pypy.jit.metainterp.optimizeutil import InvalidLoop from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt +from pypy.jit.metainterp.jitprof import EmptyProfiler from pypy.jit.metainterp import executor, compile, resume from pypy.jit.metainterp.resoperation import rop, opname, ResOperation from pypy.jit.metainterp.test.oparser import pure_parse @@ -22,13 +23,19 @@ self.jitcode = code self.pc = pc self.exception_target = exc_target + +class FakeMetaInterpStaticData(object): + + def __init__(self, cpu): + self.cpu = cpu + self.profiler = EmptyProfiler() def test_store_final_boxes_in_guard(): from pypy.jit.metainterp.compile import ResumeGuardDescr from pypy.jit.metainterp.resume import tag, TAGBOX b0 = BoxInt() b1 = BoxInt() - opt = optimizeopt.Optimizer(None, None) + opt = optimizeopt.Optimizer(FakeMetaInterpStaticData(None), None) fdescr = ResumeGuardDescr(None) op = ResOperation(rop.GUARD_TRUE, [], None, descr=fdescr) # setup rd data @@ -179,7 +186,7 @@ loop.token.specnodes = self.unpack_specnodes(spectext) # self.loop = loop - optimize_loop_1(self.cpu, loop) + optimize_loop_1(FakeMetaInterpStaticData(self.cpu), loop) # expected = self.parse(optops) self.assert_equal(loop, expected) From arigo at codespeak.net Fri Oct 9 11:38:10 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Oct 2009 11:38:10 +0200 (CEST) Subject: [pypy-svn] r68265 - in pypy/branch/gc-compress/pypy: rpython/lltypesystem rpython/memory translator/c translator/c/src translator/c/test Message-ID: <20091009093810.E14B049844D@codespeak.net> Author: arigo Date: Fri Oct 9 11:38:09 2009 New Revision: 68265 Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llarena.py pypy/branch/gc-compress/pypy/rpython/memory/lltypelayout.py pypy/branch/gc-compress/pypy/translator/c/primitive.py pypy/branch/gc-compress/pypy/translator/c/src/mem.h pypy/branch/gc-compress/pypy/translator/c/test/test_lltyped.py Log: Enhance round_up_for_allocation() to take a second parameter, which is a constant giving a minimum size. Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/llarena.py Fri Oct 9 11:38:09 2009 @@ -240,12 +240,15 @@ """A size that is rounded up in order to preserve alignment of objects following it. For arenas containing heterogenous objects. """ - def __init__(self, basesize): + def __init__(self, basesize, minsize): assert isinstance(basesize, llmemory.AddressOffset) + assert isinstance(minsize, llmemory.AddressOffset) or minsize == 0 self.basesize = basesize + self.minsize = minsize def __repr__(self): - return '< RoundedUpForAllocation %r >' % (self.basesize,) + return '< RoundedUpForAllocation %r %r >' % (self.basesize, + self.minsize) def known_nonneg(self): return self.basesize.known_nonneg() @@ -299,10 +302,14 @@ % (addr.offset,)) addr.arena.allocate_object(addr.offset, size) -def round_up_for_allocation(size): +def round_up_for_allocation(size, minsize=0): """Round up the size in order to preserve alignment of objects - following an object. For arenas containing heterogenous objects.""" - return RoundedUpForAllocation(size) + following an object. For arenas containing heterogenous objects. + If minsize is specified, it gives a minimum on the resulting size.""" + return _round_up_for_allocation(size, minsize) + +def _round_up_for_allocation(size, minsize): # internal + return RoundedUpForAllocation(size, minsize) def arena_new_view(ptr): """Return a fresh memory view on an arena @@ -401,10 +408,11 @@ sandboxsafe=True) llimpl_round_up_for_allocation = rffi.llexternal('ROUND_UP_FOR_ALLOCATION', - [lltype.Signed], lltype.Signed, + [lltype.Signed, lltype.Signed], + lltype.Signed, sandboxsafe=True, _nowrapper=True) -register_external(round_up_for_allocation, [int], int, +register_external(_round_up_for_allocation, [int, int], int, 'll_arena.round_up_for_allocation', llimpl=llimpl_round_up_for_allocation, llfakeimpl=round_up_for_allocation, Modified: pypy/branch/gc-compress/pypy/rpython/memory/lltypelayout.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/lltypelayout.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/lltypelayout.py Fri Oct 9 11:38:09 2009 @@ -118,6 +118,10 @@ return 0 elif isinstance(offset, llarena.RoundedUpForAllocation): basesize = convert_offset_to_int(offset.basesize) + if isinstance(offset.minsize, llmemory.AddressOffset): + minsize = convert_offset_to_int(offset.minsize) + if minsize > basesize: + basesize = minsize mask = memory_alignment - 1 return (basesize + mask) & ~ mask else: Modified: pypy/branch/gc-compress/pypy/translator/c/primitive.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/primitive.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/primitive.py Fri Oct 9 11:38:09 2009 @@ -50,8 +50,9 @@ elif type(value) == GCHeaderOffset: return '0' elif type(value) == RoundedUpForAllocation: - return 'ROUND_UP_FOR_ALLOCATION(%s)' % ( - name_signed(value.basesize, db)) + return 'ROUND_UP_FOR_ALLOCATION(%s, %s)' % ( + name_signed(value.basesize, db), + name_signed(value.minsize, db)) elif isinstance(value, CDefinedIntSymbolic): return str(value.expr) elif isinstance(value, ComputedIntSymbolic): Modified: pypy/branch/gc-compress/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/src/mem.h (original) +++ pypy/branch/gc-compress/pypy/translator/c/src/mem.h Fri Oct 9 11:38:09 2009 @@ -14,8 +14,9 @@ struct rpy_memory_alignment_test1 s; }; #define MEMORY_ALIGNMENT offsetof(struct rpy_memory_alignment_test2, s) -#define ROUND_UP_FOR_ALLOCATION(x) \ - (((x) + (MEMORY_ALIGNMENT-1)) & ~(MEMORY_ALIGNMENT-1)) +#define ROUND_UP_FOR_ALLOCATION(x, minsize) \ + ((((x)>=(minsize)?(x):(minsize)) \ + + (MEMORY_ALIGNMENT-1)) & ~(MEMORY_ALIGNMENT-1)) extern char __gcmapstart; extern char __gcmapend; Modified: pypy/branch/gc-compress/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/test/test_lltyped.py Fri Oct 9 11:38:09 2009 @@ -741,3 +741,34 @@ fn = self.getcompiled(f, [int]) res = fn(-1) assert res == 5 + + def test_round_up_for_allocation(self): + from pypy.rpython.lltypesystem import llmemory, llarena + S = Struct('S', ('x', Char), ('y', Char)) + M = Struct('M', ('x', Char), ('y', Signed)) + # + def g(): + ssize = llarena.round_up_for_allocation(llmemory.sizeof(S)) + msize = llarena.round_up_for_allocation(llmemory.sizeof(M)) + smsize = llarena.round_up_for_allocation(llmemory.sizeof(S), + llmemory.sizeof(M)) + mssize = llarena.round_up_for_allocation(llmemory.sizeof(M), + llmemory.sizeof(S)) + return ssize, msize, smsize, mssize + # + glob_sizes = g() + # + def check((ssize, msize, smsize, mssize)): + assert ssize == llmemory.sizeof(Signed) + assert msize == llmemory.sizeof(Signed) * 2 + assert smsize == msize + assert mssize == msize + # + def f(): + check(glob_sizes) + check(g()) + return 42 + # + fn = self.getcompiled(f, []) + res = fn() + assert res == 42 From arigo at codespeak.net Fri Oct 9 11:56:08 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Oct 2009 11:56:08 +0200 (CEST) Subject: [pypy-svn] r68266 - in pypy/branch/gc-compress/pypy: rpython/lltypesystem rpython/lltypesystem/test translator/c translator/c/src Message-ID: <20091009095608.4416E49844D@codespeak.net> Author: arigo Date: Fri Oct 9 11:56:07 2009 New Revision: 68266 Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llgroup.py pypy/branch/gc-compress/pypy/translator/c/primitive.py pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h Log: Yet Another Symbolic type, to combine a USHORT and some flags into a single Signed value. Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py Fri Oct 9 11:56:07 2009 @@ -69,3 +69,26 @@ assert skipoffset.TYPE == lltype.typeOf(self.member).TO assert skipoffset.repeat == 1 return self.grpptr._as_obj().members[self.index + 1]._as_ptr() + + +class CombinedSymbolic(llmemory.Symbolic): + """A general-purpose Signed symbolic that combines a USHORT and the + rest of the word (typically flags). Only supports extracting the USHORT + with 'llop.extract_ushort', and extracting the rest of the word with + '&~0xFFFF' or with a direct masking like '&0x10000'. + """ + def annotation(self): + from pypy.annotation import model + return model.SomeInteger() + + def lltype(self): + return lltype.Signed + + def __init__(self, lowpart, rest): + assert (rest & 0xFFFF) == 0 + self.lowpart = lowpart + self.rest = rest + + def __and__(self, other): + assert (other & 0xFFFF) == 0 + return self.rest & other Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py Fri Oct 9 11:56:07 2009 @@ -413,6 +413,7 @@ 'get_group_member': LLOp(canfold=True), 'get_next_group_member':LLOp(canfold=True), 'is_group_member_zero': LLOp(canfold=True), + 'extract_ushort': LLOp(canfold=True), # __________ used by the JIT ________ Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py Fri Oct 9 11:56:07 2009 @@ -175,6 +175,13 @@ assert isinstance(y, (int, llmemory.AddressOffset)) return intmask(x + y) +def op_int_and(x, y): + if not isinstance(x, int): + from pypy.rpython.lltypesystem import llgroup + assert isinstance(x, llgroup.CombinedSymbolic) + assert isinstance(y, int) + return x & y + def op_int_mul(x, y): assert isinstance(x, (int, llmemory.AddressOffset)) assert isinstance(y, (int, llmemory.AddressOffset)) @@ -410,6 +417,11 @@ assert isinstance(memberoffset, int) return memberoffset == 0 +def op_extract_ushort(combinedoffset): + from pypy.rpython.lltypesystem import llgroup + assert isinstance(combinedoffset, llgroup.CombinedSymbolic) + return combinedoffset.lowpart + # ____________________________________________________________ def get_op_impl(opname): Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llgroup.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llgroup.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llgroup.py Fri Oct 9 11:56:07 2009 @@ -67,6 +67,12 @@ test.build() grpptr = test.grpptr g1x = [test.g1a, test.g1b] + cs1 = CombinedSymbolic(test.g1b, 0x450000) + cs2 = CombinedSymbolic(test.g1b, 0x410000) + assert llop.extract_ushort(rffi.USHORT, cs1) is test.g1b + assert cs1 & ~0xFFFF == 0x450000 + cslist = [cs1, cs2] + # def f(): p = llop.get_group_member(lltype.Ptr(test.S1), grpptr, test.g1a) assert p == test.p1a @@ -92,5 +98,12 @@ p = llop.get_group_member(lltype.Ptr(test.S1), grpptr, g1x[i]) assert p.x == expected[i] # + for i in range(2): + s = llop.extract_ushort(rffi.USHORT, cslist[i]) + p = llop.get_group_member(lltype.Ptr(test.S1), grpptr, s) + assert p == test.p1b + assert cslist[0] & ~0xFFFF == 0x450000 + assert cslist[1] & ~0xFFFF == 0x410000 + # return 42 return f Modified: pypy/branch/gc-compress/pypy/translator/c/primitive.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/primitive.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/primitive.py Fri Oct 9 11:56:07 2009 @@ -57,6 +57,8 @@ return str(value.expr) elif isinstance(value, ComputedIntSymbolic): value = value.compute_fn() + elif isinstance(value, llgroup.CombinedSymbolic): + return '(%s|%dL)' % (name_ushort(value.lowpart, db), value.rest) else: raise Exception("unimplemented symbolic %r"%value) if value is None: Modified: pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h (original) +++ pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h Fri Oct 9 11:56:07 2009 @@ -14,6 +14,9 @@ #define OP_IS_GROUP_MEMBER_ZERO(compactoffset, r) \ r = (compactoffset == 0) +#define OP_EXTRACT_USHORT(value, r) \ + r = (unsigned short)value + /* A macro to crash at compile-time if sizeof(group) is too large. Uses a hack that I've found on some random forum. Haaaaaaaaaackish. */ #define PYPY_GROUP_CHECK_SIZE(groupname) \ From arigo at codespeak.net Fri Oct 9 12:00:19 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Oct 2009 12:00:19 +0200 (CEST) Subject: [pypy-svn] r68267 - in pypy/branch/gc-compress/pypy/rpython/memory: . gc gc/test gctransform test Message-ID: <20091009100019.A9DD349844D@codespeak.net> Author: arigo Date: Fri Oct 9 12:00:17 2009 New Revision: 68267 Modified: pypy/branch/gc-compress/pypy/rpython/memory/gc/base.py pypy/branch/gc-compress/pypy/rpython/memory/gc/semispace.py pypy/branch/gc-compress/pypy/rpython/memory/gc/test/test_direct.py pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py pypy/branch/gc-compress/pypy/rpython/memory/test/test_gctypelayout.py Log: Use the "minimal size" argument of round_up_for_allocation. Modified: pypy/branch/gc-compress/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gc/base.py Fri Oct 9 12:00:17 2009 @@ -13,6 +13,7 @@ malloc_zero_filled = False prebuilt_gc_objects_are_static_roots = True can_realloc = False + object_minimal_size = 0 def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE): self.gcheaderbuilder = GCHeaderBuilder(self.HDR) Modified: pypy/branch/gc-compress/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gc/semispace.py Fri Oct 9 12:00:17 2009 @@ -58,6 +58,8 @@ ('forw', llmemory.Address)) FORWARDSTUBPTR = lltype.Ptr(FORWARDSTUB) + object_minimal_size = llmemory.sizeof(FORWARDSTUB) + # the following values override the default arguments of __init__ when # translating to a real backend. TRANSLATION_PARAMS = {'space_size': 8*1024*1024} # XXX adjust Modified: pypy/branch/gc-compress/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gc/test/test_direct.py Fri Oct 9 12:00:17 2009 @@ -68,7 +68,7 @@ self.gc.DEBUG = True self.rootwalker = DirectRootWalker(self) self.gc.set_root_walker(self.rootwalker) - self.layoutbuilder = TypeLayoutBuilder() + self.layoutbuilder = TypeLayoutBuilder(self.GCClass) self.get_type_id = self.layoutbuilder.get_type_id self.layoutbuilder.initialize_gc_query_function(self.gc) self.gc.setup() Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py Fri Oct 9 12:00:17 2009 @@ -130,7 +130,7 @@ if hasattr(translator, '_jit2gc'): self.layoutbuilder = translator._jit2gc['layoutbuilder'] else: - self.layoutbuilder = TransformerLayoutBuilder() + self.layoutbuilder = TransformerLayoutBuilder(GCClass) self.layoutbuilder.transformer = self self.get_type_id = self.layoutbuilder.get_type_id Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py Fri Oct 9 12:00:17 2009 @@ -23,6 +23,7 @@ ("finalizer", FINALIZERTYPE), ("fixedsize", lltype.Signed), ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), + hints={'immutable': True}, ) VARSIZE_TYPE_INFO = lltype.Struct("varsize_type_info", ("header", TYPE_INFO), @@ -30,6 +31,7 @@ ("ofstovar", lltype.Signed), ("ofstolength", lltype.Signed), ("varofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), + hints={'immutable': True}, ) TYPE_INFO_PTR = lltype.Ptr(TYPE_INFO) VARSIZE_TYPE_INFO_PTR = lltype.Ptr(VARSIZE_TYPE_INFO) @@ -121,7 +123,7 @@ info.finalizer = builder.make_finalizer_funcptr_for_type(TYPE) if not TYPE._is_varsize(): info.fixedsize = llarena.round_up_for_allocation( - llmemory.sizeof(TYPE)) + llmemory.sizeof(TYPE), builder.GCClass.object_minimal_size) # note about round_up_for_allocation(): in the 'info' table # we put a rounded-up size only for fixed-size objects. For # varsize ones, the GC must anyway compute the size at run-time @@ -163,7 +165,8 @@ can_add_new_types = True can_encode_type_shape = True # set to False initially by the JIT - def __init__(self): + def __init__(self, GCClass): + self.GCClass = GCClass self.make_type_info_group() self.id_of_type = {} # {LLTYPE: type_id} self.seen_roots = {} Modified: pypy/branch/gc-compress/pypy/rpython/memory/test/test_gctypelayout.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/test/test_gctypelayout.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/test/test_gctypelayout.py Fri Oct 9 12:00:17 2009 @@ -4,6 +4,9 @@ from pypy.rpython.test.test_llinterp import get_interpreter from pypy.objspace.flow.model import Constant +class FakeGC: + object_minimal_size = 0 + def getname(T): try: return "field:" + T._name @@ -34,7 +37,7 @@ def test_layout_builder(): # XXX a very minimal test - layoutbuilder = TypeLayoutBuilder() + layoutbuilder = TypeLayoutBuilder(FakeGC) for T1, T2 in [(GC_A, GC_S), (GC_A2, GC_S2), (GC_S3, GC_S2)]: tid1 = layoutbuilder.get_type_id(T1) tid2 = layoutbuilder.get_type_id(T2) @@ -44,7 +47,7 @@ assert len(lst1) == len(lst2) def test_constfold(): - layoutbuilder = TypeLayoutBuilder() + layoutbuilder = TypeLayoutBuilder(FakeGC) tid1 = layoutbuilder.get_type_id(GC_A) tid2 = layoutbuilder.get_type_id(GC_S3) class MockGC: From cfbolz at codespeak.net Fri Oct 9 12:30:32 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 9 Oct 2009 12:30:32 +0200 (CEST) Subject: [pypy-svn] r68268 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091009103032.DE30C168074@codespeak.net> Author: cfbolz Date: Fri Oct 9 12:30:32 2009 New Revision: 68268 Modified: pypy/trunk/pypy/jit/metainterp/jitprof.py Log: nonsense Modified: pypy/trunk/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/jitprof.py (original) +++ pypy/trunk/pypy/jit/metainterp/jitprof.py Fri Oct 9 12:30:32 2009 @@ -167,7 +167,8 @@ self._print_intline("opt ops", cnt[OPT_OPS]) self._print_intline("opt guards", cnt[OPT_GUARDS]) self._print_intline("forcings", cnt[OPT_FORCINGS]) - self._print_intline("trace too long", cnt[ABORT_BRIDGE]) + self._print_intline("trace too long", cnt[ABORT_TOO_LONG]) + self._print_intline("bridge abort", cnt[ABORT_BRIDGE]) def _print_line_time(self, string, i, tim): final = "%s:%s\t%d\t%f\n" % (string, " " * max(0, 13-len(string)), i, tim) From arigo at codespeak.net Fri Oct 9 12:34:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Oct 2009 12:34:42 +0200 (CEST) Subject: [pypy-svn] r68269 - in pypy/branch/gc-compress/pypy: rpython/lltypesystem rpython/memory/gc translator/c/src Message-ID: <20091009103442.04EBE49844D@codespeak.net> Author: arigo Date: Fri Oct 9 12:34:42 2009 New Revision: 68269 Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py pypy/branch/gc-compress/pypy/rpython/memory/gc/markcompact.py pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h Log: Phew. The mark&compact tests start passing again. Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py Fri Oct 9 12:34:42 2009 @@ -92,3 +92,7 @@ def __and__(self, other): assert (other & 0xFFFF) == 0 return self.rest & other + + def __or__(self, other): + assert (other & 0xFFFF) == 0 + return CombinedSymbolic(self.lowpart, self.rest | other) Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py Fri Oct 9 12:34:42 2009 @@ -414,6 +414,7 @@ 'get_next_group_member':LLOp(canfold=True), 'is_group_member_zero': LLOp(canfold=True), 'extract_ushort': LLOp(canfold=True), + 'combine_ushort': LLOp(canfold=True), # __________ used by the JIT ________ Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py Fri Oct 9 12:34:42 2009 @@ -182,6 +182,13 @@ assert isinstance(y, int) return x & y +def op_int_or(x, y): + if not isinstance(x, int): + from pypy.rpython.lltypesystem import llgroup + assert isinstance(x, llgroup.CombinedSymbolic) + assert isinstance(y, int) + return x | y + def op_int_mul(x, y): assert isinstance(x, (int, llmemory.AddressOffset)) assert isinstance(y, (int, llmemory.AddressOffset)) @@ -422,6 +429,10 @@ assert isinstance(combinedoffset, llgroup.CombinedSymbolic) return combinedoffset.lowpart +def op_combine_ushort(ushort, rest): + from pypy.rpython.lltypesystem import llgroup + return llgroup.CombinedSymbolic(ushort, rest) + # ____________________________________________________________ def get_op_impl(opname): Modified: pypy/branch/gc-compress/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gc/markcompact.py Fri Oct 9 12:34:42 2009 @@ -1,7 +1,7 @@ import time -from pypy.rpython.lltypesystem import lltype, llmemory, llarena +from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup from pypy.rpython.memory.gc.base import MovingGCBase from pypy.rlib.debug import ll_assert from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE @@ -12,11 +12,10 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.objectmodel import we_are_translated from pypy.rpython.lltypesystem import rffi +from pypy.rpython.memory.gcheader import GCHeaderBuilder -TYPEID_MASK = 0xffff0000 -first_gcflag = 2 +first_gcflag = 1 << 16 GCFLAG_MARKBIT = first_gcflag << 0 -GCFLAG_EXTERNAL = first_gcflag << 1 memoryError = MemoryError() @@ -68,8 +67,9 @@ TID_TYPE = rffi.USHORT BYTES_PER_TID = rffi.sizeof(TID_TYPE) + class MarkCompactGC(MovingGCBase): - HDR = lltype.Struct('header', ('forward_ptr', llmemory.Address)) + HDR = lltype.Struct('header', ('tid', lltype.Signed)) TID_BACKUP = lltype.Array(TID_TYPE, hints={'nolength':True}) WEAKREF_OFFSETS = lltype.Array(lltype.Signed) @@ -79,7 +79,7 @@ malloc_zero_filled = True inline_simple_malloc = True inline_simple_malloc_varsize = True - first_unused_gcflag = first_gcflag << 2 + first_unused_gcflag = first_gcflag << 1 total_collection_time = 0.0 total_collection_count = 0 @@ -100,21 +100,18 @@ self.objects_with_weakrefs = self.AddressStack() self.tid_backup = lltype.nullptr(self.TID_BACKUP) - # flags = 1 to make lltype & llmemory happy about odd/even pointers - - def init_gc_object(self, addr, typeid, flags=1): + def init_gc_object(self, addr, typeid16, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.forward_ptr = llmemory.cast_int_to_adr((typeid << 16) | flags) + hdr.tid = self.combine(typeid16, flags) - def init_gc_object_immortal(self, addr, typeid, flags=1): + def init_gc_object_immortal(self, addr, typeid16, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - flags |= GCFLAG_EXTERNAL - hdr.forward_ptr = llmemory.cast_int_to_adr((typeid << 16) | flags) + hdr.tid = self.combine(typeid16, flags) # XXX we can store forward_ptr to itself, if we fix C backend # so that get_forwarding_address(obj) returns # obj itself if obj is a prebuilt object - def malloc_fixedsize_clear(self, typeid, size, can_collect, + def malloc_fixedsize_clear(self, typeid16, size, can_collect, has_finalizer=False, contains_weakptr=False): size_gc_header = self.gcheaderbuilder.size_gc_header totalsize = size_gc_header + size @@ -122,7 +119,7 @@ if raw_malloc_usage(totalsize) > self.top_of_space - result: result = self.obtain_free_space(totalsize) llarena.arena_reserve(result, totalsize) - self.init_gc_object(result, typeid) + self.init_gc_object(result, typeid16) self.free += totalsize if has_finalizer: self.objects_with_finalizers.append(result + size_gc_header) @@ -130,7 +127,7 @@ self.objects_with_weakrefs.append(result + size_gc_header) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) - def malloc_varsize_clear(self, typeid, length, size, itemsize, + def malloc_varsize_clear(self, typeid16, length, size, itemsize, offset_to_length, can_collect): size_gc_header = self.gcheaderbuilder.size_gc_header nonvarsize = size_gc_header + size @@ -143,7 +140,7 @@ if raw_malloc_usage(totalsize) > self.top_of_space - result: result = self.obtain_free_space(totalsize) llarena.arena_reserve(result, totalsize) - self.init_gc_object(result, typeid) + self.init_gc_object(result, typeid16) (result + size_gc_header + offset_to_length).signed[0] = length self.free = result + llarena.round_up_for_allocation(totalsize) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) @@ -319,11 +316,26 @@ self.objects_with_finalizers.delete() self.objects_with_finalizers = objects_with_finalizers - def get_tid(self, addr): - return llmemory.cast_adr_to_int(self.header(addr).forward_ptr) + def header(self, addr): + # like header(), but asserts that we have a normal header + hdr = MovingGCBase.header(self, addr) + if not we_are_translated(): + assert isinstance(hdr.tid, llgroup.CombinedSymbolic) + return hdr + + def header_forwarded(self, addr): + # like header(), but asserts that we have a forwarding header + hdr = MovingGCBase.header(self, addr) + if not we_are_translated(): + assert isinstance(hdr.tid, int) + return hdr + + def combine(self, typeid16, flags): + return llop.combine_ushort(lltype.Signed, typeid16, flags) def get_type_id(self, addr): - return self.get_tid(addr) >> 16 + tid = self.header(addr).tid + return llop.extract_ushort(rffi.USHORT, tid) def mark_roots_recursively(self): self.root_walker.walk_roots( @@ -350,13 +362,13 @@ self.to_see.append(root.address[0]) def mark(self, obj): - previous = self.get_tid(obj) - self.header(obj).forward_ptr = llmemory.cast_int_to_adr(previous | GCFLAG_MARKBIT) + self.header(obj).tid |= GCFLAG_MARKBIT def marked(self, obj): - return self.get_tid(obj) & GCFLAG_MARKBIT + return self.header(obj).tid & GCFLAG_MARKBIT def update_forward_pointers(self, toaddr, num_of_alive_objs): + self.base_forwarding_addr = toaddr fromaddr = self.space size_gc_header = self.gcheaderbuilder.size_gc_header i = 0 @@ -366,8 +378,7 @@ objsize = self.get_size(obj) totalsize = size_gc_header + objsize if not self.marked(obj): - self.set_forwarding_address(obj, NULL, i) - hdr.forward_ptr = NULL + self.set_null_forwarding_address(obj, i) else: llarena.arena_reserve(toaddr, totalsize) self.set_forwarding_address(obj, toaddr, i) @@ -438,30 +449,44 @@ if pointer.address[0] != NULL: pointer.address[0] = self.get_forwarding_address(pointer.address[0]) - def is_forwarded(self, addr): - return self.header(addr).forward_ptr != NULL - def _is_external(self, obj): - tid = self.get_tid(obj) - return (tid & 1) and (tid & GCFLAG_EXTERNAL) + return not (self.space <= obj < self.top_of_space) def get_forwarding_address(self, obj): if self._is_external(obj): return obj - return self.header(obj).forward_ptr + self.size_gc_header() + return self.get_header_forwarded_addr(obj) - def set_forwarding_address(self, obj, newaddr, num): + def set_null_forwarding_address(self, obj, num): self.backup_typeid(num, obj) - self.header(obj).forward_ptr = newaddr + hdr = self.header(obj) + hdr.tid = -1 # make the object forwarded to NULL + + def set_forwarding_address(self, obj, newobjhdr, num): + self.backup_typeid(num, obj) + forward_offset = newobjhdr - self.base_forwarding_addr + hdr = self.header(obj) + hdr.tid = forward_offset # make the object forwarded to newobj + + def restore_normal_header(self, obj, num): + # Reverse of set_forwarding_address(). + typeid16 = self.get_typeid_from_backup(num) + hdr = self.header_forwarded(obj) + hdr.tid = self.combine(typeid16, 0) # restore the normal header + + def get_header_forwarded_addr(self, obj): + return (self.base_forwarding_addr + + self.header_forwarded(obj).tid + + self.gcheaderbuilder.size_gc_header) def surviving(self, obj): - return self._is_external(obj) or self.header(obj).forward_ptr != NULL + return self._is_external(obj) or self.header_forwarded(obj).tid != -1 def backup_typeid(self, num, obj): - self.tid_backup[num] = rffi.cast(rffi.USHORT, self.get_type_id(obj)) + self.tid_backup[num] = self.get_type_id(obj) def get_typeid_from_backup(self, num): - return rffi.cast(lltype.Signed, self.tid_backup[num]) + return self.tid_backup[num] def get_size_from_backup(self, obj, num): typeid = self.get_typeid_from_backup(num) @@ -484,7 +509,6 @@ num = 0 while fromaddr < self.free: obj = fromaddr + size_gc_header - hdr = llmemory.cast_adr_to_ptr(fromaddr, lltype.Ptr(self.HDR)) objsize = self.get_size_from_backup(obj, num) totalsize = size_gc_header + objsize if not self.surviving(obj): @@ -492,16 +516,16 @@ # we clear it to make debugging easier llarena.arena_reset(fromaddr, totalsize, False) else: - ll_assert(self.is_forwarded(obj), "not forwarded, surviving obj") - forward_ptr = hdr.forward_ptr if resizing: end = fromaddr - val = (self.get_typeid_from_backup(num) << 16) + 1 - hdr.forward_ptr = llmemory.cast_int_to_adr(val) - if fromaddr != forward_ptr: + forward_obj = self.get_header_forwarded_addr(obj) + self.restore_normal_header(obj, num) + if obj != forward_obj: #llop.debug_print(lltype.Void, "Copying from to", # fromaddr, forward_ptr, totalsize) - llmemory.raw_memmove(fromaddr, forward_ptr, totalsize) + llmemory.raw_memmove(fromaddr, + forward_obj - size_gc_header, + totalsize) if resizing and end - start > GC_CLEARANCE: diff = end - start #llop.debug_print(lltype.Void, "Cleaning", start, diff) Modified: pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h (original) +++ pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h Fri Oct 9 12:34:42 2009 @@ -11,12 +11,15 @@ #define OP_GET_NEXT_GROUP_MEMBER(groupptr, compactoffset, skipoffset, r) \ r = ((char*)groupptr) + ((long)compactoffset)*sizeof(long) + skipoffset -#define OP_IS_GROUP_MEMBER_ZERO(compactoffset, r) \ +#define OP_IS_GROUP_MEMBER_ZERO(compactoffset, r) \ r = (compactoffset == 0) -#define OP_EXTRACT_USHORT(value, r) \ +#define OP_EXTRACT_USHORT(value, r) \ r = (unsigned short)value +#define OP_COMBINE_USHORT(ushort, rest, r) \ + r = ((long)ushort) | rest; + /* A macro to crash at compile-time if sizeof(group) is too large. Uses a hack that I've found on some random forum. Haaaaaaaaaackish. */ #define PYPY_GROUP_CHECK_SIZE(groupname) \ From arigo at codespeak.net Fri Oct 9 12:38:31 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Oct 2009 12:38:31 +0200 (CEST) Subject: [pypy-svn] r68270 - pypy/branch/gc-compress/pypy/rpython/memory Message-ID: <20091009103831.B38AD168074@codespeak.net> Author: arigo Date: Fri Oct 9 12:38:31 2009 New Revision: 68270 Modified: pypy/branch/gc-compress/pypy/rpython/memory/gcwrapper.py Log: Fix for test_gc. Modified: pypy/branch/gc-compress/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gcwrapper.py Fri Oct 9 12:38:31 2009 @@ -18,7 +18,8 @@ self.gc.setup() def prepare_graphs(self, flowgraphs): - layoutbuilder = DirectRunLayoutBuilder(self.llinterp) + layoutbuilder = DirectRunLayoutBuilder(self.gc.__class__, + self.llinterp) self.get_type_id = layoutbuilder.get_type_id layoutbuilder.initialize_gc_query_function(self.gc) @@ -159,9 +160,9 @@ class DirectRunLayoutBuilder(gctypelayout.TypeLayoutBuilder): - def __init__(self, llinterp): + def __init__(self, GCClass, llinterp): self.llinterp = llinterp - super(DirectRunLayoutBuilder, self).__init__() + super(DirectRunLayoutBuilder, self).__init__(GCClass) def make_finalizer_funcptr_for_type(self, TYPE): from pypy.rpython.memory.gctransform.support import get_rtti, \ From arigo at codespeak.net Fri Oct 9 13:05:57 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Oct 2009 13:05:57 +0200 (CEST) Subject: [pypy-svn] r68271 - in pypy/branch/gc-compress/pypy: rpython/memory/gctransform translator/c Message-ID: <20091009110557.34C40168076@codespeak.net> Author: arigo Date: Fri Oct 9 13:05:56 2009 New Revision: 68271 Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py pypy/branch/gc-compress/pypy/translator/c/database.py pypy/branch/gc-compress/pypy/translator/c/node.py Log: Fix translation. A lot of efforts for what is finally just one line... Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py Fri Oct 9 13:05:56 2009 @@ -457,6 +457,10 @@ newgcdependencies = [] newgcdependencies.append(ll_static_roots_inside) + # also add the type_info_group's members into newgcdependencies, + # to make sure that they are all followed (only a part of them + # might have been followed by a previous enum_dependencies()). + newgcdependencies.extend(self.layoutbuilder.type_info_group.members) self.write_typeid_list() return newgcdependencies Modified: pypy/branch/gc-compress/pypy/translator/c/database.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/database.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/database.py Fri Oct 9 13:05:56 2009 @@ -143,7 +143,7 @@ return 'struct %s @' % ( valid_identifier('pypy_opaque_' + T.tag),) elif isinstance(T, llgroup.GroupType): - return 'XXX-dont-use-me @' + return "/*don't use me*/ void @" else: raise Exception("don't know about type %r" % (T,)) Modified: pypy/branch/gc-compress/pypy/translator/c/node.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/node.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/node.py Fri Oct 9 13:05:56 2009 @@ -914,6 +914,9 @@ return self.obj.name def enum_dependencies(self): + # note: for the group used by the GC, it can grow during this phase, + # which means that we might not return all members yet. This is + # fixed by finish_tables() in rpython/memory/gctransform/framework.py for member in self.obj.members: yield member._as_ptr() From antocuni at codespeak.net Fri Oct 9 14:58:13 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 9 Oct 2009 14:58:13 +0200 (CEST) Subject: [pypy-svn] r68272 - in pypy/trunk/pypy/rlib: . test Message-ID: <20091009125813.79AB5168076@codespeak.net> Author: antocuni Date: Fri Oct 9 14:58:12 2009 New Revision: 68272 Modified: pypy/trunk/pypy/rlib/nonconst.py pypy/trunk/pypy/rlib/test/test_nonconst.py Log: make sure we can use the idiom "if NonConstant(False): ..." to make the annotator seeing some code without executing it Modified: pypy/trunk/pypy/rlib/nonconst.py ============================================================================== --- pypy/trunk/pypy/rlib/nonconst.py (original) +++ pypy/trunk/pypy/rlib/nonconst.py Fri Oct 9 14:58:12 2009 @@ -17,6 +17,9 @@ def __setattr__(self, attr, value): setattr(self.__dict__['constant'], attr, value) + def __nonzero__(self): + return bool(self.__dict__['constant']) + class EntryNonConstant(ExtRegistryEntry): _about_ = NonConstant Modified: pypy/trunk/pypy/rlib/test/test_nonconst.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_nonconst.py (original) +++ pypy/trunk/pypy/rlib/test/test_nonconst.py Fri Oct 9 14:58:12 2009 @@ -47,3 +47,15 @@ if option.view: a.translator.view() assert isinstance(s, SomeInstance) + +def test_bool_nonconst(): + def fn(): + return bool(NonConstant(False)) + + assert not fn() + + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert s.knowntype is bool + assert not hasattr(s, 'const') + From antocuni at codespeak.net Fri Oct 9 15:00:33 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 9 Oct 2009 15:00:33 +0200 (CEST) Subject: [pypy-svn] r68273 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091009130033.01F1D168076@codespeak.net> Author: antocuni Date: Fri Oct 9 15:00:32 2009 New Revision: 68273 Modified: pypy/trunk/pypy/jit/metainterp/warmspot.py Log: the hack introduced in r68258 does not work reliably, as staticmethods are not supported as default values for fields (e.g., it breaks ootype translation and has unclear behaviour on lltype). Revert it and introduce a new hack to make the annotator seeing the full optimizer Modified: pypy/trunk/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmspot.py Fri Oct 9 15:00:32 2009 @@ -23,6 +23,7 @@ from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper from pypy.jit.metainterp.jitprof import Profiler, EmptyProfiler from pypy.rlib.jit import DEBUG_STEPS, DEBUG_DETAILED, DEBUG_OFF, DEBUG_PROFILE +from pypy.rlib.nonconst import NonConstant # ____________________________________________________________ # Bootstrapping @@ -751,13 +752,6 @@ set_future_value._annspecialcase_ = 'specialize:ll_and_arg(2)' class WarmEnterState: - # make sure we always see the saner optimizer from an annotation - # point of view, otherwise we get lots of blocked ops - from pypy.jit.metainterp import optimize as _optimize - optimize_loop = staticmethod(_optimize.optimize_loop) - optimize_bridge = staticmethod(_optimize.optimize_bridge) - del _optimize - def __init__(self): # initialize the state with the default values of the # parameters specified in rlib/jit.py @@ -802,7 +796,7 @@ elif optimizer == OPTIMIZER_FULL: from pypy.jit.metainterp import optimize self.optimize_loop = optimize.optimize_loop - self.optimize_bridge = optimize.optimize_bridge + self.optimize_bridge = optimize.optimize_bridge else: raise ValueError("unknown optimizer") @@ -822,6 +816,11 @@ # not too bad. def maybe_compile_and_run(self, *args): + if NonConstant(False): + # make sure we always see the saner optimizer from an annotation + # point of view, otherwise we get lots of blocked ops + self.set_param_optimizer(OPTIMIZER_FULL) + # get the greenargs and look for the cell corresponding to the hash greenargs = args[:num_green_args] argshash = self.getkeyhash(*greenargs) & self.hashtablemask From antocuni at codespeak.net Fri Oct 9 15:02:44 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 9 Oct 2009 15:02:44 +0200 (CEST) Subject: [pypy-svn] r68274 - pypy/trunk/pypy/jit/backend/cli Message-ID: <20091009130244.E5D4A168076@codespeak.net> Author: antocuni Date: Fri Oct 9 15:02:42 2009 New Revision: 68274 Modified: pypy/trunk/pypy/jit/backend/cli/runner.py Log: fix cli jit tests Modified: pypy/trunk/pypy/jit/backend/cli/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/runner.py (original) +++ pypy/trunk/pypy/jit/backend/cli/runner.py Fri Oct 9 15:02:42 2009 @@ -463,6 +463,7 @@ selfclass = ootype.nullruntimeclass fieldname = '' _is_pointer_field = False + _is_float_field = False def __init__(self, TYPE, fieldname): DescrWithKey.__init__(self, (TYPE, fieldname)) @@ -488,6 +489,9 @@ def is_pointer_field(self): return self._is_pointer_field + def is_float_field(self): + return self._is_float_field + def equals(self, other): assert isinstance(other, FieldDescr) return self.key == other.key From pedronis at codespeak.net Fri Oct 9 15:03:56 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 9 Oct 2009 15:03:56 +0200 (CEST) Subject: [pypy-svn] r68275 - in pypy/trunk/pypy/interpreter: . test Message-ID: <20091009130356.8A596168002@codespeak.net> Author: pedronis Date: Fri Oct 9 15:03:56 2009 New Revision: 68275 Modified: pypy/trunk/pypy/interpreter/eval.py pypy/trunk/pypy/interpreter/function.py pypy/trunk/pypy/interpreter/gateway.py pypy/trunk/pypy/interpreter/pycode.py pypy/trunk/pypy/interpreter/test/test_function.py Log: (cfbolz, pedronis) fast path for flat calls and default args moving toward killing ArgumentsFromValueStack Modified: pypy/trunk/pypy/interpreter/eval.py ============================================================================== --- pypy/trunk/pypy/interpreter/eval.py (original) +++ pypy/trunk/pypy/interpreter/eval.py Fri Oct 9 15:03:56 2009 @@ -13,9 +13,14 @@ hidden_applevel = False # n >= 0 : arity - # -n: special cases - # -99: hopeless - fast_natural_arity = -99 + # FLATPYCALL = 0x100 + # n|FLATPYCALL: pycode flat case + # FLATPYCALL<=1): special cases + # HOPELESS: hopeless + FLATPYCALL = 0x100 + PASSTHROUGHARGS1 = 0x200 + HOPELESS = 0x400 + fast_natural_arity = HOPELESS def __init__(self, co_name): self.co_name = co_name Modified: pypy/trunk/pypy/interpreter/function.py ============================================================================== --- pypy/trunk/pypy/interpreter/function.py (original) +++ pypy/trunk/pypy/interpreter/function.py Fri Oct 9 15:03:56 2009 @@ -82,7 +82,7 @@ if i < nargs: new_frame.fastlocals_w[i] = args_w[i] return new_frame.run() - elif nargs >= 1 and fast_natural_arity == -1: + elif nargs >= 1 and fast_natural_arity == Code.PASSTHROUGHARGS1: assert isinstance(code, gateway.BuiltinCodePassThroughArguments1) return code.funcrun_obj(self, args_w[0], Arguments(self.space, @@ -115,10 +115,16 @@ return code.fastcall_4(self.space, self, frame.peekvalue(3), frame.peekvalue(2), frame.peekvalue(1), frame.peekvalue(0)) - elif (nargs|PyCode.FLATPYCALL) == fast_natural_arity: + elif (nargs|Code.FLATPYCALL) == fast_natural_arity: assert isinstance(code, PyCode) - return self._flat_pycall(code, nargs, frame) - elif fast_natural_arity == -1 and nargs >= 1: + return self._flat_pycall(code, nargs, frame, 0) + elif fast_natural_arity&Code.FLATPYCALL: + natural_arity = fast_natural_arity&0xff + if natural_arity > nargs >= natural_arity-len(self.defs_w): + assert isinstance(code, PyCode) + return self._flat_pycall(code, nargs, frame, + natural_arity-nargs) + elif fast_natural_arity == Code.PASSTHROUGHARGS1 and nargs >= 1: assert isinstance(code, gateway.BuiltinCodePassThroughArguments1) w_obj = frame.peekvalue(nargs-1) args = frame.make_arguments(nargs-1) @@ -135,13 +141,22 @@ if isinstance(args, ArgumentsFromValuestack): args.frame = None - def _flat_pycall(self, code, nargs, frame): + def _flat_pycall(self, code, nargs, frame, defs_to_load): # code is a PyCode new_frame = self.space.createframe(code, self.w_func_globals, self.closure) for i in xrange(nargs): w_arg = frame.peekvalue(nargs-1-i) new_frame.fastlocals_w[i] = w_arg + + if defs_to_load: + defs_w = self.defs_w + ndefs = len(defs_w) + start = ndefs-defs_to_load + i = nargs + for j in xrange(start, ndefs): + new_frame.fastlocals_w[i] = defs_w[j] + i += 1 return new_frame.run() def getdict(self): Modified: pypy/trunk/pypy/interpreter/gateway.py ============================================================================== --- pypy/trunk/pypy/interpreter/gateway.py (original) +++ pypy/trunk/pypy/interpreter/gateway.py Fri Oct 9 15:03:56 2009 @@ -559,7 +559,7 @@ class BuiltinCodePassThroughArguments1(BuiltinCode): _immutable_ = True - fast_natural_arity = -1 + fast_natural_arity = eval.Code.PASSTHROUGHARGS1 def funcrun_obj(self, func, w_obj, args): space = func.space Modified: pypy/trunk/pypy/interpreter/pycode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pycode.py (original) +++ pypy/trunk/pypy/interpreter/pycode.py Fri Oct 9 15:03:56 2009 @@ -170,11 +170,10 @@ _code_new_w = staticmethod(_code_new_w) - FLATPYCALL = 0x100 def _compute_flatcall(self): # Speed hack! - self.fast_natural_arity = -99 + self.fast_natural_arity = eval.Code.HOPELESS if self.co_flags & (CO_VARARGS | CO_VARKEYWORDS): return if len(self._args_as_cellvars) > 0: @@ -182,7 +181,7 @@ if self.co_argcount > 0xff: return - self.fast_natural_arity = PyCode.FLATPYCALL | self.co_argcount + self.fast_natural_arity = eval.Code.FLATPYCALL | self.co_argcount def funcrun(self, func, args): frame = self.space.createframe(self, func.w_func_globals, Modified: pypy/trunk/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_function.py (original) +++ pypy/trunk/pypy/interpreter/test/test_function.py Fri Oct 9 15:03:56 2009 @@ -1,5 +1,6 @@ import unittest +from pypy.interpreter import eval from pypy.interpreter.function import Function, Method, descr_function_get from pypy.interpreter.pycode import PyCode from pypy.interpreter.argument import Arguments @@ -77,6 +78,25 @@ assert res[0] == 23 assert res[1] == 42 + def test_simple_call_default(self): + def func(arg1, arg2=11, arg3=111): + return arg1, arg2, arg3 + res = func(1) + assert res[0] == 1 + assert res[1] == 11 + assert res[2] == 111 + res = func(1, 22) + assert res[0] == 1 + assert res[1] == 22 + assert res[2] == 111 + res = func(1, 22, 333) + assert res[0] == 1 + assert res[1] == 22 + assert res[2] == 333 + + raises(TypeError, func) + raises(TypeError, func, 1, 2, 3, 4) + def test_simple_varargs(self): def func(arg1, *args): return arg1, args @@ -564,3 +584,63 @@ """) assert space.is_true(w_res) + + def test_flatcall_default_arg(self): + space = self.space + + def f(a, b): + return a+b + code = PyCode._from_code(self.space, f.func_code) + fn = Function(self.space, code, self.space.newdict(), + defs_w=[space.newint(1)]) + + assert fn.code.fast_natural_arity == 2|eval.Code.FLATPYCALL + + def bomb(*args): + assert False, "shortcutting should have avoided this" + + code.funcrun = bomb + code.funcrun_obj = bomb + + w_3 = space.newint(3) + w_4 = space.newint(4) + # ignore this for now + #w_res = space.call_function(fn, w_3) + # assert space.eq_w(w_res, w_4) + + w_res = space.appexec([fn, w_3], """(f, x): + return f(x) + """) + + assert space.eq_w(w_res, w_4) + + def test_flatcall_default_arg_method(self): + space = self.space + + def f(self, a, b): + return a+b + code = PyCode._from_code(self.space, f.func_code) + fn = Function(self.space, code, self.space.newdict(), + defs_w=[space.newint(1)]) + + assert fn.code.fast_natural_arity == 3|eval.Code.FLATPYCALL + + def bomb(*args): + assert False, "shortcutting should have avoided this" + + code.funcrun = bomb + code.funcrun_obj = bomb + + w_3 = space.newint(3) + + w_res = space.appexec([fn, w_3], """(f, x): + class A(object): + m = f + y = A().m(x) + b = A().m + z = b(x) + return y+10*z + """) + + assert space.eq_w(w_res, space.wrap(44)) + From arigo at codespeak.net Fri Oct 9 16:09:41 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Oct 2009 16:09:41 +0200 (CEST) Subject: [pypy-svn] r68276 - pypy/trunk/pypy/translator/c Message-ID: <20091009140941.D7E56168074@codespeak.net> Author: arigo Date: Fri Oct 9 16:09:41 2009 New Revision: 68276 Modified: pypy/trunk/pypy/translator/c/genc.py Log: Oups. "make clean" did not remove the .s files, if you have some left around from an explicit command like "make implement.s". Modified: pypy/trunk/pypy/translator/c/genc.py ============================================================================== --- pypy/trunk/pypy/translator/c/genc.py (original) +++ pypy/trunk/pypy/translator/c/genc.py Fri Oct 9 16:09:41 2009 @@ -463,8 +463,8 @@ mk.definition('PROFOPT', profopt) rules = [ - ('clean', '', 'rm -f $(OBJECTS) $(TARGET) $(GCMAPFILES) *.gc?? ../module_cache/*.gc??'), - ('clean_noprof', '', 'rm -f $(OBJECTS) $(TARGET) $(GCMAPFILES)'), + ('clean', '', 'rm -f $(OBJECTS) $(TARGET) $(GCMAPFILES) $(ASMFILES) *.gc?? ../module_cache/*.gc??'), + ('clean_noprof', '', 'rm -f $(OBJECTS) $(TARGET) $(GCMAPFILES) $(ASMFILES)'), ('debug', '', '$(MAKE) CFLAGS="-g -O1 -DRPY_ASSERT" $(TARGET)'), ('debug_exc', '', '$(MAKE) CFLAGS="-g -O1 -DRPY_ASSERT -DDO_LOG_EXC" $(TARGET)'), ('debug_mem', '', '$(MAKE) CFLAGS="-g -O1 -DRPY_ASSERT -DTRIVIAL_MALLOC_DEBUG" $(TARGET)'), @@ -486,8 +486,10 @@ mk.rule(*rule) if self.config.translation.gcrootfinder == 'asmgcc': + sfiles = ['%s.s' % (cfile[:-2],) for cfile in mk.cfiles] lblsfiles = ['%s.lbl.s' % (cfile[:-2],) for cfile in mk.cfiles] gcmapfiles = ['%s.gcmap' % (cfile[:-2],) for cfile in mk.cfiles] + mk.definition('ASMFILES', sfiles) mk.definition('ASMLBLFILES', lblsfiles) mk.definition('GCMAPFILES', gcmapfiles) mk.definition('OBJECTS', '$(ASMLBLFILES) gcmaptable.s') From pedronis at codespeak.net Fri Oct 9 16:41:15 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 9 Oct 2009 16:41:15 +0200 (CEST) Subject: [pypy-svn] r68277 - in pypy/trunk/pypy: interpreter objspace/std Message-ID: <20091009144115.05FB849844D@codespeak.net> Author: pedronis Date: Fri Oct 9 16:41:15 2009 New Revision: 68277 Modified: pypy/trunk/pypy/interpreter/argument.py pypy/trunk/pypy/interpreter/baseobjspace.py pypy/trunk/pypy/interpreter/function.py pypy/trunk/pypy/interpreter/pyframe.py pypy/trunk/pypy/objspace/std/objspace.py Log: (cfbolz, pedronis): kill ArgumentsFromValuestack. Modified: pypy/trunk/pypy/interpreter/argument.py ============================================================================== --- pypy/trunk/pypy/interpreter/argument.py (original) +++ pypy/trunk/pypy/interpreter/argument.py Fri Oct 9 16:41:15 2009 @@ -186,127 +186,6 @@ raise NotImplementedError() -class ArgumentsFromValuestack(AbstractArguments): - """ - Collects the arguments of a function call as stored on a PyFrame - valuestack. - - Only for the case of purely positional arguments, for now. - """ - - def __init__(self, space, frame, nargs=0): - self.space = space - self.frame = frame - self.nargs = nargs - - def firstarg(self): - if self.nargs <= 0: - return None - return self.frame.peekvalue(self.nargs - 1) - - def prepend(self, w_firstarg): - "Return a new Arguments with a new argument inserted first." - args_w = self.frame.peekvalues(self.nargs) - return Arguments(self.space, [w_firstarg] + args_w) - - def __repr__(self): - return 'ArgumentsFromValuestack(%r, %r)' % (self.frame, self.nargs) - - def has_keywords(self): - return False - - def unpack(self): - args_w = [None] * self.nargs - for i in range(self.nargs): - args_w[i] = self.frame.peekvalue(self.nargs - 1 - i) - return args_w, {} - - def fixedunpack(self, argcount): - if self.nargs > argcount: - raise ValueError, "too many arguments (%d expected)" % argcount - elif self.nargs < argcount: - raise ValueError, "not enough arguments (%d expected)" % argcount - data_w = [None] * self.nargs - nargs = self.nargs - for i in range(nargs): - data_w[i] = self.frame.peekvalue(nargs - 1 - i) - return data_w - - def _rawshape(self, nextra=0): - return nextra + self.nargs, (), False, False - - def _match_signature(self, w_firstarg, scope_w, argnames, has_vararg=False, has_kwarg=False, defaults_w=[], blindargs=0): - """Parse args and kwargs according to the signature of a code object, - or raise an ArgErr in case of failure. - Return the number of arguments filled in. - """ - co_argcount = len(argnames) - extravarargs = None - input_argcount = 0 - - if w_firstarg is not None: - upfront = 1 - if co_argcount > 0: - scope_w[0] = w_firstarg - input_argcount = 1 - else: - extravarargs = [ w_firstarg ] - else: - upfront = 0 - - avail = upfront + self.nargs - - if avail + len(defaults_w) < co_argcount: - raise ArgErrCount(self.nargs , 0, - (co_argcount, has_vararg, has_kwarg), - defaults_w, co_argcount - avail - len(defaults_w)) - if avail > co_argcount and not has_vararg: - raise ArgErrCount(self.nargs, 0, - (co_argcount, has_vararg, has_kwarg), - defaults_w, 0) - - if avail >= co_argcount: - for i in range(co_argcount - input_argcount): - scope_w[i + input_argcount] = self.frame.peekvalue(self.nargs - 1 - i) - if has_vararg: - if upfront > co_argcount: - assert extravarargs is not None - stararg_w = extravarargs + [None] * self.nargs - for i in range(self.nargs): - stararg_w[i + len(extravarargs)] = self.frame.peekvalue(self.nargs - 1 - i) - else: - args_left = co_argcount - upfront - stararg_w = [None] * (avail - co_argcount) - for i in range(args_left, self.nargs): - stararg_w[i - args_left] = self.frame.peekvalue(self.nargs - 1 - i) - scope_w[co_argcount] = self.space.newtuple(stararg_w) - else: - for i in range(self.nargs): - scope_w[i + input_argcount] = self.frame.peekvalue(self.nargs - 1 - i) - ndefaults = len(defaults_w) - missing = co_argcount - avail - first_default = ndefaults - missing - for i in range(missing): - scope_w[avail + i] = defaults_w[first_default + i] - if has_vararg: - scope_w[co_argcount] = self.space.newtuple([]) - - if has_kwarg: - scope_w[co_argcount + has_vararg] = self.space.newdict() - return co_argcount + has_vararg + has_kwarg - - def flatten(self): - data_w = [None] * self.nargs - for i in range(self.nargs): - data_w[i] = self.frame.peekvalue(self.nargs - 1 - i) - return nextra + self.nargs, (), False, False, data_w - - def num_args(self): - return self.nargs - - def num_kwds(self): - return 0 - class Arguments(AbstractArguments): """ Collects the arguments of a function call. Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Fri Oct 9 16:41:15 2009 @@ -1,7 +1,7 @@ from pypy.interpreter.executioncontext import ExecutionContext, ActionFlag from pypy.interpreter.executioncontext import UserDelAction, FrameTraceAction from pypy.interpreter.error import OperationError -from pypy.interpreter.argument import Arguments, ArgumentsFromValuestack +from pypy.interpreter.argument import Arguments from pypy.interpreter.pycompiler import CPythonCompiler, PythonAstCompiler from pypy.interpreter.miscutils import ThreadLocals from pypy.tool.cache import Cache @@ -734,11 +734,7 @@ # XXX: this code is copied&pasted :-( from the slow path below # call_valuestack(). args = frame.make_arguments(nargs) - try: - return self.call_args_and_c_profile(frame, w_func, args) - finally: - if isinstance(args, ArgumentsFromValuestack): - args.frame = None + return self.call_args_and_c_profile(frame, w_func, args) if not self.config.objspace.disable_call_speedhacks: # XXX start of hack for performance @@ -759,11 +755,7 @@ # XXX end of hack for performance args = frame.make_arguments(nargs) - try: - return self.call_args(w_func, args) - finally: - if isinstance(args, ArgumentsFromValuestack): - args.frame = None + return self.call_args(w_func, args) @dont_look_inside def call_args_and_c_profile(self, frame, w_func, args): Modified: pypy/trunk/pypy/interpreter/function.py ============================================================================== --- pypy/trunk/pypy/interpreter/function.py (original) +++ pypy/trunk/pypy/interpreter/function.py Fri Oct 9 16:41:15 2009 @@ -10,7 +10,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.eval import Code -from pypy.interpreter.argument import Arguments, ArgumentsFromValuestack +from pypy.interpreter.argument import Arguments from pypy.rlib.jit import hint funccallunrolling = unrolling_iterable(range(4)) @@ -128,18 +128,10 @@ assert isinstance(code, gateway.BuiltinCodePassThroughArguments1) w_obj = frame.peekvalue(nargs-1) args = frame.make_arguments(nargs-1) - try: - return code.funcrun_obj(self, w_obj, args) - finally: - if isinstance(args, ArgumentsFromValuestack): - args.frame = None + return code.funcrun_obj(self, w_obj, args) args = frame.make_arguments(nargs) - try: - return self.call_args(args) - finally: - if isinstance(args, ArgumentsFromValuestack): - args.frame = None + return self.call_args(args) def _flat_pycall(self, code, nargs, frame, defs_to_load): # code is a PyCode Modified: pypy/trunk/pypy/interpreter/pyframe.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyframe.py (original) +++ pypy/trunk/pypy/interpreter/pyframe.py Fri Oct 9 16:41:15 2009 @@ -3,7 +3,7 @@ from pypy.tool.pairtype import extendabletype from pypy.interpreter import eval, baseobjspace, pycode -from pypy.interpreter.argument import Arguments, ArgumentsFromValuestack +from pypy.interpreter.argument import Arguments from pypy.interpreter.error import OperationError from pypy.interpreter.executioncontext import ExecutionContext from pypy.interpreter import pytraceback @@ -280,10 +280,7 @@ self.dropvaluesuntil(len(items_w)) def make_arguments(self, nargs): - if we_are_jitted(): - return Arguments(self.space, self.peekvalues(nargs)) - else: - return ArgumentsFromValuestack(self.space, self, nargs) + return Arguments(self.space, self.peekvalues(nargs)) @jit.dont_look_inside def descr__reduce__(self, space): Modified: pypy/trunk/pypy/objspace/std/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/std/objspace.py (original) +++ pypy/trunk/pypy/objspace/std/objspace.py Fri Oct 9 16:41:15 2009 @@ -2,7 +2,6 @@ from pypy.interpreter.baseobjspace import ObjSpace, Wrappable, UnpackValueError from pypy.interpreter.error import OperationError, debug_print from pypy.interpreter.typedef import get_unique_interplevel_subclass -from pypy.interpreter import argument from pypy.interpreter import pyframe from pypy.interpreter import function from pypy.interpreter.pyopcode import unrolling_compare_dispatch_table, \ @@ -173,11 +172,7 @@ executioncontext.c_return_trace(f, w_function) return res args = f.make_arguments(nargs) - try: - return f.space.call_args(w_function, args) - finally: - if isinstance(args, argument.ArgumentsFromValuestack): - args.frame = None + return f.space.call_args(w_function, args) if self.config.objspace.opcodes.CALL_METHOD: # def LOOKUP_METHOD(...): From pedronis at codespeak.net Fri Oct 9 16:43:11 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 9 Oct 2009 16:43:11 +0200 (CEST) Subject: [pypy-svn] r68278 - pypy/branch/improve-kwd-args Message-ID: <20091009144311.B0A7E168076@codespeak.net> Author: pedronis Date: Fri Oct 9 16:43:11 2009 New Revision: 68278 Added: pypy/branch/improve-kwd-args/ - copied from r68277, pypy/trunk/ Log: (cfbolz, pedronis): a branch where we want to refactor the Arguments class for fun and profit. From arigo at codespeak.net Fri Oct 9 16:50:11 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Oct 2009 16:50:11 +0200 (CEST) Subject: [pypy-svn] r68279 - in pypy/branch/gc-compress/pypy: rpython/lltypesystem translator/c Message-ID: <20091009145011.4CB4749844D@codespeak.net> Author: arigo Date: Fri Oct 9 16:50:10 2009 New Revision: 68279 Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py pypy/branch/gc-compress/pypy/translator/c/node.py Log: Don't raise immediately "structure is part of two groups". Needed for tests. Now it just marks the old group as outdated when a structure it contains is also inserted into a new group, and translator/c complains if writing an outdated group. Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py Fri Oct 9 16:50:10 2009 @@ -16,6 +16,7 @@ class group(lltype._container): _TYPE = Group + outdated = None def __init__(self, name): self.name = name @@ -26,7 +27,10 @@ assert isinstance(TYPE.TO, lltype.Struct) assert TYPE.TO._gckind == 'raw' struct = structptr._as_obj() - assert struct not in _membership,"cannot be a member of several groups" + prevgroup = _membership.get(struct) + if prevgroup is not None: + prevgroup.outdated = ( + "structure %s was inserted into another group" % (struct,)) assert struct._parentstructure() is None index = len(self.members) self.members.append(struct) Modified: pypy/branch/gc-compress/pypy/translator/c/node.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/node.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/node.py Fri Oct 9 16:50:10 2009 @@ -921,6 +921,8 @@ yield member._as_ptr() def _fix_members(self): + if self.obj.outdated: + raise Exception(self.obj.outdated) if self.count_members is None: self.count_members = len(self.obj.members) else: From pedronis at codespeak.net Fri Oct 9 16:50:59 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 9 Oct 2009 16:50:59 +0200 (CEST) Subject: [pypy-svn] r68280 - in pypy/branch/improve-kwd-args/pypy: interpreter interpreter/test objspace/fake Message-ID: <20091009145059.9290449844D@codespeak.net> Author: pedronis Date: Fri Oct 9 16:50:58 2009 New Revision: 68280 Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py pypy/branch/improve-kwd-args/pypy/interpreter/gateway.py pypy/branch/improve-kwd-args/pypy/interpreter/test/test_gateway.py pypy/branch/improve-kwd-args/pypy/objspace/fake/checkmodule.py Log: (pedronis, cfbolz): kill the now useless abstract base class Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/argument.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/argument.py Fri Oct 9 16:50:58 2009 @@ -4,189 +4,7 @@ from pypy.interpreter.error import OperationError -class AbstractArguments: - - def parse_into_scope(self, w_firstarg, - scope_w, fnname, signature, defaults_w=[]): - """Parse args and kwargs to initialize a frame - according to the signature of code object. - Store the argumentvalues into scope_w. - scope_w must be big enough for signature. - """ - argnames, varargname, kwargname = signature - has_vararg = varargname is not None - has_kwarg = kwargname is not None - try: - return self._match_signature(w_firstarg, - scope_w, argnames, has_vararg, - has_kwarg, defaults_w, 0) - except ArgErr, e: - raise OperationError(self.space.w_TypeError, - self.space.wrap(e.getmsg(fnname))) - - def _parse(self, w_firstarg, signature, defaults_w, blindargs=0): - """Parse args and kwargs according to the signature of a code object, - or raise an ArgErr in case of failure. - """ - argnames, varargname, kwargname = signature - scopelen = len(argnames) - has_vararg = varargname is not None - has_kwarg = kwargname is not None - if has_vararg: - scopelen += 1 - if has_kwarg: - scopelen += 1 - scope_w = [None] * scopelen - self._match_signature(w_firstarg, scope_w, argnames, has_vararg, has_kwarg, defaults_w, blindargs) - return scope_w - - def parse(self, fnname, signature, defaults_w=[], blindargs=0): - """Parse args and kwargs to initialize a frame - according to the signature of code object. - """ - try: - return self._parse(None, signature, defaults_w, blindargs) - except ArgErr, e: - raise OperationError(self.space.w_TypeError, - self.space.wrap(e.getmsg(fnname))) - - # xxx have only this one - def parse_obj(self, w_firstarg, - fnname, signature, defaults_w=[], blindargs=0): - """Parse args and kwargs to initialize a frame - according to the signature of code object. - """ - try: - return self._parse(w_firstarg, signature, defaults_w, blindargs) - except ArgErr, e: - raise OperationError(self.space.w_TypeError, - self.space.wrap(e.getmsg(fnname))) - - def frompacked(space, w_args=None, w_kwds=None): - """Convenience static method to build an Arguments - from a wrapped sequence and a wrapped dictionary.""" - return Arguments(space, [], w_stararg=w_args, w_starstararg=w_kwds) - frompacked = staticmethod(frompacked) - - def topacked(self): - """Express the Argument object as a pair of wrapped w_args, w_kwds.""" - space = self.space - args_w, kwds_w = self.unpack() - w_args = space.newtuple(args_w) - w_kwds = space.newdict() - for key, w_value in kwds_w.items(): - space.setitem(w_kwds, space.wrap(key), w_value) - return w_args, w_kwds - - def fromshape(space, (shape_cnt,shape_keys,shape_star,shape_stst), data_w): - args_w = data_w[:shape_cnt] - p = shape_cnt - kwds_w = {} - for i in range(len(shape_keys)): - kwds_w[shape_keys[i]] = data_w[p] - p += 1 - if shape_star: - w_star = data_w[p] - p += 1 - else: - w_star = None - if shape_stst: - w_starstar = data_w[p] - p += 1 - else: - w_starstar = None - return Arguments(space, args_w, kwds_w, w_star, w_starstar) - fromshape = staticmethod(fromshape) - - def match_signature(self, signature, defaults_w): - """Parse args and kwargs according to the signature of a code object, - or raise an ArgErr in case of failure. - """ - return self._parse(None, signature, defaults_w) - - def unmatch_signature(self, signature, data_w): - """kind of inverse of match_signature""" - args_w, kwds_w = self.unpack() - need_cnt = len(args_w) - need_kwds = kwds_w.keys() - space = self.space - argnames, varargname, kwargname = signature - cnt = len(argnames) - data_args_w = data_w[:cnt] - if varargname: - data_w_stararg = data_w[cnt] - cnt += 1 - else: - data_w_stararg = space.newtuple([]) - - unfiltered_kwds_w = {} - if kwargname: - data_w_starargarg = data_w[cnt] - for w_key in space.unpackiterable(data_w_starargarg): - key = space.str_w(w_key) - w_value = space.getitem(data_w_starargarg, w_key) - unfiltered_kwds_w[key] = w_value - cnt += 1 - assert len(data_w) == cnt - - ndata_args_w = len(data_args_w) - if ndata_args_w >= need_cnt: - args_w = data_args_w[:need_cnt] - for argname, w_arg in zip(argnames[need_cnt:], data_args_w[need_cnt:]): - unfiltered_kwds_w[argname] = w_arg - assert not space.is_true(data_w_stararg) - else: - stararg_w = space.unpackiterable(data_w_stararg) - datalen = len(data_args_w) - args_w = [None] * (datalen + len(stararg_w)) - for i in range(0, datalen): - args_w[i] = data_args_w[i] - for i in range(0, len(stararg_w)): - args_w[i + datalen] = stararg_w[i] - assert len(args_w) == need_cnt - - kwds_w = {} - for key in need_kwds: - kwds_w[key] = unfiltered_kwds_w[key] - - return Arguments(self.space, args_w, kwds_w) - - def normalize(self): - """Return an instance of the Arguments class. (Instances of other - classes may not be suitable for long-term storage or multiple - usage.) Also force the type and validity of the * and ** arguments - to be checked now. - """ - args_w, kwds_w = self.unpack() - return Arguments(self.space, args_w, kwds_w) - - def unpack(self): - """ Purely abstract - """ - raise NotImplementedError() - - def firstarg(self): - """ Purely abstract - """ - raise NotImplementedError() - - def prepend(self, w_firstarg): - """ Purely abstract - """ - raise NotImplementedError() - - def _match_signature(self, w_firstarg, scope_w, argnames, has_vararg=False, has_kwarg=False, defaults_w=[], blindargs=0): - """ Purely abstract - """ - raise NotImplementedError() - - def fixedunpack(self, argcount): - """ Purely abstract - """ - raise NotImplementedError() - - -class Arguments(AbstractArguments): +class Arguments(object): """ Collects the arguments of a function call. @@ -230,6 +48,7 @@ self.kwds_w, self.w_stararg) + ### Manipulation ### def unpack(self): @@ -469,6 +288,162 @@ data_w.append(self.w_starstararg) return (shape_cnt, shape_keys, shape_star, shape_stst), data_w + + def parse_into_scope(self, w_firstarg, + scope_w, fnname, signature, defaults_w=[]): + """Parse args and kwargs to initialize a frame + according to the signature of code object. + Store the argumentvalues into scope_w. + scope_w must be big enough for signature. + """ + argnames, varargname, kwargname = signature + has_vararg = varargname is not None + has_kwarg = kwargname is not None + try: + return self._match_signature(w_firstarg, + scope_w, argnames, has_vararg, + has_kwarg, defaults_w, 0) + except ArgErr, e: + raise OperationError(self.space.w_TypeError, + self.space.wrap(e.getmsg(fnname))) + + def _parse(self, w_firstarg, signature, defaults_w, blindargs=0): + """Parse args and kwargs according to the signature of a code object, + or raise an ArgErr in case of failure. + """ + argnames, varargname, kwargname = signature + scopelen = len(argnames) + has_vararg = varargname is not None + has_kwarg = kwargname is not None + if has_vararg: + scopelen += 1 + if has_kwarg: + scopelen += 1 + scope_w = [None] * scopelen + self._match_signature(w_firstarg, scope_w, argnames, has_vararg, has_kwarg, defaults_w, blindargs) + return scope_w + + def parse(self, fnname, signature, defaults_w=[], blindargs=0): + """Parse args and kwargs to initialize a frame + according to the signature of code object. + """ + try: + return self._parse(None, signature, defaults_w, blindargs) + except ArgErr, e: + raise OperationError(self.space.w_TypeError, + self.space.wrap(e.getmsg(fnname))) + + # xxx have only this one + def parse_obj(self, w_firstarg, + fnname, signature, defaults_w=[], blindargs=0): + """Parse args and kwargs to initialize a frame + according to the signature of code object. + """ + try: + return self._parse(w_firstarg, signature, defaults_w, blindargs) + except ArgErr, e: + raise OperationError(self.space.w_TypeError, + self.space.wrap(e.getmsg(fnname))) + + def frompacked(space, w_args=None, w_kwds=None): + """Convenience static method to build an Arguments + from a wrapped sequence and a wrapped dictionary.""" + return Arguments(space, [], w_stararg=w_args, w_starstararg=w_kwds) + frompacked = staticmethod(frompacked) + + def topacked(self): + """Express the Argument object as a pair of wrapped w_args, w_kwds.""" + space = self.space + args_w, kwds_w = self.unpack() + w_args = space.newtuple(args_w) + w_kwds = space.newdict() + for key, w_value in kwds_w.items(): + space.setitem(w_kwds, space.wrap(key), w_value) + return w_args, w_kwds + + def fromshape(space, (shape_cnt,shape_keys,shape_star,shape_stst), data_w): + args_w = data_w[:shape_cnt] + p = shape_cnt + kwds_w = {} + for i in range(len(shape_keys)): + kwds_w[shape_keys[i]] = data_w[p] + p += 1 + if shape_star: + w_star = data_w[p] + p += 1 + else: + w_star = None + if shape_stst: + w_starstar = data_w[p] + p += 1 + else: + w_starstar = None + return Arguments(space, args_w, kwds_w, w_star, w_starstar) + fromshape = staticmethod(fromshape) + + def match_signature(self, signature, defaults_w): + """Parse args and kwargs according to the signature of a code object, + or raise an ArgErr in case of failure. + """ + return self._parse(None, signature, defaults_w) + + def unmatch_signature(self, signature, data_w): + """kind of inverse of match_signature""" + args_w, kwds_w = self.unpack() + need_cnt = len(args_w) + need_kwds = kwds_w.keys() + space = self.space + argnames, varargname, kwargname = signature + cnt = len(argnames) + data_args_w = data_w[:cnt] + if varargname: + data_w_stararg = data_w[cnt] + cnt += 1 + else: + data_w_stararg = space.newtuple([]) + + unfiltered_kwds_w = {} + if kwargname: + data_w_starargarg = data_w[cnt] + for w_key in space.unpackiterable(data_w_starargarg): + key = space.str_w(w_key) + w_value = space.getitem(data_w_starargarg, w_key) + unfiltered_kwds_w[key] = w_value + cnt += 1 + assert len(data_w) == cnt + + ndata_args_w = len(data_args_w) + if ndata_args_w >= need_cnt: + args_w = data_args_w[:need_cnt] + for argname, w_arg in zip(argnames[need_cnt:], data_args_w[need_cnt:]): + unfiltered_kwds_w[argname] = w_arg + assert not space.is_true(data_w_stararg) + else: + stararg_w = space.unpackiterable(data_w_stararg) + datalen = len(data_args_w) + args_w = [None] * (datalen + len(stararg_w)) + for i in range(0, datalen): + args_w[i] = data_args_w[i] + for i in range(0, len(stararg_w)): + args_w[i + datalen] = stararg_w[i] + assert len(args_w) == need_cnt + + kwds_w = {} + for key in need_kwds: + kwds_w[key] = unfiltered_kwds_w[key] + + return Arguments(self.space, args_w, kwds_w) + + def normalize(self): + """Return an instance of the Arguments class. (Instances of other + classes may not be suitable for long-term storage or multiple + usage.) Also force the type and validity of the * and ** arguments + to be checked now. + """ + args_w, kwds_w = self.unpack() + return Arguments(self.space, args_w, kwds_w) + + def rawshape(args, nextra=0): return args._rawshape(nextra) Modified: pypy/branch/improve-kwd-args/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/gateway.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/gateway.py Fri Oct 9 16:50:58 2009 @@ -18,7 +18,7 @@ from pypy.interpreter.function import Function, Method, ClassMethod from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable from pypy.interpreter.baseobjspace import Wrappable, SpaceCache, DescrMismatch -from pypy.interpreter.argument import Arguments, AbstractArguments +from pypy.interpreter.argument import Arguments from pypy.tool.sourcetools import NiceCompile, compile2 from pypy.rlib.rarithmetic import r_longlong, r_int, r_ulonglong, r_uint @@ -865,7 +865,7 @@ else: args = args_w[-1] assert args is not None - if not isinstance(args, AbstractArguments): + if not isinstance(args, Arguments): args = Arguments(space, list(args_w)) else: # ...which is merged with the previous arguments, if any Modified: pypy/branch/improve-kwd-args/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/test/test_gateway.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/test/test_gateway.py Fri Oct 9 16:50:58 2009 @@ -593,7 +593,7 @@ w_res = space.call_function(w_g, w_self) assert space.is_true(space.eq(w_res, space.wrap(('g', 'self')))) assert len(called) == 1 - assert isinstance(called[0], argument.AbstractArguments) + assert isinstance(called[0], argument.Arguments) called = [] w_res = space.appexec([w_g], """(g): @@ -601,7 +601,7 @@ """) assert space.is_true(space.eq(w_res, space.wrap(('g', 'self', 11)))) assert len(called) == 1 - assert isinstance(called[0], argument.AbstractArguments) + assert isinstance(called[0], argument.Arguments) called = [] w_res = space.appexec([w_g], """(g): @@ -617,7 +617,7 @@ """) assert space.is_true(w_res) assert len(called) == 1 - assert isinstance(called[0], argument.AbstractArguments) + assert isinstance(called[0], argument.Arguments) class TestPassThroughArguments_CALL_METHOD(TestPassThroughArguments): Modified: pypy/branch/improve-kwd-args/pypy/objspace/fake/checkmodule.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/objspace/fake/checkmodule.py (original) +++ pypy/branch/improve-kwd-args/pypy/objspace/fake/checkmodule.py Fri Oct 9 16:50:58 2009 @@ -1,7 +1,7 @@ import re from copy import copy from pypy.tool.error import debug -from pypy.interpreter.argument import AbstractArguments +from pypy.interpreter.argument import Arguments from pypy.interpreter.gateway import interp2app from pypy.rlib.nonconst import NonConstant @@ -85,7 +85,7 @@ gateways = find_gateways(modname, basepath, module) functions = [gw.__spacebind__(space) for gw in gateways] - arguments = AbstractArguments.frompacked(space, W_Object(), W_Object()) + arguments = Arguments.frompacked(space, W_Object(), W_Object()) dummy_function = copy(functions[0]) def main(argv): # use the standalone mode not to allow SomeObject From antocuni at codespeak.net Fri Oct 9 17:02:24 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 9 Oct 2009 17:02:24 +0200 (CEST) Subject: [pypy-svn] r68281 - in pypy/trunk/pypy: jit/backend/cli jit/backend/cli/test jit/metainterp/test translator/cli/src Message-ID: <20091009150224.A193A49844D@codespeak.net> Author: antocuni Date: Fri Oct 9 17:02:24 2009 New Revision: 68281 Modified: pypy/trunk/pypy/jit/backend/cli/method.py pypy/trunk/pypy/jit/backend/cli/methodfactory.py pypy/trunk/pypy/jit/backend/cli/runner.py pypy/trunk/pypy/jit/backend/cli/test/test_basic.py pypy/trunk/pypy/jit/backend/cli/test/test_runner.py pypy/trunk/pypy/jit/metainterp/test/test_basic.py pypy/trunk/pypy/translator/cli/src/pypylib.cs Log: add float supports to cli jit backend Modified: pypy/trunk/pypy/jit/backend/cli/method.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/method.py (original) +++ pypy/trunk/pypy/jit/backend/cli/method.py Fri Oct 9 17:02:24 2009 @@ -6,7 +6,7 @@ from pypy.translator.cli.dotnet import CLR from pypy.translator.cli import opcodes from pypy.jit.metainterp import history -from pypy.jit.metainterp.history import (AbstractValue, Const, ConstInt, +from pypy.jit.metainterp.history import (AbstractValue, Const, ConstInt, ConstFloat, ConstObj, BoxInt, LoopToken) from pypy.jit.metainterp.resoperation import rop, opname from pypy.jit.metainterp.typesystem import oohelper @@ -32,6 +32,8 @@ if self.type == history.INT: return dotnet.typeof(System.Int32) + elif self.type == history.FLOAT: + return dotnet.typeof(System.Double) elif self.type == history.REF: return dotnet.typeof(System.Object) else: @@ -68,6 +70,16 @@ meth.il.Emit(OpCodes.Ldc_I4, self.value) +class __extend__(ConstFloat): + __metaclass__ = extendabletype + + def load(self, meth): + # we cannot invoke il.Emit(Ldc_R8, self.value) directly because + # pythonnet would select the wrong overload. The C# version works + # arond it + Utils.Emit_Ldc_R8(meth.il, self.value); + + class ConstFunction(Const): def __init__(self, name): @@ -274,6 +286,8 @@ t = dotnet.typeof(InputArgs) if type == history.INT: fieldname = 'ints' + elif type == history.FLOAT: + fieldname = 'floats' elif type == history.REF: fieldname = 'objs' else: @@ -739,6 +753,8 @@ lines.append('self.store_result(op)') elif isinstance(instr, opcodes.PushArg): lines.append('self.push_arg(op, %d)' % instr.n) + elif instr == 'ldc.r8 0': + lines.append('Utils.Emit_Ldc_R8(self.il, 0.0)') else: assert isinstance(instr, str), 'unknown instruction %s' % instr if instr.startswith('call '): @@ -751,6 +767,7 @@ src = body.putaround('def %s(self, op):' % methname) dic = {'OpCodes': OpCodes, 'System': System, + 'Utils': Utils, 'dotnet': dotnet} exec src.compile() in dic return dic[methname] Modified: pypy/trunk/pypy/jit/backend/cli/methodfactory.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/methodfactory.py (original) +++ pypy/trunk/pypy/jit/backend/cli/methodfactory.py Fri Oct 9 17:02:24 2009 @@ -71,7 +71,7 @@ def create_delegate(self, delegatetype, consts): t = self.typeBuilder.CreateType() methinfo = t.GetMethod("invoke") -## if self.name == 'Loop #0(r1)_2': +## if self.name == 'Loop1(r0)_1': ## assemblyData.auto_save_assembly.Save() return System.Delegate.CreateDelegate(delegatetype, consts, Modified: pypy/trunk/pypy/jit/backend/cli/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/runner.py (original) +++ pypy/trunk/pypy/jit/backend/cli/runner.py Fri Oct 9 17:02:24 2009 @@ -39,7 +39,8 @@ class CliCPU(model.AbstractCPU): - + + supports_floats = True ts = oohelper def __init__(self, rtyper, stats, translate_support_code=False, @@ -139,6 +140,9 @@ def set_future_value_int(self, index, intvalue): self.get_inputargs().set_int(index, intvalue) + def set_future_value_float(self, index, intvalue): + self.get_inputargs().set_float(index, intvalue) + def set_future_value_ref(self, index, objvalue): obj = dotnet.cast_to_native_object(objvalue) self.get_inputargs().set_obj(index, obj) @@ -146,6 +150,9 @@ def get_latest_value_int(self, index): return self.get_inputargs().get_int(index) + def get_latest_value_float(self, index): + return self.get_inputargs().get_float(index) + def get_latest_value_ref(self, index): obj = self.get_inputargs().get_obj(index) return dotnet.cast_from_native_object(obj) Modified: pypy/trunk/pypy/jit/backend/cli/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/backend/cli/test/test_basic.py Fri Oct 9 17:02:24 2009 @@ -15,9 +15,6 @@ def skip(self): py.test.skip("works only after translation") - def _skip(self): - py.test.skip("in-progress") - test_string = skip test_chr2str = skip test_unicode = skip Modified: pypy/trunk/pypy/jit/backend/cli/test/test_runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/test/test_runner.py (original) +++ pypy/trunk/pypy/jit/backend/cli/test/test_runner.py Fri Oct 9 17:02:24 2009 @@ -30,6 +30,9 @@ test_field_basic = skip test_ooops = skip + def test_unused_result_float(self): + py.test.skip('fixme! max 32 inputargs so far') + def test_ovf_operations(self, reversed=False): self.skip() Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Fri Oct 9 17:02:24 2009 @@ -493,6 +493,25 @@ res = self.meta_interp(f, [-5]) assert res == 5+4+3+2+1+0+1+2+3+4+5+6+7+8+9 + def test_float(self): + myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'res']) + def f(x, y): + x = float(x) + y = float(y) + res = 0.0 + while y > 0.0: + myjitdriver.can_enter_jit(x=x, y=y, res=res) + myjitdriver.jit_merge_point(x=x, y=y, res=res) + res += x + y -= 1.0 + return res + res = self.meta_interp(f, [6, 7]) + assert res == 42.0 + self.check_loop_count(1) + self.check_loops({'guard_true': 1, + 'float_add': 1, 'float_sub': 1, 'float_gt': 1, + 'jump': 1}) + def test_print(self): myjitdriver = JitDriver(greens = [], reds = ['n']) def f(n): Modified: pypy/trunk/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/trunk/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/trunk/pypy/translator/cli/src/pypylib.cs Fri Oct 9 17:02:24 2009 @@ -114,7 +114,7 @@ public class InputArgs { public int[] ints = new int[32]; - public float[] floats = new float[32]; + public double[] floats = new double[32]; public object[] objs = new object[32]; public object exc_value = null; public int failed_op = -1; @@ -129,6 +129,16 @@ ints[i] = n; } + public double get_float(int i) + { + return floats[i]; + } + + public void set_float(int i, double n) + { + floats[i] = n; + } + public object get_obj(int i) { return objs[i]; @@ -163,7 +173,7 @@ public void ensure_floats(int n) { if (floats.Length < n) - floats = new float[n]; + floats = new double[n]; } public void ensure_objs(int n) @@ -402,6 +412,14 @@ return new DynamicMethod(name, res, args, typeof(Utils).Module); } + // if you call il.Emit(OpCodes.Ldc_R8, mydouble) from pythonnet, it + // selects the wrong overload. To work around it, we call it from C# and + // live happy + public static void Emit_Ldc_R8(ILGenerator il, double val) + { + il.Emit(OpCodes.Ldc_R8, val); + } + public static object RuntimeNew(Type t) { return t.GetConstructor(new Type[0]).Invoke(new object[0]); From pedronis at codespeak.net Fri Oct 9 17:04:17 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 9 Oct 2009 17:04:17 +0200 (CEST) Subject: [pypy-svn] r68282 - in pypy/branch/improve-kwd-args/pypy: config objspace objspace/test Message-ID: <20091009150417.26E0149844D@codespeak.net> Author: pedronis Date: Fri Oct 9 17:04:16 2009 New Revision: 68282 Removed: pypy/branch/improve-kwd-args/pypy/objspace/reflective.py pypy/branch/improve-kwd-args/pypy/objspace/test/test_reflective.py Modified: pypy/branch/improve-kwd-args/pypy/config/pypyoption.py Log: (pedronis, cfbolz): kill the reflective object space, while we are at it (it is one of the few users of some strange stuff in argument.py) Modified: pypy/branch/improve-kwd-args/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/config/pypyoption.py (original) +++ pypy/branch/improve-kwd-args/pypy/config/pypyoption.py Fri Oct 9 17:04:16 2009 @@ -102,11 +102,8 @@ pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [ ChoiceOption("name", "Object Space name", - ["std", "flow", "thunk", "dump", "taint", "reflective"], + ["std", "flow", "thunk", "dump", "taint"], "std", - requires={"reflective": - [("objspace.disable_call_speedhacks", True)] - }, cmdline='--objspace -o'), ChoiceOption("parser", "which parser to use for app-level code", From pedronis at codespeak.net Fri Oct 9 17:14:00 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 9 Oct 2009 17:14:00 +0200 (CEST) Subject: [pypy-svn] r68283 - pypy/branch/improve-kwd-args/pypy/interpreter Message-ID: <20091009151400.3F1FE49844D@codespeak.net> Author: pedronis Date: Fri Oct 9 17:13:59 2009 New Revision: 68283 Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py pypy/branch/improve-kwd-args/pypy/interpreter/eval.py Log: (pedronis, cfbolz): reorder methods a bit to group those used only by the translation toolchain. annotate with use places a bit all methods. kill has_keywords. Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/argument.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/argument.py Fri Oct 9 17:13:59 2009 @@ -24,11 +24,11 @@ self.w_stararg = w_stararg self.w_starstararg = w_starstararg - def num_args(self): + def num_args(self): # only used in module/__builtin__/interp_classobj.py self._unpack() return len(self.arguments_w) - def num_kwds(self): + def num_kwds(self): # only used in module/__builtin__/interp_classobj.py self._unpack() return len(self.kwds_w) @@ -51,12 +51,12 @@ ### Manipulation ### - def unpack(self): + def unpack(self): # used often "Return a ([w1,w2...], {'kw':w3...}) pair." self._unpack() return self.arguments_w, self.kwds_w - def prepend(self, w_firstarg): + def prepend(self, w_firstarg): # used often "Return a new Arguments with a new argument inserted first." return Arguments(self.space, [w_firstarg] + self.arguments_w, self.kwds_w, self.w_stararg, self.w_starstararg) @@ -99,14 +99,14 @@ self.kwds_w = d self.w_starstararg = None - def has_keywords(self): - return bool(self.kwds_w) or (self.w_starstararg is not None and - self.space.is_true(self.w_starstararg)) - def fixedunpack(self, argcount): + # used by ./objspace/std/typeobject.py + # ./objspace/std/objecttype.py + # ./annotation/description.py """The simplest argument parsing: get the 'argcount' arguments, or raise a real ValueError if the length is wrong.""" - if self.has_keywords(): + if bool(self.kwds_w) or (self.w_starstararg is not None and + self.space.is_true(self.w_starstararg)): raise ValueError, "no keyword arguments expected" if len(self.arguments_w) > argcount: raise ValueError, "too many arguments (%d expected)" % argcount @@ -120,6 +120,10 @@ return self.arguments_w def firstarg(self): + # used by + # ./module/_random/interp_random.py + # ./interpreter/gateway.py + # ./interpreter/function.py "Return the first argument for inspection." if self.arguments_w: return self.arguments_w[0] @@ -266,27 +270,6 @@ return co_argcount + has_vararg + has_kwarg - ### Argument <-> list of w_objects together with "shape" information - - def _rawshape(self, nextra=0): - shape_cnt = len(self.arguments_w)+nextra # Number of positional args - if self.kwds_w: - shape_keys = self.kwds_w.keys() # List of keywords (strings) - else: - shape_keys = [] - shape_star = self.w_stararg is not None # Flag: presence of *arg - shape_stst = self.w_starstararg is not None # Flag: presence of **kwds - shape_keys.sort() - return shape_cnt, tuple(shape_keys), shape_star, shape_stst # shape_keys are sorted - - def flatten(self): - shape_cnt, shape_keys, shape_star, shape_stst = self._rawshape() - data_w = self.arguments_w + [self.kwds_w[key] for key in shape_keys] - if shape_star: - data_w.append(self.w_stararg) - if shape_stst: - data_w.append(self.w_starstararg) - return (shape_cnt, shape_keys, shape_star, shape_stst), data_w def parse_into_scope(self, w_firstarg, @@ -324,18 +307,17 @@ return scope_w def parse(self, fnname, signature, defaults_w=[], blindargs=0): + # used by geninterped code """Parse args and kwargs to initialize a frame according to the signature of code object. """ - try: - return self._parse(None, signature, defaults_w, blindargs) - except ArgErr, e: - raise OperationError(self.space.w_TypeError, - self.space.wrap(e.getmsg(fnname))) + return self.parse_obj(None, fnname, signature, defaults_w, + blindargs) # xxx have only this one def parse_obj(self, w_firstarg, fnname, signature, defaults_w=[], blindargs=0): + # used by ./interpreter/gateway.py """Parse args and kwargs to initialize a frame according to the signature of code object. """ @@ -346,12 +328,19 @@ self.space.wrap(e.getmsg(fnname))) def frompacked(space, w_args=None, w_kwds=None): + # used by + # ./module/_stackless/interp_coroutine.py + # ./module/thread/os_thread.py + # ./objspace/fake/checkmodule.py + # ./interpreter/gateway.py + # ./interpreter/baseobjspace.py """Convenience static method to build an Arguments from a wrapped sequence and a wrapped dictionary.""" return Arguments(space, [], w_stararg=w_args, w_starstararg=w_kwds) frompacked = staticmethod(frompacked) def topacked(self): + # used by ./module/_stackless/interp_coroutine.py """Express the Argument object as a pair of wrapped w_args, w_kwds.""" space = self.space args_w, kwds_w = self.unpack() @@ -362,6 +351,11 @@ return w_args, w_kwds def fromshape(space, (shape_cnt,shape_keys,shape_star,shape_stst), data_w): + # used by + # geninterped code + # ./rpython/callparse.py + # ./rpython/rbuiltin.py + # ./annotation/bookkeeper.py args_w = data_w[:shape_cnt] p = shape_cnt kwds_w = {} @@ -381,6 +375,18 @@ return Arguments(space, args_w, kwds_w, w_star, w_starstar) fromshape = staticmethod(fromshape) + + def normalize(self): + """Return an instance of the Arguments class. (Instances of other + classes may not be suitable for long-term storage or multiple + usage.) Also force the type and validity of the * and ** arguments + to be checked now. + """ + args_w, kwds_w = self.unpack() + return Arguments(self.space, args_w, kwds_w) + + # ------------- used only by the translation toolchain ------------------- + def match_signature(self, signature, defaults_w): """Parse args and kwargs according to the signature of a code object, or raise an ArgErr in case of failure. @@ -434,17 +440,30 @@ return Arguments(self.space, args_w, kwds_w) - def normalize(self): - """Return an instance of the Arguments class. (Instances of other - classes may not be suitable for long-term storage or multiple - usage.) Also force the type and validity of the * and ** arguments - to be checked now. - """ - args_w, kwds_w = self.unpack() - return Arguments(self.space, args_w, kwds_w) + def flatten(self): + # used by ./objspace/flow/objspace.py + """ Argument <-> list of w_objects together with "shape" information """ + shape_cnt, shape_keys, shape_star, shape_stst = self._rawshape() + data_w = self.arguments_w + [self.kwds_w[key] for key in shape_keys] + if shape_star: + data_w.append(self.w_stararg) + if shape_stst: + data_w.append(self.w_starstararg) + return (shape_cnt, shape_keys, shape_star, shape_stst), data_w + def _rawshape(self, nextra=0): + shape_cnt = len(self.arguments_w)+nextra # Number of positional args + if self.kwds_w: + shape_keys = self.kwds_w.keys() # List of keywords (strings) + else: + shape_keys = [] + shape_star = self.w_stararg is not None # Flag: presence of *arg + shape_stst = self.w_starstararg is not None # Flag: presence of **kwds + shape_keys.sort() + return shape_cnt, tuple(shape_keys), shape_star, shape_stst # shape_keys are sorted def rawshape(args, nextra=0): + # used by ./annotation/description.py return args._rawshape(nextra) Modified: pypy/branch/improve-kwd-args/pypy/interpreter/eval.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/eval.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/eval.py Fri Oct 9 17:13:59 2009 @@ -59,12 +59,7 @@ return space.w_None def funcrun(self, func, args): - frame = func.space.createframe(self, func.w_func_globals, - func.closure) - sig = self.signature() - scope_w = args.parse(func.name, sig, func.defs_w) - frame.setfastscope(scope_w) - return frame.run() + raise NotImplementedError("purely abstract") def funcrun_obj(self, func, w_obj, args): return self.funcrun(func, args.prepend(w_obj)) From pedronis at codespeak.net Fri Oct 9 17:52:30 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 9 Oct 2009 17:52:30 +0200 (CEST) Subject: [pypy-svn] r68284 - in pypy/branch/improve-kwd-args/pypy: annotation interpreter objspace/flow rpython Message-ID: <20091009155230.1CE1849844D@codespeak.net> Author: pedronis Date: Fri Oct 9 17:52:29 2009 New Revision: 68284 Modified: pypy/branch/improve-kwd-args/pypy/annotation/bookkeeper.py pypy/branch/improve-kwd-args/pypy/interpreter/argument.py pypy/branch/improve-kwd-args/pypy/interpreter/pyframe.py pypy/branch/improve-kwd-args/pypy/interpreter/pyopcode.py pypy/branch/improve-kwd-args/pypy/objspace/flow/flowcontext.py pypy/branch/improve-kwd-args/pypy/rpython/callparse.py Log: (pedronis, cfbolz): make a subclass of Arguments, ArgumentsForTranslation, which is used only in the translation toolchain (flow space, annotator, rtyper to be precise). Modified: pypy/branch/improve-kwd-args/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/improve-kwd-args/pypy/annotation/bookkeeper.py Fri Oct 9 17:52:29 2009 @@ -16,7 +16,7 @@ from pypy.annotation.dictdef import DictDef from pypy.annotation import description from pypy.annotation.signature import annotationoftype -from pypy.interpreter.argument import Arguments +from pypy.interpreter.argument import ArgumentsForTranslation from pypy.rlib.objectmodel import r_dict, Symbolic from pypy.tool.algo.unionfind import UnionFind from pypy.rpython.lltypesystem import lltype, llmemory @@ -695,10 +695,11 @@ def build_args(self, op, args_s): space = RPythonCallsSpace() if op == "simple_call": - return Arguments(space, list(args_s)) + return ArgumentsForTranslation(space, list(args_s)) elif op == "call_args": - return Arguments.fromshape(space, args_s[0].const, # shape - list(args_s[1:])) + return ArgumentsForTranslation.fromshape( + space, args_s[0].const, # shape + list(args_s[1:])) def ondegenerated(self, what, s_value, where=None, called_from_graph=None): self.annotator.ondegenerated(what, s_value, where=where, Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/argument.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/argument.py Fri Oct 9 17:52:29 2009 @@ -24,6 +24,12 @@ self.w_stararg = w_stararg self.w_starstararg = w_starstararg + @staticmethod + def factory(space, args_w, kwds_w=None, + w_stararg=None, w_starstararg=None): + return Argument(space, args_w, kwds_w, + w_stararg, w_starstararg) + def num_args(self): # only used in module/__builtin__/interp_classobj.py self._unpack() return len(self.arguments_w) @@ -33,20 +39,22 @@ return len(self.kwds_w) def __repr__(self): + """ NOT_RPYTHON """ if self.w_starstararg is not None: - return 'Arguments(%s, %s, %s, %s)' % (self.arguments_w, - self.kwds_w, - self.w_stararg, - self.w_starstararg) + return '%s(%s, %s, %s, %s)' % (self.__class__, + self.arguments_w, + self.kwds_w, + self.w_stararg, + self.w_starstararg) if self.w_stararg is None: if not self.kwds_w: - return 'Arguments(%s)' % (self.arguments_w,) + return '%s(%s)' % (self.__class__, self.arguments_w,) else: - return 'Arguments(%s, %s)' % (self.arguments_w, self.kwds_w) + return '%s(%s, %s)' % (self.__class__, self.arguments_w, + self.kwds_w) else: - return 'Arguments(%s, %s, %s)' % (self.arguments_w, - self.kwds_w, - self.w_stararg) + return '%s(%s, %s, %s)' % (self.__class__, self.arguments_w, + self.kwds_w, self.w_stararg) ### Manipulation ### @@ -58,8 +66,8 @@ def prepend(self, w_firstarg): # used often "Return a new Arguments with a new argument inserted first." - return Arguments(self.space, [w_firstarg] + self.arguments_w, - self.kwds_w, self.w_stararg, self.w_starstararg) + return self.factory(self.space, [w_firstarg] + self.arguments_w, + self.kwds_w, self.w_stararg, self.w_starstararg) def _unpack(self): "unpack the *arg and **kwd into w_arguments and kwds_w" @@ -336,7 +344,7 @@ # ./interpreter/baseobjspace.py """Convenience static method to build an Arguments from a wrapped sequence and a wrapped dictionary.""" - return Arguments(space, [], w_stararg=w_args, w_starstararg=w_kwds) + return self.factory(space, [], w_stararg=w_args, w_starstararg=w_kwds) frompacked = staticmethod(frompacked) def topacked(self): @@ -351,11 +359,7 @@ return w_args, w_kwds def fromshape(space, (shape_cnt,shape_keys,shape_star,shape_stst), data_w): - # used by - # geninterped code - # ./rpython/callparse.py - # ./rpython/rbuiltin.py - # ./annotation/bookkeeper.py + # used by geninterped code args_w = data_w[:shape_cnt] p = shape_cnt kwds_w = {} @@ -385,7 +389,13 @@ args_w, kwds_w = self.unpack() return Arguments(self.space, args_w, kwds_w) - # ------------- used only by the translation toolchain ------------------- + +class ArgumentsForTranslation(Arguments): + @staticmethod + def factory(space, args_w, kwds_w=None, + w_stararg=None, w_starstararg=None): + return ArgumentsForTranslation(space, args_w, kwds_w, + w_stararg, w_starstararg) def match_signature(self, signature, defaults_w): """Parse args and kwargs according to the signature of a code object, @@ -438,7 +448,31 @@ for key in need_kwds: kwds_w[key] = unfiltered_kwds_w[key] - return Arguments(self.space, args_w, kwds_w) + return ArgumentsForTranslation(self.space, args_w, kwds_w) + + def fromshape(space, (shape_cnt,shape_keys,shape_star,shape_stst), data_w): + # used by + # ./rpython/callparse.py + # ./rpython/rbuiltin.py + # ./annotation/bookkeeper.py + args_w = data_w[:shape_cnt] + p = shape_cnt + kwds_w = {} + for i in range(len(shape_keys)): + kwds_w[shape_keys[i]] = data_w[p] + p += 1 + if shape_star: + w_star = data_w[p] + p += 1 + else: + w_star = None + if shape_stst: + w_starstar = data_w[p] + p += 1 + else: + w_starstar = None + return ArgumentsForTranslation(space, args_w, kwds_w, w_star, w_starstar) + fromshape = staticmethod(fromshape) def flatten(self): # used by ./objspace/flow/objspace.py Modified: pypy/branch/improve-kwd-args/pypy/interpreter/pyframe.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/pyframe.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/pyframe.py Fri Oct 9 17:52:29 2009 @@ -282,6 +282,9 @@ def make_arguments(self, nargs): return Arguments(self.space, self.peekvalues(nargs)) + def argument_factory(self, arguments, keywords, w_star, w_starstar): + return Arguments(self.space, arguments, keywords, w_star, w_starstar) + @jit.dont_look_inside def descr__reduce__(self, space): from pypy.interpreter.mixedmodule import MixedModule Modified: pypy/branch/improve-kwd-args/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/pyopcode.py Fri Oct 9 17:52:29 2009 @@ -900,7 +900,7 @@ if n_keywords: keywords = f.popstrdictvalues(n_keywords) arguments = f.popvalues(n_arguments) - args = Arguments(f.space, arguments, keywords, w_star, w_starstar) + args = f.argument_factory(arguments, keywords, w_star, w_starstar) w_function = f.popvalue() if we_are_jitted(): w_result = f.space.call_args(w_function, args) Modified: pypy/branch/improve-kwd-args/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/objspace/flow/flowcontext.py (original) +++ pypy/branch/improve-kwd-args/pypy/objspace/flow/flowcontext.py Fri Oct 9 17:52:29 2009 @@ -2,6 +2,7 @@ from pypy.interpreter.executioncontext import ExecutionContext from pypy.interpreter.error import OperationError from pypy.interpreter import pyframe +from pypy.interpreter.argument import ArgumentsForTranslation from pypy.objspace.flow.model import * from pypy.objspace.flow.framestate import FrameState @@ -219,8 +220,8 @@ # create an empty frame suitable for the code object # while ignoring any operation like the creation of the locals dict self.recorder = [] - frame = pyframe.PyFrame(self.space, self.code, - self.w_globals, self.closure) + frame = FlowSpaceFrame(self.space, self.code, + self.w_globals, self.closure) frame.last_instr = 0 return frame @@ -392,3 +393,9 @@ if isinstance(w_v, Constant): if w_v.value is oldvalue: stack_items_w[i] = w_new + +class FlowSpaceFrame(pyframe.PyFrame): + def make_arguments(self, nargs): + return ArgumentsForTranslation(self.space, self.peekvalues(nargs)) + def argument_factory(self, arguments, keywords, w_star, w_starstar): + return ArgumentsForTranslation(self.space, arguments, keywords, w_star, w_starstar) Modified: pypy/branch/improve-kwd-args/pypy/rpython/callparse.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/rpython/callparse.py (original) +++ pypy/branch/improve-kwd-args/pypy/rpython/callparse.py Fri Oct 9 17:52:29 2009 @@ -1,4 +1,4 @@ -from pypy.interpreter.argument import Arguments, ArgErr +from pypy.interpreter.argument import ArgumentsForTranslation, ArgErr from pypy.annotation import model as annmodel from pypy.rpython import rtuple from pypy.rpython.error import TyperError @@ -40,9 +40,9 @@ start = 0 rinputs[0] = r_self if opname == "simple_call": - arguments = Arguments(space, args_h(start)) + arguments = ArgumentsForTranslation(space, args_h(start)) elif opname == "call_args": - arguments = Arguments.fromshape(space, + arguments = ArgumentsForTranslation.fromshape(space, hop.args_s[start].const, # shape args_h(start+1)) # parse the arguments according to the function we are calling From arigo at codespeak.net Fri Oct 9 18:01:50 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Oct 2009 18:01:50 +0200 (CEST) Subject: [pypy-svn] r68285 - in pypy/branch/gc-compress/pypy: jit/backend/llsupport jit/backend/x86 jit/metainterp rpython rpython/lltypesystem rpython/memory rpython/memory/gc rpython/memory/gc/test rpython/memory/gctransform rpython/memory/test translator/c Message-ID: <20091009160150.AEA1249844D@codespeak.net> Author: arigo Date: Fri Oct 9 18:01:49 2009 New Revision: 68285 Modified: pypy/branch/gc-compress/pypy/jit/backend/llsupport/gc.py pypy/branch/gc-compress/pypy/jit/backend/x86/runner.py pypy/branch/gc-compress/pypy/jit/metainterp/policy.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/rclass.py pypy/branch/gc-compress/pypy/rpython/memory/gc/markcompact.py pypy/branch/gc-compress/pypy/rpython/memory/gc/marksweep.py pypy/branch/gc-compress/pypy/rpython/memory/gc/semispace.py pypy/branch/gc-compress/pypy/rpython/memory/gc/test/test_direct.py pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py pypy/branch/gc-compress/pypy/rpython/memory/gctransform/transform.py pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py pypy/branch/gc-compress/pypy/rpython/memory/gcwrapper.py pypy/branch/gc-compress/pypy/rpython/memory/test/test_gctypelayout.py pypy/branch/gc-compress/pypy/rpython/rtyper.py pypy/branch/gc-compress/pypy/translator/c/gc.py pypy/branch/gc-compress/pypy/translator/c/node.py Log: Intermediate check-in (huge!). - Change an interface on rtyper. - Don't generate the 'typeptr' field in genc. - Add an operation to mean 'I need to do a getfield of the typeptr'. - Will fix the C backend next... Modified: pypy/branch/gc-compress/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/gc-compress/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/gc-compress/pypy/jit/backend/llsupport/gc.py Fri Oct 9 18:01:49 2009 @@ -322,14 +322,16 @@ # make a TransformerLayoutBuilder and save it on the translator # where it can be fished and reused by the FrameworkGCTransformer - self.layoutbuilder = framework.TransformerLayoutBuilder() + self.GCClass, _ = choose_gc_from_config(gcdescr.config) + lltype2vtable = translator.rtyper.lltype2vtable + self.layoutbuilder = framework.TransformerLayoutBuilder(self.GCClass, + lltype2vtable) self.layoutbuilder.delay_encoding() self.translator._jit2gc = { 'layoutbuilder': self.layoutbuilder, 'gcmapstart': lambda: gcrootmap.gcmapstart(), 'gcmapend': lambda: gcrootmap.gcmapend(), } - self.GCClass, _ = choose_gc_from_config(gcdescr.config) self.moving_gc = self.GCClass.moving_gc self.HDRPTR = lltype.Ptr(self.GCClass.HDR) self.gcheaderbuilder = GCHeaderBuilder(self.HDRPTR.TO) Modified: pypy/branch/gc-compress/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/gc-compress/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/gc-compress/pypy/jit/backend/x86/runner.py Fri Oct 9 18:01:49 2009 @@ -22,8 +22,6 @@ gcdescr) self._bootstrap_cache = {} self._faildescr_list = [] - if rtyper is not None: # for tests - self.lltype2vtable = rtyper.lltype_to_vtable_mapping() def setup(self): self.assembler = Assembler386(self, self.translate_support_code) Modified: pypy/branch/gc-compress/pypy/jit/metainterp/policy.py ============================================================================== --- pypy/branch/gc-compress/pypy/jit/metainterp/policy.py (original) +++ pypy/branch/gc-compress/pypy/jit/metainterp/policy.py Fri Oct 9 18:01:49 2009 @@ -63,7 +63,7 @@ return None def _graphs_of_all_instantiate(self, rtyper): - for vtable in rtyper.lltype_to_vtable_mapping().itervalues(): + for vtable in rtyper.lltype2vtable.values(): if vtable.instantiate: yield vtable.instantiate._obj.graph Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py Fri Oct 9 18:01:49 2009 @@ -415,6 +415,7 @@ 'is_group_member_zero': LLOp(canfold=True), 'extract_ushort': LLOp(canfold=True), 'combine_ushort': LLOp(canfold=True), + 'getfield_typeptr_group':LLOp(canfold=True), # __________ used by the JIT ________ Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py Fri Oct 9 18:01:49 2009 @@ -433,6 +433,20 @@ from pypy.rpython.lltypesystem import llgroup return llgroup.CombinedSymbolic(ushort, rest) +def op_getfield_typeptr_group(TYPE, obj, grpptr, vtableinfo): + HDR = vtableinfo[0] + size_gc_header = vtableinfo[1] + fieldname = vtableinfo[2] + skipoffset = vtableinfo[3] + objaddr = llmemory.cast_ptr_to_adr(obj) + hdraddr = objaddr - size_gc_header + hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR)) + typeid = getattr(hdr, fieldname) + if lltype.typeOf(typeid) == lltype.Signed: + typeid = op_extract_ushort(typeid) + return op_get_next_group_member(TYPE, grpptr, typeid, skipoffset) +op_getfield_typeptr_group.need_result_type = True + # ____________________________________________________________ def get_op_impl(opname): Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/rclass.py Fri Oct 9 18:01:49 2009 @@ -389,6 +389,7 @@ OBJECT, destrptr) vtable = self.rclass.getvtable() self.rtyper.type_for_typeptr[vtable._obj] = self.lowleveltype.TO + self.rtyper.lltype2vtable[self.lowleveltype.TO] = vtable def common_repr(self): # -> object or nongcobject reprs return getinstancerepr(self.rtyper, None, self.gcflavor) Modified: pypy/branch/gc-compress/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gc/markcompact.py Fri Oct 9 18:01:49 2009 @@ -70,6 +70,7 @@ class MarkCompactGC(MovingGCBase): HDR = lltype.Struct('header', ('tid', lltype.Signed)) + typeid_is_in_field = 'tid' TID_BACKUP = lltype.Array(TID_TYPE, hints={'nolength':True}) WEAKREF_OFFSETS = lltype.Array(lltype.Signed) Modified: pypy/branch/gc-compress/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gc/marksweep.py Fri Oct 9 18:01:49 2009 @@ -29,6 +29,7 @@ ('mark', lltype.Bool), ('curpool_flag', lltype.Bool), ('next', HDRPTR))) + typeid_is_in_field = 'typeid16' POOL = lltype.GcStruct('gc_pool') POOLPTR = lltype.Ptr(POOL) Modified: pypy/branch/gc-compress/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gc/semispace.py Fri Oct 9 18:01:49 2009 @@ -54,6 +54,7 @@ 'addflag': _hdr_addflag, 'delflag': _hdr_delflag, }) + typeid_is_in_field = 'typeid16' FORWARDSTUB = lltype.GcStruct('forwarding_stub', ('forw', llmemory.Address)) FORWARDSTUBPTR = lltype.Ptr(FORWARDSTUB) Modified: pypy/branch/gc-compress/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gc/test/test_direct.py Fri Oct 9 18:01:49 2009 @@ -68,7 +68,7 @@ self.gc.DEBUG = True self.rootwalker = DirectRootWalker(self) self.gc.set_root_walker(self.rootwalker) - self.layoutbuilder = TypeLayoutBuilder(self.GCClass) + self.layoutbuilder = TypeLayoutBuilder(self.GCClass, {}) self.get_type_id = self.layoutbuilder.get_type_id self.layoutbuilder.initialize_gc_query_function(self.gc) self.gc.setup() Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py Fri Oct 9 18:01:49 2009 @@ -130,7 +130,9 @@ if hasattr(translator, '_jit2gc'): self.layoutbuilder = translator._jit2gc['layoutbuilder'] else: - self.layoutbuilder = TransformerLayoutBuilder(GCClass) + lltype2vtable = translator.rtyper.lltype2vtable + self.layoutbuilder = TransformerLayoutBuilder(GCClass, + lltype2vtable) self.layoutbuilder.transformer = self self.get_type_id = self.layoutbuilder.get_type_id @@ -403,6 +405,13 @@ FLDTYPE = getattr(HDR, fldname) fields.append(('_' + fldname, FLDTYPE)) + size_gc_header = self.gcdata.gc.gcheaderbuilder.size_gc_header + vtableinfo = (HDR, size_gc_header, self.gcdata.gc.typeid_is_in_field, + llmemory.sizeof(gcdata.TYPE_INFO)) + self.c_vtableinfo = rmodel.inputconst(lltype.Void, vtableinfo) + tig = self.layoutbuilder.type_info_group._as_ptr() + self.c_type_info_group = rmodel.inputconst(lltype.typeOf(tig), tig) + def build_root_walker(self): return ShadowStackRootWalker(self) @@ -769,6 +778,33 @@ v_structaddr]) hop.rename('bare_' + opname) + def transform_getfield_typeptr(self, hop): + # this would become quite a lot of operations, even if it compiles + # to C code that is just as efficient as "obj->typeptr". To avoid + # that, we just generate a single custom operation instead. + hop.genop('getfield_typeptr_group', [hop.spaceop.args[0], + self.c_type_info_group, + self.c_vtableinfo], + resultvar = hop.spaceop.result) + + def transform_setfield_typeptr(self, hop): + # replace such a setfield with an assertion that the typeptr is right + pass # XXX later + + def gct_getfield(self, hop): + if (hop.spaceop.args[1].value == 'typeptr' and + hop.spaceop.args[0].concretetype.TO._hints.get('typeptr')): + self.transform_getfield_typeptr(hop) + else: + GCTransformer.gct_getfield(self, hop) + + def gct_setfield(self, hop): + if (hop.spaceop.args[1].value == 'typeptr' and + hop.spaceop.args[0].concretetype.TO._hints.get('typeptr')): + self.transform_setfield_typeptr(hop) + else: + GCTransformer.gct_setfield(self, hop) + def var_needs_set_transform(self, var): return var_needsgc(var) Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gctransform/transform.py Fri Oct 9 18:01:49 2009 @@ -367,6 +367,8 @@ gct_setarrayitem = gct_setfield gct_setinteriorfield = gct_setfield + gct_getfield = default + def gct_zero_gc_pointers_inside(self, hop): pass Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py Fri Oct 9 18:01:49 2009 @@ -165,8 +165,11 @@ can_add_new_types = True can_encode_type_shape = True # set to False initially by the JIT - def __init__(self, GCClass): + size_of_fixed_type_info = llmemory.sizeof(GCData.TYPE_INFO) + + def __init__(self, GCClass, lltype2vtable): self.GCClass = GCClass + self.lltype2vtable = lltype2vtable self.make_type_info_group() self.id_of_type = {} # {LLTYPE: type_id} self.seen_roots = {} @@ -213,6 +216,13 @@ # store it type_id = self.type_info_group.add_member(fullinfo) self.id_of_type[TYPE] = type_id + # store the vtable of the type (if any) immediately thereafter + vtable = self.lltype2vtable.get(TYPE, None) + if vtable is not None: + # check that if we have a vtable, we are not varsize + assert lltype.typeOf(fullinfo) == GCData.TYPE_INFO_PTR + vtable = lltype.normalizeptr(vtable) + self.type_info_group.add_member(vtable) return type_id def get_info(self, type_id): Modified: pypy/branch/gc-compress/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gcwrapper.py Fri Oct 9 18:01:49 2009 @@ -18,7 +18,9 @@ self.gc.setup() def prepare_graphs(self, flowgraphs): + lltype2vtable = self.llinterp.typer.lltype2vtable layoutbuilder = DirectRunLayoutBuilder(self.gc.__class__, + lltype2vtable, self.llinterp) self.get_type_id = layoutbuilder.get_type_id layoutbuilder.initialize_gc_query_function(self.gc) @@ -160,9 +162,9 @@ class DirectRunLayoutBuilder(gctypelayout.TypeLayoutBuilder): - def __init__(self, GCClass, llinterp): + def __init__(self, GCClass, lltype2vtable, llinterp): self.llinterp = llinterp - super(DirectRunLayoutBuilder, self).__init__(GCClass) + super(DirectRunLayoutBuilder, self).__init__(GCClass, lltype2vtable) def make_finalizer_funcptr_for_type(self, TYPE): from pypy.rpython.memory.gctransform.support import get_rtti, \ Modified: pypy/branch/gc-compress/pypy/rpython/memory/test/test_gctypelayout.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/test/test_gctypelayout.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/test/test_gctypelayout.py Fri Oct 9 18:01:49 2009 @@ -1,6 +1,7 @@ +import py from pypy.rpython.memory.gctypelayout import TypeLayoutBuilder, GCData from pypy.rpython.memory.gctypelayout import offsets_to_gc_pointers -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, rclass from pypy.rpython.test.test_llinterp import get_interpreter from pypy.objspace.flow.model import Constant @@ -35,9 +36,9 @@ for T, c in [(GC_S, 0), (GC_S2, 2), (GC_A, 0), (GC_A2, 0), (GC_S3, 2)]: assert len(offsets_to_gc_pointers(T)) == c -def test_layout_builder(): +def test_layout_builder(lltype2vtable={}): # XXX a very minimal test - layoutbuilder = TypeLayoutBuilder(FakeGC) + layoutbuilder = TypeLayoutBuilder(FakeGC, lltype2vtable) for T1, T2 in [(GC_A, GC_S), (GC_A2, GC_S2), (GC_S3, GC_S2)]: tid1 = layoutbuilder.get_type_id(T1) tid2 = layoutbuilder.get_type_id(T2) @@ -45,9 +46,27 @@ lst1 = gcdata.q_varsize_offsets_to_gcpointers_in_var_part(tid1) lst2 = gcdata.q_offsets_to_gc_pointers(tid2) assert len(lst1) == len(lst2) + return layoutbuilder + +def test_layout_builder_with_vtable(): + from pypy.rpython.lltypesystem.lloperation import llop + vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) + layoutbuilder = test_layout_builder({GC_S: vtable}) + tid1 = layoutbuilder.get_type_id(GC_S) + tid2 = layoutbuilder.get_type_id(GC_S2) + tid3 = layoutbuilder.get_type_id(GC_S3) + group = layoutbuilder.type_info_group + vt = llop.get_next_group_member(rclass.CLASSTYPE, group._as_ptr(), tid1, + layoutbuilder.size_of_fixed_type_info) + assert vt == vtable + for tid in [tid2, tid3]: + py.test.raises((lltype.InvalidCast, AssertionError), + llop.get_next_group_member, + rclass.CLASSTYPE, group._as_ptr(), tid, + layoutbuilder.size_of_fixed_type_info) def test_constfold(): - layoutbuilder = TypeLayoutBuilder(FakeGC) + layoutbuilder = TypeLayoutBuilder(FakeGC, {}) tid1 = layoutbuilder.get_type_id(GC_A) tid2 = layoutbuilder.get_type_id(GC_S3) class MockGC: Modified: pypy/branch/gc-compress/pypy/rpython/rtyper.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/rtyper.py (original) +++ pypy/branch/gc-compress/pypy/rpython/rtyper.py Fri Oct 9 18:01:49 2009 @@ -65,6 +65,7 @@ self.class_pbc_attributes = {} self.oo_meth_impls = {} self.cache_dummy_values = {} + self.lltype2vtable = {} self.typererrors = [] self.typererror_count = 0 # make the primitive_to_repr constant mapping @@ -131,12 +132,6 @@ result[repr.lowleveltype] = classdef return result - def lltype_to_vtable_mapping(self): - result = {} - for repr in self.instance_reprs.itervalues(): - result[repr.lowleveltype.TO] = repr.rclass.getvtable() - return result - def get_type_for_typeptr(self, typeptr): try: return self.type_for_typeptr[typeptr._obj] Modified: pypy/branch/gc-compress/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/gc.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/gc.py Fri Oct 9 18:01:49 2009 @@ -11,6 +11,7 @@ class BasicGcPolicy(object): requires_stackless = False + need_no_typeptr = False def __init__(self, db, thread_enabled=False): self.db = db @@ -277,6 +278,7 @@ class FrameworkGcPolicy(BasicGcPolicy): transformerclass = framework.FrameworkGCTransformer + need_no_typeptr = True def struct_setup(self, structdefnode, rtti): if rtti is not None and hasattr(rtti._obj, 'destructor_funcptr'): Modified: pypy/branch/gc-compress/pypy/translator/c/node.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/node.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/node.py Fri Oct 9 18:01:49 2009 @@ -67,6 +67,12 @@ bare=True) self.prefix = somelettersfrom(STRUCT._name) + '_' self.dependencies = {} + # + self.fieldnames = STRUCT._names + if STRUCT._hints.get('typeptr', False): + if db.gcpolicy.need_no_typeptr: + assert self.fieldnames == ('typeptr',) + self.fieldnames = () def setup(self): # this computes self.fields @@ -80,7 +86,7 @@ if needs_gcheader(self.STRUCT): for fname, T in db.gcpolicy.struct_gcheader_definition(self): self.fields.append((fname, db.gettype(T, who_asks=self))) - for name in STRUCT._names: + for name in self.fieldnames: T = self.c_struct_field_type(name) if name == STRUCT._arrayfld: typename = db.gettype(T, varlength=self.varlength, @@ -147,8 +153,7 @@ yield line def visitor_lines(self, prefix, on_field): - STRUCT = self.STRUCT - for name in STRUCT._names: + for name in self.fieldnames: FIELD_T = self.c_struct_field_type(name) cname = self.c_struct_field_name(name) for line in on_field('%s.%s' % (prefix, cname), @@ -157,8 +162,7 @@ def debug_offsets(self): # generate number exprs giving the offset of the elements in the struct - STRUCT = self.STRUCT - for name in STRUCT._names: + for name in self.fieldnames: FIELD_T = self.c_struct_field_type(name) if FIELD_T is Void: yield '-1' @@ -518,7 +522,7 @@ for i, thing in enumerate(self.db.gcpolicy.struct_gcheader_initdata(self)): data.append(('gcheader%d'%i, thing)) - for name in self.T._names: + for name in defnode.fieldnames: data.append((name, getattr(self.obj, name))) # Reasonably, you should only initialise one of the fields of a union From arigo at codespeak.net Fri Oct 9 18:14:12 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Oct 2009 18:14:12 +0200 (CEST) Subject: [pypy-svn] r68286 - pypy/branch/gc-compress/pypy/rpython/memory/test Message-ID: <20091009161412.19A3649844D@codespeak.net> Author: arigo Date: Fri Oct 9 18:14:11 2009 New Revision: 68286 Modified: pypy/branch/gc-compress/pypy/rpython/memory/test/test_transformed_gc.py Log: Fix tests. Modified: pypy/branch/gc-compress/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/test/test_transformed_gc.py Fri Oct 9 18:14:11 2009 @@ -195,7 +195,6 @@ assert heap_size < 16000 * INT_SIZE / 4 # xxx def test_nongc_static_root(self): - from pypy.rpython.lltypesystem import lltype T1 = lltype.GcStruct("C", ('x', lltype.Signed)) T2 = lltype.Struct("C", ('p', lltype.Ptr(T1))) static = lltype.malloc(T2, immortal=True) @@ -552,7 +551,7 @@ class A(object): pass def f(): - from pypy.rpython.lltypesystem import lltype, rffi + from pypy.rpython.lltypesystem import rffi alist = [A() for i in range(50)] idarray = lltype.malloc(rffi.INTP.TO, len(alist), flavor='raw') # Compute the id of all the elements of the list. The goal is @@ -592,7 +591,11 @@ def fix_graph_of_g(translator): from pypy.translator.translator import graphof from pypy.objspace.flow.model import Constant - layoutbuilder = framework.TransformerLayoutBuilder() + from pypy.rpython.lltypesystem import rffi + GCClass = self.gcpolicy.transformerclass.GCClass + lltype2vtable = translator.rtyper.lltype2vtable + layoutbuilder = framework.TransformerLayoutBuilder(GCClass, + lltype2vtable) layoutbuilder.delay_encoding() translator._jit2gc = { 'layoutbuilder': layoutbuilder, @@ -603,7 +606,7 @@ graph = graphof(translator, g) for op in graph.startblock.operations: if op.opname == 'do_malloc_fixedsize_clear': - op.args = [Constant(type_id, lltype.Signed), + op.args = [Constant(type_id, rffi.USHORT), Constant(llmemory.sizeof(P), lltype.Signed), Constant(True, lltype.Bool), # can_collect Constant(False, lltype.Bool), # has_finalizer @@ -628,7 +631,11 @@ def fix_graph_of_g(translator): from pypy.translator.translator import graphof from pypy.objspace.flow.model import Constant - layoutbuilder = framework.TransformerLayoutBuilder() + from pypy.rpython.lltypesystem import rffi + GCClass = self.gcpolicy.transformerclass.GCClass + lltype2vtable = translator.rtyper.lltype2vtable + layoutbuilder = framework.TransformerLayoutBuilder(GCClass, + lltype2vtable) layoutbuilder.delay_encoding() translator._jit2gc = { 'layoutbuilder': layoutbuilder, @@ -639,7 +646,7 @@ graph = graphof(translator, g) for op in graph.startblock.operations: if op.opname == 'do_malloc_fixedsize_clear': - op.args = [Constant(type_id, lltype.Signed), + op.args = [Constant(type_id, rffi.USHORT), Constant(llmemory.sizeof(P), lltype.Signed), Constant(True, lltype.Bool), # can_collect Constant(False, lltype.Bool), # has_finalizer @@ -934,7 +941,6 @@ assert res == 20 + 20 def test_nongc_static_root_minor_collect(self): - from pypy.rpython.lltypesystem import lltype T1 = lltype.GcStruct("C", ('x', lltype.Signed)) T2 = lltype.Struct("C", ('p', lltype.Ptr(T1))) static = lltype.malloc(T2, immortal=True) @@ -957,7 +963,6 @@ def test_static_root_minor_collect(self): - from pypy.rpython.lltypesystem import lltype class A: pass class B: From arigo at codespeak.net Fri Oct 9 18:19:44 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Oct 2009 18:19:44 +0200 (CEST) Subject: [pypy-svn] r68287 - pypy/branch/gc-compress/pypy/rpython/memory/gctransform Message-ID: <20091009161944.5EDFE49844D@codespeak.net> Author: arigo Date: Fri Oct 9 18:19:43 2009 New Revision: 68287 Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py Log: Write a debug_assert for setfield('typeptr'). Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py Fri Oct 9 18:19:43 2009 @@ -789,7 +789,16 @@ def transform_setfield_typeptr(self, hop): # replace such a setfield with an assertion that the typeptr is right - pass # XXX later + v_new = hop.spaceop.args[2] + v_old = hop.genop('getfield_typeptr_group', [hop.spaceop.args[0], + self.c_type_info_group, + self.c_vtableinfo], + resulttype = v_new.concretetype) + v_eq = hop.genop("ptr_eq", [v_old, v_new], + resulttype = lltype.Bool) + c_errmsg = rmodel.inputconst(lltype.Void, + "setfield_typeptr: wrong type") + hop.genop('debug_assert', [v_eq, c_errmsg]) def gct_getfield(self, hop): if (hop.spaceop.args[1].value == 'typeptr' and From pedronis at codespeak.net Fri Oct 9 18:34:41 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 9 Oct 2009 18:34:41 +0200 (CEST) Subject: [pypy-svn] r68288 - pypy/branch/improve-kwd-args/pypy/interpreter Message-ID: <20091009163441.2478B49845F@codespeak.net> Author: pedronis Date: Fri Oct 9 18:34:40 2009 New Revision: 68288 Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py Log: (cfbolz, pedronis) fix some breakage, time to go home Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/argument.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/argument.py Fri Oct 9 18:34:40 2009 @@ -27,8 +27,8 @@ @staticmethod def factory(space, args_w, kwds_w=None, w_stararg=None, w_starstararg=None): - return Argument(space, args_w, kwds_w, - w_stararg, w_starstararg) + return Arguments(space, args_w, kwds_w, + w_stararg, w_starstararg) def num_args(self): # only used in module/__builtin__/interp_classobj.py self._unpack() @@ -335,6 +335,7 @@ raise OperationError(self.space.w_TypeError, self.space.wrap(e.getmsg(fnname))) + @staticmethod def frompacked(space, w_args=None, w_kwds=None): # used by # ./module/_stackless/interp_coroutine.py @@ -344,8 +345,7 @@ # ./interpreter/baseobjspace.py """Convenience static method to build an Arguments from a wrapped sequence and a wrapped dictionary.""" - return self.factory(space, [], w_stararg=w_args, w_starstararg=w_kwds) - frompacked = staticmethod(frompacked) + return Arguments(space, [], w_stararg=w_args, w_starstararg=w_kwds) def topacked(self): # used by ./module/_stackless/interp_coroutine.py @@ -450,6 +450,10 @@ return ArgumentsForTranslation(self.space, args_w, kwds_w) + @staticmethod + def frompacked(space, w_args=None, w_kwds=None): + raise NotImplementedError("go away") + def fromshape(space, (shape_cnt,shape_keys,shape_star,shape_stst), data_w): # used by # ./rpython/callparse.py From arigo at codespeak.net Fri Oct 9 18:39:28 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Oct 2009 18:39:28 +0200 (CEST) Subject: [pypy-svn] r68289 - in pypy/branch/gc-compress/pypy: rpython/lltypesystem rpython/memory/gctransform translator/c Message-ID: <20091009163928.D9F61498455@codespeak.net> Author: arigo Date: Fri Oct 9 18:39:28 2009 New Revision: 68289 Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py pypy/branch/gc-compress/pypy/translator/c/gc.py Log: Fix genc. Seems to work :-) Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py Fri Oct 9 18:39:28 2009 @@ -415,7 +415,7 @@ 'is_group_member_zero': LLOp(canfold=True), 'extract_ushort': LLOp(canfold=True), 'combine_ushort': LLOp(canfold=True), - 'getfield_typeptr_group':LLOp(canfold=True), + 'gc_gettypeptr_group': LLOp(canfold=True), # __________ used by the JIT ________ Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py Fri Oct 9 18:39:28 2009 @@ -433,11 +433,10 @@ from pypy.rpython.lltypesystem import llgroup return llgroup.CombinedSymbolic(ushort, rest) -def op_getfield_typeptr_group(TYPE, obj, grpptr, vtableinfo): +def op_gc_gettypeptr_group(TYPE, obj, grpptr, skipoffset, vtableinfo): HDR = vtableinfo[0] size_gc_header = vtableinfo[1] fieldname = vtableinfo[2] - skipoffset = vtableinfo[3] objaddr = llmemory.cast_ptr_to_adr(obj) hdraddr = objaddr - size_gc_header hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR)) @@ -445,7 +444,7 @@ if lltype.typeOf(typeid) == lltype.Signed: typeid = op_extract_ushort(typeid) return op_get_next_group_member(TYPE, grpptr, typeid, skipoffset) -op_getfield_typeptr_group.need_result_type = True +op_gc_gettypeptr_group.need_result_type = True # ____________________________________________________________ Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py Fri Oct 9 18:39:28 2009 @@ -406,11 +406,12 @@ fields.append(('_' + fldname, FLDTYPE)) size_gc_header = self.gcdata.gc.gcheaderbuilder.size_gc_header - vtableinfo = (HDR, size_gc_header, self.gcdata.gc.typeid_is_in_field, - llmemory.sizeof(gcdata.TYPE_INFO)) + vtableinfo = (HDR, size_gc_header, self.gcdata.gc.typeid_is_in_field) self.c_vtableinfo = rmodel.inputconst(lltype.Void, vtableinfo) tig = self.layoutbuilder.type_info_group._as_ptr() self.c_type_info_group = rmodel.inputconst(lltype.typeOf(tig), tig) + sko = llmemory.sizeof(gcdata.TYPE_INFO) + self.c_vtinfo_skip_offset = rmodel.inputconst(lltype.typeOf(sko), sko) def build_root_walker(self): return ShadowStackRootWalker(self) @@ -782,17 +783,19 @@ # this would become quite a lot of operations, even if it compiles # to C code that is just as efficient as "obj->typeptr". To avoid # that, we just generate a single custom operation instead. - hop.genop('getfield_typeptr_group', [hop.spaceop.args[0], - self.c_type_info_group, - self.c_vtableinfo], + hop.genop('gc_gettypeptr_group', [hop.spaceop.args[0], + self.c_type_info_group, + self.c_vtinfo_skip_offset, + self.c_vtableinfo], resultvar = hop.spaceop.result) def transform_setfield_typeptr(self, hop): # replace such a setfield with an assertion that the typeptr is right v_new = hop.spaceop.args[2] - v_old = hop.genop('getfield_typeptr_group', [hop.spaceop.args[0], - self.c_type_info_group, - self.c_vtableinfo], + v_old = hop.genop('gc_gettypeptr_group', [hop.spaceop.args[0], + self.c_type_info_group, + self.c_vtinfo_skip_offset, + self.c_vtableinfo], resulttype = v_new.concretetype) v_eq = hop.genop("ptr_eq", [v_old, v_new], resulttype = lltype.Bool) Modified: pypy/branch/gc-compress/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/gc.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/gc.py Fri Oct 9 18:39:28 2009 @@ -324,6 +324,19 @@ o = top_container(defnode.obj) return defnode.db.gctransformer.gc_field_values_for(o) + def OP_GC_GETTYPEPTR_GROUP(self, funcgen, op): + # expands to a number of steps, as per rpython/lltypesystem/opimpl.py, + # all implemented by a single call to a C macro. + [v_obj, c_grpptr, c_skipoffset, c_vtableinfo] = op.args + fieldname = c_vtableinfo.value[2] + return ( + 'OP_GET_NEXT_GROUP_MEMBER(%s, (unsigned short)%s->_%s, %s, %s);' + % (funcgen.expr(c_grpptr), + funcgen.expr(v_obj), + fieldname, + funcgen.expr(c_skipoffset), + funcgen.expr(op.result))) + class AsmGcRootFrameworkGcPolicy(FrameworkGcPolicy): transformerclass = asmgcroot.AsmGcRootFrameworkGCTransformer From arigo at codespeak.net Fri Oct 9 19:22:17 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Oct 2009 19:22:17 +0200 (CEST) Subject: [pypy-svn] r68290 - in pypy/trunk/pypy/lib: . test2 Message-ID: <20091009172217.1895A498465@codespeak.net> Author: arigo Date: Fri Oct 9 19:22:16 2009 New Revision: 68290 Added: pypy/trunk/pypy/lib/test2/test_grp_extra.py (contents, props changed) Modified: pypy/trunk/pypy/lib/grp.py Log: issue457 resolved Test and fix for grp.getgrnam(). Thanks haypo. Modified: pypy/trunk/pypy/lib/grp.py ============================================================================== --- pypy/trunk/pypy/lib/grp.py (original) +++ pypy/trunk/pypy/lib/grp.py Fri Oct 9 19:22:16 2009 @@ -71,10 +71,12 @@ raise KeyError(gid) return _group_from_gstruct(res) -def getgrnam(gid): - res = libc.getgrnam(gid) +def getgrnam(name): + if not isinstance(name, str): + raise TypeError("expected string") + res = libc.getgrnam(name) if not res: - raise KeyError(gid) + raise KeyError(name) return _group_from_gstruct(res) def getgrall(): Added: pypy/trunk/pypy/lib/test2/test_grp_extra.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/lib/test2/test_grp_extra.py Fri Oct 9 19:22:16 2009 @@ -0,0 +1,5 @@ +import py +from pypy.lib import grp + +def test_extra(): + py.test.raises(TypeError, grp.getgrnam, None) From arigo at codespeak.net Fri Oct 9 19:24:39 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Oct 2009 19:24:39 +0200 (CEST) Subject: [pypy-svn] r68291 - pypy/trunk/pypy/lib/test2 Message-ID: <20091009172439.04BC9498465@codespeak.net> Author: arigo Date: Fri Oct 9 19:24:39 2009 New Revision: 68291 Modified: pypy/trunk/pypy/lib/test2/test_grp_extra.py Log: Also mention the original line that showed the problem. Modified: pypy/trunk/pypy/lib/test2/test_grp_extra.py ============================================================================== --- pypy/trunk/pypy/lib/test2/test_grp_extra.py (original) +++ pypy/trunk/pypy/lib/test2/test_grp_extra.py Fri Oct 9 19:24:39 2009 @@ -2,4 +2,5 @@ from pypy.lib import grp def test_extra(): + py.test.raises(TypeError, grp.getgrnam, False) py.test.raises(TypeError, grp.getgrnam, None) From arigo at codespeak.net Fri Oct 9 19:59:33 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Oct 2009 19:59:33 +0200 (CEST) Subject: [pypy-svn] r68292 - in pypy/branch/gc-compress/pypy: rpython/lltypesystem rpython/memory translator/c/src Message-ID: <20091009175933.DE0BB498464@codespeak.net> Author: arigo Date: Fri Oct 9 19:59:33 2009 New Revision: 68292 Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h Log: Invert this operation for now. Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/lloperation.py Fri Oct 9 19:59:33 2009 @@ -410,9 +410,10 @@ 'cast_adr_to_ptr': LLOp(canfold=True), 'cast_adr_to_int': LLOp(sideeffects=False), 'cast_int_to_adr': LLOp(canfold=True), # not implemented in llinterp + 'get_group_member': LLOp(canfold=True), 'get_next_group_member':LLOp(canfold=True), - 'is_group_member_zero': LLOp(canfold=True), + 'is_group_member_nonzero':LLOp(canfold=True), 'extract_ushort': LLOp(canfold=True), 'combine_ushort': LLOp(canfold=True), 'gc_gettypeptr_group': LLOp(canfold=True), Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py Fri Oct 9 19:59:33 2009 @@ -416,13 +416,13 @@ return lltype.cast_pointer(TYPE, member) op_get_next_group_member.need_result_type = True -def op_is_group_member_zero(memberoffset): +def op_is_group_member_nonzero(memberoffset): from pypy.rpython.lltypesystem import llgroup if isinstance(memberoffset, llgroup.GroupMemberOffset): - return memberoffset.index == 0 + return memberoffset.index != 0 else: assert isinstance(memberoffset, int) - return memberoffset == 0 + return memberoffset != 0 def op_extract_ushort(combinedoffset): from pypy.rpython.lltypesystem import llgroup Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py Fri Oct 9 19:59:33 2009 @@ -42,14 +42,14 @@ self.type_info_group_ptr = type_info_group._as_ptr() def get(self, typeid): - ll_assert(not llop.is_group_member_zero(lltype.Bool, typeid), + ll_assert(llop.is_group_member_nonzero(lltype.Bool, typeid), "invalid type_id") return llop.get_group_member(GCData.TYPE_INFO_PTR, self.type_info_group_ptr, typeid) def get_varsize(self, typeid): - ll_assert(not llop.is_group_member_zero(lltype.Bool, typeid), + ll_assert(llop.is_group_member_nonzero(lltype.Bool, typeid), "invalid type_id") return llop.get_group_member(GCData.VARSIZE_TYPE_INFO_PTR, self.type_info_group_ptr, Modified: pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h (original) +++ pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h Fri Oct 9 19:59:33 2009 @@ -11,8 +11,8 @@ #define OP_GET_NEXT_GROUP_MEMBER(groupptr, compactoffset, skipoffset, r) \ r = ((char*)groupptr) + ((long)compactoffset)*sizeof(long) + skipoffset -#define OP_IS_GROUP_MEMBER_ZERO(compactoffset, r) \ - r = (compactoffset == 0) +#define OP_IS_GROUP_MEMBER_NONZERO(compactoffset, r) \ + r = (compactoffset != 0) #define OP_EXTRACT_USHORT(value, r) \ r = (unsigned short)value From arigo at codespeak.net Fri Oct 9 20:16:00 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Oct 2009 20:16:00 +0200 (CEST) Subject: [pypy-svn] r68293 - pypy/branch/gc-compress/pypy/translator/backendopt Message-ID: <20091009181600.B2641168032@codespeak.net> Author: arigo Date: Fri Oct 9 20:16:00 2009 New Revision: 68293 Modified: pypy/branch/gc-compress/pypy/translator/backendopt/inline.py Log: Uh? This seemed to be broken. No test cares... Modified: pypy/branch/gc-compress/pypy/translator/backendopt/inline.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/backendopt/inline.py (original) +++ pypy/branch/gc-compress/pypy/translator/backendopt/inline.py Fri Oct 9 20:16:00 2009 @@ -753,7 +753,7 @@ subcount = inline_function(translator, graph, parentgraph, lltype_to_classdef, raise_analyzer, call_count_pred, cleanup=False) - to_cleanup[graph] = True + to_cleanup[parentgraph] = True res = bool(subcount) except CannotInline: try_again[graph] = True From cfbolz at codespeak.net Fri Oct 9 21:35:59 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 9 Oct 2009 21:35:59 +0200 (CEST) Subject: [pypy-svn] r68294 - pypy/trunk/pypy/interpreter Message-ID: <20091009193559.928A5498464@codespeak.net> Author: cfbolz Date: Fri Oct 9 21:35:56 2009 New Revision: 68294 Modified: pypy/trunk/pypy/interpreter/argument.py pypy/trunk/pypy/interpreter/function.py Log: (pedronis around, cfbolz): do things slightly differently to get some more speed Modified: pypy/trunk/pypy/interpreter/argument.py ============================================================================== --- pypy/trunk/pypy/interpreter/argument.py (original) +++ pypy/trunk/pypy/interpreter/argument.py Fri Oct 9 21:35:56 2009 @@ -382,7 +382,7 @@ # which were put into place by prepend(). This way, keywords do # not conflict with the hidden extra argument bound by methods. if kwds_w and input_argcount > blindargs: - for name in argnames[blindargs:input_argcount]: + for name in argnames[blindargs:input_argcount]: # XXX if name in kwds_w: raise ArgErrMultipleValues(name) Modified: pypy/trunk/pypy/interpreter/function.py ============================================================================== --- pypy/trunk/pypy/interpreter/function.py (original) +++ pypy/trunk/pypy/interpreter/function.py Fri Oct 9 21:35:56 2009 @@ -117,13 +117,13 @@ frame.peekvalue(0)) elif (nargs|Code.FLATPYCALL) == fast_natural_arity: assert isinstance(code, PyCode) - return self._flat_pycall(code, nargs, frame, 0) + return self._flat_pycall(code, nargs, frame) elif fast_natural_arity&Code.FLATPYCALL: natural_arity = fast_natural_arity&0xff if natural_arity > nargs >= natural_arity-len(self.defs_w): assert isinstance(code, PyCode) - return self._flat_pycall(code, nargs, frame, - natural_arity-nargs) + return self._flat_pycall_defaults(code, nargs, frame, + natural_arity-nargs) elif fast_natural_arity == Code.PASSTHROUGHARGS1 and nargs >= 1: assert isinstance(code, gateway.BuiltinCodePassThroughArguments1) w_obj = frame.peekvalue(nargs-1) @@ -133,7 +133,7 @@ args = frame.make_arguments(nargs) return self.call_args(args) - def _flat_pycall(self, code, nargs, frame, defs_to_load): + def _flat_pycall(self, code, nargs, frame): # code is a PyCode new_frame = self.space.createframe(code, self.w_func_globals, self.closure) @@ -141,14 +141,23 @@ w_arg = frame.peekvalue(nargs-1-i) new_frame.fastlocals_w[i] = w_arg - if defs_to_load: - defs_w = self.defs_w - ndefs = len(defs_w) - start = ndefs-defs_to_load - i = nargs - for j in xrange(start, ndefs): - new_frame.fastlocals_w[i] = defs_w[j] - i += 1 + return new_frame.run() + + def _flat_pycall_defaults(self, code, nargs, frame, defs_to_load): + # code is a PyCode + new_frame = self.space.createframe(code, self.w_func_globals, + self.closure) + for i in xrange(nargs): + w_arg = frame.peekvalue(nargs-1-i) + new_frame.fastlocals_w[i] = w_arg + + defs_w = self.defs_w + ndefs = len(defs_w) + start = ndefs-defs_to_load + i = nargs + for j in xrange(start, ndefs): + new_frame.fastlocals_w[i] = defs_w[j] + i += 1 return new_frame.run() def getdict(self): From arigo at codespeak.net Fri Oct 9 21:38:04 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Oct 2009 21:38:04 +0200 (CEST) Subject: [pypy-svn] r68295 - in pypy/branch/gc-compress/pypy/translator/c: . src Message-ID: <20091009193804.9ADE8498464@codespeak.net> Author: arigo Date: Fri Oct 9 21:38:04 2009 New Revision: 68295 Modified: pypy/branch/gc-compress/pypy/translator/c/funcgen.py pypy/branch/gc-compress/pypy/translator/c/gc.py pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h Log: Fix warnings. Modified: pypy/branch/gc-compress/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/funcgen.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/funcgen.py Fri Oct 9 21:38:04 2009 @@ -781,5 +781,22 @@ def OP_PROMOTE_VIRTUALIZABLE(self, op): return '/* PROMOTE_VIRTUALIZABLE %s */' % op + def OP_GET_GROUP_MEMBER(self, op): + typename = self.db.gettype(op.result.concretetype) + return '%s = (%s)_OP_GET_GROUP_MEMBER(%s, %s);' % ( + self.expr(op.result), + cdecl(typename, ''), + self.expr(op.args[0]), + self.expr(op.args[1])) + + def OP_GET_NEXT_GROUP_MEMBER(self, op): + typename = self.db.gettype(op.result.concretetype) + return '%s = (%s)_OP_GET_NEXT_GROUP_MEMBER(%s, %s, %s);' % ( + self.expr(op.result), + cdecl(typename, ''), + self.expr(op.args[0]), + self.expr(op.args[1]), + self.expr(op.args[2])) + assert not USESLOTS or '__dict__' not in dir(FunctionCodeGenerator) Modified: pypy/branch/gc-compress/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/gc.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/gc.py Fri Oct 9 21:38:04 2009 @@ -328,14 +328,16 @@ # expands to a number of steps, as per rpython/lltypesystem/opimpl.py, # all implemented by a single call to a C macro. [v_obj, c_grpptr, c_skipoffset, c_vtableinfo] = op.args + typename = funcgen.db.gettype(op.result.concretetype) fieldname = c_vtableinfo.value[2] return ( - 'OP_GET_NEXT_GROUP_MEMBER(%s, (unsigned short)%s->_%s, %s, %s);' - % (funcgen.expr(c_grpptr), + '%s = (%s)_OP_GET_NEXT_GROUP_MEMBER(%s, (unsigned short)%s->_%s, %s);' + % (funcgen.expr(op.result), + cdecl(typename, ''), + funcgen.expr(c_grpptr), funcgen.expr(v_obj), fieldname, - funcgen.expr(c_skipoffset), - funcgen.expr(op.result))) + funcgen.expr(c_skipoffset))) class AsmGcRootFrameworkGcPolicy(FrameworkGcPolicy): transformerclass = asmgcroot.AsmGcRootFrameworkGCTransformer Modified: pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h (original) +++ pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h Fri Oct 9 21:38:04 2009 @@ -5,11 +5,11 @@ #define GROUP_MEMBER_OFFSET(group, membername) \ ((unsigned short)((((char*)&membername) - ((char*)&group)) / sizeof(long))) -#define OP_GET_GROUP_MEMBER(groupptr, compactoffset, r) \ - r = ((char*)groupptr) + ((long)compactoffset)*sizeof(long) +#define _OP_GET_GROUP_MEMBER(groupptr, compactoffset) \ + (((char*)groupptr) + ((long)compactoffset)*sizeof(long)) -#define OP_GET_NEXT_GROUP_MEMBER(groupptr, compactoffset, skipoffset, r) \ - r = ((char*)groupptr) + ((long)compactoffset)*sizeof(long) + skipoffset +#define _OP_GET_NEXT_GROUP_MEMBER(groupptr, compactoffset, skipoffset) \ + (((char*)groupptr) + ((long)compactoffset)*sizeof(long) + skipoffset) #define OP_IS_GROUP_MEMBER_NONZERO(compactoffset, r) \ r = (compactoffset != 0) From arigo at codespeak.net Fri Oct 9 21:43:25 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 9 Oct 2009 21:43:25 +0200 (CEST) Subject: [pypy-svn] r68296 - pypy/branch/gc-compress/pypy/rpython/memory/gctransform Message-ID: <20091009194325.7A2C8498464@codespeak.net> Author: arigo Date: Fri Oct 9 21:43:24 2009 New Revision: 68296 Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctransform/transform.py Log: Experimental: constant-folding here appears to be needed to remove a number of expressions. No clue how much it slows down a PyPy translation. Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gctransform/transform.py Fri Oct 9 21:43:24 2009 @@ -10,6 +10,7 @@ from pypy.translator.backendopt import graphanalyze from pypy.translator.backendopt.canraise import RaiseAnalyzer from pypy.translator.backendopt.ssa import DataFlowFamilyBuilder +from pypy.translator.backendopt.constfold import constant_fold_graph from pypy.annotation import model as annmodel from pypy.rpython import rmodel, annlowlevel from pypy.rpython.memory import gc @@ -144,16 +145,20 @@ if self.inline: raise_analyzer = RaiseAnalyzer(self.translator) to_enum = self.graph_dependencies.get(graph, self.graphs_to_inline) + must_constfold = False for inline_graph in to_enum: try: inline.inline_function(self.translator, inline_graph, graph, self.lltype_to_classdef, raise_analyzer, cleanup=False) + must_constfold = True except inline.CannotInline, e: print 'CANNOT INLINE:', e print '\t%s into %s' % (inline_graph, graph) cleanup_graph(graph) + if must_constfold: + constant_fold_graph(graph) def compute_borrowed_vars(self, graph): # the input args are borrowed, and stay borrowed for as long as they From arigo at codespeak.net Sat Oct 10 14:00:48 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 10 Oct 2009 14:00:48 +0200 (CEST) Subject: [pypy-svn] r68297 - pypy/branch/gc-compress/pypy/translator/c/src Message-ID: <20091010120048.33BEB168079@codespeak.net> Author: arigo Date: Sat Oct 10 14:00:47 2009 New Revision: 68297 Modified: pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h Log: Tentatively try to force gcc to do the right thing. Modified: pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h (original) +++ pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h Sat Oct 10 14:00:47 2009 @@ -9,7 +9,7 @@ (((char*)groupptr) + ((long)compactoffset)*sizeof(long)) #define _OP_GET_NEXT_GROUP_MEMBER(groupptr, compactoffset, skipoffset) \ - (((char*)groupptr) + ((long)compactoffset)*sizeof(long) + skipoffset) + ((((char*)groupptr) + skipoffset) + ((long)compactoffset)*sizeof(long)) #define OP_IS_GROUP_MEMBER_NONZERO(compactoffset, r) \ r = (compactoffset != 0) From arigo at codespeak.net Sat Oct 10 14:09:27 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 10 Oct 2009 14:09:27 +0200 (CEST) Subject: [pypy-svn] r68298 - in pypy/branch/gc-compress/pypy: rpython/memory/gctransform translator/c Message-ID: <20091010120927.13D04168077@codespeak.net> Author: arigo Date: Sat Oct 10 14:09:26 2009 New Revision: 68298 Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py pypy/branch/gc-compress/pypy/translator/c/database.py Log: - don't generate any code for setfield('typeptr'). - tweak the dependency tracking... Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py Sat Oct 10 14:09:26 2009 @@ -467,13 +467,15 @@ newgcdependencies = [] newgcdependencies.append(ll_static_roots_inside) - # also add the type_info_group's members into newgcdependencies, - # to make sure that they are all followed (only a part of them - # might have been followed by a previous enum_dependencies()). - newgcdependencies.extend(self.layoutbuilder.type_info_group.members) self.write_typeid_list() return newgcdependencies + def get_final_dependencies(self): + # returns an iterator enumerating the type_info_group's members, + # to make sure that they are all followed (only a part of them + # might have been followed by a previous enum_dependencies()). + return iter(self.layoutbuilder.type_info_group.members) + def write_typeid_list(self): """write out the list of type ids together with some info""" from pypy.tool.udir import udir @@ -791,17 +793,19 @@ def transform_setfield_typeptr(self, hop): # replace such a setfield with an assertion that the typeptr is right - v_new = hop.spaceop.args[2] - v_old = hop.genop('gc_gettypeptr_group', [hop.spaceop.args[0], - self.c_type_info_group, - self.c_vtinfo_skip_offset, - self.c_vtableinfo], - resulttype = v_new.concretetype) - v_eq = hop.genop("ptr_eq", [v_old, v_new], - resulttype = lltype.Bool) - c_errmsg = rmodel.inputconst(lltype.Void, - "setfield_typeptr: wrong type") - hop.genop('debug_assert', [v_eq, c_errmsg]) + # (xxx not very useful right now, so disabled) + if 0: + v_new = hop.spaceop.args[2] + v_old = hop.genop('gc_gettypeptr_group', [hop.spaceop.args[0], + self.c_type_info_group, + self.c_vtinfo_skip_offset, + self.c_vtableinfo], + resulttype = v_new.concretetype) + v_eq = hop.genop("ptr_eq", [v_old, v_new], + resulttype = lltype.Bool) + c_errmsg = rmodel.inputconst(lltype.Void, + "setfield_typeptr: wrong type") + hop.genop('debug_assert', [v_eq, c_errmsg]) def gct_getfield(self, hop): if (hop.spaceop.args[1].value == 'typeptr' and Modified: pypy/branch/gc-compress/pypy/translator/c/database.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/database.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/database.py Sat Oct 10 14:09:26 2009 @@ -288,6 +288,8 @@ finish_callbacks.append(('Stackless transformer: finished', self.stacklesstransformer.finish)) if self.gctransformer: + finish_callbacks.append(('GC transformer: tracking vtables', + self.gctransformer.get_final_dependencies)) finish_callbacks.append(('GC transformer: finished tables', self.gctransformer.finish_tables)) From arigo at codespeak.net Sat Oct 10 14:30:48 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 10 Oct 2009 14:30:48 +0200 (CEST) Subject: [pypy-svn] r68299 - pypy/trunk/demo Message-ID: <20091010123048.1FC22168079@codespeak.net> Author: arigo Date: Sat Oct 10 14:30:45 2009 New Revision: 68299 Modified: pypy/trunk/demo/bpnn.py Log: Allow an optional command-line parameter, both in the translated and non-translated versions. Modified: pypy/trunk/demo/bpnn.py ============================================================================== --- pypy/trunk/demo/bpnn.py (original) +++ pypy/trunk/demo/bpnn.py Sat Oct 10 14:30:45 2009 @@ -191,7 +191,10 @@ import time def entry_point(argv): - N = 200 + if len(argv) > 1: + N = int(argv[1]) + else: + N = 200 T = time.time() for i in range(N): demo() @@ -205,5 +208,7 @@ return entry_point, None if __name__ == '__main__': - demo() + if len(sys.argv) == 1: + sys.argv.append('1') + entry_point(sys.argv) print __doc__ From arigo at codespeak.net Sat Oct 10 15:24:04 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 10 Oct 2009 15:24:04 +0200 (CEST) Subject: [pypy-svn] r68300 - in pypy/branch/gc-compress/pypy: rpython rpython/lltypesystem rpython/memory/gc translator/c Message-ID: <20091010132404.B0E2816807A@codespeak.net> Author: arigo Date: Sat Oct 10 15:24:04 2009 New Revision: 68300 Modified: pypy/branch/gc-compress/pypy/rpython/llinterp.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py pypy/branch/gc-compress/pypy/rpython/memory/gc/generation.py pypy/branch/gc-compress/pypy/rpython/memory/gc/hybrid.py pypy/branch/gc-compress/pypy/rpython/memory/gc/semispace.py pypy/branch/gc-compress/pypy/translator/c/gc.py Log: Use the CombinedSymbolic in semispace and related GCs too. Simplifies things a tiny bit. Need to hack on gc_reload_possibly_moved to detect when we are trying to reload a value into a constant(!). Modified: pypy/branch/gc-compress/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/llinterp.py (original) +++ pypy/branch/gc-compress/pypy/rpython/llinterp.py Sat Oct 10 15:24:04 2009 @@ -869,7 +869,10 @@ assert v_ptr.concretetype.TO._gckind == 'gc' newaddr = self.getval(v_newaddr) p = llmemory.cast_adr_to_ptr(newaddr, v_ptr.concretetype) - self.setvar(v_ptr, p) + if isinstance(v_ptr, Constant): + assert v_ptr.value == p + else: + self.setvar(v_ptr, p) op_gc_reload_possibly_moved.specialform = True def op_gc_id(self, v_ptr): Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py Sat Oct 10 15:24:04 2009 @@ -94,9 +94,20 @@ self.rest = rest def __and__(self, other): - assert (other & 0xFFFF) == 0 - return self.rest & other + if (other & 0xFFFF) == 0: + return self.rest & other + if (other & 0xFFFF) == 0xFFFF: + return CombinedSymbolic(self.lowpart, self.rest & other) + raise Exception("other=0x%x" % other) def __or__(self, other): assert (other & 0xFFFF) == 0 return CombinedSymbolic(self.lowpart, self.rest | other) + + def __add__(self, other): + assert (other & 0xFFFF) == 0 + return CombinedSymbolic(self.lowpart, self.rest + other) + + def __sub__(self, other): + assert (other & 0xFFFF) == 0 + return CombinedSymbolic(self.lowpart, self.rest - other) Modified: pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/gc-compress/pypy/rpython/lltypesystem/opimpl.py Sat Oct 10 15:24:04 2009 @@ -171,10 +171,19 @@ return not b def op_int_add(x, y): - assert isinstance(x, (int, llmemory.AddressOffset)) + if not isinstance(x, (int, llmemory.AddressOffset)): + from pypy.rpython.lltypesystem import llgroup + assert isinstance(x, llgroup.CombinedSymbolic) assert isinstance(y, (int, llmemory.AddressOffset)) return intmask(x + y) +def op_int_sub(x, y): + if not isinstance(x, int): + from pypy.rpython.lltypesystem import llgroup + assert isinstance(x, llgroup.CombinedSymbolic) + assert isinstance(y, int) + return intmask(x - y) + def op_int_and(x, y): if not isinstance(x, int): from pypy.rpython.lltypesystem import llgroup Modified: pypy/branch/gc-compress/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gc/generation.py Sat Oct 10 15:24:04 2009 @@ -244,7 +244,7 @@ newobj = SemiSpaceGC.make_a_copy(self, obj, objsize) # During a full collect, all copied objects might implicitly come # from the nursery. In case they do, we must add this flag: - self.header(newobj).addflag(GCFLAG_NO_YOUNG_PTRS) + self.header(newobj).tid |= GCFLAG_NO_YOUNG_PTRS return newobj # history: this was missing and caused an object to become old but without the # flag set. Such an object is bogus in the sense that the write_barrier doesn't @@ -263,7 +263,7 @@ while oldlist.non_empty(): obj = oldlist.pop() hdr = self.header(obj) - hdr.addflag(GCFLAG_NO_YOUNG_PTRS) + hdr.tid |= GCFLAG_NO_YOUNG_PTRS def weakrefs_grow_older(self): while self.young_objects_with_weakrefs.non_empty(): @@ -293,7 +293,7 @@ self.last_generation_root_objects = self.AddressStack() while stack.non_empty(): obj = stack.pop() - self.header(obj).addflag(GCFLAG_NO_HEAP_PTRS) + self.header(obj).tid |= GCFLAG_NO_HEAP_PTRS # ^^^ the flag we just added will be removed immediately if # the object still contains pointers to younger objects self.trace(obj, self._trace_external_obj, obj) @@ -360,7 +360,7 @@ count += 1 obj = oldlist.pop() hdr = self.header(obj) - hdr.addflag(GCFLAG_NO_YOUNG_PTRS) + hdr.tid |= GCFLAG_NO_YOUNG_PTRS self.trace_and_drag_out_of_nursery(obj) if self.config.gcconfig.debugprint: llop.debug_print(lltype.Void, "collect_oldrefs_to_nursery", count) @@ -426,7 +426,7 @@ JIT_WB_IF_FLAG = "XXXX" # GCFLAG_NO_YOUNG_PTRS def write_barrier(self, newvalue, addr_struct): - if self.header(addr_struct).getflags() & GCFLAG_NO_YOUNG_PTRS: + if self.header(addr_struct).tid & GCFLAG_NO_YOUNG_PTRS: self.remember_young_pointer(addr_struct, newvalue) def _setup_wb(self): @@ -444,7 +444,7 @@ "nursery object with GCFLAG_NO_YOUNG_PTRS") if self.is_in_nursery(addr): self.old_objects_pointing_to_young.append(addr_struct) - self.header(addr_struct).delflag(GCFLAG_NO_YOUNG_PTRS) + self.header(addr_struct).tid &= ~GCFLAG_NO_YOUNG_PTRS elif addr == NULL: return self.write_into_last_generation_obj(addr_struct, addr) @@ -453,25 +453,24 @@ def assume_young_pointers(self, addr_struct): objhdr = self.header(addr_struct) - flags = objhdr.getflags() + flags = objhdr.tid if flags & GCFLAG_NO_YOUNG_PTRS: self.old_objects_pointing_to_young.append(addr_struct) - objhdr.delflag(GCFLAG_NO_YOUNG_PTRS) + objhdr.tid &= ~GCFLAG_NO_YOUNG_PTRS if flags & GCFLAG_NO_HEAP_PTRS: - objhdr.delflag(GCFLAG_NO_HEAP_PTRS) + objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS self.last_generation_root_objects.append(addr_struct) def write_into_last_generation_obj(self, addr_struct, addr): objhdr = self.header(addr_struct) - flags = objhdr.getflags() - if flags & GCFLAG_NO_HEAP_PTRS: + if objhdr.tid & GCFLAG_NO_HEAP_PTRS: if not self.is_last_generation(addr): - objhdr.delflag(GCFLAG_NO_HEAP_PTRS) + objhdr.tid &= ~GCFLAG_NO_HEAP_PTRS self.last_generation_root_objects.append(addr_struct) def is_last_generation(self, obj): # overridden by HybridGC - return (self.header(obj).getflags() & GCFLAG_EXTERNAL) != 0 + return (self.header(obj).tid & GCFLAG_EXTERNAL) != 0 def _compute_id(self, obj): if self.is_in_nursery(obj): @@ -502,7 +501,7 @@ """Check the invariants about 'obj' that should be true between collections.""" SemiSpaceGC.debug_check_object(self, obj) - flags = self.header(obj).getflags() + flags = self.header(obj).tid if flags & GCFLAG_NO_YOUNG_PTRS: ll_assert(not self.is_in_nursery(obj), "nursery object with GCFLAG_NO_YOUNG_PTRS") @@ -539,10 +538,10 @@ self._debug_check_flag_2, None) def _debug_check_flag_1(self, obj, ignored): - ll_assert(not (self.header(obj).getflags() & GCFLAG_NO_YOUNG_PTRS), + ll_assert(not (self.header(obj).tid & GCFLAG_NO_YOUNG_PTRS), "unexpected GCFLAG_NO_YOUNG_PTRS") def _debug_check_flag_2(self, obj, ignored): - ll_assert(not (self.header(obj).getflags() & GCFLAG_NO_HEAP_PTRS), + ll_assert(not (self.header(obj).tid & GCFLAG_NO_HEAP_PTRS), "unexpected GCFLAG_NO_HEAP_PTRS") def debug_check_can_copy(self, obj): Modified: pypy/branch/gc-compress/pypy/rpython/memory/gc/hybrid.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gc/hybrid.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gc/hybrid.py Sat Oct 10 15:24:04 2009 @@ -252,7 +252,7 @@ return llmemory.cast_adr_to_ptr(result + size_gc_header, llmemory.GCREF) def can_move(self, addr): - return not (self.header(addr).getflags() & GCFLAG_EXTERNAL) + return not (self.header(addr).tid & GCFLAG_EXTERNAL) def malloc_varsize_collecting_nursery(self, totalsize): result = self.collect_nursery() @@ -341,9 +341,9 @@ self._nonmoving_copy_size = 0 def _set_gcflag_unvisited(self, obj, ignored): - ll_assert(not (self.header(obj).getflags() & GCFLAG_UNVISITED), + ll_assert(not (self.header(obj).tid & GCFLAG_UNVISITED), "bogus GCFLAG_UNVISITED on gen3 obj") - self.header(obj).addflag(GCFLAG_UNVISITED) + self.header(obj).tid |= GCFLAG_UNVISITED def collect_roots(self): if not self.is_collecting_gen3(): @@ -362,20 +362,20 @@ # ones with GCFLAG_FORWARDED set and GCFLAG_UNVISITED not set. # This is equivalent to self.is_forwarded() for all objects except # the ones obtained by raw_malloc. - flags = self.header(obj).getflags() + flags = self.header(obj).tid return ((flags & (GCFLAG_FORWARDED|GCFLAG_UNVISITED)) == GCFLAG_FORWARDED) def is_last_generation(self, obj): - flags = self.header(obj).getflags() + flags = self.header(obj).tid return ((flags & (GCFLAG_EXTERNAL|GCFLAG_AGE_MASK)) == (GCFLAG_EXTERNAL|GCFLAG_AGE_MAX)) def visit_external_object(self, obj): hdr = self.header(obj) - if hdr.getflags() & GCFLAG_UNVISITED: + if hdr.tid & GCFLAG_UNVISITED: # This is a not-visited-yet raw_malloced object. - hdr.delflag(GCFLAG_UNVISITED) + hdr.tid &= ~GCFLAG_UNVISITED self.rawmalloced_objects_to_trace.append(obj) def make_a_copy(self, obj, objsize): @@ -384,7 +384,7 @@ # If they don't, we count how many times they are copied and when # some threshold is reached we make the copy a non-movable "external" # object. The threshold is MAX_SEMISPACE_AGE. - flags = self.header(obj).getflags() + flags = self.header(obj).tid # XXX the following logic is not doing exactly what is explained # above: any object without GCFLAG_NO_YOUNG_PTRS has its age not # incremented. This is accidental: it means that objects that @@ -403,7 +403,7 @@ # skip GenerationGC.make_a_copy() as we already did the right # thing about GCFLAG_NO_YOUNG_PTRS newobj = SemiSpaceGC.make_a_copy(self, obj, objsize) - self.header(newobj).setflags(flags) + self.header(newobj).tid = flags return newobj def make_a_nonmoving_copy(self, obj, objsize): @@ -420,7 +420,7 @@ llmemory.raw_memcopy(obj - self.size_gc_header(), newaddr, totalsize) newobj = newaddr + self.size_gc_header() hdr = self.header(newobj) - hdr.addflag(self.GCFLAGS_FOR_NEW_EXTERNAL_OBJECTS) + hdr.tid |= self.GCFLAGS_FOR_NEW_EXTERNAL_OBJECTS # GCFLAG_UNVISITED is not set # GCFLAG_NO_HEAP_PTRS is not set either, conservatively. It may be # set by the next collection's collect_last_generation_roots(). @@ -485,7 +485,7 @@ newgen3roots = self.AddressStack() while gen3roots.non_empty(): obj = gen3roots.pop() - if not (self.header(obj).getflags() & GCFLAG_UNVISITED): + if not (self.header(obj).tid & GCFLAG_UNVISITED): newgen3roots.append(obj) gen3roots.delete() self.last_generation_root_objects = newgen3roots @@ -500,7 +500,7 @@ alive_count = alive_size = dead_count = dead_size = 0 while objects.non_empty(): obj = objects.pop() - flags = self.header(obj).getflags() + flags = self.header(obj).tid if flags & GCFLAG_UNVISITED: if self.config.gcconfig.debugprint: dead_count+=1 @@ -526,12 +526,12 @@ # the object stays in generation 2 flags |= GCFLAG_UNVISITED surviving_objects.append(obj) - self.header(obj).setflags(flags) + self.header(obj).tid = flags elif generation == -2: # the object stays in generation -2 flags |= GCFLAG_UNVISITED surviving_objects.append(obj) - self.header(obj).setflags(flags) + self.header(obj).tid = flags objects.delete() if generation == 2: self.gen2_rawmalloced_objects = surviving_objects @@ -575,7 +575,7 @@ """Check the invariants about 'obj' that should be true between collections.""" GenerationGC.debug_check_object(self, obj) - flags = self.header(obj).getflags() + flags = self.header(obj).tid if flags & GCFLAG_UNVISITED: ll_assert(self._d_gen2ro.contains(obj), "GCFLAG_UNVISITED on non-gen2 object") @@ -589,7 +589,7 @@ self.gen3_rawmalloced_objects.foreach(self._debug_check_gen3, None) def _debug_check_gen2(self, obj, ignored): - flags = self.header(obj).getflags() + flags = self.header(obj).tid ll_assert(bool(flags & GCFLAG_EXTERNAL), "gen2: missing GCFLAG_EXTERNAL") ll_assert(bool(flags & GCFLAG_UNVISITED), @@ -597,7 +597,7 @@ ll_assert((flags & GCFLAG_AGE_MASK) < GCFLAG_AGE_MAX, "gen2: age field too large") def _debug_check_gen3(self, obj, ignored): - flags = self.header(obj).getflags() + flags = self.header(obj).tid ll_assert(bool(flags & GCFLAG_EXTERNAL), "gen3: missing GCFLAG_EXTERNAL") ll_assert(not (flags & GCFLAG_UNVISITED), Modified: pypy/branch/gc-compress/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gc/semispace.py Sat Oct 10 15:24:04 2009 @@ -13,7 +13,7 @@ import sys, os, time -first_gcflag = 1 +first_gcflag = 1 << 16 GCFLAG_FORWARDED = first_gcflag # GCFLAG_EXTERNAL is set on objects not living in the semispace: # either immortal objects or (for HybridGC) externally raw_malloc'ed @@ -22,20 +22,6 @@ memoryError = MemoryError() -# Handlers for the adt methods getflags() etc. on the HDR. -# These are mostly just workarounds for the limited support -# for 16-bits integer, providing the necessary casts. -def _hdr_getflags(hdr): - return lltype.cast_primitive(lltype.Signed, hdr.flags16) -def _hdr_setflags(hdr, flags): - hdr.flags16 = lltype.cast_primitive(rffi.USHORT, flags) -def _hdr_addflag(hdr, flag): - flags = lltype.cast_primitive(lltype.Signed, hdr.flags16) - hdr.flags16 = lltype.cast_primitive(rffi.USHORT, flags | flag) -def _hdr_delflag(hdr, flag): - flags = lltype.cast_primitive(lltype.Signed, hdr.flags16) - hdr.flags16 = lltype.cast_primitive(rffi.USHORT, flags & ~flag) - class SemiSpaceGC(MovingGCBase): _alloc_flavor_ = "raw" @@ -46,15 +32,8 @@ total_collection_time = 0.0 total_collection_count = 0 - HDR = lltype.Struct('header', ('typeid16', rffi.USHORT), - ('flags16', rffi.USHORT), - adtmeths = { - 'getflags': _hdr_getflags, - 'setflags': _hdr_setflags, - 'addflag': _hdr_addflag, - 'delflag': _hdr_delflag, - }) - typeid_is_in_field = 'typeid16' + HDR = lltype.Struct('header', ('tid', lltype.Signed)) # XXX or rffi.INT? + typeid_is_in_field = 'tid' FORWARDSTUB = lltype.GcStruct('forwarding_stub', ('forw', llmemory.Address)) FORWARDSTUBPTR = lltype.Ptr(FORWARDSTUB) @@ -374,11 +353,11 @@ return self.is_forwarded(obj) def is_forwarded(self, obj): - return self.header(obj).getflags() & GCFLAG_FORWARDED != 0 + return self.header(obj).tid & GCFLAG_FORWARDED != 0 # note: all prebuilt objects also have this flag set def get_forwarding_address(self, obj): - flags = self.header(obj).getflags() + flags = self.header(obj).tid if flags & GCFLAG_EXTERNAL: self.visit_external_object(obj) return obj # external or prebuilt objects are "forwarded" @@ -397,7 +376,7 @@ # writes after translation to C. size_gc_header = self.size_gc_header() stubsize = llmemory.sizeof(self.FORWARDSTUB) - flags = self.header(obj).getflags() + flags = self.header(obj).tid ll_assert(flags & GCFLAG_EXTERNAL == 0, "unexpected GCFLAG_EXTERNAL") ll_assert(flags & GCFLAG_FORWARDED == 0, "unexpected GCFLAG_FORWARDED") # replace the object at 'obj' with a FORWARDSTUB. @@ -405,30 +384,32 @@ llarena.arena_reset(hdraddr, size_gc_header + objsize, False) llarena.arena_reserve(hdraddr, size_gc_header + stubsize) hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(self.HDR)) - hdr.addflag(GCFLAG_FORWARDED) + hdr.tid |= GCFLAG_FORWARDED stub = llmemory.cast_adr_to_ptr(obj, self.FORWARDSTUBPTR) stub.forw = newobj + def combine(self, typeid16, flags): + return llop.combine_ushort(lltype.Signed, typeid16, flags) + def get_type_id(self, addr): hdr = self.header(addr) - flg = hdr.getflags() + flg = hdr.tid ll_assert(flg & (GCFLAG_FORWARDED|GCFLAG_EXTERNAL) != GCFLAG_FORWARDED, "get_type_id on forwarded obj") # Non-prebuilt forwarded objects are overwritten with a FORWARDSTUB. # Although calling get_type_id() on a forwarded object works by itself, # we catch it as an error because it's likely that what is then # done with the typeid is bogus. - return hdr.typeid16 + return llop.extract_ushort(rffi.USHORT, hdr.tid) def init_gc_object(self, addr, typeid16, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.typeid16 = typeid16 - hdr.setflags(flags) + hdr.tid = self.combine(typeid16, flags) def init_gc_object_immortal(self, addr, typeid16, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.typeid16 = typeid16 - hdr.setflags(flags | GCFLAG_EXTERNAL | GCFLAG_FORWARDED) + flags |= GCFLAG_EXTERNAL | GCFLAG_FORWARDED + hdr.tid = self.combine(typeid16, flags) # immortal objects always have GCFLAG_FORWARDED set; # see get_forwarding_address(). @@ -493,13 +474,13 @@ if self.surviving(obj): newobj = self.get_forwarding_address(obj) hdr = self.header(newobj) - if hdr.getflags() & GCFLAG_FINALIZATION_ORDERING: + if hdr.tid & GCFLAG_FINALIZATION_ORDERING: return 2 else: return 3 else: hdr = self.header(obj) - if hdr.getflags() & GCFLAG_FINALIZATION_ORDERING: + if hdr.tid & GCFLAG_FINALIZATION_ORDERING: return 1 else: return 0 @@ -508,7 +489,7 @@ ll_assert(self._finalization_state(obj) == 0, "unexpected finalization state != 0") hdr = self.header(obj) - hdr.addflag(GCFLAG_FINALIZATION_ORDERING) + hdr.tid |= GCFLAG_FINALIZATION_ORDERING def _recursively_bump_finalization_state_from_2_to_3(self, obj): ll_assert(self._finalization_state(obj) == 2, @@ -520,10 +501,8 @@ while pending.non_empty(): y = pending.pop() hdr = self.header(y) - flags = hdr.getflags() - if flags & GCFLAG_FINALIZATION_ORDERING: # state 2 ? - flags &= ~GCFLAG_FINALIZATION_ORDERING # change to state 3 - hdr.setflags(flags) + if hdr.tid & GCFLAG_FINALIZATION_ORDERING: # state 2 ? + hdr.tid &= ~GCFLAG_FINALIZATION_ORDERING # change to state 3 self.trace(y, self._append_if_nonnull, pending) def _recursively_bump_finalization_state_from_1_to_2(self, obj, scan): @@ -569,12 +548,12 @@ self.run_finalizers = new_run_finalizer def _is_external(self, obj): - return (self.header(obj).getflags() & GCFLAG_EXTERNAL) != 0 + return (self.header(obj).tid & GCFLAG_EXTERNAL) != 0 def debug_check_object(self, obj): """Check the invariants about 'obj' that should be true between collections.""" - flags = self.header(obj).getflags() + flags = self.header(obj).tid if flags & GCFLAG_EXTERNAL: ll_assert(flags & GCFLAG_FORWARDED, "bug: external+!forwarded") ll_assert(not (self.tospace <= obj < self.free), Modified: pypy/branch/gc-compress/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/gc.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/gc.py Sat Oct 10 15:24:04 2009 @@ -1,4 +1,5 @@ import sys +from pypy.objspace.flow.model import Constant from pypy.translator.c.support import cdecl from pypy.translator.c.node import ContainerNode from pypy.rpython.lltypesystem.lltype import \ @@ -314,8 +315,11 @@ return framework.convert_weakref_to(ptarget) def OP_GC_RELOAD_POSSIBLY_MOVED(self, funcgen, op): - args = [funcgen.expr(v) for v in op.args] - return '%s = %s; /* for moving GCs */' % (args[1], args[0]) + if isinstance(op.args[1], Constant): + return '/* %s */' % (op,) + else: + args = [funcgen.expr(v) for v in op.args] + return '%s = %s; /* for moving GCs */' % (args[1], args[0]) def common_gcheader_definition(self, defnode): return defnode.db.gctransformer.gc_fields() From arigo at codespeak.net Sat Oct 10 16:11:11 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 10 Oct 2009 16:11:11 +0200 (CEST) Subject: [pypy-svn] r68301 - in pypy/trunk/pypy/rpython: lltypesystem memory/gc Message-ID: <20091010141111.D3A03168077@codespeak.net> Author: arigo Date: Sat Oct 10 16:11:10 2009 New Revision: 68301 Modified: pypy/trunk/pypy/rpython/lltypesystem/llarena.py pypy/trunk/pypy/rpython/memory/gc/generation.py Log: Don't use /dev/zero just to clear the nursery. That's a bit nonsense and going into system calls actually takes more time. Modified: pypy/trunk/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/llarena.py Sat Oct 10 16:11:10 2009 @@ -282,8 +282,12 @@ def arena_reset(arena_addr, size, zero): """Free all objects in the arena, which can then be reused. - The arena is filled with zeroes if 'zero' is True. This can also - be used on a subrange of the arena.""" + This can also be used on a subrange of the arena. + The value of 'zero' is: + * 0: don't fill the area with zeroes + * 1: clear, optimized for a very large area of memory + * 2: clear, optimized for a small or medium area of memory + """ arena_addr = _getfakearenaaddress(arena_addr) arena_addr.arena.reset(zero, arena_addr.offset, size) @@ -385,8 +389,11 @@ def llimpl_arena_reset(arena_addr, size, zero): if zero: - clear_large_memory_chunk(arena_addr, size) -register_external(arena_reset, [llmemory.Address, int, bool], None, + if zero == 1: + clear_large_memory_chunk(arena_addr, size) + else: + llmemory.raw_memclear(arena_addr, size) +register_external(arena_reset, [llmemory.Address, int, int], None, 'll_arena.arena_reset', llimpl=llimpl_arena_reset, llfakeimpl=arena_reset, Modified: pypy/trunk/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/generation.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/generation.py Sat Oct 10 16:11:10 2009 @@ -332,7 +332,7 @@ if self.young_objects_with_id.length() > 0: self.update_young_objects_with_id() # mark the nursery as free and fill it with zeroes again - llarena.arena_reset(self.nursery, self.nursery_size, True) + llarena.arena_reset(self.nursery, self.nursery_size, 2) if self.config.gcconfig.debugprint: llop.debug_print(lltype.Void, "survived (fraction of the size):", From benjamin at codespeak.net Sun Oct 11 00:57:50 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 11 Oct 2009 00:57:50 +0200 (CEST) Subject: [pypy-svn] r68305 - in pypy/trunk/pypy/module/posix: . test Message-ID: <20091010225750.E9E981680AB@codespeak.net> Author: benjamin Date: Sun Oct 11 00:57:49 2009 New Revision: 68305 Modified: pypy/trunk/pypy/module/posix/interp_posix.py pypy/trunk/pypy/module/posix/test/test_posix2.py Log: os.getpgid() can raise an OSError #468 Modified: pypy/trunk/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/trunk/pypy/module/posix/interp_posix.py (original) +++ pypy/trunk/pypy/module/posix/interp_posix.py Sun Oct 11 00:57:49 2009 @@ -737,7 +737,11 @@ Call the system call getpgid(). """ - return space.wrap(os.getpgid(pid)) + try: + pgid = os.getpgid(pid) + except OSError, e: + raise wrap_oserror(space, e) + return space.wrap(pgid) getpgid.unwrap_spec = [ObjSpace, int] def setpgid(space, pid, pgrp): Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/trunk/pypy/module/posix/test/test_posix2.py (original) +++ pypy/trunk/pypy/module/posix/test/test_posix2.py Sun Oct 11 00:57:49 2009 @@ -47,6 +47,8 @@ cls.w_geteuid = space.wrap(os.geteuid()) if hasattr(os, 'getgid'): cls.w_getgid = space.wrap(os.getgid()) + if hasattr(os, 'getpgid'): + cls.w_getpgid = space.wrap(os.getpgid(os.getpid())) if hasattr(os, 'getsid'): cls.w_getsid0 = space.wrap(os.getsid(0)) if hasattr(os, 'sysconf'): @@ -364,6 +366,12 @@ os = self.posix assert os.getgid() == self.getgid + if hasattr(os, 'getpgid'): + def test_os_getpgid(self): + os = self.posix + assert os.getpgid(os.getpid()) == self.getpgid + raises(OSError, os.getpgid, 1234567) + if hasattr(os, 'setgid'): def test_os_setgid_error(self): os = self.posix From arigo at codespeak.net Sun Oct 11 11:23:05 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 11 Oct 2009 11:23:05 +0200 (CEST) Subject: [pypy-svn] r68308 - in pypy/branch/gc-compress/pypy: config jit/backend/llsupport rpython/memory rpython/memory/gc rpython/memory/gctransform rpython/memory/test translator/c translator/c/test Message-ID: <20091011092305.0D43116805C@codespeak.net> Author: arigo Date: Sun Oct 11 11:23:04 2009 New Revision: 68308 Modified: pypy/branch/gc-compress/pypy/config/translationoption.py pypy/branch/gc-compress/pypy/jit/backend/llsupport/gc.py pypy/branch/gc-compress/pypy/rpython/memory/gc/generation.py pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py pypy/branch/gc-compress/pypy/rpython/memory/test/test_transformed_gc.py pypy/branch/gc-compress/pypy/translator/c/gc.py pypy/branch/gc-compress/pypy/translator/c/node.py pypy/branch/gc-compress/pypy/translator/c/test/test_newgc.py Log: Make the saving of one word per object configurable. Modified: pypy/branch/gc-compress/pypy/config/translationoption.py ============================================================================== --- pypy/branch/gc-compress/pypy/config/translationoption.py (original) +++ pypy/branch/gc-compress/pypy/config/translationoption.py Sun Oct 11 11:23:04 2009 @@ -69,7 +69,9 @@ }), OptionDescription("gcconfig", "Configure garbage collectors", [ BoolOption("debugprint", "Turn on debug printing for the GC", - default=False) + default=False), + BoolOption("removetypeptr", "Remove the typeptr from every object", + default=False, cmdline="--gcremovetypeptr"), ]), ChoiceOption("gcrootfinder", "Strategy for finding GC Roots (framework GCs only)", @@ -95,7 +97,8 @@ # JIT generation: use -Ojit to enable it BoolOption("jit", "generate a JIT", default=False, - requires=[("translation.thread", False)], + requires=[("translation.thread", False), + ("translation.gcconfig.removetypeptr", False)], suggests=[("translation.gc", "hybrid"), # or "boehm" ("translation.gcrootfinder", "asmgcc"), ("translation.list_comprehension_operations", True)]), @@ -315,7 +318,7 @@ '0': 'boehm nobackendopt', '1': 'boehm lowinline', 'size': 'boehm lowinline remove_asserts', - 'mem': 'markcompact lowinline remove_asserts', + 'mem': 'markcompact lowinline remove_asserts removetypeptr', '2': 'hybrid extraopts', '3': 'hybrid extraopts remove_asserts', 'jit': 'hybrid extraopts jit', @@ -355,6 +358,8 @@ config.translation.suggest(withsmallfuncsets=5) elif word == 'jit': config.translation.suggest(jit=True) + elif word == 'removetypeptr': + config.translation.gcconfig.suggest(removetypeptr=True) else: raise ValueError(word) Modified: pypy/branch/gc-compress/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/gc-compress/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/gc-compress/pypy/jit/backend/llsupport/gc.py Sun Oct 11 11:23:04 2009 @@ -295,7 +295,7 @@ class GcLLDescr_framework(GcLLDescription): def __init__(self, gcdescr, translator, llop1=llop): - from pypy.rpython.memory.gc.base import choose_gc_from_config + from pypy.rpython.memory.gctypelayout import _check_typeid from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rpython.memory.gctransform import framework GcLLDescription.__init__(self, gcdescr, translator) @@ -322,16 +322,15 @@ # make a TransformerLayoutBuilder and save it on the translator # where it can be fished and reused by the FrameworkGCTransformer - self.GCClass, _ = choose_gc_from_config(gcdescr.config) - lltype2vtable = translator.rtyper.lltype2vtable - self.layoutbuilder = framework.TransformerLayoutBuilder(self.GCClass, - lltype2vtable) + self.layoutbuilder = framework.JITTransformerLayoutBuilder( + gcdescr.config) self.layoutbuilder.delay_encoding() self.translator._jit2gc = { 'layoutbuilder': self.layoutbuilder, 'gcmapstart': lambda: gcrootmap.gcmapstart(), 'gcmapend': lambda: gcrootmap.gcmapend(), } + self.GCClass = self.layoutbuilder.GCClass self.moving_gc = self.GCClass.moving_gc self.HDRPTR = lltype.Ptr(self.GCClass.HDR) self.gcheaderbuilder = GCHeaderBuilder(self.HDRPTR.TO) @@ -345,6 +344,7 @@ # make a malloc function, with three arguments def malloc_basic(size, type_id, has_finalizer): + _check_typeid(type_id) res = llop1.do_malloc_fixedsize_clear(llmemory.GCREF, type_id, size, True, has_finalizer, False) @@ -358,6 +358,7 @@ [llmemory.Address, llmemory.Address], lltype.Void)) # def malloc_array(itemsize, type_id, num_elem): + _check_typeid(type_id) return llop1.do_malloc_varsize_clear( llmemory.GCREF, type_id, num_elem, self.array_basesize, itemsize, @@ -393,10 +394,9 @@ self.gcrootmap.initialize() def init_size_descr(self, S, descr): - from pypy.rpython.memory.gctypelayout import weakpointer_offset type_id = self.layoutbuilder.get_type_id(S) has_finalizer = bool(self.layoutbuilder.has_finalizer(S)) - assert weakpointer_offset(S) == -1 # XXX + assert not self.layoutbuilder.is_weakref(type_id) descr.type_id = type_id descr.has_finalizer = has_finalizer @@ -409,14 +409,12 @@ size = sizedescr.size type_id = sizedescr.type_id has_finalizer = sizedescr.has_finalizer - assert type_id > 0 return self.malloc_basic(size, type_id, has_finalizer) def gc_malloc_array(self, arraydescr, num_elem): assert isinstance(arraydescr, BaseArrayDescr) itemsize = arraydescr.get_item_size(self.translate_support_code) type_id = arraydescr.type_id - assert type_id > 0 return self.malloc_array(itemsize, type_id, num_elem) def gc_malloc_str(self, num_elem): Modified: pypy/branch/gc-compress/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gc/generation.py Sun Oct 11 11:23:04 2009 @@ -423,7 +423,7 @@ # for the JIT: a minimal description of the write_barrier() method # (the JIT assumes it is of the shape # "if newvalue.int0 & JIT_WB_IF_FLAG: remember_young_pointer()") - JIT_WB_IF_FLAG = "XXXX" # GCFLAG_NO_YOUNG_PTRS + JIT_WB_IF_FLAG = GCFLAG_NO_YOUNG_PTRS def write_barrier(self, newvalue, addr_struct): if self.header(addr_struct).tid & GCFLAG_NO_YOUNG_PTRS: Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gctransform/framework.py Sun Oct 11 11:23:04 2009 @@ -130,7 +130,10 @@ if hasattr(translator, '_jit2gc'): self.layoutbuilder = translator._jit2gc['layoutbuilder'] else: - lltype2vtable = translator.rtyper.lltype2vtable + if translator.config.translation.gcconfig.removetypeptr: + lltype2vtable = translator.rtyper.lltype2vtable + else: + lltype2vtable = {} self.layoutbuilder = TransformerLayoutBuilder(GCClass, lltype2vtable) self.layoutbuilder.transformer = self @@ -809,14 +812,16 @@ def gct_getfield(self, hop): if (hop.spaceop.args[1].value == 'typeptr' and - hop.spaceop.args[0].concretetype.TO._hints.get('typeptr')): + hop.spaceop.args[0].concretetype.TO._hints.get('typeptr') and + self.translator.config.translation.gcconfig.removetypeptr): self.transform_getfield_typeptr(hop) else: GCTransformer.gct_getfield(self, hop) def gct_setfield(self, hop): if (hop.spaceop.args[1].value == 'typeptr' and - hop.spaceop.args[0].concretetype.TO._hints.get('typeptr')): + hop.spaceop.args[0].concretetype.TO._hints.get('typeptr') and + self.translator.config.translation.gcconfig.removetypeptr): self.transform_setfield_typeptr(hop) else: GCTransformer.gct_setfield(self, hop) @@ -910,6 +915,14 @@ return fptr +class JITTransformerLayoutBuilder(TransformerLayoutBuilder): + # for the JIT: currently does not support removetypeptr + def __init__(self, config): + from pypy.rpython.memory.gc.base import choose_gc_from_config + GCClass, _ = choose_gc_from_config(config) + TransformerLayoutBuilder.__init__(self, GCClass, {}) + + def gen_zero_gc_pointers(TYPE, v, llops, previous_steps=None): if previous_steps is None: previous_steps = [] Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gctypelayout.py Sun Oct 11 11:23:04 2009 @@ -42,15 +42,13 @@ self.type_info_group_ptr = type_info_group._as_ptr() def get(self, typeid): - ll_assert(llop.is_group_member_nonzero(lltype.Bool, typeid), - "invalid type_id") + _check_typeid(typeid) return llop.get_group_member(GCData.TYPE_INFO_PTR, self.type_info_group_ptr, typeid) def get_varsize(self, typeid): - ll_assert(llop.is_group_member_nonzero(lltype.Bool, typeid), - "invalid type_id") + _check_typeid(typeid) return llop.get_group_member(GCData.VARSIZE_TYPE_INFO_PTR, self.type_info_group_ptr, typeid) @@ -114,6 +112,10 @@ T_IS_GCARRAY_OF_GCPTR = 0x04 T_IS_WEAKREF = 0x08 +def _check_typeid(typeid): + ll_assert(llop.is_group_member_nonzero(lltype.Bool, typeid), + "invalid type_id") + def encode_type_shape(builder, info, TYPE): """Encode the shape of the TYPE into the TYPE_INFO structure 'info'.""" @@ -217,6 +219,8 @@ type_id = self.type_info_group.add_member(fullinfo) self.id_of_type[TYPE] = type_id # store the vtable of the type (if any) immediately thereafter + # (note that if gcconfig.removetypeptr is False, lltype2vtable + # is empty) vtable = self.lltype2vtable.get(TYPE, None) if vtable is not None: # check that if we have a vtable, we are not varsize @@ -235,6 +239,9 @@ self.type_info_group._as_ptr(), type_id) + def is_weakref(self, type_id): + return self.get_info(type_id).infobits & T_IS_WEAKREF + def encode_type_shapes_now(self): if not self.can_encode_type_shape: self.can_encode_type_shape = True Modified: pypy/branch/gc-compress/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/test/test_transformed_gc.py Sun Oct 11 11:23:04 2009 @@ -21,6 +21,7 @@ t = TranslationContext() # XXX XXX XXX mess t.config.translation.gc = gcname + t.config.translation.gcconfig.removetypeptr = True if stacklessgc: t.config.translation.gcrootfinder = "stackless" t.config.set(**extraconfigopts) Modified: pypy/branch/gc-compress/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/gc.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/gc.py Sun Oct 11 11:23:04 2009 @@ -12,8 +12,7 @@ class BasicGcPolicy(object): requires_stackless = False - need_no_typeptr = False - + def __init__(self, db, thread_enabled=False): self.db = db self.thread_enabled = thread_enabled @@ -51,6 +50,9 @@ post_include_bits=['typedef void *GC_hidden_pointer;'] ) + def need_no_typeptr(self): + return False + def gc_startup_code(self): return [] @@ -279,7 +281,6 @@ class FrameworkGcPolicy(BasicGcPolicy): transformerclass = framework.FrameworkGCTransformer - need_no_typeptr = True def struct_setup(self, structdefnode, rtti): if rtti is not None and hasattr(rtti._obj, 'destructor_funcptr'): @@ -328,6 +329,10 @@ o = top_container(defnode.obj) return defnode.db.gctransformer.gc_field_values_for(o) + def need_no_typeptr(self): + config = self.db.translator.config + return config.translation.gcconfig.removetypeptr + def OP_GC_GETTYPEPTR_GROUP(self, funcgen, op): # expands to a number of steps, as per rpython/lltypesystem/opimpl.py, # all implemented by a single call to a C macro. Modified: pypy/branch/gc-compress/pypy/translator/c/node.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/node.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/node.py Sun Oct 11 11:23:04 2009 @@ -70,7 +70,7 @@ # self.fieldnames = STRUCT._names if STRUCT._hints.get('typeptr', False): - if db.gcpolicy.need_no_typeptr: + if db.gcpolicy.need_no_typeptr(): assert self.fieldnames == ('typeptr',) self.fieldnames = () Modified: pypy/branch/gc-compress/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/gc-compress/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/gc-compress/pypy/translator/c/test/test_newgc.py Sun Oct 11 11:23:04 2009 @@ -16,6 +16,7 @@ class TestUsingFramework(object): gcpolicy = "marksweep" should_be_moving = False + removetypeptr = False GC_CAN_MOVE = False GC_CANNOT_MALLOC_NONMOVABLE = False @@ -25,6 +26,7 @@ def _makefunc2(cls, f): t = Translation(f, [int, int], gc=cls.gcpolicy, policy=annpolicy.StrictAnnotatorPolicy()) + t.config.translation.gcconfig.removetypeptr = cls.removetypeptr t.disable(['backendopt']) t.set_backend_extra_options(c_isolated=True, c_debug_defines=True) t.rtype() @@ -796,6 +798,9 @@ def test_gc_set_max_heap_size(self): py.test.skip("not implemented") +class TestHybridGCRemoveTypePtr(TestHybridGC): + removetypeptr = True + class TestMarkCompactGC(TestSemiSpaceGC): gcpolicy = "markcompact" should_be_moving = True From arigo at codespeak.net Sun Oct 11 11:31:26 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 11 Oct 2009 11:31:26 +0200 (CEST) Subject: [pypy-svn] r68309 - pypy/branch/gc-compress/pypy/rpython/memory/gctransform Message-ID: <20091011093126.CF520168059@codespeak.net> Author: arigo Date: Sun Oct 11 11:31:25 2009 New Revision: 68309 Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctransform/transform.py Log: Test fix. Modified: pypy/branch/gc-compress/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/gc-compress/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/gc-compress/pypy/rpython/memory/gctransform/transform.py Sun Oct 11 11:31:25 2009 @@ -312,6 +312,9 @@ newgcdependencies = self.ll_finalizers_ptrs return newgcdependencies + def get_final_dependencies(self): + pass + def finish_tables(self): pass From arigo at codespeak.net Sun Oct 11 14:03:40 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 11 Oct 2009 14:03:40 +0200 (CEST) Subject: [pypy-svn] r68310 - in pypy/branch/gc-compress/pypy/jit/backend/llsupport: . test Message-ID: <20091011120340.80CD616805B@codespeak.net> Author: arigo Date: Sun Oct 11 14:03:39 2009 New Revision: 68310 Modified: pypy/branch/gc-compress/pypy/jit/backend/llsupport/gc.py pypy/branch/gc-compress/pypy/jit/backend/llsupport/test/test_gc.py Log: Test and fix: args_for_new() must return a list of plain integers, not a list containing a USHORT. Modified: pypy/branch/gc-compress/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/gc-compress/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/gc-compress/pypy/jit/backend/llsupport/gc.py Sun Oct 11 14:03:39 2009 @@ -343,7 +343,9 @@ symbolic.get_array_token(lltype.GcArray(lltype.Signed), True) # make a malloc function, with three arguments - def malloc_basic(size, type_id, has_finalizer): + def malloc_basic(size, combined): + type_id = llop.extract_ushort(rffi.USHORT, combined) + has_finalizer = bool(combined & (1<<16)) _check_typeid(type_id) res = llop1.do_malloc_fixedsize_clear(llmemory.GCREF, type_id, size, True, @@ -353,11 +355,12 @@ return res self.malloc_basic = malloc_basic self.GC_MALLOC_BASIC = lltype.Ptr(lltype.FuncType( - [lltype.Signed, lltype.Signed, lltype.Bool], llmemory.GCREF)) + [lltype.Signed, lltype.Signed], llmemory.GCREF)) self.WB_FUNCPTR = lltype.Ptr(lltype.FuncType( [llmemory.Address, llmemory.Address], lltype.Void)) # - def malloc_array(itemsize, type_id, num_elem): + def malloc_array(itemsize, combined, num_elem): + type_id = llop.extract_ushort(rffi.USHORT, combined) _check_typeid(type_id) return llop1.do_malloc_varsize_clear( llmemory.GCREF, @@ -409,13 +412,16 @@ size = sizedescr.size type_id = sizedescr.type_id has_finalizer = sizedescr.has_finalizer - return self.malloc_basic(size, type_id, has_finalizer) + combined = llop.combine_ushort(lltype.Signed, type_id, + int(has_finalizer) << 16) + return self.malloc_basic(size, combined) def gc_malloc_array(self, arraydescr, num_elem): assert isinstance(arraydescr, BaseArrayDescr) itemsize = arraydescr.get_item_size(self.translate_support_code) type_id = arraydescr.type_id - return self.malloc_array(itemsize, type_id, num_elem) + combined = llop.combine_ushort(lltype.Signed, type_id, 0) + return self.malloc_array(itemsize, combined, num_elem) def gc_malloc_str(self, num_elem): return self.malloc_str(num_elem) @@ -426,15 +432,18 @@ def args_for_new(self, sizedescr): assert isinstance(sizedescr, BaseSizeDescr) size = sizedescr.size - type_id = sizedescr.type_id - has_finalizer = sizedescr.has_finalizer - return [size, type_id, has_finalizer] + type_id = sizedescr.type_id # a USHORT + has_finalizer = sizedescr.has_finalizer # a Bool + combined = llop.combine_ushort(lltype.Signed, type_id, + int(has_finalizer) << 16) + return [size, combined] def args_for_new_array(self, arraydescr): assert isinstance(arraydescr, BaseArrayDescr) itemsize = arraydescr.get_item_size(self.translate_support_code) type_id = arraydescr.type_id - return [itemsize, type_id] + combined = llop.combine_ushort(lltype.Signed, type_id, 0) + return [itemsize, combined] def get_funcptr_for_new(self): return llhelper(self.GC_MALLOC_BASIC, self.malloc_basic) Modified: pypy/branch/gc-compress/pypy/jit/backend/llsupport/test/test_gc.py ============================================================================== --- pypy/branch/gc-compress/pypy/jit/backend/llsupport/test/test_gc.py (original) +++ pypy/branch/gc-compress/pypy/jit/backend/llsupport/test/test_gc.py Sun Oct 11 14:03:39 2009 @@ -165,6 +165,18 @@ self.gc_ll_descr = gc_ll_descr self.fake_cpu = FakeCPU() + def test_args_for_new(self): + S = lltype.GcStruct('S', ('x', lltype.Signed)) + sizedescr = get_size_descr(self.gc_ll_descr, S) + args = self.gc_ll_descr.args_for_new(sizedescr) + for x in args: + assert lltype.typeOf(x) == lltype.Signed + A = lltype.GcArray(lltype.Signed) + arraydescr = get_array_descr(self.gc_ll_descr, A) + args = self.gc_ll_descr.args_for_new(sizedescr) + for x in args: + assert lltype.typeOf(x) == lltype.Signed + def test_gc_malloc(self): S = lltype.GcStruct('S', ('x', lltype.Signed)) sizedescr = get_size_descr(self.gc_ll_descr, S) @@ -172,7 +184,8 @@ assert self.llop1.record == [("fixedsize", sizedescr.type_id, repr(sizedescr.size), False, p)] assert repr(self.gc_ll_descr.args_for_new(sizedescr)) == repr( - [sizedescr.size, sizedescr.type_id, False]) + [sizedescr.size, + llgroup.CombinedSymbolic(sizedescr.type_id, 0)]) def test_gc_malloc_array(self): A = lltype.GcArray(lltype.Signed) @@ -184,7 +197,8 @@ repr(arraydescr.get_ofs_length(True)), p)] assert repr(self.gc_ll_descr.args_for_new_array(arraydescr)) == repr( - [arraydescr.get_item_size(True), arraydescr.type_id]) + [arraydescr.get_item_size(True), + llgroup.CombinedSymbolic(arraydescr.type_id, 0)]) def test_gc_malloc_str(self): p = self.gc_ll_descr.gc_malloc_str(10) From arigo at codespeak.net Sun Oct 11 14:19:48 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 11 Oct 2009 14:19:48 +0200 (CEST) Subject: [pypy-svn] r68311 - in pypy/branch/gc-compress/pypy/jit/backend/llsupport: . test Message-ID: <20091011121948.0C6EF16805B@codespeak.net> Author: arigo Date: Sun Oct 11 14:19:48 2009 New Revision: 68311 Modified: pypy/branch/gc-compress/pypy/jit/backend/llsupport/gc.py pypy/branch/gc-compress/pypy/jit/backend/llsupport/test/test_gc.py Log: Unifies the type_id and the flags into a single Signed, as early as in the SizeDescr. Modified: pypy/branch/gc-compress/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/gc-compress/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/gc-compress/pypy/jit/backend/llsupport/gc.py Sun Oct 11 14:19:48 2009 @@ -343,9 +343,9 @@ symbolic.get_array_token(lltype.GcArray(lltype.Signed), True) # make a malloc function, with three arguments - def malloc_basic(size, combined): - type_id = llop.extract_ushort(rffi.USHORT, combined) - has_finalizer = bool(combined & (1<<16)) + def malloc_basic(size, tid): + type_id = llop.extract_ushort(rffi.USHORT, tid) + has_finalizer = bool(tid & (1<<16)) _check_typeid(type_id) res = llop1.do_malloc_fixedsize_clear(llmemory.GCREF, type_id, size, True, @@ -359,8 +359,8 @@ self.WB_FUNCPTR = lltype.Ptr(lltype.FuncType( [llmemory.Address, llmemory.Address], lltype.Void)) # - def malloc_array(itemsize, combined, num_elem): - type_id = llop.extract_ushort(rffi.USHORT, combined) + def malloc_array(itemsize, tid, num_elem): + type_id = llop.extract_ushort(rffi.USHORT, tid) _check_typeid(type_id) return llop1.do_malloc_varsize_clear( llmemory.GCREF, @@ -398,30 +398,23 @@ def init_size_descr(self, S, descr): type_id = self.layoutbuilder.get_type_id(S) - has_finalizer = bool(self.layoutbuilder.has_finalizer(S)) assert not self.layoutbuilder.is_weakref(type_id) - descr.type_id = type_id - descr.has_finalizer = has_finalizer + has_finalizer = bool(self.layoutbuilder.has_finalizer(S)) + flags = int(has_finalizer) << 16 + descr.tid = llop.combine_ushort(lltype.Signed, type_id, flags) def init_array_descr(self, A, descr): type_id = self.layoutbuilder.get_type_id(A) - descr.type_id = type_id + descr.tid = llop.combine_ushort(lltype.Signed, type_id, 0) def gc_malloc(self, sizedescr): assert isinstance(sizedescr, BaseSizeDescr) - size = sizedescr.size - type_id = sizedescr.type_id - has_finalizer = sizedescr.has_finalizer - combined = llop.combine_ushort(lltype.Signed, type_id, - int(has_finalizer) << 16) - return self.malloc_basic(size, combined) + return self.malloc_basic(sizedescr.size, sizedescr.tid) def gc_malloc_array(self, arraydescr, num_elem): assert isinstance(arraydescr, BaseArrayDescr) itemsize = arraydescr.get_item_size(self.translate_support_code) - type_id = arraydescr.type_id - combined = llop.combine_ushort(lltype.Signed, type_id, 0) - return self.malloc_array(itemsize, combined, num_elem) + return self.malloc_array(itemsize, arraydescr.tid, num_elem) def gc_malloc_str(self, num_elem): return self.malloc_str(num_elem) @@ -431,19 +424,12 @@ def args_for_new(self, sizedescr): assert isinstance(sizedescr, BaseSizeDescr) - size = sizedescr.size - type_id = sizedescr.type_id # a USHORT - has_finalizer = sizedescr.has_finalizer # a Bool - combined = llop.combine_ushort(lltype.Signed, type_id, - int(has_finalizer) << 16) - return [size, combined] + return [sizedescr.size, sizedescr.tid] def args_for_new_array(self, arraydescr): assert isinstance(arraydescr, BaseArrayDescr) itemsize = arraydescr.get_item_size(self.translate_support_code) - type_id = arraydescr.type_id - combined = llop.combine_ushort(lltype.Signed, type_id, 0) - return [itemsize, combined] + return [itemsize, arraydescr.tid] def get_funcptr_for_new(self): return llhelper(self.GC_MALLOC_BASIC, self.malloc_basic) Modified: pypy/branch/gc-compress/pypy/jit/backend/llsupport/test/test_gc.py ============================================================================== --- pypy/branch/gc-compress/pypy/jit/backend/llsupport/test/test_gc.py (original) +++ pypy/branch/gc-compress/pypy/jit/backend/llsupport/test/test_gc.py Sun Oct 11 14:19:48 2009 @@ -1,5 +1,6 @@ import random from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rstr +from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper from pypy.jit.backend.llsupport.descr import * from pypy.jit.backend.llsupport.gc import * @@ -119,8 +120,9 @@ assert not contains_weakptr p = llmemory.raw_malloc(size) p = llmemory.cast_adr_to_ptr(p, RESTYPE) - self.record.append(("fixedsize", type_id, repr(size), - has_finalizer, p)) + flags = int(has_finalizer) << 16 + tid = llop.combine_ushort(lltype.Signed, type_id, flags) + self.record.append(("fixedsize", repr(size), tid, p)) return p def do_malloc_varsize_clear(self, RESTYPE, type_id, length, size, @@ -129,7 +131,8 @@ p = llmemory.raw_malloc(size + itemsize * length) (p + offset_to_length).signed[0] = length p = llmemory.cast_adr_to_ptr(p, RESTYPE) - self.record.append(("varsize", type_id, length, + tid = llop.combine_ushort(lltype.Signed, type_id, 0) + self.record.append(("varsize", tid, length, repr(size), repr(itemsize), repr(offset_to_length), p)) return p @@ -181,40 +184,41 @@ S = lltype.GcStruct('S', ('x', lltype.Signed)) sizedescr = get_size_descr(self.gc_ll_descr, S) p = self.gc_ll_descr.gc_malloc(sizedescr) - assert self.llop1.record == [("fixedsize", sizedescr.type_id, - repr(sizedescr.size), False, p)] + assert self.llop1.record == [("fixedsize", + repr(sizedescr.size), + sizedescr.tid, p)] assert repr(self.gc_ll_descr.args_for_new(sizedescr)) == repr( - [sizedescr.size, - llgroup.CombinedSymbolic(sizedescr.type_id, 0)]) + [sizedescr.size, sizedescr.tid]) def test_gc_malloc_array(self): A = lltype.GcArray(lltype.Signed) arraydescr = get_array_descr(self.gc_ll_descr, A) p = self.gc_ll_descr.gc_malloc_array(arraydescr, 10) - assert self.llop1.record == [("varsize", arraydescr.type_id, 10, + assert self.llop1.record == [("varsize", arraydescr.tid, 10, repr(arraydescr.get_base_size(True)), repr(arraydescr.get_item_size(True)), repr(arraydescr.get_ofs_length(True)), p)] assert repr(self.gc_ll_descr.args_for_new_array(arraydescr)) == repr( - [arraydescr.get_item_size(True), - llgroup.CombinedSymbolic(arraydescr.type_id, 0)]) + [arraydescr.get_item_size(True), arraydescr.tid]) def test_gc_malloc_str(self): p = self.gc_ll_descr.gc_malloc_str(10) type_id = self.gc_ll_descr.layoutbuilder.get_type_id(rstr.STR) + tid = llop.combine_ushort(lltype.Signed, type_id, 0) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, True) - assert self.llop1.record == [("varsize", type_id, 10, + assert self.llop1.record == [("varsize", tid, 10, repr(basesize), repr(itemsize), repr(ofs_length), p)] def test_gc_malloc_unicode(self): p = self.gc_ll_descr.gc_malloc_unicode(10) type_id = self.gc_ll_descr.layoutbuilder.get_type_id(rstr.UNICODE) + tid = llop.combine_ushort(lltype.Signed, type_id, 0) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, True) - assert self.llop1.record == [("varsize", type_id, 10, + assert self.llop1.record == [("varsize", tid, 10, repr(basesize), repr(itemsize), repr(ofs_length), p)] From arigo at codespeak.net Sun Oct 11 15:32:07 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 11 Oct 2009 15:32:07 +0200 (CEST) Subject: [pypy-svn] r68312 - pypy/branch/gc-compress/pypy/doc/config Message-ID: <20091011133207.859E116805B@codespeak.net> Author: arigo Date: Sun Oct 11 15:32:04 2009 New Revision: 68312 Added: pypy/branch/gc-compress/pypy/doc/config/translation.gcconfig.removetypeptr.txt (contents, props changed) Log: Add missing doc for the option. Added: pypy/branch/gc-compress/pypy/doc/config/translation.gcconfig.removetypeptr.txt ============================================================================== --- (empty file) +++ pypy/branch/gc-compress/pypy/doc/config/translation.gcconfig.removetypeptr.txt Sun Oct 11 15:32:04 2009 @@ -0,0 +1 @@ +If set, save one word in every object. Framework GC only. From arigo at codespeak.net Sun Oct 11 15:37:48 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 11 Oct 2009 15:37:48 +0200 (CEST) Subject: [pypy-svn] r68313 - pypy/build/testrunner Message-ID: <20091011133748.AA11916805B@codespeak.net> Author: arigo Date: Sun Oct 11 15:37:48 2009 New Revision: 68313 Modified: pypy/build/testrunner/runner.py Log: Attempt to run directories in alphabetical order in our nightly test runs. Modified: pypy/build/testrunner/runner.py ============================================================================== --- pypy/build/testrunner/runner.py (original) +++ pypy/build/testrunner/runner.py Sun Oct 11 15:37:48 2009 @@ -264,6 +264,7 @@ reldir = self.reltoroot(p) entries = [p1 for p1 in p.listdir() if p1.check(dotfile=0)] + entries.sort() if p != self.root: for p1 in entries: From arigo at codespeak.net Sun Oct 11 16:32:29 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 11 Oct 2009 16:32:29 +0200 (CEST) Subject: [pypy-svn] r68314 - in pypy/trunk/pypy: config doc/config jit/backend/llsupport jit/backend/llsupport/test jit/backend/x86 jit/metainterp rpython rpython/lltypesystem rpython/lltypesystem/test rpython/memory rpython/memory/gc rpython/memory/gc/test rpython/memory/gctransform rpython/memory/test translator/backendopt translator/c translator/c/src translator/c/test Message-ID: <20091011143229.CFD8E16805B@codespeak.net> Author: arigo Date: Sun Oct 11 16:32:27 2009 New Revision: 68314 Added: pypy/trunk/pypy/doc/config/translation.gcconfig.removetypeptr.txt - copied unchanged from r68313, pypy/branch/gc-compress/pypy/doc/config/translation.gcconfig.removetypeptr.txt pypy/trunk/pypy/rpython/lltypesystem/llgroup.py - copied unchanged from r68313, pypy/branch/gc-compress/pypy/rpython/lltypesystem/llgroup.py pypy/trunk/pypy/rpython/lltypesystem/test/__init__.py - copied unchanged from r68313, pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/__init__.py pypy/trunk/pypy/rpython/lltypesystem/test/test_llgroup.py - copied unchanged from r68313, pypy/branch/gc-compress/pypy/rpython/lltypesystem/test/test_llgroup.py pypy/trunk/pypy/translator/c/src/llgroup.h - copied unchanged from r68313, pypy/branch/gc-compress/pypy/translator/c/src/llgroup.h Modified: pypy/trunk/pypy/config/translationoption.py pypy/trunk/pypy/jit/backend/llsupport/gc.py pypy/trunk/pypy/jit/backend/llsupport/test/test_gc.py pypy/trunk/pypy/jit/backend/x86/runner.py pypy/trunk/pypy/jit/metainterp/policy.py pypy/trunk/pypy/rpython/llinterp.py pypy/trunk/pypy/rpython/lltypesystem/llarena.py pypy/trunk/pypy/rpython/lltypesystem/lloperation.py pypy/trunk/pypy/rpython/lltypesystem/opimpl.py pypy/trunk/pypy/rpython/lltypesystem/rclass.py pypy/trunk/pypy/rpython/memory/gc/base.py pypy/trunk/pypy/rpython/memory/gc/hybrid.py pypy/trunk/pypy/rpython/memory/gc/markcompact.py pypy/trunk/pypy/rpython/memory/gc/marksweep.py pypy/trunk/pypy/rpython/memory/gc/semispace.py pypy/trunk/pypy/rpython/memory/gc/test/test_direct.py pypy/trunk/pypy/rpython/memory/gctransform/framework.py pypy/trunk/pypy/rpython/memory/gctransform/transform.py pypy/trunk/pypy/rpython/memory/gctypelayout.py pypy/trunk/pypy/rpython/memory/gcwrapper.py pypy/trunk/pypy/rpython/memory/lltypelayout.py pypy/trunk/pypy/rpython/memory/test/test_gctypelayout.py pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py pypy/trunk/pypy/rpython/rtyper.py pypy/trunk/pypy/translator/backendopt/inline.py pypy/trunk/pypy/translator/c/database.py pypy/trunk/pypy/translator/c/funcgen.py pypy/trunk/pypy/translator/c/gc.py pypy/trunk/pypy/translator/c/node.py pypy/trunk/pypy/translator/c/primitive.py pypy/trunk/pypy/translator/c/src/g_include.h pypy/trunk/pypy/translator/c/src/mem.h pypy/trunk/pypy/translator/c/test/test_lltyped.py pypy/trunk/pypy/translator/c/test/test_newgc.py Log: Merge the gc-compress branch. This adds a new option, "--gcremovetypeptr", which compiles a framework GC in such a way as needing only one word to store the GC headers (including the GC-specific type data pointer) and the 'typeptr' field of all regular RPython instances. The option is False by default for now, but with a bit more testing it should become True. It seems to give no noticeable speed-up, but no slow-down either, and it saves memory. Implemented with a new concept in rpython/lltypesystem, a "group", which regroups otherwise-independent prebuilt structures in a lazy manner, and offers unsigned 16-bits symbolics representing offsets of one structure in the group. Used for the type info table, and if "--gcremovetypeptr", for all the vtables of the program as well. The JIT does not support yet "--gcremovetypeptr" but that could be fixed. Modified: pypy/trunk/pypy/config/translationoption.py ============================================================================== --- pypy/trunk/pypy/config/translationoption.py (original) +++ pypy/trunk/pypy/config/translationoption.py Sun Oct 11 16:32:27 2009 @@ -69,7 +69,9 @@ }), OptionDescription("gcconfig", "Configure garbage collectors", [ BoolOption("debugprint", "Turn on debug printing for the GC", - default=False) + default=False), + BoolOption("removetypeptr", "Remove the typeptr from every object", + default=False, cmdline="--gcremovetypeptr"), ]), ChoiceOption("gcrootfinder", "Strategy for finding GC Roots (framework GCs only)", @@ -95,7 +97,8 @@ # JIT generation: use -Ojit to enable it BoolOption("jit", "generate a JIT", default=False, - requires=[("translation.thread", False)], + requires=[("translation.thread", False), + ("translation.gcconfig.removetypeptr", False)], suggests=[("translation.gc", "hybrid"), # or "boehm" ("translation.gcrootfinder", "asmgcc"), ("translation.list_comprehension_operations", True)]), @@ -315,7 +318,7 @@ '0': 'boehm nobackendopt', '1': 'boehm lowinline', 'size': 'boehm lowinline remove_asserts', - 'mem': 'markcompact lowinline remove_asserts', + 'mem': 'markcompact lowinline remove_asserts removetypeptr', '2': 'hybrid extraopts', '3': 'hybrid extraopts remove_asserts', 'jit': 'hybrid extraopts jit', @@ -355,6 +358,8 @@ config.translation.suggest(withsmallfuncsets=5) elif word == 'jit': config.translation.suggest(jit=True) + elif word == 'removetypeptr': + config.translation.gcconfig.suggest(removetypeptr=True) else: raise ValueError(word) Modified: pypy/trunk/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/gc.py Sun Oct 11 16:32:27 2009 @@ -295,7 +295,7 @@ class GcLLDescr_framework(GcLLDescription): def __init__(self, gcdescr, translator, llop1=llop): - from pypy.rpython.memory.gc.base import choose_gc_from_config + from pypy.rpython.memory.gctypelayout import _check_typeid from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rpython.memory.gctransform import framework GcLLDescription.__init__(self, gcdescr, translator) @@ -322,14 +322,15 @@ # make a TransformerLayoutBuilder and save it on the translator # where it can be fished and reused by the FrameworkGCTransformer - self.layoutbuilder = framework.TransformerLayoutBuilder() + self.layoutbuilder = framework.JITTransformerLayoutBuilder( + gcdescr.config) self.layoutbuilder.delay_encoding() self.translator._jit2gc = { 'layoutbuilder': self.layoutbuilder, 'gcmapstart': lambda: gcrootmap.gcmapstart(), 'gcmapend': lambda: gcrootmap.gcmapend(), } - self.GCClass, _ = choose_gc_from_config(gcdescr.config) + self.GCClass = self.layoutbuilder.GCClass self.moving_gc = self.GCClass.moving_gc self.HDRPTR = lltype.Ptr(self.GCClass.HDR) self.gcheaderbuilder = GCHeaderBuilder(self.HDRPTR.TO) @@ -342,7 +343,10 @@ symbolic.get_array_token(lltype.GcArray(lltype.Signed), True) # make a malloc function, with three arguments - def malloc_basic(size, type_id, has_finalizer): + def malloc_basic(size, tid): + type_id = llop.extract_ushort(rffi.USHORT, tid) + has_finalizer = bool(tid & (1<<16)) + _check_typeid(type_id) res = llop1.do_malloc_fixedsize_clear(llmemory.GCREF, type_id, size, True, has_finalizer, False) @@ -351,11 +355,13 @@ return res self.malloc_basic = malloc_basic self.GC_MALLOC_BASIC = lltype.Ptr(lltype.FuncType( - [lltype.Signed, lltype.Signed, lltype.Bool], llmemory.GCREF)) + [lltype.Signed, lltype.Signed], llmemory.GCREF)) self.WB_FUNCPTR = lltype.Ptr(lltype.FuncType( [llmemory.Address, llmemory.Address], lltype.Void)) # - def malloc_array(itemsize, type_id, num_elem): + def malloc_array(itemsize, tid, num_elem): + type_id = llop.extract_ushort(rffi.USHORT, tid) + _check_typeid(type_id) return llop1.do_malloc_varsize_clear( llmemory.GCREF, type_id, num_elem, self.array_basesize, itemsize, @@ -391,31 +397,24 @@ self.gcrootmap.initialize() def init_size_descr(self, S, descr): - from pypy.rpython.memory.gctypelayout import weakpointer_offset type_id = self.layoutbuilder.get_type_id(S) + assert not self.layoutbuilder.is_weakref(type_id) has_finalizer = bool(self.layoutbuilder.has_finalizer(S)) - assert weakpointer_offset(S) == -1 # XXX - descr.type_id = type_id - descr.has_finalizer = has_finalizer + flags = int(has_finalizer) << 16 + descr.tid = llop.combine_ushort(lltype.Signed, type_id, flags) def init_array_descr(self, A, descr): type_id = self.layoutbuilder.get_type_id(A) - descr.type_id = type_id + descr.tid = llop.combine_ushort(lltype.Signed, type_id, 0) def gc_malloc(self, sizedescr): assert isinstance(sizedescr, BaseSizeDescr) - size = sizedescr.size - type_id = sizedescr.type_id - has_finalizer = sizedescr.has_finalizer - assert type_id > 0 - return self.malloc_basic(size, type_id, has_finalizer) + return self.malloc_basic(sizedescr.size, sizedescr.tid) def gc_malloc_array(self, arraydescr, num_elem): assert isinstance(arraydescr, BaseArrayDescr) itemsize = arraydescr.get_item_size(self.translate_support_code) - type_id = arraydescr.type_id - assert type_id > 0 - return self.malloc_array(itemsize, type_id, num_elem) + return self.malloc_array(itemsize, arraydescr.tid, num_elem) def gc_malloc_str(self, num_elem): return self.malloc_str(num_elem) @@ -425,16 +424,12 @@ def args_for_new(self, sizedescr): assert isinstance(sizedescr, BaseSizeDescr) - size = sizedescr.size - type_id = sizedescr.type_id - has_finalizer = sizedescr.has_finalizer - return [size, type_id, has_finalizer] + return [sizedescr.size, sizedescr.tid] def args_for_new_array(self, arraydescr): assert isinstance(arraydescr, BaseArrayDescr) itemsize = arraydescr.get_item_size(self.translate_support_code) - type_id = arraydescr.type_id - return [itemsize, type_id] + return [itemsize, arraydescr.tid] def get_funcptr_for_new(self): return llhelper(self.GC_MALLOC_BASIC, self.malloc_basic) Modified: pypy/trunk/pypy/jit/backend/llsupport/test/test_gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/test/test_gc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/test/test_gc.py Sun Oct 11 16:32:27 2009 @@ -1,5 +1,6 @@ import random from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rstr +from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper from pypy.jit.backend.llsupport.descr import * from pypy.jit.backend.llsupport.gc import * @@ -119,8 +120,9 @@ assert not contains_weakptr p = llmemory.raw_malloc(size) p = llmemory.cast_adr_to_ptr(p, RESTYPE) - self.record.append(("fixedsize", type_id, repr(size), - has_finalizer, p)) + flags = int(has_finalizer) << 16 + tid = llop.combine_ushort(lltype.Signed, type_id, flags) + self.record.append(("fixedsize", repr(size), tid, p)) return p def do_malloc_varsize_clear(self, RESTYPE, type_id, length, size, @@ -129,7 +131,8 @@ p = llmemory.raw_malloc(size + itemsize * length) (p + offset_to_length).signed[0] = length p = llmemory.cast_adr_to_ptr(p, RESTYPE) - self.record.append(("varsize", type_id, length, + tid = llop.combine_ushort(lltype.Signed, type_id, 0) + self.record.append(("varsize", tid, length, repr(size), repr(itemsize), repr(offset_to_length), p)) return p @@ -165,42 +168,57 @@ self.gc_ll_descr = gc_ll_descr self.fake_cpu = FakeCPU() + def test_args_for_new(self): + S = lltype.GcStruct('S', ('x', lltype.Signed)) + sizedescr = get_size_descr(self.gc_ll_descr, S) + args = self.gc_ll_descr.args_for_new(sizedescr) + for x in args: + assert lltype.typeOf(x) == lltype.Signed + A = lltype.GcArray(lltype.Signed) + arraydescr = get_array_descr(self.gc_ll_descr, A) + args = self.gc_ll_descr.args_for_new(sizedescr) + for x in args: + assert lltype.typeOf(x) == lltype.Signed + def test_gc_malloc(self): S = lltype.GcStruct('S', ('x', lltype.Signed)) sizedescr = get_size_descr(self.gc_ll_descr, S) p = self.gc_ll_descr.gc_malloc(sizedescr) - assert self.llop1.record == [("fixedsize", sizedescr.type_id, - repr(sizedescr.size), False, p)] + assert self.llop1.record == [("fixedsize", + repr(sizedescr.size), + sizedescr.tid, p)] assert repr(self.gc_ll_descr.args_for_new(sizedescr)) == repr( - [sizedescr.size, sizedescr.type_id, False]) + [sizedescr.size, sizedescr.tid]) def test_gc_malloc_array(self): A = lltype.GcArray(lltype.Signed) arraydescr = get_array_descr(self.gc_ll_descr, A) p = self.gc_ll_descr.gc_malloc_array(arraydescr, 10) - assert self.llop1.record == [("varsize", arraydescr.type_id, 10, + assert self.llop1.record == [("varsize", arraydescr.tid, 10, repr(arraydescr.get_base_size(True)), repr(arraydescr.get_item_size(True)), repr(arraydescr.get_ofs_length(True)), p)] assert repr(self.gc_ll_descr.args_for_new_array(arraydescr)) == repr( - [arraydescr.get_item_size(True), arraydescr.type_id]) + [arraydescr.get_item_size(True), arraydescr.tid]) def test_gc_malloc_str(self): p = self.gc_ll_descr.gc_malloc_str(10) type_id = self.gc_ll_descr.layoutbuilder.get_type_id(rstr.STR) + tid = llop.combine_ushort(lltype.Signed, type_id, 0) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, True) - assert self.llop1.record == [("varsize", type_id, 10, + assert self.llop1.record == [("varsize", tid, 10, repr(basesize), repr(itemsize), repr(ofs_length), p)] def test_gc_malloc_unicode(self): p = self.gc_ll_descr.gc_malloc_unicode(10) type_id = self.gc_ll_descr.layoutbuilder.get_type_id(rstr.UNICODE) + tid = llop.combine_ushort(lltype.Signed, type_id, 0) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, True) - assert self.llop1.record == [("varsize", type_id, 10, + assert self.llop1.record == [("varsize", tid, 10, repr(basesize), repr(itemsize), repr(ofs_length), p)] Modified: pypy/trunk/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/runner.py Sun Oct 11 16:32:27 2009 @@ -22,8 +22,6 @@ gcdescr) self._bootstrap_cache = {} self._faildescr_list = [] - if rtyper is not None: # for tests - self.lltype2vtable = rtyper.lltype_to_vtable_mapping() def setup(self): self.assembler = Assembler386(self, self.translate_support_code) Modified: pypy/trunk/pypy/jit/metainterp/policy.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/policy.py (original) +++ pypy/trunk/pypy/jit/metainterp/policy.py Sun Oct 11 16:32:27 2009 @@ -63,7 +63,7 @@ return None def _graphs_of_all_instantiate(self, rtyper): - for vtable in rtyper.lltype_to_vtable_mapping().itervalues(): + for vtable in rtyper.lltype2vtable.values(): if vtable.instantiate: yield vtable.instantiate._obj.graph Modified: pypy/trunk/pypy/rpython/llinterp.py ============================================================================== --- pypy/trunk/pypy/rpython/llinterp.py (original) +++ pypy/trunk/pypy/rpython/llinterp.py Sun Oct 11 16:32:27 2009 @@ -869,7 +869,10 @@ assert v_ptr.concretetype.TO._gckind == 'gc' newaddr = self.getval(v_newaddr) p = llmemory.cast_adr_to_ptr(newaddr, v_ptr.concretetype) - self.setvar(v_ptr, p) + if isinstance(v_ptr, Constant): + assert v_ptr.value == p + else: + self.setvar(v_ptr, p) op_gc_reload_possibly_moved.specialform = True def op_gc_id(self, v_ptr): Modified: pypy/trunk/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/llarena.py Sun Oct 11 16:32:27 2009 @@ -240,12 +240,15 @@ """A size that is rounded up in order to preserve alignment of objects following it. For arenas containing heterogenous objects. """ - def __init__(self, basesize): + def __init__(self, basesize, minsize): assert isinstance(basesize, llmemory.AddressOffset) + assert isinstance(minsize, llmemory.AddressOffset) or minsize == 0 self.basesize = basesize + self.minsize = minsize def __repr__(self): - return '< RoundedUpForAllocation %r >' % (self.basesize,) + return '< RoundedUpForAllocation %r %r >' % (self.basesize, + self.minsize) def known_nonneg(self): return self.basesize.known_nonneg() @@ -303,10 +306,14 @@ % (addr.offset,)) addr.arena.allocate_object(addr.offset, size) -def round_up_for_allocation(size): +def round_up_for_allocation(size, minsize=0): """Round up the size in order to preserve alignment of objects - following an object. For arenas containing heterogenous objects.""" - return RoundedUpForAllocation(size) + following an object. For arenas containing heterogenous objects. + If minsize is specified, it gives a minimum on the resulting size.""" + return _round_up_for_allocation(size, minsize) + +def _round_up_for_allocation(size, minsize): # internal + return RoundedUpForAllocation(size, minsize) def arena_new_view(ptr): """Return a fresh memory view on an arena @@ -408,10 +415,11 @@ sandboxsafe=True) llimpl_round_up_for_allocation = rffi.llexternal('ROUND_UP_FOR_ALLOCATION', - [lltype.Signed], lltype.Signed, + [lltype.Signed, lltype.Signed], + lltype.Signed, sandboxsafe=True, _nowrapper=True) -register_external(round_up_for_allocation, [int], int, +register_external(_round_up_for_allocation, [int, int], int, 'll_arena.round_up_for_allocation', llimpl=llimpl_round_up_for_allocation, llfakeimpl=round_up_for_allocation, Modified: pypy/trunk/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/lloperation.py Sun Oct 11 16:32:27 2009 @@ -411,6 +411,13 @@ 'cast_adr_to_int': LLOp(sideeffects=False), 'cast_int_to_adr': LLOp(canfold=True), # not implemented in llinterp + 'get_group_member': LLOp(canfold=True), + 'get_next_group_member':LLOp(canfold=True), + 'is_group_member_nonzero':LLOp(canfold=True), + 'extract_ushort': LLOp(canfold=True), + 'combine_ushort': LLOp(canfold=True), + 'gc_gettypeptr_group': LLOp(canfold=True), + # __________ used by the JIT ________ 'jit_marker': LLOp(), Modified: pypy/trunk/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/opimpl.py Sun Oct 11 16:32:27 2009 @@ -171,10 +171,33 @@ return not b def op_int_add(x, y): - assert isinstance(x, (int, llmemory.AddressOffset)) + if not isinstance(x, (int, llmemory.AddressOffset)): + from pypy.rpython.lltypesystem import llgroup + assert isinstance(x, llgroup.CombinedSymbolic) assert isinstance(y, (int, llmemory.AddressOffset)) return intmask(x + y) +def op_int_sub(x, y): + if not isinstance(x, int): + from pypy.rpython.lltypesystem import llgroup + assert isinstance(x, llgroup.CombinedSymbolic) + assert isinstance(y, int) + return intmask(x - y) + +def op_int_and(x, y): + if not isinstance(x, int): + from pypy.rpython.lltypesystem import llgroup + assert isinstance(x, llgroup.CombinedSymbolic) + assert isinstance(y, int) + return x & y + +def op_int_or(x, y): + if not isinstance(x, int): + from pypy.rpython.lltypesystem import llgroup + assert isinstance(x, llgroup.CombinedSymbolic) + assert isinstance(y, int) + return x | y + def op_int_mul(x, y): assert isinstance(x, (int, llmemory.AddressOffset)) assert isinstance(y, (int, llmemory.AddressOffset)) @@ -388,6 +411,50 @@ def op_promote_virtualizable(object, fieldname, flags): pass # XXX should do something +def op_get_group_member(TYPE, grpptr, memberoffset): + from pypy.rpython.lltypesystem import llgroup + assert isinstance(memberoffset, llgroup.GroupMemberOffset) + member = memberoffset._get_group_member(grpptr) + return lltype.cast_pointer(TYPE, member) +op_get_group_member.need_result_type = True + +def op_get_next_group_member(TYPE, grpptr, memberoffset, skipoffset): + from pypy.rpython.lltypesystem import llgroup + assert isinstance(memberoffset, llgroup.GroupMemberOffset) + member = memberoffset._get_next_group_member(grpptr, skipoffset) + return lltype.cast_pointer(TYPE, member) +op_get_next_group_member.need_result_type = True + +def op_is_group_member_nonzero(memberoffset): + from pypy.rpython.lltypesystem import llgroup + if isinstance(memberoffset, llgroup.GroupMemberOffset): + return memberoffset.index != 0 + else: + assert isinstance(memberoffset, int) + return memberoffset != 0 + +def op_extract_ushort(combinedoffset): + from pypy.rpython.lltypesystem import llgroup + assert isinstance(combinedoffset, llgroup.CombinedSymbolic) + return combinedoffset.lowpart + +def op_combine_ushort(ushort, rest): + from pypy.rpython.lltypesystem import llgroup + return llgroup.CombinedSymbolic(ushort, rest) + +def op_gc_gettypeptr_group(TYPE, obj, grpptr, skipoffset, vtableinfo): + HDR = vtableinfo[0] + size_gc_header = vtableinfo[1] + fieldname = vtableinfo[2] + objaddr = llmemory.cast_ptr_to_adr(obj) + hdraddr = objaddr - size_gc_header + hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR)) + typeid = getattr(hdr, fieldname) + if lltype.typeOf(typeid) == lltype.Signed: + typeid = op_extract_ushort(typeid) + return op_get_next_group_member(TYPE, grpptr, typeid, skipoffset) +op_gc_gettypeptr_group.need_result_type = True + # ____________________________________________________________ def get_op_impl(opname): Modified: pypy/trunk/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rclass.py Sun Oct 11 16:32:27 2009 @@ -389,6 +389,7 @@ OBJECT, destrptr) vtable = self.rclass.getvtable() self.rtyper.type_for_typeptr[vtable._obj] = self.lowleveltype.TO + self.rtyper.lltype2vtable[self.lowleveltype.TO] = vtable def common_repr(self): # -> object or nongcobject reprs return getinstancerepr(self.rtyper, None, self.gcflavor) Modified: pypy/trunk/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/base.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/base.py Sun Oct 11 16:32:27 2009 @@ -13,6 +13,7 @@ malloc_zero_filled = False prebuilt_gc_objects_are_static_roots = True can_realloc = False + object_minimal_size = 0 def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE): self.gcheaderbuilder = GCHeaderBuilder(self.HDR) Modified: pypy/trunk/pypy/rpython/memory/gc/hybrid.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/hybrid.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/hybrid.py Sun Oct 11 16:32:27 2009 @@ -222,7 +222,6 @@ def realloc(self, ptr, newlength, fixedsize, itemsize, lengthofs, grow): size_gc_header = self.size_gc_header() addr = llmemory.cast_ptr_to_adr(ptr) - tid = self.get_type_id(addr) nonvarsize = size_gc_header + fixedsize try: varsize = ovfcheck(itemsize * newlength) @@ -375,7 +374,7 @@ hdr = self.header(obj) if hdr.tid & GCFLAG_UNVISITED: # This is a not-visited-yet raw_malloced object. - hdr.tid -= GCFLAG_UNVISITED + hdr.tid &= ~GCFLAG_UNVISITED self.rawmalloced_objects_to_trace.append(obj) def make_a_copy(self, obj, objsize): Modified: pypy/trunk/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/markcompact.py Sun Oct 11 16:32:27 2009 @@ -1,7 +1,7 @@ import time -from pypy.rpython.lltypesystem import lltype, llmemory, llarena +from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup from pypy.rpython.memory.gc.base import MovingGCBase from pypy.rlib.debug import ll_assert from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE @@ -12,11 +12,10 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.objectmodel import we_are_translated from pypy.rpython.lltypesystem import rffi +from pypy.rpython.memory.gcheader import GCHeaderBuilder -TYPEID_MASK = 0xffff0000 -first_gcflag = 2 +first_gcflag = 1 << 16 GCFLAG_MARKBIT = first_gcflag << 0 -GCFLAG_EXTERNAL = first_gcflag << 1 memoryError = MemoryError() @@ -68,8 +67,10 @@ TID_TYPE = rffi.USHORT BYTES_PER_TID = rffi.sizeof(TID_TYPE) + class MarkCompactGC(MovingGCBase): - HDR = lltype.Struct('header', ('forward_ptr', llmemory.Address)) + HDR = lltype.Struct('header', ('tid', lltype.Signed)) + typeid_is_in_field = 'tid' TID_BACKUP = lltype.Array(TID_TYPE, hints={'nolength':True}) WEAKREF_OFFSETS = lltype.Array(lltype.Signed) @@ -79,7 +80,7 @@ malloc_zero_filled = True inline_simple_malloc = True inline_simple_malloc_varsize = True - first_unused_gcflag = first_gcflag << 2 + first_unused_gcflag = first_gcflag << 1 total_collection_time = 0.0 total_collection_count = 0 @@ -100,21 +101,18 @@ self.objects_with_weakrefs = self.AddressStack() self.tid_backup = lltype.nullptr(self.TID_BACKUP) - # flags = 1 to make lltype & llmemory happy about odd/even pointers - - def init_gc_object(self, addr, typeid, flags=1): + def init_gc_object(self, addr, typeid16, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.forward_ptr = llmemory.cast_int_to_adr((typeid << 16) | flags) + hdr.tid = self.combine(typeid16, flags) - def init_gc_object_immortal(self, addr, typeid, flags=1): + def init_gc_object_immortal(self, addr, typeid16, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - flags |= GCFLAG_EXTERNAL - hdr.forward_ptr = llmemory.cast_int_to_adr((typeid << 16) | flags) + hdr.tid = self.combine(typeid16, flags) # XXX we can store forward_ptr to itself, if we fix C backend # so that get_forwarding_address(obj) returns # obj itself if obj is a prebuilt object - def malloc_fixedsize_clear(self, typeid, size, can_collect, + def malloc_fixedsize_clear(self, typeid16, size, can_collect, has_finalizer=False, contains_weakptr=False): size_gc_header = self.gcheaderbuilder.size_gc_header totalsize = size_gc_header + size @@ -122,7 +120,7 @@ if raw_malloc_usage(totalsize) > self.top_of_space - result: result = self.obtain_free_space(totalsize) llarena.arena_reserve(result, totalsize) - self.init_gc_object(result, typeid) + self.init_gc_object(result, typeid16) self.free += totalsize if has_finalizer: self.objects_with_finalizers.append(result + size_gc_header) @@ -130,7 +128,7 @@ self.objects_with_weakrefs.append(result + size_gc_header) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) - def malloc_varsize_clear(self, typeid, length, size, itemsize, + def malloc_varsize_clear(self, typeid16, length, size, itemsize, offset_to_length, can_collect): size_gc_header = self.gcheaderbuilder.size_gc_header nonvarsize = size_gc_header + size @@ -143,7 +141,7 @@ if raw_malloc_usage(totalsize) > self.top_of_space - result: result = self.obtain_free_space(totalsize) llarena.arena_reserve(result, totalsize) - self.init_gc_object(result, typeid) + self.init_gc_object(result, typeid16) (result + size_gc_header + offset_to_length).signed[0] = length self.free = result + llarena.round_up_for_allocation(totalsize) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) @@ -319,11 +317,26 @@ self.objects_with_finalizers.delete() self.objects_with_finalizers = objects_with_finalizers - def get_tid(self, addr): - return llmemory.cast_adr_to_int(self.header(addr).forward_ptr) + def header(self, addr): + # like header(), but asserts that we have a normal header + hdr = MovingGCBase.header(self, addr) + if not we_are_translated(): + assert isinstance(hdr.tid, llgroup.CombinedSymbolic) + return hdr + + def header_forwarded(self, addr): + # like header(), but asserts that we have a forwarding header + hdr = MovingGCBase.header(self, addr) + if not we_are_translated(): + assert isinstance(hdr.tid, int) + return hdr + + def combine(self, typeid16, flags): + return llop.combine_ushort(lltype.Signed, typeid16, flags) def get_type_id(self, addr): - return self.get_tid(addr) >> 16 + tid = self.header(addr).tid + return llop.extract_ushort(rffi.USHORT, tid) def mark_roots_recursively(self): self.root_walker.walk_roots( @@ -350,13 +363,13 @@ self.to_see.append(root.address[0]) def mark(self, obj): - previous = self.get_tid(obj) - self.header(obj).forward_ptr = llmemory.cast_int_to_adr(previous | GCFLAG_MARKBIT) + self.header(obj).tid |= GCFLAG_MARKBIT def marked(self, obj): - return self.get_tid(obj) & GCFLAG_MARKBIT + return self.header(obj).tid & GCFLAG_MARKBIT def update_forward_pointers(self, toaddr, num_of_alive_objs): + self.base_forwarding_addr = toaddr fromaddr = self.space size_gc_header = self.gcheaderbuilder.size_gc_header i = 0 @@ -366,8 +379,7 @@ objsize = self.get_size(obj) totalsize = size_gc_header + objsize if not self.marked(obj): - self.set_forwarding_address(obj, NULL, i) - hdr.forward_ptr = NULL + self.set_null_forwarding_address(obj, i) else: llarena.arena_reserve(toaddr, totalsize) self.set_forwarding_address(obj, toaddr, i) @@ -438,30 +450,44 @@ if pointer.address[0] != NULL: pointer.address[0] = self.get_forwarding_address(pointer.address[0]) - def is_forwarded(self, addr): - return self.header(addr).forward_ptr != NULL - def _is_external(self, obj): - tid = self.get_tid(obj) - return (tid & 1) and (tid & GCFLAG_EXTERNAL) + return not (self.space <= obj < self.top_of_space) def get_forwarding_address(self, obj): if self._is_external(obj): return obj - return self.header(obj).forward_ptr + self.size_gc_header() + return self.get_header_forwarded_addr(obj) - def set_forwarding_address(self, obj, newaddr, num): + def set_null_forwarding_address(self, obj, num): self.backup_typeid(num, obj) - self.header(obj).forward_ptr = newaddr + hdr = self.header(obj) + hdr.tid = -1 # make the object forwarded to NULL + + def set_forwarding_address(self, obj, newobjhdr, num): + self.backup_typeid(num, obj) + forward_offset = newobjhdr - self.base_forwarding_addr + hdr = self.header(obj) + hdr.tid = forward_offset # make the object forwarded to newobj + + def restore_normal_header(self, obj, num): + # Reverse of set_forwarding_address(). + typeid16 = self.get_typeid_from_backup(num) + hdr = self.header_forwarded(obj) + hdr.tid = self.combine(typeid16, 0) # restore the normal header + + def get_header_forwarded_addr(self, obj): + return (self.base_forwarding_addr + + self.header_forwarded(obj).tid + + self.gcheaderbuilder.size_gc_header) def surviving(self, obj): - return self._is_external(obj) or self.header(obj).forward_ptr != NULL + return self._is_external(obj) or self.header_forwarded(obj).tid != -1 def backup_typeid(self, num, obj): - self.tid_backup[num] = rffi.cast(rffi.USHORT, self.get_type_id(obj)) + self.tid_backup[num] = self.get_type_id(obj) def get_typeid_from_backup(self, num): - return rffi.cast(lltype.Signed, self.tid_backup[num]) + return self.tid_backup[num] def get_size_from_backup(self, obj, num): typeid = self.get_typeid_from_backup(num) @@ -484,7 +510,6 @@ num = 0 while fromaddr < self.free: obj = fromaddr + size_gc_header - hdr = llmemory.cast_adr_to_ptr(fromaddr, lltype.Ptr(self.HDR)) objsize = self.get_size_from_backup(obj, num) totalsize = size_gc_header + objsize if not self.surviving(obj): @@ -492,16 +517,16 @@ # we clear it to make debugging easier llarena.arena_reset(fromaddr, totalsize, False) else: - ll_assert(self.is_forwarded(obj), "not forwarded, surviving obj") - forward_ptr = hdr.forward_ptr if resizing: end = fromaddr - val = (self.get_typeid_from_backup(num) << 16) + 1 - hdr.forward_ptr = llmemory.cast_int_to_adr(val) - if fromaddr != forward_ptr: + forward_obj = self.get_header_forwarded_addr(obj) + self.restore_normal_header(obj, num) + if obj != forward_obj: #llop.debug_print(lltype.Void, "Copying from to", # fromaddr, forward_ptr, totalsize) - llmemory.raw_memmove(fromaddr, forward_ptr, totalsize) + llmemory.raw_memmove(fromaddr, + forward_obj - size_gc_header, + totalsize) if resizing and end - start > GC_CLEARANCE: diff = end - start #llop.debug_print(lltype.Void, "Cleaning", start, diff) Modified: pypy/trunk/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/marksweep.py Sun Oct 11 16:32:27 2009 @@ -4,7 +4,7 @@ from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE from pypy.rpython.memory.support import get_address_stack from pypy.rpython.memory.gcheader import GCHeaderBuilder -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rlib.objectmodel import free_non_gc_object from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck @@ -25,8 +25,11 @@ HDRPTR = lltype.Ptr(HDR) # need to maintain a linked list of malloced objects, since we used the # systems allocator and can't walk the heap - HDR.become(lltype.Struct('header', ('typeid', lltype.Signed), + HDR.become(lltype.Struct('header', ('typeid16', rffi.USHORT), + ('mark', lltype.Bool), + ('curpool_flag', lltype.Bool), ('next', HDRPTR))) + typeid_is_in_field = 'typeid16' POOL = lltype.GcStruct('gc_pool') POOLPTR = lltype.Ptr(POOL) @@ -75,14 +78,14 @@ if self.bytes_malloced > self.bytes_malloced_threshold: self.collect() - def write_malloc_statistics(self, typeid, size, result, varsize): + def write_malloc_statistics(self, typeid16, size, result, varsize): pass - def write_free_statistics(self, typeid, result): + def write_free_statistics(self, typeid16, result): pass - def malloc_fixedsize(self, typeid, size, can_collect, has_finalizer=False, - contains_weakptr=False): + def malloc_fixedsize(self, typeid16, size, can_collect, + has_finalizer=False, contains_weakptr=False): if can_collect: self.maybe_collect() size_gc_header = self.gcheaderbuilder.size_gc_header @@ -97,7 +100,9 @@ if not result: raise memoryError hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) - hdr.typeid = typeid << 1 + hdr.typeid16 = typeid16 + hdr.mark = False + hdr.curpool_flag = False if has_finalizer: hdr.next = self.malloced_objects_with_finalizer self.malloced_objects_with_finalizer = hdr @@ -109,13 +114,13 @@ self.malloced_objects = hdr self.bytes_malloced = bytes_malloced result += size_gc_header - #llop.debug_print(lltype.Void, 'malloc typeid', typeid, + #llop.debug_print(lltype.Void, 'malloc typeid', typeid16, # '->', llmemory.cast_adr_to_int(result)) - self.write_malloc_statistics(typeid, tot_size, result, False) + self.write_malloc_statistics(typeid16, tot_size, result, False) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) malloc_fixedsize._dont_inline_ = True - def malloc_fixedsize_clear(self, typeid, size, can_collect, + def malloc_fixedsize_clear(self, typeid16, size, can_collect, has_finalizer=False, contains_weakptr=False): if can_collect: self.maybe_collect() @@ -132,7 +137,9 @@ raise memoryError raw_memclear(result, tot_size) hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) - hdr.typeid = typeid << 1 + hdr.typeid16 = typeid16 + hdr.mark = False + hdr.curpool_flag = False if has_finalizer: hdr.next = self.malloced_objects_with_finalizer self.malloced_objects_with_finalizer = hdr @@ -144,14 +151,14 @@ self.malloced_objects = hdr self.bytes_malloced = bytes_malloced result += size_gc_header - #llop.debug_print(lltype.Void, 'malloc typeid', typeid, + #llop.debug_print(lltype.Void, 'malloc typeid', typeid16, # '->', llmemory.cast_adr_to_int(result)) - self.write_malloc_statistics(typeid, tot_size, result, False) + self.write_malloc_statistics(typeid16, tot_size, result, False) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) malloc_fixedsize_clear._dont_inline_ = True - def malloc_varsize(self, typeid, length, size, itemsize, offset_to_length, - can_collect): + def malloc_varsize(self, typeid16, length, size, itemsize, + offset_to_length, can_collect): if can_collect: self.maybe_collect() size_gc_header = self.gcheaderbuilder.size_gc_header @@ -169,20 +176,22 @@ raise memoryError (result + size_gc_header + offset_to_length).signed[0] = length hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) - hdr.typeid = typeid << 1 + hdr.typeid16 = typeid16 + hdr.mark = False + hdr.curpool_flag = False hdr.next = self.malloced_objects self.malloced_objects = hdr self.bytes_malloced = bytes_malloced result += size_gc_header #llop.debug_print(lltype.Void, 'malloc_varsize length', length, - # 'typeid', typeid, + # 'typeid', typeid16, # '->', llmemory.cast_adr_to_int(result)) - self.write_malloc_statistics(typeid, tot_size, result, True) + self.write_malloc_statistics(typeid16, tot_size, result, True) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) malloc_varsize._dont_inline_ = True - def malloc_varsize_clear(self, typeid, length, size, itemsize, + def malloc_varsize_clear(self, typeid16, length, size, itemsize, offset_to_length, can_collect): if can_collect: self.maybe_collect() @@ -202,16 +211,18 @@ raw_memclear(result, tot_size) (result + size_gc_header + offset_to_length).signed[0] = length hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) - hdr.typeid = typeid << 1 + hdr.typeid16 = typeid16 + hdr.mark = False + hdr.curpool_flag = False hdr.next = self.malloced_objects self.malloced_objects = hdr self.bytes_malloced = bytes_malloced result += size_gc_header #llop.debug_print(lltype.Void, 'malloc_varsize length', length, - # 'typeid', typeid, + # 'typeid', typeid16, # '->', llmemory.cast_adr_to_int(result)) - self.write_malloc_statistics(typeid, tot_size, result, True) + self.write_malloc_statistics(typeid16, tot_size, result, True) return llmemory.cast_adr_to_ptr(result, llmemory.GCREF) malloc_varsize_clear._dont_inline_ = True @@ -251,10 +262,10 @@ hdr = self.malloced_objects_with_finalizer while hdr: next = hdr.next - typeid = hdr.typeid >> 1 + typeid = hdr.typeid16 gc_info = llmemory.cast_ptr_to_adr(hdr) obj = gc_info + size_gc_header - if not hdr.typeid & 1: + if not hdr.mark: self.add_reachable_to_stack(obj, objects) addr = llmemory.cast_ptr_to_adr(hdr) size = self.fixed_size(typeid) @@ -271,31 +282,30 @@ curr = objects.pop() gc_info = curr - size_gc_header hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - if hdr.typeid & 1: + if hdr.mark: continue self.add_reachable_to_stack(curr, objects) - hdr.typeid = hdr.typeid | 1 + hdr.mark = True objects.delete() # also mark self.curpool if self.curpool: gc_info = llmemory.cast_ptr_to_adr(self.curpool) - size_gc_header hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - hdr.typeid = hdr.typeid | 1 + hdr.mark = True # go through the list of objects containing weak pointers # and kill the links if they go to dead objects # if the object itself is not marked, free it hdr = self.objects_with_weak_pointers surviving = lltype.nullptr(self.HDR) while hdr: - typeid = hdr.typeid >> 1 + typeid = hdr.typeid16 next = hdr.next addr = llmemory.cast_ptr_to_adr(hdr) size = self.fixed_size(typeid) estimate = raw_malloc_usage(size_gc_header + size) - if hdr.typeid & 1: - typeid = hdr.typeid >> 1 + if hdr.mark: offset = self.weakpointer_offset(typeid) - hdr.typeid = hdr.typeid & (~1) + hdr.mark = False gc_info = llmemory.cast_ptr_to_adr(hdr) weakref_obj = gc_info + size_gc_header pointing_to = (weakref_obj + offset).address[0] @@ -306,7 +316,7 @@ # pointed to object will die # XXX what to do if the object has a finalizer which resurrects # the object? - if not hdr_pointing_to.typeid & 1: + if not hdr_pointing_to.mark: (weakref_obj + offset).address[0] = NULL hdr.next = surviving surviving = hdr @@ -331,7 +341,7 @@ ppnext += llmemory.offsetof(self.POOLNODE, 'linkedlist') hdr = poolnode.linkedlist while hdr: #sweep - typeid = hdr.typeid >> 1 + typeid = hdr.typeid16 next = hdr.next addr = llmemory.cast_ptr_to_adr(hdr) size = self.fixed_size(typeid) @@ -339,8 +349,8 @@ length = (addr + size_gc_header + self.varsize_offset_to_length(typeid)).signed[0] size += self.varsize_item_sizes(typeid) * length estimate = raw_malloc_usage(size_gc_header + size) - if hdr.typeid & 1: - hdr.typeid = hdr.typeid & (~1) + if hdr.mark: + hdr.mark = False ppnext.address[0] = addr ppnext = llmemory.cast_ptr_to_adr(hdr) ppnext += llmemory.offsetof(self.HDR, 'next') @@ -423,17 +433,17 @@ last = lltype.nullptr(self.HDR) while hdr: next = hdr.next - if hdr.typeid & 1: + if hdr.mark: hdr.next = lltype.nullptr(self.HDR) if not self.malloced_objects_with_finalizer: self.malloced_objects_with_finalizer = hdr else: last.next = hdr - hdr.typeid = hdr.typeid & (~1) + hdr.mark = False last = hdr else: obj = llmemory.cast_ptr_to_adr(hdr) + size_gc_header - finalizer = self.getfinalizer(hdr.typeid >> 1) + finalizer = self.getfinalizer(hdr.typeid16) # make malloced_objects_with_finalizer consistent # for the sake of a possible collection caused by finalizer if not self.malloced_objects_with_finalizer: @@ -473,7 +483,7 @@ size_gc_header = self.gcheaderbuilder.size_gc_header gc_info = gcobjectaddr - size_gc_header hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - hdr.typeid = hdr.typeid & (~1) + hdr.mark = False STAT_HEAP_USAGE = 0 STAT_BYTES_MALLOCED = 1 @@ -483,7 +493,7 @@ size_gc_header = self.gcheaderbuilder.size_gc_header gc_info = obj - size_gc_header hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) - return hdr.typeid >> 1 + return hdr.typeid16 def add_reachable_to_stack(self, obj, objects): self.trace(obj, self._add_reachable, objects) @@ -504,13 +514,17 @@ def init_gc_object(self, addr, typeid): hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) - hdr.typeid = typeid << 1 + hdr.typeid16 = typeid + hdr.mark = False + hdr.curpool_flag = False def init_gc_object_immortal(self, addr, typeid, flags=0): # prebuilt gc structures always have the mark bit set # ignore flags hdr = llmemory.cast_adr_to_ptr(addr, self.HDRPTR) - hdr.typeid = (typeid << 1) | 1 + hdr.typeid16 = typeid + hdr.mark = True + hdr.curpool_flag = False # experimental support for thread cloning def x_swap_pool(self, newpool): @@ -566,7 +580,6 @@ # in the specified pool. A new pool is built to contain the # copies, and the 'gcobjectptr' and 'pool' fields of clonedata # are adjusted to refer to the result. - CURPOOL_FLAG = sys.maxint // 2 + 1 # install a new pool into which all the mallocs go curpool = self.x_swap_pool(lltype.nullptr(X_POOL)) @@ -583,7 +596,7 @@ hdr = hdr.next # skip the POOL object itself while hdr: next = hdr.next - hdr.typeid |= CURPOOL_FLAG # mark all objects from malloced_list + hdr.curpool_flag = True # mark all objects from malloced_list hdr.next = lltype.nullptr(self.HDR) # abused to point to the copy oldobjects.append(llmemory.cast_ptr_to_adr(hdr)) hdr = next @@ -600,12 +613,11 @@ continue # pointer is NULL oldhdr = llmemory.cast_adr_to_ptr(oldobj_addr - size_gc_header, self.HDRPTR) - typeid = oldhdr.typeid - if not (typeid & CURPOOL_FLAG): + if not oldhdr.curpool_flag: continue # ignore objects that were not in the malloced_list newhdr = oldhdr.next # abused to point to the copy if not newhdr: - typeid = (typeid & ~CURPOOL_FLAG) >> 1 + typeid = oldhdr.typeid16 size = self.fixed_size(typeid) # XXX! collect() at the beginning if the free heap is low if self.is_varsize(typeid): @@ -631,11 +643,15 @@ newhdr_addr = newobj_addr - size_gc_header newhdr = llmemory.cast_adr_to_ptr(newhdr_addr, self.HDRPTR) - saved_id = newhdr.typeid # XXX hack needed for genc + saved_id = newhdr.typeid16 # XXX hack needed for genc + saved_flg1 = newhdr.mark + saved_flg2 = newhdr.curpool_flag saved_next = newhdr.next # where size_gc_header == 0 raw_memcopy(oldobj_addr, newobj_addr, size) - newhdr.typeid = saved_id - newhdr.next = saved_next + newhdr.typeid16 = saved_id + newhdr.mark = saved_flg1 + newhdr.curpool_flag = saved_flg2 + newhdr.next = saved_next offsets = self.offsets_to_gc_pointers(typeid) i = 0 @@ -669,7 +685,7 @@ next = lltype.nullptr(self.HDR) while oldobjects.non_empty(): hdr = llmemory.cast_adr_to_ptr(oldobjects.pop(), self.HDRPTR) - hdr.typeid &= ~CURPOOL_FLAG # reset the flag + hdr.curpool_flag = False # reset the flag hdr.next = next next = hdr oldobjects.delete() Modified: pypy/trunk/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/semispace.py Sun Oct 11 16:32:27 2009 @@ -4,7 +4,7 @@ from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE from pypy.rpython.memory.support import get_address_stack, get_address_deque from pypy.rpython.memory.support import AddressDict -from pypy.rpython.lltypesystem import lltype, llmemory, llarena +from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi from pypy.rlib.objectmodel import free_non_gc_object from pypy.rlib.debug import ll_assert from pypy.rpython.lltypesystem.lloperation import llop @@ -13,7 +13,6 @@ import sys, os, time -TYPEID_MASK = 0xffff first_gcflag = 1 << 16 GCFLAG_FORWARDED = first_gcflag # GCFLAG_EXTERNAL is set on objects not living in the semispace: @@ -23,6 +22,7 @@ memoryError = MemoryError() + class SemiSpaceGC(MovingGCBase): _alloc_flavor_ = "raw" inline_simple_malloc = True @@ -32,11 +32,14 @@ total_collection_time = 0.0 total_collection_count = 0 - HDR = lltype.Struct('header', ('tid', lltype.Signed)) + HDR = lltype.Struct('header', ('tid', lltype.Signed)) # XXX or rffi.INT? + typeid_is_in_field = 'tid' FORWARDSTUB = lltype.GcStruct('forwarding_stub', ('forw', llmemory.Address)) FORWARDSTUBPTR = lltype.Ptr(FORWARDSTUB) + object_minimal_size = llmemory.sizeof(FORWARDSTUB) + # the following values override the default arguments of __init__ when # translating to a real backend. TRANSLATION_PARAMS = {'space_size': 8*1024*1024} # XXX adjust @@ -64,7 +67,7 @@ # This class only defines the malloc_{fixed,var}size_clear() methods # because the spaces are filled with zeroes in advance. - def malloc_fixedsize_clear(self, typeid, size, can_collect, + def malloc_fixedsize_clear(self, typeid16, size, can_collect, has_finalizer=False, contains_weakptr=False): size_gc_header = self.gcheaderbuilder.size_gc_header totalsize = size_gc_header + size @@ -74,7 +77,7 @@ raise memoryError result = self.obtain_free_space(totalsize) llarena.arena_reserve(result, totalsize) - self.init_gc_object(result, typeid) + self.init_gc_object(result, typeid16) self.free = result + totalsize if has_finalizer: self.objects_with_finalizers.append(result + size_gc_header) @@ -82,7 +85,7 @@ self.objects_with_weakrefs.append(result + size_gc_header) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) - def malloc_varsize_clear(self, typeid, length, size, itemsize, + def malloc_varsize_clear(self, typeid16, length, size, itemsize, offset_to_length, can_collect): size_gc_header = self.gcheaderbuilder.size_gc_header nonvarsize = size_gc_header + size @@ -97,7 +100,7 @@ raise memoryError result = self.obtain_free_space(totalsize) llarena.arena_reserve(result, totalsize) - self.init_gc_object(result, typeid) + self.init_gc_object(result, typeid16) (result + size_gc_header + offset_to_length).signed[0] = length self.free = result + llarena.round_up_for_allocation(totalsize) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) @@ -385,6 +388,9 @@ stub = llmemory.cast_adr_to_ptr(obj, self.FORWARDSTUBPTR) stub.forw = newobj + def combine(self, typeid16, flags): + return llop.combine_ushort(lltype.Signed, typeid16, flags) + def get_type_id(self, addr): tid = self.header(addr).tid ll_assert(tid & (GCFLAG_FORWARDED|GCFLAG_EXTERNAL) != GCFLAG_FORWARDED, @@ -393,15 +399,16 @@ # Although calling get_type_id() on a forwarded object works by itself, # we catch it as an error because it's likely that what is then # done with the typeid is bogus. - return tid & TYPEID_MASK + return llop.extract_ushort(rffi.USHORT, tid) - def init_gc_object(self, addr, typeid, flags=0): + def init_gc_object(self, addr, typeid16, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.tid = typeid | flags + hdr.tid = self.combine(typeid16, flags) - def init_gc_object_immortal(self, addr, typeid, flags=0): + def init_gc_object_immortal(self, addr, typeid16, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) - hdr.tid = typeid | flags | GCFLAG_EXTERNAL | GCFLAG_FORWARDED + flags |= GCFLAG_EXTERNAL | GCFLAG_FORWARDED + hdr.tid = self.combine(typeid16, flags) # immortal objects always have GCFLAG_FORWARDED set; # see get_forwarding_address(). Modified: pypy/trunk/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/test/test_direct.py Sun Oct 11 16:32:27 2009 @@ -68,7 +68,7 @@ self.gc.DEBUG = True self.rootwalker = DirectRootWalker(self) self.gc.set_root_walker(self.rootwalker) - self.layoutbuilder = TypeLayoutBuilder() + self.layoutbuilder = TypeLayoutBuilder(self.GCClass, {}) self.get_type_id = self.layoutbuilder.get_type_id self.layoutbuilder.initialize_gc_query_function(self.gc) self.gc.setup() Modified: pypy/trunk/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/framework.py Sun Oct 11 16:32:27 2009 @@ -1,7 +1,7 @@ from pypy.rpython.memory.gctransform.transform import GCTransformer from pypy.rpython.memory.gctransform.support import find_gc_ptrs_in_type, \ get_rtti, ll_call_destructor, type_contains_pyobjs, var_ispyobj -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython import rmodel from pypy.rpython.memory import gctypelayout from pypy.rpython.memory.gc import marksweep @@ -22,6 +22,9 @@ import sys, types +TYPE_ID = rffi.USHORT + + class CollectAnalyzer(graphanalyze.BoolGraphAnalyzer): def analyze_direct_call(self, graph, seen=None): @@ -127,15 +130,18 @@ if hasattr(translator, '_jit2gc'): self.layoutbuilder = translator._jit2gc['layoutbuilder'] else: - self.layoutbuilder = TransformerLayoutBuilder() + if translator.config.translation.gcconfig.removetypeptr: + lltype2vtable = translator.rtyper.lltype2vtable + else: + lltype2vtable = {} + self.layoutbuilder = TransformerLayoutBuilder(GCClass, + lltype2vtable) self.layoutbuilder.transformer = self self.get_type_id = self.layoutbuilder.get_type_id - # set up dummy a table, to be overwritten with the real one in finish() - type_info_table = lltype._ptr( - lltype.Ptr(gctypelayout.GCData.TYPE_INFO_TABLE), - "delayed!type_info_table", solid=True) - gcdata = gctypelayout.GCData(type_info_table) + # set up GCData with the llgroup from the layoutbuilder, which + # will grow as more TYPE_INFO members are added to it + gcdata = gctypelayout.GCData(self.layoutbuilder.type_info_group) # initialize the following two fields with a random non-NULL address, # to make the annotator happy. The fields are patched in finish() @@ -163,6 +169,8 @@ gcdata.gc.setup() bk = self.translator.annotator.bookkeeper + r_typeid16 = rffi.platform.numbertype_to_rclass[TYPE_ID] + s_typeid16 = annmodel.SomeInteger(knowntype=r_typeid16) # the point of this little dance is to not annotate # self.gcdata.static_root_xyz as constants. XXX is it still needed?? @@ -212,7 +220,7 @@ malloc_fixedsize_clear_meth = GCClass.malloc_fixedsize_clear.im_func self.malloc_fixedsize_clear_ptr = getfn( malloc_fixedsize_clear_meth, - [s_gc, annmodel.SomeInteger(nonneg=True), + [s_gc, s_typeid16, annmodel.SomeInteger(nonneg=True), annmodel.SomeBool(), annmodel.SomeBool(), annmodel.SomeBool()], s_gcref, @@ -221,7 +229,7 @@ malloc_fixedsize_meth = GCClass.malloc_fixedsize.im_func self.malloc_fixedsize_ptr = getfn( malloc_fixedsize_meth, - [s_gc, annmodel.SomeInteger(nonneg=True), + [s_gc, s_typeid16, annmodel.SomeInteger(nonneg=True), annmodel.SomeBool(), annmodel.SomeBool(), annmodel.SomeBool()], s_gcref, @@ -235,7 +243,8 @@ ## + [annmodel.SomeBool(), annmodel.SomeBool()], s_gcref) self.malloc_varsize_clear_ptr = getfn( GCClass.malloc_varsize_clear.im_func, - [s_gc] + [annmodel.SomeInteger(nonneg=True) for i in range(5)] + [s_gc, s_typeid16] + + [annmodel.SomeInteger(nonneg=True) for i in range(4)] + [annmodel.SomeBool()], s_gcref) self.collect_ptr = getfn(GCClass.collect.im_func, [s_gc, annmodel.SomeInteger()], annmodel.s_None) @@ -268,7 +277,7 @@ s_True = annmodel.SomeBool(); s_True .const = True self.malloc_fast_ptr = getfn( malloc_fast, - [s_gc, annmodel.SomeInteger(nonneg=True), + [s_gc, s_typeid16, annmodel.SomeInteger(nonneg=True), s_True, s_False, s_False], s_gcref, @@ -288,7 +297,7 @@ s_True = annmodel.SomeBool(); s_True .const = True self.malloc_varsize_clear_fast_ptr = getfn( malloc_varsize_clear_fast, - [s_gc, annmodel.SomeInteger(nonneg=True), + [s_gc, s_typeid16, annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True), @@ -304,7 +313,7 @@ "malloc_varsize_nonmovable") self.malloc_varsize_nonmovable_ptr = getfn( malloc_nonmovable, - [s_gc, annmodel.SomeInteger(nonneg=True), + [s_gc, s_typeid16, annmodel.SomeInteger(nonneg=True)], s_gcref) else: self.malloc_varsize_nonmovable_ptr = None @@ -315,7 +324,7 @@ "malloc_varsize_resizable") self.malloc_varsize_resizable_ptr = getfn( malloc_resizable, - [s_gc, annmodel.SomeInteger(nonneg=True), + [s_gc, s_typeid16, annmodel.SomeInteger(nonneg=True)], s_gcref) else: self.malloc_varsize_resizable_ptr = None @@ -399,6 +408,14 @@ FLDTYPE = getattr(HDR, fldname) fields.append(('_' + fldname, FLDTYPE)) + size_gc_header = self.gcdata.gc.gcheaderbuilder.size_gc_header + vtableinfo = (HDR, size_gc_header, self.gcdata.gc.typeid_is_in_field) + self.c_vtableinfo = rmodel.inputconst(lltype.Void, vtableinfo) + tig = self.layoutbuilder.type_info_group._as_ptr() + self.c_type_info_group = rmodel.inputconst(lltype.typeOf(tig), tig) + sko = llmemory.sizeof(gcdata.TYPE_INFO) + self.c_vtinfo_skip_offset = rmodel.inputconst(lltype.typeOf(sko), sko) + def build_root_walker(self): return ShadowStackRootWalker(self) @@ -421,21 +438,14 @@ return [getattr(hdr, fldname) for fldname in HDR._names] def finish_tables(self): - table = self.layoutbuilder.flatten_table() - log.info("assigned %s typeids" % (len(table), )) + group = self.layoutbuilder.close_table() + log.info("assigned %s typeids" % (len(group.members), )) log.info("added %s push/pop stack root instructions" % ( self.num_pushs, )) if self.write_barrier_ptr: log.info("inserted %s write barrier calls" % ( self.write_barrier_calls, )) - # replace the type_info_table pointer in gcdata -- at this point, - # the database is in principle complete, so it has already seen - # the delayed pointer. We need to force it to consider the new - # array now. - - self.gcdata.type_info_table._become(table) - # XXX because we call inputconst already in replace_malloc, we can't # modify the instance, we have to modify the 'rtyped instance' # instead. horrors. is there a better way? @@ -463,16 +473,24 @@ self.write_typeid_list() return newgcdependencies + def get_final_dependencies(self): + # returns an iterator enumerating the type_info_group's members, + # to make sure that they are all followed (only a part of them + # might have been followed by a previous enum_dependencies()). + return iter(self.layoutbuilder.type_info_group.members) + def write_typeid_list(self): """write out the list of type ids together with some info""" from pypy.tool.udir import udir # XXX not ideal since it is not per compilation, but per run + # XXX argh argh, this only gives the member index but not the + # real typeid, which is a complete mess to obtain now... + all_ids = self.layoutbuilder.id_of_type.items() + all_ids = [(typeinfo.index, TYPE) for (TYPE, typeinfo) in all_ids] + all_ids = dict(all_ids) f = udir.join("typeids.txt").open("w") - all = [(typeid, TYPE) - for TYPE, typeid in self.layoutbuilder.id_of_type.iteritems()] - all.sort() - for typeid, TYPE in all: - f.write("%s %s\n" % (typeid, TYPE)) + for index in range(len(self.layoutbuilder.type_info_group.members)): + f.write("member%-4d %s\n" % (index, all_ids.get(index, '?'))) f.close() def transform_graph(self, graph): @@ -502,8 +520,8 @@ assert PTRTYPE.TO == TYPE type_id = self.get_type_id(TYPE) - c_type_id = rmodel.inputconst(lltype.Signed, type_id) - info = self.layoutbuilder.type_info_list[type_id] + c_type_id = rmodel.inputconst(TYPE_ID, type_id) + info = self.layoutbuilder.get_info(type_id) c_size = rmodel.inputconst(lltype.Signed, info.fixedsize) has_finalizer = bool(self.finalizer_funcptr_for_type(TYPE)) c_has_finalizer = rmodel.inputconst(lltype.Bool, has_finalizer) @@ -523,9 +541,12 @@ c_has_finalizer, rmodel.inputconst(lltype.Bool, False)] else: assert not c_has_finalizer.value + info_varsize = self.layoutbuilder.get_info_varsize(type_id) v_length = op.args[-1] - c_ofstolength = rmodel.inputconst(lltype.Signed, info.ofstolength) - c_varitemsize = rmodel.inputconst(lltype.Signed, info.varitemsize) + c_ofstolength = rmodel.inputconst(lltype.Signed, + info_varsize.ofstolength) + c_varitemsize = rmodel.inputconst(lltype.Signed, + info_varsize.varitemsize) if flags.get('resizable') and self.malloc_varsize_resizable_ptr: assert c_can_collect.value malloc_ptr = self.malloc_varsize_resizable_ptr @@ -656,8 +677,8 @@ type_id = self.get_type_id(WEAKREF) - c_type_id = rmodel.inputconst(lltype.Signed, type_id) - info = self.layoutbuilder.type_info_list[type_id] + c_type_id = rmodel.inputconst(TYPE_ID, type_id) + info = self.layoutbuilder.get_info(type_id) c_size = rmodel.inputconst(lltype.Signed, info.fixedsize) malloc_ptr = self.malloc_fixedsize_ptr c_has_finalizer = rmodel.inputconst(lltype.Bool, False) @@ -763,6 +784,48 @@ v_structaddr]) hop.rename('bare_' + opname) + def transform_getfield_typeptr(self, hop): + # this would become quite a lot of operations, even if it compiles + # to C code that is just as efficient as "obj->typeptr". To avoid + # that, we just generate a single custom operation instead. + hop.genop('gc_gettypeptr_group', [hop.spaceop.args[0], + self.c_type_info_group, + self.c_vtinfo_skip_offset, + self.c_vtableinfo], + resultvar = hop.spaceop.result) + + def transform_setfield_typeptr(self, hop): + # replace such a setfield with an assertion that the typeptr is right + # (xxx not very useful right now, so disabled) + if 0: + v_new = hop.spaceop.args[2] + v_old = hop.genop('gc_gettypeptr_group', [hop.spaceop.args[0], + self.c_type_info_group, + self.c_vtinfo_skip_offset, + self.c_vtableinfo], + resulttype = v_new.concretetype) + v_eq = hop.genop("ptr_eq", [v_old, v_new], + resulttype = lltype.Bool) + c_errmsg = rmodel.inputconst(lltype.Void, + "setfield_typeptr: wrong type") + hop.genop('debug_assert', [v_eq, c_errmsg]) + + def gct_getfield(self, hop): + if (hop.spaceop.args[1].value == 'typeptr' and + hop.spaceop.args[0].concretetype.TO._hints.get('typeptr') and + self.translator.config.translation.gcconfig.removetypeptr): + self.transform_getfield_typeptr(hop) + else: + GCTransformer.gct_getfield(self, hop) + + def gct_setfield(self, hop): + if (hop.spaceop.args[1].value == 'typeptr' and + hop.spaceop.args[0].concretetype.TO._hints.get('typeptr') and + self.translator.config.translation.gcconfig.removetypeptr): + self.transform_setfield_typeptr(hop) + else: + GCTransformer.gct_setfield(self, hop) + def var_needs_set_transform(self, var): return var_needsgc(var) @@ -852,6 +915,15 @@ return fptr +class JITTransformerLayoutBuilder(TransformerLayoutBuilder): + # for the JIT: currently does not support removetypeptr + def __init__(self, config): + from pypy.rpython.memory.gc.base import choose_gc_from_config + assert not config.translation.gcconfig.removetypeptr + GCClass, _ = choose_gc_from_config(config) + TransformerLayoutBuilder.__init__(self, GCClass, {}) + + def gen_zero_gc_pointers(TYPE, v, llops, previous_steps=None): if previous_steps is None: previous_steps = [] Modified: pypy/trunk/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/transform.py Sun Oct 11 16:32:27 2009 @@ -10,6 +10,7 @@ from pypy.translator.backendopt import graphanalyze from pypy.translator.backendopt.canraise import RaiseAnalyzer from pypy.translator.backendopt.ssa import DataFlowFamilyBuilder +from pypy.translator.backendopt.constfold import constant_fold_graph from pypy.annotation import model as annmodel from pypy.rpython import rmodel, annlowlevel from pypy.rpython.memory import gc @@ -144,16 +145,20 @@ if self.inline: raise_analyzer = RaiseAnalyzer(self.translator) to_enum = self.graph_dependencies.get(graph, self.graphs_to_inline) + must_constfold = False for inline_graph in to_enum: try: inline.inline_function(self.translator, inline_graph, graph, self.lltype_to_classdef, raise_analyzer, cleanup=False) + must_constfold = True except inline.CannotInline, e: print 'CANNOT INLINE:', e print '\t%s into %s' % (inline_graph, graph) cleanup_graph(graph) + if must_constfold: + constant_fold_graph(graph) def compute_borrowed_vars(self, graph): # the input args are borrowed, and stay borrowed for as long as they @@ -307,6 +312,9 @@ newgcdependencies = self.ll_finalizers_ptrs return newgcdependencies + def get_final_dependencies(self): + pass + def finish_tables(self): pass @@ -367,6 +375,8 @@ gct_setarrayitem = gct_setfield gct_setinteriorfield = gct_setfield + gct_getfield = default + def gct_zero_gc_pointers_inside(self, hop): pass Modified: pypy/trunk/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/trunk/pypy/rpython/memory/gctypelayout.py Sun Oct 11 16:32:27 2009 @@ -1,4 +1,6 @@ -from pypy.rpython.lltypesystem import lltype, llmemory, llarena +from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup +from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import ll_assert @@ -17,67 +19,78 @@ # structure describing the layout of a typeid TYPE_INFO = lltype.Struct("type_info", + ("infobits", lltype.Signed), # combination of the T_xxx consts ("finalizer", FINALIZERTYPE), ("fixedsize", lltype.Signed), ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), + hints={'immutable': True}, + ) + VARSIZE_TYPE_INFO = lltype.Struct("varsize_type_info", + ("header", TYPE_INFO), ("varitemsize", lltype.Signed), ("ofstovar", lltype.Signed), ("ofstolength", lltype.Signed), ("varofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), - ("weakptrofs", lltype.Signed), + hints={'immutable': True}, ) - TYPE_INFO_TABLE = lltype.Array(TYPE_INFO) + TYPE_INFO_PTR = lltype.Ptr(TYPE_INFO) + VARSIZE_TYPE_INFO_PTR = lltype.Ptr(VARSIZE_TYPE_INFO) - def __init__(self, type_info_table): - self.type_info_table = type_info_table - # 'type_info_table' is a list of TYPE_INFO structures when - # running with gcwrapper, or a real TYPE_INFO_TABLE after - # the gctransformer. + def __init__(self, type_info_group): + assert isinstance(type_info_group, llgroup.group) + self.type_info_group = type_info_group + self.type_info_group_ptr = type_info_group._as_ptr() + + def get(self, typeid): + _check_typeid(typeid) + return llop.get_group_member(GCData.TYPE_INFO_PTR, + self.type_info_group_ptr, + typeid) + + def get_varsize(self, typeid): + _check_typeid(typeid) + return llop.get_group_member(GCData.VARSIZE_TYPE_INFO_PTR, + self.type_info_group_ptr, + typeid) def q_is_varsize(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return (typeid & T_IS_FIXSIZE) == 0 + infobits = self.get(typeid).infobits + return (infobits & T_IS_VARSIZE) != 0 def q_has_gcptr_in_varsize(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return (typeid & (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE)) == 0 + infobits = self.get(typeid).infobits + return (infobits & T_HAS_GCPTR_IN_VARSIZE) != 0 def q_is_gcarrayofgcptr(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return (typeid & - (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE|T_NOT_SIMPLE_GCARRAY)) == 0 + infobits = self.get(typeid).infobits + return (infobits & T_IS_GCARRAY_OF_GCPTR) != 0 def q_finalizer(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].finalizer + return self.get(typeid).finalizer def q_offsets_to_gc_pointers(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].ofstoptrs + return self.get(typeid).ofstoptrs def q_fixed_size(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].fixedsize + return self.get(typeid).fixedsize def q_varsize_item_sizes(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].varitemsize + return self.get_varsize(typeid).varitemsize def q_varsize_offset_to_variable_part(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].ofstovar + return self.get_varsize(typeid).ofstovar def q_varsize_offset_to_length(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].ofstolength + return self.get_varsize(typeid).ofstolength def q_varsize_offsets_to_gcpointers_in_var_part(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].varofstoptrs + return self.get_varsize(typeid).varofstoptrs def q_weakpointer_offset(self, typeid): - ll_assert(typeid > 0, "invalid type_id") - return self.type_info_table[typeid].weakptrofs + infobits = self.get(typeid).infobits + if infobits & T_IS_WEAKREF: + return weakptr_offset + return -1 def set_query_functions(self, gc): gc.set_query_functions( @@ -93,88 +106,59 @@ self.q_varsize_offsets_to_gcpointers_in_var_part, self.q_weakpointer_offset) -# For the q_xxx functions that return flags, we use bit patterns -# in the typeid instead of entries in the type_info_table. The -# following flag combinations are used (the idea being that it's -# very fast on CPUs to check if all flags in a set are all zero): - -# * if T_IS_FIXSIZE is set, the gc object is not var-sized -# * if T_IS_FIXSIZE and T_NO_GCPTR_IN_VARSIZE are both cleared, -# there are gc ptrs in the var-sized part -# * if T_IS_FIXSIZE, T_NO_GCPTR_IN_VARSIZE and T_NOT_SIMPLE_GCARRAY -# are all cleared, the shape is just like GcArray(gcptr) - -T_IS_FIXSIZE = 0x4 -T_NO_GCPTR_IN_VARSIZE = 0x2 -T_NOT_SIMPLE_GCARRAY = 0x1 - -def get_typeid_bitmask(TYPE): - """Return the bits that we would like to be set or cleared in the type_id - corresponding to TYPE. This returns (mask, expected_value), where - the condition is that 'type_id & mask == expected_value'. - """ - if not TYPE._is_varsize(): - return (T_IS_FIXSIZE, T_IS_FIXSIZE) # not var-sized - if (isinstance(TYPE, lltype.GcArray) - and isinstance(TYPE.OF, lltype.Ptr) - and TYPE.OF.TO._gckind == 'gc'): - # a simple GcArray(gcptr) - return (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE|T_NOT_SIMPLE_GCARRAY, 0) - - if isinstance(TYPE, lltype.Struct): - ARRAY = TYPE._flds[TYPE._arrayfld] - else: - ARRAY = TYPE - assert isinstance(ARRAY, lltype.Array) - if ARRAY.OF != lltype.Void and len(offsets_to_gc_pointers(ARRAY.OF)) > 0: - # var-sized, with gc pointers in the variable part - return (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE|T_NOT_SIMPLE_GCARRAY, - T_NOT_SIMPLE_GCARRAY) - else: - # var-sized, but no gc pointer in the variable part - return (T_IS_FIXSIZE|T_NO_GCPTR_IN_VARSIZE, T_NO_GCPTR_IN_VARSIZE) +T_IS_VARSIZE = 0x01 +T_HAS_GCPTR_IN_VARSIZE = 0x02 +T_IS_GCARRAY_OF_GCPTR = 0x04 +T_IS_WEAKREF = 0x08 + +def _check_typeid(typeid): + ll_assert(llop.is_group_member_nonzero(lltype.Bool, typeid), + "invalid type_id") def encode_type_shape(builder, info, TYPE): """Encode the shape of the TYPE into the TYPE_INFO structure 'info'.""" offsets = offsets_to_gc_pointers(TYPE) + infobits = 0 info.ofstoptrs = builder.offsets2table(offsets, TYPE) info.finalizer = builder.make_finalizer_funcptr_for_type(TYPE) - info.weakptrofs = weakpointer_offset(TYPE) if not TYPE._is_varsize(): - #info.isvarsize = False - #info.gcptrinvarsize = False info.fixedsize = llarena.round_up_for_allocation( - llmemory.sizeof(TYPE)) - info.ofstolength = -1 + llmemory.sizeof(TYPE), builder.GCClass.object_minimal_size) # note about round_up_for_allocation(): in the 'info' table # we put a rounded-up size only for fixed-size objects. For # varsize ones, the GC must anyway compute the size at run-time # and round up that result. else: - #info.isvarsize = True + infobits |= T_IS_VARSIZE + varinfo = lltype.cast_pointer(GCData.VARSIZE_TYPE_INFO_PTR, info) info.fixedsize = llmemory.sizeof(TYPE, 0) if isinstance(TYPE, lltype.Struct): ARRAY = TYPE._flds[TYPE._arrayfld] ofs1 = llmemory.offsetof(TYPE, TYPE._arrayfld) - info.ofstolength = ofs1 + llmemory.ArrayLengthOffset(ARRAY) - info.ofstovar = ofs1 + llmemory.itemoffsetof(ARRAY, 0) + varinfo.ofstolength = ofs1 + llmemory.ArrayLengthOffset(ARRAY) + varinfo.ofstovar = ofs1 + llmemory.itemoffsetof(ARRAY, 0) else: + assert isinstance(TYPE, lltype.GcArray) ARRAY = TYPE - info.ofstolength = llmemory.ArrayLengthOffset(ARRAY) - info.ofstovar = llmemory.itemoffsetof(TYPE, 0) + if (isinstance(ARRAY.OF, lltype.Ptr) + and ARRAY.OF.TO._gckind == 'gc'): + infobits |= T_IS_GCARRAY_OF_GCPTR + varinfo.ofstolength = llmemory.ArrayLengthOffset(ARRAY) + varinfo.ofstovar = llmemory.itemoffsetof(TYPE, 0) assert isinstance(ARRAY, lltype.Array) if ARRAY.OF != lltype.Void: offsets = offsets_to_gc_pointers(ARRAY.OF) else: offsets = () - info.varofstoptrs = builder.offsets2table(offsets, ARRAY.OF) - info.varitemsize = llmemory.sizeof(ARRAY.OF) - #info.gcptrinvarsize = len(offsets) > 0 - #info.gcarrayofgcptr = (isinstance(TYPE, lltype.GcArray) - # and isinstance(TYPE.OF, lltype.Ptr) - # and TYPE.OF.TO._gckind == 'gc') + if len(offsets) > 0: + infobits |= T_HAS_GCPTR_IN_VARSIZE + varinfo.varofstoptrs = builder.offsets2table(offsets, ARRAY.OF) + varinfo.varitemsize = llmemory.sizeof(ARRAY.OF) + if TYPE == WEAKREF: + infobits |= T_IS_WEAKREF + info.infobits = infobits # ____________________________________________________________ @@ -183,8 +167,12 @@ can_add_new_types = True can_encode_type_shape = True # set to False initially by the JIT - def __init__(self): - self.type_info_list = [None] # don't use typeid 0, helps debugging + size_of_fixed_type_info = llmemory.sizeof(GCData.TYPE_INFO) + + def __init__(self, GCClass, lltype2vtable): + self.GCClass = GCClass + self.lltype2vtable = lltype2vtable + self.make_type_info_group() self.id_of_type = {} # {LLTYPE: type_id} self.seen_roots = {} # the following are lists of addresses of gc pointers living inside the @@ -199,7 +187,13 @@ self.all_prebuilt_gc = [] self.finalizer_funcptrs = {} self.offsettable_cache = {} - self.next_typeid_cache = {} + + def make_type_info_group(self): + self.type_info_group = llgroup.group("typeinfo") + # don't use typeid 0, may help debugging + DUMMY = lltype.Struct("dummy", ('x', lltype.Signed)) + dummy = lltype.malloc(DUMMY, immortal=True, zero=True) + self.type_info_group.add_member(dummy) def get_type_id(self, TYPE): try: @@ -208,33 +202,46 @@ assert self.can_add_new_types assert isinstance(TYPE, (lltype.GcStruct, lltype.GcArray)) # Record the new type_id description as a TYPE_INFO structure. - # It goes into a list for now, which will be turned into a - # TYPE_INFO_TABLE in flatten_table() by the gc transformer. - - # pick the next type_id with the correct bits set or cleared - mask, expected = get_typeid_bitmask(TYPE) - type_id = self.next_typeid_cache.get((mask, expected), 1) - while True: - if type_id == len(self.type_info_list): - self.type_info_list.append(None) - if (self.type_info_list[type_id] is None and - (type_id & mask) == expected): - break # can use this type_id - else: - type_id += 1 # continue searching - self.next_typeid_cache[mask, expected] = type_id + 1 - assert type_id & 0xffff == type_id # make sure it fits into 2 bytes - # build the TYPE_INFO structure - info = lltype.malloc(GCData.TYPE_INFO, immortal=True, zero=True) + if not TYPE._is_varsize(): + fullinfo = lltype.malloc(GCData.TYPE_INFO, + immortal=True, zero=True) + info = fullinfo + else: + fullinfo = lltype.malloc(GCData.VARSIZE_TYPE_INFO, + immortal=True, zero=True) + info = fullinfo.header if self.can_encode_type_shape: encode_type_shape(self, info, TYPE) else: self._pending_type_shapes.append((info, TYPE)) - self.type_info_list[type_id] = info + # store it + type_id = self.type_info_group.add_member(fullinfo) self.id_of_type[TYPE] = type_id + # store the vtable of the type (if any) immediately thereafter + # (note that if gcconfig.removetypeptr is False, lltype2vtable + # is empty) + vtable = self.lltype2vtable.get(TYPE, None) + if vtable is not None: + # check that if we have a vtable, we are not varsize + assert lltype.typeOf(fullinfo) == GCData.TYPE_INFO_PTR + vtable = lltype.normalizeptr(vtable) + self.type_info_group.add_member(vtable) return type_id + def get_info(self, type_id): + return llop.get_group_member(GCData.TYPE_INFO_PTR, + self.type_info_group._as_ptr(), + type_id) + + def get_info_varsize(self, type_id): + return llop.get_group_member(GCData.VARSIZE_TYPE_INFO_PTR, + self.type_info_group._as_ptr(), + type_id) + + def is_weakref(self, type_id): + return self.get_info(type_id).infobits & T_IS_WEAKREF + def encode_type_shapes_now(self): if not self.can_encode_type_shape: self.can_encode_type_shape = True @@ -260,20 +267,11 @@ self.offsettable_cache[TYPE] = cachedarray return cachedarray - def flatten_table(self): + def close_table(self): + # make sure we no longer add members to the type_info_group. self.can_add_new_types = False self.offsettable_cache = None - table = lltype.malloc(GCData.TYPE_INFO_TABLE, len(self.type_info_list), - immortal=True) - fieldnames = GCData.TYPE_INFO._names - for tableentry, newcontent in zip(table, self.type_info_list): - if newcontent is None: # empty entry - tableentry.weakptrofs = -1 - tableentry.ofstolength = -1 - else: - for name in fieldnames: - setattr(tableentry, name, getattr(newcontent, name)) - return table + return self.type_info_group def finalizer_funcptr_for_type(self, TYPE): if TYPE in self.finalizer_funcptrs: @@ -287,7 +285,7 @@ return lltype.nullptr(GCData.ADDRESS_VOID_FUNC) def initialize_gc_query_function(self, gc): - return GCData(self.type_info_list).set_query_functions(gc) + return GCData(self.type_info_group).set_query_functions(gc) def consider_constant(self, TYPE, value, gc): if value is not lltype.top_container(value): @@ -349,11 +347,6 @@ offsets.append(0) return offsets -def weakpointer_offset(TYPE): - if TYPE == WEAKREF: - return llmemory.offsetof(WEAKREF, "weakptr") - return -1 - def gc_pointers_inside(v, adr, mutable_only=False): t = lltype.typeOf(v) if isinstance(t, lltype.Struct): @@ -409,6 +402,7 @@ sizeof_weakref= llmemory.sizeof(WEAKREF) empty_weakref = lltype.malloc(WEAKREF, immortal=True) empty_weakref.weakptr = llmemory.NULL +weakptr_offset = llmemory.offsetof(WEAKREF, "weakptr") def ll_weakref_deref(wref): wref = llmemory.cast_weakrefptr_to_ptr(WEAKREFPTR, wref) Modified: pypy/trunk/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/trunk/pypy/rpython/memory/gcwrapper.py Sun Oct 11 16:32:27 2009 @@ -18,7 +18,10 @@ self.gc.setup() def prepare_graphs(self, flowgraphs): - layoutbuilder = DirectRunLayoutBuilder(self.llinterp) + lltype2vtable = self.llinterp.typer.lltype2vtable + layoutbuilder = DirectRunLayoutBuilder(self.gc.__class__, + lltype2vtable, + self.llinterp) self.get_type_id = layoutbuilder.get_type_id layoutbuilder.initialize_gc_query_function(self.gc) @@ -159,9 +162,9 @@ class DirectRunLayoutBuilder(gctypelayout.TypeLayoutBuilder): - def __init__(self, llinterp): + def __init__(self, GCClass, lltype2vtable, llinterp): self.llinterp = llinterp - super(DirectRunLayoutBuilder, self).__init__() + super(DirectRunLayoutBuilder, self).__init__(GCClass, lltype2vtable) def make_finalizer_funcptr_for_type(self, TYPE): from pypy.rpython.memory.gctransform.support import get_rtti, \ Modified: pypy/trunk/pypy/rpython/memory/lltypelayout.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/lltypelayout.py (original) +++ pypy/trunk/pypy/rpython/memory/lltypelayout.py Sun Oct 11 16:32:27 2009 @@ -118,6 +118,10 @@ return 0 elif isinstance(offset, llarena.RoundedUpForAllocation): basesize = convert_offset_to_int(offset.basesize) + if isinstance(offset.minsize, llmemory.AddressOffset): + minsize = convert_offset_to_int(offset.minsize) + if minsize > basesize: + basesize = minsize mask = memory_alignment - 1 return (basesize + mask) & ~ mask else: Modified: pypy/trunk/pypy/rpython/memory/test/test_gctypelayout.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/test/test_gctypelayout.py (original) +++ pypy/trunk/pypy/rpython/memory/test/test_gctypelayout.py Sun Oct 11 16:32:27 2009 @@ -1,6 +1,12 @@ +import py from pypy.rpython.memory.gctypelayout import TypeLayoutBuilder, GCData from pypy.rpython.memory.gctypelayout import offsets_to_gc_pointers -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, rclass +from pypy.rpython.test.test_llinterp import get_interpreter +from pypy.objspace.flow.model import Constant + +class FakeGC: + object_minimal_size = 0 def getname(T): try: @@ -30,13 +36,57 @@ for T, c in [(GC_S, 0), (GC_S2, 2), (GC_A, 0), (GC_A2, 0), (GC_S3, 2)]: assert len(offsets_to_gc_pointers(T)) == c -def test_layout_builder(): +def test_layout_builder(lltype2vtable={}): # XXX a very minimal test - layoutbuilder = TypeLayoutBuilder() + layoutbuilder = TypeLayoutBuilder(FakeGC, lltype2vtable) for T1, T2 in [(GC_A, GC_S), (GC_A2, GC_S2), (GC_S3, GC_S2)]: tid1 = layoutbuilder.get_type_id(T1) tid2 = layoutbuilder.get_type_id(T2) - gcdata = GCData(layoutbuilder.type_info_list) + gcdata = GCData(layoutbuilder.type_info_group) lst1 = gcdata.q_varsize_offsets_to_gcpointers_in_var_part(tid1) lst2 = gcdata.q_offsets_to_gc_pointers(tid2) assert len(lst1) == len(lst2) + return layoutbuilder + +def test_layout_builder_with_vtable(): + from pypy.rpython.lltypesystem.lloperation import llop + vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) + layoutbuilder = test_layout_builder({GC_S: vtable}) + tid1 = layoutbuilder.get_type_id(GC_S) + tid2 = layoutbuilder.get_type_id(GC_S2) + tid3 = layoutbuilder.get_type_id(GC_S3) + group = layoutbuilder.type_info_group + vt = llop.get_next_group_member(rclass.CLASSTYPE, group._as_ptr(), tid1, + layoutbuilder.size_of_fixed_type_info) + assert vt == vtable + for tid in [tid2, tid3]: + py.test.raises((lltype.InvalidCast, AssertionError), + llop.get_next_group_member, + rclass.CLASSTYPE, group._as_ptr(), tid, + layoutbuilder.size_of_fixed_type_info) + +def test_constfold(): + layoutbuilder = TypeLayoutBuilder(FakeGC, {}) + tid1 = layoutbuilder.get_type_id(GC_A) + tid2 = layoutbuilder.get_type_id(GC_S3) + class MockGC: + def set_query_functions(self, is_varsize, + has_gcptr_in_varsize, + is_gcarrayofgcptr, + *rest): + self.is_varsize = is_varsize + self.has_gcptr_in_varsize = has_gcptr_in_varsize + self.is_gcarrayofgcptr = is_gcarrayofgcptr + gc = MockGC() + layoutbuilder.initialize_gc_query_function(gc) + # + def f(): + return (1 * gc.is_varsize(tid1) + + 10 * gc.has_gcptr_in_varsize(tid1) + + 100 * gc.is_gcarrayofgcptr(tid1) + + 1000 * gc.is_varsize(tid2) + + 10000 * gc.has_gcptr_in_varsize(tid2) + + 100000 * gc.is_gcarrayofgcptr(tid2)) + interp, graph = get_interpreter(f, [], backendopt=True) + assert interp.eval_graph(graph, []) == 11001 + assert graph.startblock.exits[0].args == [Constant(11001, lltype.Signed)] Modified: pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py Sun Oct 11 16:32:27 2009 @@ -21,6 +21,7 @@ t = TranslationContext() # XXX XXX XXX mess t.config.translation.gc = gcname + t.config.translation.gcconfig.removetypeptr = True if stacklessgc: t.config.translation.gcrootfinder = "stackless" t.config.set(**extraconfigopts) @@ -195,7 +196,6 @@ assert heap_size < 16000 * INT_SIZE / 4 # xxx def test_nongc_static_root(self): - from pypy.rpython.lltypesystem import lltype T1 = lltype.GcStruct("C", ('x', lltype.Signed)) T2 = lltype.Struct("C", ('p', lltype.Ptr(T1))) static = lltype.malloc(T2, immortal=True) @@ -552,7 +552,7 @@ class A(object): pass def f(): - from pypy.rpython.lltypesystem import lltype, rffi + from pypy.rpython.lltypesystem import rffi alist = [A() for i in range(50)] idarray = lltype.malloc(rffi.INTP.TO, len(alist), flavor='raw') # Compute the id of all the elements of the list. The goal is @@ -592,7 +592,11 @@ def fix_graph_of_g(translator): from pypy.translator.translator import graphof from pypy.objspace.flow.model import Constant - layoutbuilder = framework.TransformerLayoutBuilder() + from pypy.rpython.lltypesystem import rffi + GCClass = self.gcpolicy.transformerclass.GCClass + lltype2vtable = translator.rtyper.lltype2vtable + layoutbuilder = framework.TransformerLayoutBuilder(GCClass, + lltype2vtable) layoutbuilder.delay_encoding() translator._jit2gc = { 'layoutbuilder': layoutbuilder, @@ -603,7 +607,7 @@ graph = graphof(translator, g) for op in graph.startblock.operations: if op.opname == 'do_malloc_fixedsize_clear': - op.args = [Constant(type_id, lltype.Signed), + op.args = [Constant(type_id, rffi.USHORT), Constant(llmemory.sizeof(P), lltype.Signed), Constant(True, lltype.Bool), # can_collect Constant(False, lltype.Bool), # has_finalizer @@ -628,7 +632,11 @@ def fix_graph_of_g(translator): from pypy.translator.translator import graphof from pypy.objspace.flow.model import Constant - layoutbuilder = framework.TransformerLayoutBuilder() + from pypy.rpython.lltypesystem import rffi + GCClass = self.gcpolicy.transformerclass.GCClass + lltype2vtable = translator.rtyper.lltype2vtable + layoutbuilder = framework.TransformerLayoutBuilder(GCClass, + lltype2vtable) layoutbuilder.delay_encoding() translator._jit2gc = { 'layoutbuilder': layoutbuilder, @@ -639,7 +647,7 @@ graph = graphof(translator, g) for op in graph.startblock.operations: if op.opname == 'do_malloc_fixedsize_clear': - op.args = [Constant(type_id, lltype.Signed), + op.args = [Constant(type_id, rffi.USHORT), Constant(llmemory.sizeof(P), lltype.Signed), Constant(True, lltype.Bool), # can_collect Constant(False, lltype.Bool), # has_finalizer @@ -934,7 +942,6 @@ assert res == 20 + 20 def test_nongc_static_root_minor_collect(self): - from pypy.rpython.lltypesystem import lltype T1 = lltype.GcStruct("C", ('x', lltype.Signed)) T2 = lltype.Struct("C", ('p', lltype.Ptr(T1))) static = lltype.malloc(T2, immortal=True) @@ -957,7 +964,6 @@ def test_static_root_minor_collect(self): - from pypy.rpython.lltypesystem import lltype class A: pass class B: Modified: pypy/trunk/pypy/rpython/rtyper.py ============================================================================== --- pypy/trunk/pypy/rpython/rtyper.py (original) +++ pypy/trunk/pypy/rpython/rtyper.py Sun Oct 11 16:32:27 2009 @@ -65,6 +65,7 @@ self.class_pbc_attributes = {} self.oo_meth_impls = {} self.cache_dummy_values = {} + self.lltype2vtable = {} self.typererrors = [] self.typererror_count = 0 # make the primitive_to_repr constant mapping @@ -131,12 +132,6 @@ result[repr.lowleveltype] = classdef return result - def lltype_to_vtable_mapping(self): - result = {} - for repr in self.instance_reprs.itervalues(): - result[repr.lowleveltype.TO] = repr.rclass.getvtable() - return result - def get_type_for_typeptr(self, typeptr): try: return self.type_for_typeptr[typeptr._obj] Modified: pypy/trunk/pypy/translator/backendopt/inline.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/inline.py (original) +++ pypy/trunk/pypy/translator/backendopt/inline.py Sun Oct 11 16:32:27 2009 @@ -753,7 +753,7 @@ subcount = inline_function(translator, graph, parentgraph, lltype_to_classdef, raise_analyzer, call_count_pred, cleanup=False) - to_cleanup[graph] = True + to_cleanup[parentgraph] = True res = bool(subcount) except CannotInline: try_again[graph] = True Modified: pypy/trunk/pypy/translator/c/database.py ============================================================================== --- pypy/trunk/pypy/translator/c/database.py (original) +++ pypy/trunk/pypy/translator/c/database.py Sun Oct 11 16:32:27 2009 @@ -5,6 +5,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem.llmemory import WeakRef, _WeakRefType, GCREF from pypy.rpython.lltypesystem.rffi import CConstant +from pypy.rpython.lltypesystem import llgroup from pypy.tool.sourcetools import valid_identifier from pypy.translator.c.primitive import PrimitiveName, PrimitiveType from pypy.translator.c.node import StructDefNode, ArrayDefNode @@ -141,6 +142,8 @@ #raise Exception("don't know about opaque type %r" % (T,)) return 'struct %s @' % ( valid_identifier('pypy_opaque_' + T.tag),) + elif isinstance(T, llgroup.GroupType): + return "/*don't use me*/ void @" else: raise Exception("don't know about type %r" % (T,)) @@ -285,6 +288,8 @@ finish_callbacks.append(('Stackless transformer: finished', self.stacklesstransformer.finish)) if self.gctransformer: + finish_callbacks.append(('GC transformer: tracking vtables', + self.gctransformer.get_final_dependencies)) finish_callbacks.append(('GC transformer: finished tables', self.gctransformer.finish_tables)) Modified: pypy/trunk/pypy/translator/c/funcgen.py ============================================================================== --- pypy/trunk/pypy/translator/c/funcgen.py (original) +++ pypy/trunk/pypy/translator/c/funcgen.py Sun Oct 11 16:32:27 2009 @@ -781,5 +781,22 @@ def OP_PROMOTE_VIRTUALIZABLE(self, op): return '/* PROMOTE_VIRTUALIZABLE %s */' % op + def OP_GET_GROUP_MEMBER(self, op): + typename = self.db.gettype(op.result.concretetype) + return '%s = (%s)_OP_GET_GROUP_MEMBER(%s, %s);' % ( + self.expr(op.result), + cdecl(typename, ''), + self.expr(op.args[0]), + self.expr(op.args[1])) + + def OP_GET_NEXT_GROUP_MEMBER(self, op): + typename = self.db.gettype(op.result.concretetype) + return '%s = (%s)_OP_GET_NEXT_GROUP_MEMBER(%s, %s, %s);' % ( + self.expr(op.result), + cdecl(typename, ''), + self.expr(op.args[0]), + self.expr(op.args[1]), + self.expr(op.args[2])) + assert not USESLOTS or '__dict__' not in dir(FunctionCodeGenerator) Modified: pypy/trunk/pypy/translator/c/gc.py ============================================================================== --- pypy/trunk/pypy/translator/c/gc.py (original) +++ pypy/trunk/pypy/translator/c/gc.py Sun Oct 11 16:32:27 2009 @@ -1,4 +1,5 @@ import sys +from pypy.objspace.flow.model import Constant from pypy.translator.c.support import cdecl from pypy.translator.c.node import ContainerNode from pypy.rpython.lltypesystem.lltype import \ @@ -11,7 +12,7 @@ class BasicGcPolicy(object): requires_stackless = False - + def __init__(self, db, thread_enabled=False): self.db = db self.thread_enabled = thread_enabled @@ -49,6 +50,9 @@ post_include_bits=['typedef void *GC_hidden_pointer;'] ) + def need_no_typeptr(self): + return False + def gc_startup_code(self): return [] @@ -312,8 +316,11 @@ return framework.convert_weakref_to(ptarget) def OP_GC_RELOAD_POSSIBLY_MOVED(self, funcgen, op): - args = [funcgen.expr(v) for v in op.args] - return '%s = %s; /* for moving GCs */' % (args[1], args[0]) + if isinstance(op.args[1], Constant): + return '/* %s */' % (op,) + else: + args = [funcgen.expr(v) for v in op.args] + return '%s = %s; /* for moving GCs */' % (args[1], args[0]) def common_gcheader_definition(self, defnode): return defnode.db.gctransformer.gc_fields() @@ -322,6 +329,25 @@ o = top_container(defnode.obj) return defnode.db.gctransformer.gc_field_values_for(o) + def need_no_typeptr(self): + config = self.db.translator.config + return config.translation.gcconfig.removetypeptr + + def OP_GC_GETTYPEPTR_GROUP(self, funcgen, op): + # expands to a number of steps, as per rpython/lltypesystem/opimpl.py, + # all implemented by a single call to a C macro. + [v_obj, c_grpptr, c_skipoffset, c_vtableinfo] = op.args + typename = funcgen.db.gettype(op.result.concretetype) + fieldname = c_vtableinfo.value[2] + return ( + '%s = (%s)_OP_GET_NEXT_GROUP_MEMBER(%s, (unsigned short)%s->_%s, %s);' + % (funcgen.expr(op.result), + cdecl(typename, ''), + funcgen.expr(c_grpptr), + funcgen.expr(v_obj), + fieldname, + funcgen.expr(c_skipoffset))) + class AsmGcRootFrameworkGcPolicy(FrameworkGcPolicy): transformerclass = asmgcroot.AsmGcRootFrameworkGCTransformer Modified: pypy/trunk/pypy/translator/c/node.py ============================================================================== --- pypy/trunk/pypy/translator/c/node.py (original) +++ pypy/trunk/pypy/translator/c/node.py Sun Oct 11 16:32:27 2009 @@ -3,7 +3,7 @@ GcStruct, GcArray, RttiStruct, ContainerType, \ parentlink, Ptr, PyObject, Void, OpaqueType, Float, \ RuntimeTypeInfo, getRuntimeTypeInfo, Char, _subarray -from pypy.rpython.lltypesystem import llmemory +from pypy.rpython.lltypesystem import llmemory, llgroup from pypy.translator.c.funcgen import FunctionCodeGenerator from pypy.translator.c.external import CExternalFunctionCodeGenerator from pypy.translator.c.support import USESLOTS # set to False if necessary while refactoring @@ -67,6 +67,12 @@ bare=True) self.prefix = somelettersfrom(STRUCT._name) + '_' self.dependencies = {} + # + self.fieldnames = STRUCT._names + if STRUCT._hints.get('typeptr', False): + if db.gcpolicy.need_no_typeptr(): + assert self.fieldnames == ('typeptr',) + self.fieldnames = () def setup(self): # this computes self.fields @@ -80,7 +86,7 @@ if needs_gcheader(self.STRUCT): for fname, T in db.gcpolicy.struct_gcheader_definition(self): self.fields.append((fname, db.gettype(T, who_asks=self))) - for name in STRUCT._names: + for name in self.fieldnames: T = self.c_struct_field_type(name) if name == STRUCT._arrayfld: typename = db.gettype(T, varlength=self.varlength, @@ -147,8 +153,7 @@ yield line def visitor_lines(self, prefix, on_field): - STRUCT = self.STRUCT - for name in STRUCT._names: + for name in self.fieldnames: FIELD_T = self.c_struct_field_type(name) cname = self.c_struct_field_name(name) for line in on_field('%s.%s' % (prefix, cname), @@ -157,8 +162,7 @@ def debug_offsets(self): # generate number exprs giving the offset of the elements in the struct - STRUCT = self.STRUCT - for name in STRUCT._names: + for name in self.fieldnames: FIELD_T = self.c_struct_field_type(name) if FIELD_T is Void: yield '-1' @@ -464,11 +468,15 @@ return hasattr(self.T, "_hints") and self.T._hints.get('thread_local') def forward_declaration(self): + if llgroup.member_of_group(self.obj): + return yield '%s;' % ( forward_cdecl(self.implementationtypename, self.name, self.db.standalone, self.is_thread_local())) def implementation(self): + if llgroup.member_of_group(self.obj): + return [] lines = list(self.initializationexpr()) lines[0] = '%s = %s' % ( cdecl(self.implementationtypename, self.name, self.is_thread_local()), @@ -514,7 +522,7 @@ for i, thing in enumerate(self.db.gcpolicy.struct_gcheader_initdata(self)): data.append(('gcheader%d'%i, thing)) - for name in self.T._names: + for name in defnode.fieldnames: data.append((name, getattr(self.obj, name))) # Reasonably, you should only initialise one of the fields of a union @@ -898,6 +906,67 @@ #obj._converted_weakref = container # hack for genllvm :-/ return db.getcontainernode(container, _dont_write_c_code=False) +class GroupNode(ContainerNode): + nodekind = 'group' + count_members = None + + def __init__(self, *args): + ContainerNode.__init__(self, *args) + self.implementationtypename = 'struct group_%s_s @' % self.name + + def basename(self): + return self.obj.name + + def enum_dependencies(self): + # note: for the group used by the GC, it can grow during this phase, + # which means that we might not return all members yet. This is + # fixed by finish_tables() in rpython/memory/gctransform/framework.py + for member in self.obj.members: + yield member._as_ptr() + + def _fix_members(self): + if self.obj.outdated: + raise Exception(self.obj.outdated) + if self.count_members is None: + self.count_members = len(self.obj.members) + else: + # make sure no new member showed up, because it's too late + assert len(self.obj.members) == self.count_members + + def forward_declaration(self): + self._fix_members() + yield '' + ctype = ['%s {' % cdecl(self.implementationtypename, '')] + for i, member in enumerate(self.obj.members): + structtypename = self.db.gettype(typeOf(member)) + ctype.append('\t%s;' % cdecl(structtypename, 'member%d' % i)) + ctype.append('} @') + ctype = '\n'.join(ctype) + yield '%s;' % ( + forward_cdecl(ctype, self.name, self.db.standalone, + self.is_thread_local())) + yield '#include "src/llgroup.h"' + yield 'PYPY_GROUP_CHECK_SIZE(%s);' % self.name + for i, member in enumerate(self.obj.members): + structnode = self.db.getcontainernode(member) + yield '#define %s %s.member%d' % (structnode.name, + self.name, i) + yield '' + + def initializationexpr(self): + self._fix_members() + lines = ['{'] + lasti = len(self.obj.members) - 1 + for i, member in enumerate(self.obj.members): + structnode = self.db.getcontainernode(member) + lines1 = list(structnode.initializationexpr()) + lines1[0] += '\t/* member%d: %s */' % (i, structnode.name) + if i != lasti: + lines1[-1] += ',' + lines.extend(lines1) + lines.append('}') + return lines + ContainerNodeFactory = { Struct: StructNode, @@ -909,4 +978,5 @@ OpaqueType: opaquenode_factory, PyObjectType: PyObjectNode, llmemory._WeakRefType: weakrefnode_factory, + llgroup.GroupType: GroupNode, } Modified: pypy/trunk/pypy/translator/c/primitive.py ============================================================================== --- pypy/trunk/pypy/translator/c/primitive.py (original) +++ pypy/trunk/pypy/translator/c/primitive.py Sun Oct 11 16:32:27 2009 @@ -3,7 +3,7 @@ from pypy.rlib.objectmodel import CDefinedIntSymbolic from pypy.rlib.rarithmetic import r_longlong, isinf, isnan from pypy.rpython.lltypesystem.lltype import * -from pypy.rpython.lltypesystem import rffi +from pypy.rpython.lltypesystem import rffi, llgroup from pypy.rpython.lltypesystem.llmemory import Address, \ AddressOffset, ItemOffset, ArrayItemsOffset, FieldOffset, \ CompositeOffset, ArrayLengthOffset, \ @@ -50,12 +50,15 @@ elif type(value) == GCHeaderOffset: return '0' elif type(value) == RoundedUpForAllocation: - return 'ROUND_UP_FOR_ALLOCATION(%s)' % ( - name_signed(value.basesize, db)) + return 'ROUND_UP_FOR_ALLOCATION(%s, %s)' % ( + name_signed(value.basesize, db), + name_signed(value.minsize, db)) elif isinstance(value, CDefinedIntSymbolic): return str(value.expr) elif isinstance(value, ComputedIntSymbolic): value = value.compute_fn() + elif isinstance(value, llgroup.CombinedSymbolic): + return '(%s|%dL)' % (name_ushort(value.lowpart, db), value.rest) else: raise Exception("unimplemented symbolic %r"%value) if value is None: @@ -136,6 +139,19 @@ else: return 'NULL' +def name_ushort(value, db): + if isinstance(value, Symbolic): + if isinstance(value, llgroup.GroupMemberOffset): + groupnode = db.getcontainernode(value.grpptr._as_obj()) + structnode = db.getcontainernode(value.member._as_obj()) + return 'GROUP_MEMBER_OFFSET(%s, %s)' % ( + groupnode.name, + structnode.name, + ) + else: + raise Exception("unimplemented symbolic %r" % value) + return str(value) + # On 64 bit machines, SignedLongLong and Signed are the same, so the # order matters, because we want the Signed implementation. PrimitiveName = { @@ -151,6 +167,7 @@ Void: name_void, Address: name_address, GCREF: name_gcref, + rffi.USHORT: name_ushort, } PrimitiveType = { @@ -166,6 +183,7 @@ Void: 'void @', Address: 'void* @', GCREF: 'void* @', + rffi.USHORT: 'unsigned short @', } def define_c_primitive(ll_type, c_name): @@ -181,7 +199,7 @@ for ll_type, c_name in [(rffi.SIGNEDCHAR, 'signed char'), (rffi.UCHAR, 'unsigned char'), (rffi.SHORT, 'short'), - (rffi.USHORT, 'unsigned short'), + #(rffi.USHORT, 'unsigned short'), (rffi.INT, 'int'), (rffi.UINT, 'unsigned int'), (rffi.LONG, 'long'), Modified: pypy/trunk/pypy/translator/c/src/g_include.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/g_include.h (original) +++ pypy/trunk/pypy/translator/c/src/g_include.h Sun Oct 11 16:32:27 2009 @@ -34,6 +34,7 @@ #ifndef AVR #include "src/unichar.h" #endif +#include "src/llgroup.h" #include "src/instrument.h" Modified: pypy/trunk/pypy/translator/c/src/mem.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/mem.h (original) +++ pypy/trunk/pypy/translator/c/src/mem.h Sun Oct 11 16:32:27 2009 @@ -14,8 +14,9 @@ struct rpy_memory_alignment_test1 s; }; #define MEMORY_ALIGNMENT offsetof(struct rpy_memory_alignment_test2, s) -#define ROUND_UP_FOR_ALLOCATION(x) \ - (((x) + (MEMORY_ALIGNMENT-1)) & ~(MEMORY_ALIGNMENT-1)) +#define ROUND_UP_FOR_ALLOCATION(x, minsize) \ + ((((x)>=(minsize)?(x):(minsize)) \ + + (MEMORY_ALIGNMENT-1)) & ~(MEMORY_ALIGNMENT-1)) extern char __gcmapstart; extern char __gcmapend; Modified: pypy/trunk/pypy/translator/c/test/test_lltyped.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_lltyped.py (original) +++ pypy/trunk/pypy/translator/c/test/test_lltyped.py Sun Oct 11 16:32:27 2009 @@ -701,5 +701,74 @@ fn = self.getcompiled(llf) fn() + def test_llgroup(self): + from pypy.rpython.lltypesystem.test import test_llgroup + f = test_llgroup.build_test() + fn = self.getcompiled(f) + res = fn() + assert res == 42 - + def test_llgroup_size_limit(self): + yield self._test_size_limit, True + yield self._test_size_limit, False + + def _test_size_limit(self, toobig): + from pypy.rpython.lltypesystem import llgroup + from pypy.rpython.lltypesystem.lloperation import llop + from pypy.translator.platform import CompilationError + grp = llgroup.group("big") + S1 = Struct('S1', ('x', Signed), ('y', Signed), + ('z', Signed), ('u', Signed), + ('x2', Signed), ('y2', Signed), + ('z2', Signed), ('u2', Signed), + ('x3', Signed), ('y3', Signed), + ('z3', Signed), ('u3', Signed), + ('x4', Signed), ('y4', Signed), + ('z4', Signed), ('u4', Signed)) + goffsets = [] + for i in range(4096 + toobig): + goffsets.append(grp.add_member(malloc(S1, immortal=True))) + grpptr = grp._as_ptr() + def f(n): + p = llop.get_group_member(Ptr(S1), grpptr, goffsets[n]) + q = llop.get_group_member(Ptr(S1), grpptr, goffsets[0]) + p.x = 5 + q.x = 666 + return p.x + if toobig: + py.test.raises(CompilationError, self.getcompiled, f, [int]) + else: + fn = self.getcompiled(f, [int]) + res = fn(-1) + assert res == 5 + + def test_round_up_for_allocation(self): + from pypy.rpython.lltypesystem import llmemory, llarena + S = Struct('S', ('x', Char), ('y', Char)) + M = Struct('M', ('x', Char), ('y', Signed)) + # + def g(): + ssize = llarena.round_up_for_allocation(llmemory.sizeof(S)) + msize = llarena.round_up_for_allocation(llmemory.sizeof(M)) + smsize = llarena.round_up_for_allocation(llmemory.sizeof(S), + llmemory.sizeof(M)) + mssize = llarena.round_up_for_allocation(llmemory.sizeof(M), + llmemory.sizeof(S)) + return ssize, msize, smsize, mssize + # + glob_sizes = g() + # + def check((ssize, msize, smsize, mssize)): + assert ssize == llmemory.sizeof(Signed) + assert msize == llmemory.sizeof(Signed) * 2 + assert smsize == msize + assert mssize == msize + # + def f(): + check(glob_sizes) + check(g()) + return 42 + # + fn = self.getcompiled(f, []) + res = fn() + assert res == 42 Modified: pypy/trunk/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_newgc.py (original) +++ pypy/trunk/pypy/translator/c/test/test_newgc.py Sun Oct 11 16:32:27 2009 @@ -16,6 +16,7 @@ class TestUsingFramework(object): gcpolicy = "marksweep" should_be_moving = False + removetypeptr = False GC_CAN_MOVE = False GC_CANNOT_MALLOC_NONMOVABLE = False @@ -25,6 +26,7 @@ def _makefunc2(cls, f): t = Translation(f, [int, int], gc=cls.gcpolicy, policy=annpolicy.StrictAnnotatorPolicy()) + t.config.translation.gcconfig.removetypeptr = cls.removetypeptr t.disable(['backendopt']) t.set_backend_extra_options(c_isolated=True, c_debug_defines=True) t.rtype() @@ -796,6 +798,9 @@ def test_gc_set_max_heap_size(self): py.test.skip("not implemented") +class TestHybridGCRemoveTypePtr(TestHybridGC): + removetypeptr = True + class TestMarkCompactGC(TestSemiSpaceGC): gcpolicy = "markcompact" should_be_moving = True From arigo at codespeak.net Sun Oct 11 16:34:29 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 11 Oct 2009 16:34:29 +0200 (CEST) Subject: [pypy-svn] r68315 - pypy/branch/gc-compress Message-ID: <20091011143429.4655D16805B@codespeak.net> Author: arigo Date: Sun Oct 11 16:34:29 2009 New Revision: 68315 Removed: pypy/branch/gc-compress/ Log: Remove merged branch. From arigo at codespeak.net Sun Oct 11 16:41:44 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 11 Oct 2009 16:41:44 +0200 (CEST) Subject: [pypy-svn] r68316 - pypy/branch/gc_hash Message-ID: <20091011144144.A688B16805B@codespeak.net> Author: arigo Date: Sun Oct 11 16:41:44 2009 New Revision: 68316 Removed: pypy/branch/gc_hash/ Log: Remove this old branch. From arigo at codespeak.net Sun Oct 11 16:45:30 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 11 Oct 2009 16:45:30 +0200 (CEST) Subject: [pypy-svn] r68317 - in pypy/trunk/pypy/rpython: lltypesystem memory/gctransform Message-ID: <20091011144530.6F57216805B@codespeak.net> Author: arigo Date: Sun Oct 11 16:45:29 2009 New Revision: 68317 Modified: pypy/trunk/pypy/rpython/lltypesystem/llgroup.py pypy/trunk/pypy/rpython/memory/gctransform/framework.py Log: Fixes for jit/backend/llsupport/test/test_gc. Modified: pypy/trunk/pypy/rpython/lltypesystem/llgroup.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/llgroup.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/llgroup.py Sun Oct 11 16:45:29 2009 @@ -93,6 +93,9 @@ self.lowpart = lowpart self.rest = rest + def __repr__(self): + return '' % (self.lowpart, self.rest) + def __and__(self, other): if (other & 0xFFFF) == 0: return self.rest & other @@ -111,3 +114,17 @@ def __sub__(self, other): assert (other & 0xFFFF) == 0 return CombinedSymbolic(self.lowpart, self.rest - other) + + def __eq__(self, other): + if (isinstance(other, CombinedSymbolic) and + self.lowpart is other.lowpart): + return self.rest == other.rest + else: + return NotImplemented + + def __ne__(self, other): + if (isinstance(other, CombinedSymbolic) and + self.lowpart is other.lowpart): + return self.rest != other.rest + else: + return NotImplemented Modified: pypy/trunk/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/framework.py Sun Oct 11 16:45:29 2009 @@ -919,7 +919,10 @@ # for the JIT: currently does not support removetypeptr def __init__(self, config): from pypy.rpython.memory.gc.base import choose_gc_from_config - assert not config.translation.gcconfig.removetypeptr + try: + assert not config.translation.gcconfig.removetypeptr + except AttributeError: # for some tests + pass GCClass, _ = choose_gc_from_config(config) TransformerLayoutBuilder.__init__(self, GCClass, {}) From arigo at codespeak.net Sun Oct 11 19:15:40 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 11 Oct 2009 19:15:40 +0200 (CEST) Subject: [pypy-svn] r68318 - in pypy/trunk/pypy/jit/backend: test x86/test Message-ID: <20091011171540.7FF69168062@codespeak.net> Author: arigo Date: Sun Oct 11 19:15:39 2009 New Revision: 68318 Modified: pypy/trunk/pypy/jit/backend/test/support.py pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_loop.py pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_recursive.py pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_slist.py pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_virtualizable.py Log: Mark some of the test_zrpy of the x86 backend as 'slow tests', to run only with the --slow option. It seems that some of them take so long than even running a single file takes more than 4000 seconds on wyvern, timing out. Modified: pypy/trunk/pypy/jit/backend/test/support.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/support.py (original) +++ pypy/trunk/pypy/jit/backend/test/support.py Sun Oct 11 19:15:39 2009 @@ -92,6 +92,13 @@ class CCompiledMixin(BaseCompiledMixin): type_system = 'lltype' + slow = False + + def setup_class(cls): + if cls.slow: + from pypy.jit.conftest import option + if not option.run_slow_tests: + py.test.skip("use --slow to execute this long-running test") def _get_TranslationContext(self): t = TranslationContext() Modified: pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_loop.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_loop.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_loop.py Sun Oct 11 19:15:39 2009 @@ -5,6 +5,7 @@ class TestLoop(Jit386Mixin, LoopTest): # for the individual tests see # ====> ../../../metainterp/test/test_loop.py + slow = True def test_interp_many_paths(self): py.test.skip('not supported: pointer as argument') Modified: pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_recursive.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_recursive.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_recursive.py Sun Oct 11 19:15:39 2009 @@ -4,6 +4,7 @@ from pypy.jit.backend.x86.test.test_zrpy_slist import Jit386Mixin class TestRecursive(Jit386Mixin, RecursiveTests): + slow = True def test_inline_faulty_can_inline(self): py.test.skip("this test is not supposed to be translated") Modified: pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_slist.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_slist.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_slist.py Sun Oct 11 19:15:39 2009 @@ -10,5 +10,4 @@ class TestSList(Jit386Mixin, ListTests): # for the individual tests see # ====> ../../../test/test_slist.py - pass - + slow = True Modified: pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_virtualizable.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_virtualizable.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_virtualizable.py Sun Oct 11 19:15:39 2009 @@ -6,4 +6,4 @@ class TestLLImplicitVirtualizable(Jit386Mixin, test_virtualizable.ImplicitVirtualizableTests): - pass + slow = True From arigo at codespeak.net Sun Oct 11 19:45:03 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 11 Oct 2009 19:45:03 +0200 (CEST) Subject: [pypy-svn] r68319 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091011174503.89D47168062@codespeak.net> Author: arigo Date: Sun Oct 11 19:45:02 2009 New Revision: 68319 Modified: pypy/trunk/pypy/jit/metainterp/executor.py Log: Obscure workaround. Modified: pypy/trunk/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/executor.py (original) +++ pypy/trunk/pypy/jit/metainterp/executor.py Sun Oct 11 19:45:02 2009 @@ -235,7 +235,9 @@ return ConstInt(box1.getfloat() >= box2.getfloat()) def do_cast_float_to_int(cpu, box1): - return ConstInt(int(box1.getfloat())) + # note: we need to call int() twice to care for the fact that + # int(-2147483648.0) returns a long :-( + return ConstInt(int(int(box1.getfloat()))) def do_cast_int_to_float(cpu, box1): return ConstFloat(float(box1.getint())) From arigo at codespeak.net Sun Oct 11 20:05:59 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 11 Oct 2009 20:05:59 +0200 (CEST) Subject: [pypy-svn] r68320 - pypy/branch/gc-hash Message-ID: <20091011180559.D7FEE168062@codespeak.net> Author: arigo Date: Sun Oct 11 20:05:59 2009 New Revision: 68320 Added: pypy/branch/gc-hash/ - copied from r68319, pypy/trunk/ Log: A branch, reviving the old 'gc_hash' branch, in which to try to give all objects a gc-supported hash. From arigo at codespeak.net Sun Oct 11 20:34:36 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 11 Oct 2009 20:34:36 +0200 (CEST) Subject: [pypy-svn] r68322 - in pypy/branch/gc-hash/pypy/rpython/lltypesystem: . test Message-ID: <20091011183436.9F58116805C@codespeak.net> Author: arigo Date: Sun Oct 11 20:34:35 2009 New Revision: 68322 Modified: pypy/branch/gc-hash/pypy/rpython/lltypesystem/lltype.py pypy/branch/gc-hash/pypy/rpython/lltypesystem/test/test_lltype.py Log: hash_cache support in lltype for GcStructs, including the possibility to ask for prebuilt gcstructs to have a specific hash value. Repeats r51644 but gives hash support to all GcStructs, not just some of them. Modified: pypy/branch/gc-hash/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/gc-hash/pypy/rpython/lltypesystem/lltype.py Sun Oct 11 20:34:35 2009 @@ -1390,7 +1390,7 @@ class _struct(_parentable): _kind = "structure" - __slots__ = () + __slots__ = ('_hash_cache_',) def __new__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): my_variety = _struct_variety(TYPE._names) @@ -1840,6 +1840,26 @@ "should have been: %s" % (p, result2, result)) return result +def hash_gc_object(p): + """Returns the lltype-level hash of the given GcStruct.""" + p = normalizeptr(p) + if not p: + return 0 # hash(NULL) + try: + return p._obj._hash_cache_ + except AttributeError: + result = p._obj._hash_cache_ = intmask(id(p._obj)) + return result + +def init_hash_gc_object(p, value): + """For a prebuilt object p, initialize its hash value to 'value'.""" + p = normalizeptr(p) + if not p: + raise ValueError("cannot change hash(NULL)!") + if hasattr(p._obj, '_hash_cache_'): + raise ValueError("the hash of %r was already computed" % (p,)) + p._obj._hash_cache_ = intmask(value) + def isCompatibleType(TYPE1, TYPE2): return TYPE1._is_compatible(TYPE2) Modified: pypy/branch/gc-hash/pypy/rpython/lltypesystem/test/test_lltype.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/lltypesystem/test/test_lltype.py (original) +++ pypy/branch/gc-hash/pypy/rpython/lltypesystem/test/test_lltype.py Sun Oct 11 20:34:35 2009 @@ -739,3 +739,30 @@ del ptr import gc; gc.collect(); gc.collect() ptr2[0] = 5 # crashes if the array was deallocated + +def test_hash_gc_object(): + S = GcStruct('S', ('x', Signed)) + S2 = GcStruct('S2', ('super', S)) + S3 = GcStruct('S3', ('super', S2)) + + s3 = malloc(S3) + hash3 = hash_gc_object(s3.super) + assert hash3 == hash_gc_object(s3) + assert hash3 == hash_gc_object(s3.super) + assert hash3 == hash_gc_object(s3.super.super) + py.test.raises(ValueError, init_hash_gc_object, s3, hash3^1) + py.test.raises(ValueError, init_hash_gc_object, s3.super, hash3^4) + py.test.raises(ValueError, init_hash_gc_object, s3.super.super, hash3^9) + + s3 = malloc(S3) + init_hash_gc_object(s3.super, -123) + assert -123 == hash_gc_object(s3) + assert -123 == hash_gc_object(s3.super) + assert -123 == hash_gc_object(s3.super.super) + py.test.raises(ValueError, init_hash_gc_object, s3, 4313) + py.test.raises(ValueError, init_hash_gc_object, s3.super, 0) + py.test.raises(ValueError, init_hash_gc_object, s3.super.super, -124) + + from pypy.rpython.lltypesystem import llmemory + p3 = cast_opaque_ptr(llmemory.GCREF, s3) + assert -123 == hash_gc_object(p3) From arigo at codespeak.net Sun Oct 11 20:50:37 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 11 Oct 2009 20:50:37 +0200 (CEST) Subject: [pypy-svn] r68323 - pypy/branch/gc-hash/pypy/jit/metainterp/test Message-ID: <20091011185037.1459816805C@codespeak.net> Author: arigo Date: Sun Oct 11 20:50:36 2009 New Revision: 68323 Modified: pypy/branch/gc-hash/pypy/jit/metainterp/test/test_basic.py Log: Rewrite this test in a saner way: ooidentityhash() cannot be directly called on an oostring, but it worked anyway for obscure reasons. Modified: pypy/branch/gc-hash/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/gc-hash/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/gc-hash/pypy/jit/metainterp/test/test_basic.py Sun Oct 11 20:50:36 2009 @@ -957,10 +957,12 @@ assert res == ootype.oohash(ootype.oostring(5, -1)) def test_ooidentityhash(self): + A = ootype.Instance("A", ootype.ROOT) def f(): - s1 = ootype.oostring(5, -1) - s2 = ootype.oostring(6, -1) - return ootype.ooidentityhash(s1) == ootype.ooidentityhash(s2) + obj1 = ootype.new(A) + obj2 = ootype.new(A) + return ootype.ooidentityhash(obj1) == ootype.ooidentityhash(obj2) + assert not f() res = self.interp_operations(f, []) assert not res From arigo at codespeak.net Sun Oct 11 22:13:54 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 11 Oct 2009 22:13:54 +0200 (CEST) Subject: [pypy-svn] r68324 - in pypy/branch/gc-hash/pypy: config objspace/std objspace/std/test Message-ID: <20091011201354.6FFFD16804F@codespeak.net> Author: arigo Date: Sun Oct 11 22:13:52 2009 New Revision: 68324 Modified: pypy/branch/gc-hash/pypy/config/pypyoption.py pypy/branch/gc-hash/pypy/objspace/std/dictmultiobject.py pypy/branch/gc-hash/pypy/objspace/std/test/test_dictmultiobject.py Log: Kill SmallDictImplementation and SmallStrDictImplementation. They are not really small, that's just a lie (so far). They have the issue of asking the hash() of an object and storing it around. We don't want to support that right now. Modified: pypy/branch/gc-hash/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/gc-hash/pypy/config/pypyoption.py (original) +++ pypy/branch/gc-hash/pypy/config/pypyoption.py Sun Oct 11 22:13:52 2009 @@ -253,11 +253,6 @@ default=False, requires=[("objspace.std.withmultidict", True)]), - BoolOption("withsmalldicts", - "handle small dictionaries differently", - default=False, - requires=[("objspace.std.withmultidict", True)]), - BoolOption("withrangelist", "enable special range list implementation that does not " "actually create the full list until the resulting " Modified: pypy/branch/gc-hash/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/branch/gc-hash/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/branch/gc-hash/pypy/objspace/std/dictmultiobject.py Sun Oct 11 22:13:52 2009 @@ -29,7 +29,6 @@ # # EmptyDictImplementation # / \ -# SmallStrDictImplementation SmallDictImplementation # | | # StrDictImplementation | # \ / @@ -163,15 +162,9 @@ def setitem(self, w_key, w_value): space = self.space if _is_str(space, w_key): - if space.config.objspace.std.withsmalldicts: - return SmallStrDictImplementation(space, w_key, w_value) - else: - return StrDictImplementation(space).setitem_str(w_key, w_value) + return StrDictImplementation(space).setitem_str(w_key, w_value) else: - if space.config.objspace.std.withsmalldicts: - return SmallDictImplementation(space, w_key, w_value) - else: - return space.DefaultDictImpl(space).setitem(w_key, w_value) + return space.DefaultDictImpl(space).setitem(w_key, w_value) def setitem_str(self, w_key, w_value, shadows_type=True): return StrDictImplementation(self.space).setitem_str(w_key, w_value) #return SmallStrDictImplementation(self.space, w_key, w_value) @@ -201,228 +194,10 @@ def items(self): return [] - class EmptyIteratorImplementation(IteratorImplementation): def next_entry(self): return None -class Entry(object): - def __init__(self): - self.hash = 0 - self.w_key = None - self.w_value = None - def __repr__(self): - return '<%r, %r, %r>'%(self.hash, self.w_key, self.w_value) - -class SmallDictImplementation(DictImplementation): - # XXX document the invariants here! - - def __init__(self, space, w_key, w_value): - self.space = space - self.entries = [Entry(), Entry(), Entry(), Entry(), Entry()] - self.entries[0].hash = space.hash_w(w_key) - self.entries[0].w_key = w_key - self.entries[0].w_value = w_value - self.valid = 1 - - def _lookup(self, w_key): - hash = self.space.hash_w(w_key) - i = 0 - last = self.entries[self.valid] - last.hash = hash - last.w_key = w_key - while 1: - look_entry = self.entries[i] - if look_entry.hash == hash and self.space.eq_w(look_entry.w_key, w_key): - return look_entry - i += 1 - - def _convert_to_rdict(self): - newimpl = self.space.DefaultDictImpl(self.space) - i = 0 - while 1: - entry = self.entries[i] - if entry.w_value is None: - break - newimpl.setitem(entry.w_key, entry.w_value) - i += 1 - return newimpl - - def setitem(self, w_key, w_value): - entry = self._lookup(w_key) - if entry.w_value is None: - if self.valid == 4: - return self._convert_to_rdict().setitem(w_key, w_value) - self.valid += 1 - entry.w_value = w_value - return self - - def setitem_str(self, w_key, w_value, shadows_type=True): - return self.setitem(w_key, w_value) - - def delitem(self, w_key): - entry = self._lookup(w_key) - if entry.w_value is not None: - for i in range(self.entries.index(entry), self.valid): - self.entries[i] = self.entries[i+1] - self.entries[self.valid] = entry - entry.w_value = None - self.valid -= 1 - if self.valid == 0: - return self.space.emptydictimpl - return self - else: - entry.w_key = None - raise KeyError - - def length(self): - return self.valid - def get(self, w_lookup): - entry = self._lookup(w_lookup) - val = entry.w_value - if val is None: - entry.w_key = None - return val - - def iteritems(self): - return self._convert_to_rdict().iteritems() - def iterkeys(self): - return self._convert_to_rdict().iterkeys() - def itervalues(self): - return self._convert_to_rdict().itervalues() - - def keys(self): - return [self.entries[i].w_key for i in range(self.valid)] - def values(self): - return [self.entries[i].w_value for i in range(self.valid)] - def items(self): - return [self.space.newtuple([e.w_key, e.w_value]) - for e in [self.entries[i] for i in range(self.valid)]] - - -class StrEntry(object): - def __init__(self): - self.key = None - self.w_value = None - def __repr__(self): - return '<%r, %r, %r>'%(self.hash, self.key, self.w_value) - -class SmallStrDictImplementation(DictImplementation): - # XXX document the invariants here! - - def __init__(self, space, w_key, w_value): - self.space = space - self.entries = [StrEntry(), StrEntry(), StrEntry(), StrEntry(), StrEntry()] - key = space.str_w(w_key) - self.entries[0].key = key - self.entries[0].w_value = w_value - self.valid = 1 - - def _lookup(self, key): - assert isinstance(key, str) - _hash = hash(key) - i = 0 - last = self.entries[self.valid] - last.key = key - while 1: - look_entry = self.entries[i] - if hash(look_entry.key) == _hash and look_entry.key == key: - return look_entry - i += 1 - - def _convert_to_rdict(self): - newimpl = self.space.DefaultDictImpl(self.space) - i = 0 - while 1: - entry = self.entries[i] - if entry.w_value is None: - break - newimpl.setitem(self.space.wrap(entry.key), entry.w_value) - i += 1 - return newimpl - - def _convert_to_sdict(self, w_value): - # this relies on the fact that the new key is in the entries - # list already. - newimpl = StrDictImplementation(self.space) - i = 0 - while 1: - entry = self.entries[i] - if entry.w_value is None: - newimpl.content[entry.key] = w_value - break - newimpl.content[entry.key] = entry.w_value - i += 1 - return newimpl - - def setitem(self, w_key, w_value): - if not _is_str(self.space, w_key): - return self._convert_to_rdict().setitem(w_key, w_value) - return self.setitem_str(w_key, w_value) - - def setitem_str(self, w_key, w_value, shadows_type=True): - entry = self._lookup(self.space.str_w(w_key)) - if entry.w_value is None: - if self.valid == 4: - return self._convert_to_sdict(w_value) - self.valid += 1 - entry.w_value = w_value - return self - - def delitem(self, w_key): - space = self.space - w_key_type = space.type(w_key) - if space.is_w(w_key_type, space.w_str): - entry = self._lookup(space.str_w(w_key)) - if entry.w_value is not None: - for i in range(self.entries.index(entry), self.valid): - self.entries[i] = self.entries[i+1] - self.entries[self.valid] = entry - entry.w_value = None - self.valid -= 1 - if self.valid == 0: - return self.space.emptydictimpl - return self - else: - entry.key = None - raise KeyError - elif _is_sane_hash(self.space, w_key_type): - raise KeyError - else: - return self._convert_to_rdict().delitem(w_key) - - def length(self): - return self.valid - - def get(self, w_lookup): - space = self.space - w_lookup_type = space.type(w_lookup) - if space.is_w(w_lookup_type, space.w_str): - entry = self._lookup(space.str_w(w_lookup)) - val = entry.w_value - if val is None: - entry.key = None - return val - elif _is_sane_hash(self.space, w_lookup_type): - return None - else: - return self._convert_to_rdict().get(w_lookup) - - def iteritems(self): - return self._convert_to_rdict().iteritems() - def iterkeys(self): - return self._convert_to_rdict().iterkeys() - def itervalues(self): - return self._convert_to_rdict().itervalues() - - def keys(self): - return [self.space.wrap(self.entries[i].key) for i in range(self.valid)] - def values(self): - return [self.entries[i].w_value for i in range(self.valid)] - def items(self): - return [self.space.newtuple([self.space.wrap(e.key), e.w_value]) - for e in [self.entries[i] for i in range(self.valid)]] - class StrDictImplementation(DictImplementation): def __init__(self, space): Modified: pypy/branch/gc-hash/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/branch/gc-hash/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/branch/gc-hash/pypy/objspace/std/test/test_dictmultiobject.py Sun Oct 11 22:13:52 2009 @@ -2,7 +2,7 @@ from pypy.objspace.std.dictmultiobject import \ W_DictMultiObject, setitem__DictMulti_ANY_ANY, getitem__DictMulti_ANY, \ EmptyDictImplementation, RDictImplementation, StrDictImplementation, \ - SmallDictImplementation, SmallStrDictImplementation, MeasuringDictImplementation + MeasuringDictImplementation from pypy.objspace.std.celldict import ModuleDictImplementation from pypy.conftest import gettestobjspace @@ -91,16 +91,6 @@ raises(KeyError, "d['def']") - -class TestW_DictSmall(test_dictobject.TestW_DictObject): - def setup_class(cls): - cls.space = gettestobjspace(**{"objspace.std.withsmalldicts": True}) - -class AppTest_DictSmall(test_dictobject.AppTest_DictObject): - def setup_class(cls): - cls.space = gettestobjspace(**{"objspace.std.withsmalldicts": True}) - - class C: pass class FakeSpace(test_dictobject.FakeSpace): @@ -250,23 +240,11 @@ class TestStrDictImplementation(TestRDictImplementation): ImplementionClass = StrDictImplementation -class TestSmallDictImplementation(TestRDictImplementation): - ImplementionClass = SmallDictImplementation - - def get_impl(self): - return self.ImplementionClass(self.space, self.string, self.string2) - class TestMeasuringDictImplementation(TestRDictImplementation): ImplementionClass = MeasuringDictImplementation DevolvedClass = MeasuringDictImplementation EmptyClass = MeasuringDictImplementation -class TestSmallStrDictImplementation(TestRDictImplementation): - ImplementionClass = SmallStrDictImplementation - - def get_impl(self): - return self.ImplementionClass(self.space, self.string, self.string2) - class TestModuleDictImplementation(TestRDictImplementation): ImplementionClass = ModuleDictImplementation EmptyClass = ModuleDictImplementation From arigo at codespeak.net Sun Oct 11 22:56:55 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 11 Oct 2009 22:56:55 +0200 (CEST) Subject: [pypy-svn] r68325 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20091011205655.AF52C16804F@codespeak.net> Author: arigo Date: Sun Oct 11 22:56:53 2009 New Revision: 68325 Modified: pypy/trunk/pypy/jit/metainterp/test/test_tl.py Log: Fix for jit.backend.x86.test.test_zrpy_tl: don't use a setup_class() method here. It's a complete mess to call a parent setup_class() that may not be here, so I just give up. Modified: pypy/trunk/pypy/jit/metainterp/test/test_tl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_tl.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_tl.py Sun Oct 11 22:56:53 2009 @@ -17,7 +17,7 @@ res = self.meta_interp(main, [1, 10]) assert res == 100 - def setup_class(cls): + def _get_main(self): from pypy.jit.tl.tl import interp_without_call from pypy.jit.tl.tlopcode import compile @@ -64,12 +64,13 @@ def main(n, inputarg): code = codes[n] return interp_without_call(code, inputarg=inputarg) - cls.main = main + return main def test_tl_base(self): # 'backendopt=True' is used on lltype to kill unneeded access # to the class, which generates spurious 'guard_class' - res = self.meta_interp(self.main.im_func, [0, 6], listops=True, + main = self._get_main() + res = self.meta_interp(main, [0, 6], listops=True, backendopt=True) assert res == 5040 self.check_loops({'int_mul':1, 'jump':1, @@ -77,9 +78,10 @@ 'guard_false':1}) def test_tl_2(self): - res = self.meta_interp(self.main.im_func, [1, 10], listops=True, + main = self._get_main() + res = self.meta_interp(main, [1, 10], listops=True, backendopt=True) - assert res == self.main.im_func(1, 10) + assert res == main(1, 10) self.check_loops({'int_sub':1, 'int_le':1, 'int_is_true':1, 'guard_false':1, 'jump':1}) From fijal at codespeak.net Sun Oct 11 23:08:07 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 11 Oct 2009 23:08:07 +0200 (CEST) Subject: [pypy-svn] r68326 - pypy/trunk/pypy/rpython/tool/test Message-ID: <20091011210807.3A44316804F@codespeak.net> Author: fijal Date: Sun Oct 11 23:08:06 2009 New Revision: 68326 Modified: pypy/trunk/pypy/rpython/tool/test/test_mkrffi.py Log: Use our own platform instead of custom distutils hacks imported from test_c, which is gone Modified: pypy/trunk/pypy/rpython/tool/test/test_mkrffi.py ============================================================================== --- pypy/trunk/pypy/rpython/tool/test/test_mkrffi.py (original) +++ pypy/trunk/pypy/rpython/tool/test/test_mkrffi.py Sun Oct 11 23:08:06 2009 @@ -1,7 +1,6 @@ import ctypes from pypy.rpython.tool.mkrffi import * -from pypy.rpython.tool.test.test_c import TestBasic import py class random_structure(ctypes.Structure): @@ -54,7 +53,30 @@ """) assert rffi_source.source.strip() == _src.strip() -class TestMkrffi(TestBasic): +class TestMkrffi(object): + def setup_class(cls): + import ctypes + from pypy.tool.udir import udir + from pypy.translator.platform import platform + from pypy.translator.tool.cbuild import ExternalCompilationInfo + + c_source = """ + void *int_to_void_p(int arg) {} + + struct random_strucutre { + int one; + int *two; + }; + + struct random_structure* int_int_to_struct_p(int one, int two) {} + """ + + c_file = udir.join('rffilib.c') + c_file.write(c_source) + libname = platform.compile([c_file], ExternalCompilationInfo(), + standalone=False) + cls.lib = ctypes.CDLL(str(libname)) + def test_single_func(self): func = self.lib.int_to_void_p func.argtypes = [ctypes.c_int] From cfbolz at codespeak.net Sun Oct 11 23:12:08 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 11 Oct 2009 23:12:08 +0200 (CEST) Subject: [pypy-svn] r68327 - pypy/trunk/pypy/rlib/rcairo Message-ID: <20091011211208.4101316804F@codespeak.net> Author: cfbolz Date: Sun Oct 11 23:12:07 2009 New Revision: 68327 Removed: pypy/trunk/pypy/rlib/rcairo/ Log: kill rcairo, tests are skipped, used by nobody, uses rctypes From arigo at codespeak.net Sun Oct 11 23:55:40 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 11 Oct 2009 23:55:40 +0200 (CEST) Subject: [pypy-svn] r68328 - pypy/branch/gc-hash/pypy/rlib Message-ID: <20091011215540.24DE516803A@codespeak.net> Author: arigo Date: Sun Oct 11 23:55:39 2009 New Revision: 68328 Modified: pypy/branch/gc-hash/pypy/rlib/rweakrefimpl.py Log: Remove a usage of gethash(). Modified: pypy/branch/gc-hash/pypy/rlib/rweakrefimpl.py ============================================================================== --- pypy/branch/gc-hash/pypy/rlib/rweakrefimpl.py (original) +++ pypy/branch/gc-hash/pypy/rlib/rweakrefimpl.py Sun Oct 11 23:55:39 2009 @@ -91,6 +91,8 @@ adtmeths=entrymeths, hints={'weakarray': 'value'}) +ll_strhash = rstr.LLHelpers.ll_strhash + @jit.dont_look_inside def ll_new_weakdict(): d = lltype.malloc(WEAKDICT) @@ -101,7 +103,8 @@ @jit.dont_look_inside def ll_get(d, llkey): - i = rdict.ll_dict_lookup(d, llkey, llkey.gethash()) + hash = ll_strhash(llkey) + i = rdict.ll_dict_lookup(d, llkey, hash) #llop.debug_print(lltype.Void, i, 'get') valueref = d.entries[i].value if valueref: @@ -118,8 +121,9 @@ @jit.dont_look_inside def ll_set_nonnull(d, llkey, llvalue): + hash = ll_strhash(llkey) valueref = weakref_create(llvalue) # GC effects here, before the rest - i = rdict.ll_dict_lookup(d, llkey, llkey.gethash()) + i = rdict.ll_dict_lookup(d, llkey, hash) everused = d.entries.everused(i) d.entries[i].key = llkey d.entries[i].value = valueref @@ -132,7 +136,8 @@ @jit.dont_look_inside def ll_set_null(d, llkey): - i = rdict.ll_dict_lookup(d, llkey, llkey.gethash()) + hash = ll_strhash(llkey) + i = rdict.ll_dict_lookup(d, llkey, hash) if d.entries.everused(i): # If the entry was ever used, clean up its key and value. # We don't store a NULL value, but a dead weakref, because From antocuni at codespeak.net Mon Oct 12 10:10:11 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 12 Oct 2009 10:10:11 +0200 (CEST) Subject: [pypy-svn] r68329 - pypy/trunk/pypy/jit/backend/cli Message-ID: <20091012081011.18A81168048@codespeak.net> Author: antocuni Date: Mon Oct 12 10:10:08 2009 New Revision: 68329 Modified: pypy/trunk/pypy/jit/backend/cli/runner.py Log: fix translation Modified: pypy/trunk/pypy/jit/backend/cli/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/runner.py (original) +++ pypy/trunk/pypy/jit/backend/cli/runner.py Mon Oct 12 10:10:08 2009 @@ -353,11 +353,16 @@ self.ooclass = get_class_for_type(TYPE) self.typename = TYPE._short_name() self._is_array_of_pointers = (history.getkind(TYPE) == 'ref') + self._is_array_of_floats = (history.getkind(TYPE) == 'float') def is_array_of_pointers(self): # for arrays, TYPE is the type of the array item. return self._is_array_of_pointers + def is_array_of_floats(self): + # for arrays, TYPE is the type of the array item. + return self._is_array_of_floats + def get_clitype(self): return dotnet.class2type(self.ooclass) @@ -492,6 +497,7 @@ self.fieldname = fieldname self.key = key_manager.getkey((TYPE, fieldname)) self._is_pointer_field = (history.getkind(T) == 'ref') + self._is_float_field = (history.getkind(T) == 'float') def is_pointer_field(self): return self._is_pointer_field From pedronis at codespeak.net Mon Oct 12 10:54:18 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 12 Oct 2009 10:54:18 +0200 (CEST) Subject: [pypy-svn] r68330 - in pypy/branch/improve-kwd-args/pypy: interpreter interpreter/test objspace/std Message-ID: <20091012085418.53E0B168042@codespeak.net> Author: pedronis Date: Mon Oct 12 10:54:14 2009 New Revision: 68330 Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py pypy/branch/improve-kwd-args/pypy/interpreter/test/test_argument.py pypy/branch/improve-kwd-args/pypy/objspace/std/fake.py Log: (cfbolz, pedronis) fix failures Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/argument.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/argument.py Mon Oct 12 10:54:14 2009 @@ -316,13 +316,13 @@ def parse(self, fnname, signature, defaults_w=[], blindargs=0): # used by geninterped code + # and ./objspace/std/fake.py """Parse args and kwargs to initialize a frame according to the signature of code object. """ return self.parse_obj(None, fnname, signature, defaults_w, blindargs) - # xxx have only this one def parse_obj(self, w_firstarg, fnname, signature, defaults_w=[], blindargs=0): # used by ./interpreter/gateway.py Modified: pypy/branch/improve-kwd-args/pypy/interpreter/test/test_argument.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/test/test_argument.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/test/test_argument.py Mon Oct 12 10:54:14 2009 @@ -1,4 +1,4 @@ -from pypy.interpreter.argument import Arguments +from pypy.interpreter.argument import Arguments, ArgumentsForTranslation class DummySpace(object): @@ -31,60 +31,64 @@ w_dict = dict -class TestArguments(object): +class TestArgumentsForTranslation(object): def test_unmatch_signature(self): space = DummySpace() - args = Arguments(space, [1,2,3]) + args = ArgumentsForTranslation(space, [1,2,3]) sig = (['a', 'b', 'c'], None, None) data = args.match_signature(sig, []) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() - args = Arguments(space, [1]) + args = ArgumentsForTranslation(space, [1]) sig = (['a', 'b', 'c'], None, None) data = args.match_signature(sig, [2, 3]) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() - args = Arguments(space, [1,2,3,4,5]) + args = ArgumentsForTranslation(space, [1,2,3,4,5]) sig = (['a', 'b', 'c'], 'r', None) data = args.match_signature(sig, []) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() - args = Arguments(space, [1], {'c': 3, 'b': 2}) + args = ArgumentsForTranslation(space, [1], {'c': 3, 'b': 2}) sig = (['a', 'b', 'c'], None, None) data = args.match_signature(sig, []) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() - args = Arguments(space, [1], {'c': 5}) + args = ArgumentsForTranslation(space, [1], {'c': 5}) sig = (['a', 'b', 'c'], None, None) data = args.match_signature(sig, [2, 3]) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() - args = Arguments(space, [1], {'c': 5, 'd': 7}) + args = ArgumentsForTranslation(space, [1], {'c': 5, 'd': 7}) sig = (['a', 'b', 'c'], None, 'kw') data = args.match_signature(sig, [2, 3]) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() - args = Arguments(space, [1,2,3,4,5], {'e': 5, 'd': 7}) + args = ArgumentsForTranslation(space, [1,2,3,4,5], {'e': 5, 'd': 7}) sig = (['a', 'b', 'c'], 'r', 'kw') data = args.match_signature(sig, [2, 3]) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() - args = Arguments(space, [], {}, w_stararg=[1], w_starstararg={'c': 5, 'd': 7}) + args = ArgumentsForTranslation(space, [], {}, + w_stararg=[1], + w_starstararg={'c': 5, 'd': 7}) sig = (['a', 'b', 'c'], None, 'kw') data = args.match_signature(sig, [2, 3]) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() - args = Arguments(space, [1,2], {'g': 9}, w_stararg=[3,4,5], w_starstararg={'e': 5, 'd': 7}) + args = ArgumentsForTranslation(space, [1,2], {'g': 9}, + w_stararg=[3,4,5], + w_starstararg={'e': 5, 'd': 7}) sig = (['a', 'b', 'c'], 'r', 'kw') data = args.match_signature(sig, [2, 3]) new_args = args.unmatch_signature(sig, data) Modified: pypy/branch/improve-kwd-args/pypy/objspace/std/fake.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/objspace/std/fake.py (original) +++ pypy/branch/improve-kwd-args/pypy/objspace/std/fake.py Mon Oct 12 10:54:14 2009 @@ -129,9 +129,19 @@ eval.Code.__init__(self, getattr(cpy_callable, '__name__', '?')) self.cpy_callable = cpy_callable assert callable(cpy_callable), cpy_callable + def signature(self): return [], 'args', 'kwds' + def funcrun(self, func, args): + frame = func.space.createframe(self, func.w_func_globals, + func.closure) + sig = self.signature() + scope_w = args.parse(func.name, sig, func.defs_w) + frame.setfastscope(scope_w) + return frame.run() + + class CPythonFakeFrame(eval.Frame): def __init__(self, space, code, w_globals=None, numlocals=-1): From pedronis at codespeak.net Mon Oct 12 12:24:47 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 12 Oct 2009 12:24:47 +0200 (CEST) Subject: [pypy-svn] r68331 - pypy/branch/improve-kwd-args/pypy/interpreter/test Message-ID: <20091012102447.47A68168047@codespeak.net> Author: pedronis Date: Mon Oct 12 12:24:46 2009 New Revision: 68331 Modified: pypy/branch/improve-kwd-args/pypy/interpreter/test/test_argument.py Log: (pedronis, cfbolz): write unittests for the argument parsing. Not much fun. Modified: pypy/branch/improve-kwd-args/pypy/interpreter/test/test_argument.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/test/test_argument.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/test/test_argument.py Mon Oct 12 12:24:46 2009 @@ -1,4 +1,7 @@ -from pypy.interpreter.argument import Arguments, ArgumentsForTranslation +import py +from pypy.interpreter.argument import Arguments, ArgumentsForTranslation, ArgErr +from pypy.interpreter.argument import ArgErrUnknownKwds +from pypy.interpreter.error import OperationError class DummySpace(object): @@ -29,10 +32,167 @@ def isinstance(self, obj, cls): return isinstance(obj, cls) + def exception_match(self, w_type1, w_type2): + return issubclass(w_type1, w_type2) + + + w_TypeError = TypeError w_dict = dict -class TestArgumentsForTranslation(object): +class TestArgumentsNormal(object): + def test_match0(self): + space = DummySpace() + args = Arguments(space, []) + l = [] + args._match_signature(None, l, []) + assert len(l) == 0 + l = [None, None] + py.test.raises(ArgErr, args._match_signature, None, l, ["a"]) + py.test.raises(ArgErr, args._match_signature, None, l, ["a"], + has_vararg=True) + l = [None] + args._match_signature(None, l, ["a"], defaults_w=[1]) + assert l == [1] + l = [None] + args._match_signature(None, l, [], has_vararg=True) + assert l == [()] + l = [None] + args._match_signature(None, l, [], has_kwarg=True) + assert l == [{}] + l = [None, None] + py.test.raises(ArgErr, args._match_signature, 41, l, []) + l = [None] + args._match_signature(1, l, ["a"]) + assert l == [1] + l = [None] + args._match_signature(1, l, [], has_vararg=True) + assert l == [(1,)] + + def test_match4(self): + space = DummySpace() + values = [4, 5, 6, 7] + for havefirstarg in [0, 1]: + for i in range(len(values)-havefirstarg): + args = values[havefirstarg:i+havefirstarg] + starargs = tuple(values[i+havefirstarg:]) + if havefirstarg: + firstarg = values[0] + else: + firstarg = None + args = Arguments(space, args, w_stararg=starargs) + l = [None, None, None, None] + args._match_signature(firstarg, l, ["a", "b", "c", "d"]) + assert l == [4, 5, 6, 7] + l = [None, None, None, None, None, None] + py.test.raises(ArgErr, args._match_signature, firstarg, l, ["a"]) + py.test.raises(ArgErr, args._match_signature, firstarg, l, ["a", "b", "c", "d", "e"]) + py.test.raises(ArgErr, args._match_signature, firstarg, l, ["a", "b", "c", "d", "e"], + has_vararg=True) + l = [None, None, None, None, None] + args._match_signature(firstarg, l, ["a", "b", "c", "d", "e"], defaults_w=[1]) + assert l == [4, 5, 6, 7, 1] + for j in range(len(values)): + l = [None] * (j + 1) + args._match_signature(firstarg, l, ["a", "b", "c", "d", "e"][:j], has_vararg=True) + assert l == values[:j] + [tuple(values[j:])] + l = [None, None, None, None, None] + args._match_signature(firstarg, l, ["a", "b", "c", "d"], has_kwarg=True) + assert l == [4, 5, 6, 7, {}] + + def test_match_kwds(self): + space = DummySpace() + for i in range(3): + kwds = [("c", 3)] + kwds_w = dict(kwds[:i]) + w_kwds = dict(kwds[i:]) + if i == 2: + w_kwds = None + args = Arguments(space, [1, 2], kwds_w, w_starstararg=w_kwds) + l = [None, None, None] + args._match_signature(None, l, ["a", "b", "c"], defaults_w=[4]) + assert l == [1, 2, 3] + l = [None, None, None, None] + args._match_signature(None, l, ["a", "b", "b1", "c"], defaults_w=[4, 5]) + assert l == [1, 2, 4, 3] + l = [None, None, None, None] + args._match_signature(None, l, ["a", "b", "c", "d"], defaults_w=[4, 5]) + assert l == [1, 2, 3, 5] + l = [None, None, None, None] + py.test.raises(ArgErr, args._match_signature, None, l, + ["c", "b", "a", "d"], defaults_w=[4, 5]) + py.test.raises(ArgErr, args._match_signature, None, l, + ["a", "b", "c1", "d"], defaults_w=[4, 5]) + l = [None, None, None] + args._match_signature(None, l, ["a", "b"], has_kwarg=True) + assert l == [1, 2, {'c': 3}] + + def test_match_kwds2(self): + space = DummySpace() + kwds = [("c", 3), ('d', 4)] + for i in range(4): + kwds_w = dict(kwds[:i]) + w_kwds = dict(kwds[i:]) + if i == 3: + w_kwds = None + args = Arguments(space, [1, 2], kwds_w, w_starstararg=w_kwds) + l = [None, None, None, None] + args._match_signature(None, l, ["a", "b", "c"], has_kwarg=True) + assert l == [1, 2, 3, {'d': 4}] + + def test_duplicate_kwds(self): + space = DummySpace() + args = Arguments(space, [], {"a": 1}, w_starstararg={"a": 2}) + excinfo = py.test.raises(OperationError, args._match_signature, None, + [None], [], has_kwarg=True) + assert excinfo.value.w_type is TypeError + + def test_starstararg_wrong_type(self): + space = DummySpace() + args = Arguments(space, [], {"a": 1}, w_starstararg="hello") + excinfo = py.test.raises(OperationError, args._match_signature, None, + [None], [], has_kwarg=True) + assert excinfo.value.w_type is TypeError + + def test_unwrap_error(self): + space = DummySpace() + valuedummy = object() + def str_w(w): + if w is None: + raise OperationError(TypeError, None) + if w is valuedummy: + raise OperationError(ValueError, None) + return str(w) + space.str_w = str_w + args = Arguments(space, [], {"a": 1}, w_starstararg={None: 1}) + excinfo = py.test.raises(OperationError, args._match_signature, None, + [None], [], has_kwarg=True) + assert excinfo.value.w_type is TypeError + assert excinfo.value.w_value is not None + args = Arguments(space, [], {"a": 1}, w_starstararg={valuedummy: 1}) + excinfo = py.test.raises(OperationError, args._match_signature, None, + [None], [], has_kwarg=True) + assert excinfo.value.w_type is ValueError + assert excinfo.value.w_value is None + + def test_blindargs(self): + space = DummySpace() + kwds = [("a", 3), ('b', 4)] + for i in range(4): + kwds_w = dict(kwds[:i]) + w_kwds = dict(kwds[i:]) + if i == 3: + w_kwds = None + args = Arguments(space, [1, 2], kwds_w, w_starstararg=w_kwds) + l = [None, None, None] + args._match_signature(None, l, ["a", "b"], has_kwarg=True, blindargs=2) + assert l == [1, 2, {'a':3, 'b': 4}] + py.test.raises(ArgErrUnknownKwds, args._match_signature, None, l, + ["a", "b"], blindargs=2) + + + +class TestArgumentsForTranslation(object): def test_unmatch_signature(self): space = DummySpace() From pedronis at codespeak.net Mon Oct 12 14:57:54 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 12 Oct 2009 14:57:54 +0200 (CEST) Subject: [pypy-svn] r68332 - in pypy/branch/improve-kwd-args/pypy/interpreter: . test Message-ID: <20091012125754.BC5D716804C@codespeak.net> Author: pedronis Date: Mon Oct 12 14:57:53 2009 New Revision: 68332 Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py pypy/branch/improve-kwd-args/pypy/interpreter/test/test_argument.py Log: (pedronis, cfbolz): more tests, this time for the interfaces that the translation toolchain uses Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/argument.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/argument.py Mon Oct 12 14:57:53 2009 @@ -493,11 +493,11 @@ shape_cnt = len(self.arguments_w)+nextra # Number of positional args if self.kwds_w: shape_keys = self.kwds_w.keys() # List of keywords (strings) + shape_keys.sort() else: shape_keys = [] shape_star = self.w_stararg is not None # Flag: presence of *arg shape_stst = self.w_starstararg is not None # Flag: presence of **kwds - shape_keys.sort() return shape_cnt, tuple(shape_keys), shape_star, shape_stst # shape_keys are sorted def rawshape(args, nextra=0): Modified: pypy/branch/improve-kwd-args/pypy/interpreter/test/test_argument.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/test/test_argument.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/test/test_argument.py Mon Oct 12 14:57:53 2009 @@ -1,6 +1,6 @@ import py from pypy.interpreter.argument import Arguments, ArgumentsForTranslation, ArgErr -from pypy.interpreter.argument import ArgErrUnknownKwds +from pypy.interpreter.argument import ArgErrUnknownKwds, rawshape from pypy.interpreter.error import OperationError @@ -254,3 +254,118 @@ new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() + def test_rawshape(self): + space = DummySpace() + args = ArgumentsForTranslation(space, [1,2,3]) + assert rawshape(args) == (3, (), False, False) + + args = ArgumentsForTranslation(space, [1]) + assert rawshape(args, 2) == (3, (), False, False) + + args = ArgumentsForTranslation(space, [1,2,3,4,5]) + assert rawshape(args) == (5, (), False, False) + + args = ArgumentsForTranslation(space, [1], {'c': 3, 'b': 2}) + assert rawshape(args) == (1, ('b', 'c'), False, False) + + args = ArgumentsForTranslation(space, [1], {'c': 5}) + assert rawshape(args) == (1, ('c', ), False, False) + + args = ArgumentsForTranslation(space, [1], {'c': 5, 'd': 7}) + assert rawshape(args) == (1, ('c', 'd'), False, False) + + args = ArgumentsForTranslation(space, [1,2,3,4,5], {'e': 5, 'd': 7}) + assert rawshape(args) == (5, ('d', 'e'), False, False) + + args = ArgumentsForTranslation(space, [], {}, + w_stararg=[1], + w_starstararg={'c': 5, 'd': 7}) + assert rawshape(args) == (0, (), True, True) + + args = ArgumentsForTranslation(space, [1,2], {'g': 9}, + w_stararg=[3,4,5], + w_starstararg={'e': 5, 'd': 7}) + assert rawshape(args) == (2, ('g', ), True, True) + + def test_flatten(self): + space = DummySpace() + args = ArgumentsForTranslation(space, [1,2,3]) + assert args.flatten() == ((3, (), False, False), [1, 2, 3]) + + args = ArgumentsForTranslation(space, [1]) + assert args.flatten() == ((1, (), False, False), [1]) + + args = ArgumentsForTranslation(space, [1,2,3,4,5]) + assert args.flatten() == ((5, (), False, False), [1,2,3,4,5]) + + args = ArgumentsForTranslation(space, [1], {'c': 3, 'b': 2}) + assert args.flatten() == ((1, ('b', 'c'), False, False), [1, 2, 3]) + + args = ArgumentsForTranslation(space, [1], {'c': 5}) + assert args.flatten() == ((1, ('c', ), False, False), [1, 5]) + + args = ArgumentsForTranslation(space, [1], {'c': 5, 'd': 7}) + assert args.flatten() == ((1, ('c', 'd'), False, False), [1, 5, 7]) + + args = ArgumentsForTranslation(space, [1,2,3,4,5], {'e': 5, 'd': 7}) + assert args.flatten() == ((5, ('d', 'e'), False, False), [1, 2, 3, 4, 5, 7, 5]) + + args = ArgumentsForTranslation(space, [], {}, + w_stararg=[1], + w_starstararg={'c': 5, 'd': 7}) + assert args.flatten() == ((0, (), True, True), [[1], {'c': 5, 'd': 7}]) + + args = ArgumentsForTranslation(space, [1,2], {'g': 9}, + w_stararg=[3,4,5], + w_starstararg={'e': 5, 'd': 7}) + assert args.flatten() == ((2, ('g', ), True, True), [1, 2, 9, [3, 4, 5], {'e': 5, 'd': 7}]) + + def test_stararg_flowspace_variable(self): + space = DummySpace() + var = object() + shape = ((2, ('g', ), True, False), [1, 2, 9, var]) + args = ArgumentsForTranslation(space, [1,2], {'g': 9}, + w_stararg=var) + assert args.flatten() == shape + + args = ArgumentsForTranslation.fromshape(space, *shape) + assert args.flatten() == shape + + + def test_fromshape(self): + space = DummySpace() + shape = ((3, (), False, False), [1, 2, 3]) + args = ArgumentsForTranslation.fromshape(space, *shape) + assert args.flatten() == shape + + shape = ((1, (), False, False), [1]) + args = ArgumentsForTranslation.fromshape(space, *shape) + assert args.flatten() == shape + + shape = ((5, (), False, False), [1,2,3,4,5]) + args = ArgumentsForTranslation.fromshape(space, *shape) + assert args.flatten() == shape + + shape = ((1, ('b', 'c'), False, False), [1, 2, 3]) + args = ArgumentsForTranslation.fromshape(space, *shape) + assert args.flatten() == shape + + shape = ((1, ('c', ), False, False), [1, 5]) + args = ArgumentsForTranslation.fromshape(space, *shape) + assert args.flatten() == shape + + shape = ((1, ('c', 'd'), False, False), [1, 5, 7]) + args = ArgumentsForTranslation.fromshape(space, *shape) + assert args.flatten() == shape + + shape = ((5, ('d', 'e'), False, False), [1, 2, 3, 4, 5, 7, 5]) + args = ArgumentsForTranslation.fromshape(space, *shape) + assert args.flatten() == shape + + shape = ((0, (), True, True), [[1], {'c': 5, 'd': 7}]) + args = ArgumentsForTranslation.fromshape(space, *shape) + assert args.flatten() == shape + + shape = ((2, ('g', ), True, True), [1, 2, 9, [3, 4, 5], {'e': 5, 'd': 7}]) + args = ArgumentsForTranslation.fromshape(space, *shape) + assert args.flatten() == shape From arigo at codespeak.net Mon Oct 12 15:23:59 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Oct 2009 15:23:59 +0200 (CEST) Subject: [pypy-svn] r68333 - in pypy/trunk/pypy/rpython: . test Message-ID: <20091012132359.C730F168048@codespeak.net> Author: arigo Date: Mon Oct 12 15:23:57 2009 New Revision: 68333 Modified: pypy/trunk/pypy/rpython/rfloat.py pypy/trunk/pypy/rpython/test/test_rdict.py Log: Test and fix. Modified: pypy/trunk/pypy/rpython/rfloat.py ============================================================================== --- pypy/trunk/pypy/rpython/rfloat.py (original) +++ pypy/trunk/pypy/rpython/rfloat.py Mon Oct 12 15:23:57 2009 @@ -152,12 +152,13 @@ This should be special-cased in W_FloatObject. In the low-level case, floats cannot be used with ints in dicts, anyway. """ + from pypy.rlib.rarithmetic import intmask v, expo = math.frexp(f) v *= TAKE_NEXT hipart = int(v) v = (v - float(hipart)) * TAKE_NEXT x = hipart + int(v) + (expo << 15) - return x + return intmask(x) # # _________________________ Conversions _________________________ Modified: pypy/trunk/pypy/rpython/test/test_rdict.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_rdict.py (original) +++ pypy/trunk/pypy/rpython/test/test_rdict.py Mon Oct 12 15:23:57 2009 @@ -556,6 +556,14 @@ return d['a'] assert self.interpret(func, []) == 42 + def test_dict_of_floats(self): + d = {3.0: 42, 3.1: 43, 3.2: 44, 3.3: 45, 3.4: 46} + def fn(f): + return d[f] + + res = self.interpret(fn, [3.0]) + assert res == 42 + class TestLLtype(BaseTestRdict, LLRtypeMixin): def test_dict_but_not_with_char_keys(self): From pedronis at codespeak.net Mon Oct 12 15:41:03 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 12 Oct 2009 15:41:03 +0200 (CEST) Subject: [pypy-svn] r68334 - in pypy/branch/improve-kwd-args/pypy/module/_weakref: . test Message-ID: <20091012134103.4E873168048@codespeak.net> Author: pedronis Date: Mon Oct 12 15:41:02 2009 New Revision: 68334 Modified: pypy/branch/improve-kwd-args/pypy/module/_weakref/interp__weakref.py pypy/branch/improve-kwd-args/pypy/module/_weakref/test/test_weakref.py Log: (pedronis, cfbolz): kill an unpack Modified: pypy/branch/improve-kwd-args/pypy/module/_weakref/interp__weakref.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/module/_weakref/interp__weakref.py (original) +++ pypy/branch/improve-kwd-args/pypy/module/_weakref/interp__weakref.py Mon Oct 12 15:41:02 2009 @@ -143,12 +143,7 @@ return w_obj -def descr__new__weakref(space, w_subtype, w_obj, __args__): - args_w, kw_w = __args__.unpack() - if args_w: - w_callable = args_w[0] - else: - w_callable = space.w_None +def descr__new__weakref(space, w_subtype, w_obj, w_callable=None): lifeline = w_obj.getweakref() if lifeline is None: lifeline = WeakrefLifeline(space) @@ -189,7 +184,7 @@ which is called with the weak reference as an argument when 'obj' is about to be finalized.""", __new__ = interp2app(descr__new__weakref, - unwrap_spec=[ObjSpace, W_Root, W_Root, Arguments]), + unwrap_spec=[ObjSpace, W_Root, W_Root, W_Root]), __eq__ = interp2app(descr__eq__, unwrap_spec=[ObjSpace, W_Weakref, W_Root]), __ne__ = interp2app(descr__ne__, Modified: pypy/branch/improve-kwd-args/pypy/module/_weakref/test/test_weakref.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/module/_weakref/test/test_weakref.py (original) +++ pypy/branch/improve-kwd-args/pypy/module/_weakref/test/test_weakref.py Mon Oct 12 15:41:02 2009 @@ -409,3 +409,11 @@ a = A() assert _weakref.ref(a) == a + def test_strange_arguments(self): + import _weakref + class A(object): + pass + + a = A() + raises(TypeError, _weakref.ref, a, lambda x: None, 3) + raises(TypeError, _weakref.ref, a, lambda x: None, b=3) From arigo at codespeak.net Mon Oct 12 15:47:41 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Oct 2009 15:47:41 +0200 (CEST) Subject: [pypy-svn] r68335 - pypy/branch/gc-hash/pypy/rlib Message-ID: <20091012134741.00FBA168048@codespeak.net> Author: arigo Date: Mon Oct 12 15:47:37 2009 New Revision: 68335 Modified: pypy/branch/gc-hash/pypy/rlib/objectmodel.py Log: The object model interface that I propose to implement. Modified: pypy/branch/gc-hash/pypy/rlib/objectmodel.py ============================================================================== --- pypy/branch/gc-hash/pypy/rlib/objectmodel.py (original) +++ pypy/branch/gc-hash/pypy/rlib/objectmodel.py Mon Oct 12 15:47:37 2009 @@ -5,6 +5,7 @@ import sys import types +import math # specialize is a decorator factory for attaching _annspecialcase_ # attributes to functions: for example @@ -127,37 +128,159 @@ obj.__dict__ = {} obj.__class__ = FREED_OBJECT -from pypy.rpython.extregistry import ExtRegistryEntry - # ____________________________________________________________ # -# id-like functions. -# In addition, RPython supports hash(x) on RPython instances, -# returning a number that is not guaranteed to be unique but -# that doesn't change over time for a given 'x'. +# id-like functions. The idea is that calling hash() or id() is not +# allowed in RPython. You have to call one of the following more +# precise functions. + +def compute_hash(x): + """RPython equivalent of hash(x), where 'x' is an immutable + RPython-level object. For strings or unicodes it computes the + hash as in Python. For tuples it calls compute_hash() + recursively. For instances it uses compute_identity_hash(). + Note that this can return 0 or -1 too. + + Behavior across translation: + + * on lltypesystem, it always returns the same number, both + before and after translation. Dictionaries don't need to + be rehashed after translation. + + * on ootypesystem, the value changes because of translation. + Dictionaries need to be rehashed. + """ + if isinstance(x, (str, unicode)): + return _hash_string(x) + if isinstance(x, int): + return x + if isinstance(x, float): + return _hash_float(x) + if isinstance(x, tuple): + return _hash_tuple(x) + if x is None: + return 0 + return compute_identity_hash(x) + +def compute_identity_hash(x): + """RPython equivalent of object.__hash__(x). This returns the + so-called 'identity hash', which is the non-overridable default + hash of Python. Can be called for any RPython-level object + that turns into a GC object, or for any low-level GC object. + The value is not guaranteed to be the same before and after + translation, except for RPython instances on the lltypesystem. + """ + result = object.__hash__(x) + try: + x.__dict__['__precomputed_identity_hash'] = result + except (TypeError, AttributeError): + pass + return result def compute_unique_id(x): - """RPython equivalent of id(x). The 'x' must be an RPython instance. - This operation can be very costly depending on the garbage collector. - To remind you of this fact, we don't support id(x) directly. + """RPython equivalent of id(x). The 'x' must be an RPython-level + object that turns into a GC object. This operation can be very + costly depending on the garbage collector. To remind you of this + fact, we don't support id(x) directly. + (XXX not implemented on ootype, falls back to compute_identity_hash) """ return id(x) # XXX need to return r_longlong on some platforms def current_object_addr_as_int(x): """A cheap version of id(x). The current memory location of an - instance can change over time for moving GCs. Also note that on + object can change over time for moving GCs. Also note that on ootypesystem this typically doesn't return the real address but - just the same as hash(x). + just the same as compute_hash(x). """ from pypy.rlib.rarithmetic import intmask return intmask(id(x)) +# ---------- + +def _hash_string(s): + """The algorithm behind compute_hash() for a string or a unicode.""" + from pypy.rlib.rarithmetic import intmask + length = len(s) + if length == 0: + x = 0 + else: + x = ord(s[0]) << 7 + i = 0 + while i < length: + x = (1000003*x) ^ ord(s[i]) + i += 1 + x ^= length + return intmask(x) + +def _hash_float(f): + """The algorithm behind compute_hash() for a float. + This implementation is identical to the CPython implementation, + except the fact that the integer case is not treated specially. + In RPython, floats cannot be used with ints in dicts, anyway. + """ + v, expo = math.frexp(f) + v *= TAKE_NEXT + hipart = int(v) + v = (v - float(hipart)) * TAKE_NEXT + x = hipart + int(v) + (expo << 15) + return x +TAKE_NEXT = float(2**31) + +def _hash_tuple(t): + """NOT_RPYTHON. The algorithm behind compute_hash() for a tuple.""" + x = 0x345678 + mult = 1000003 + length = len(t) + for item in t: + y = compute_hash(item) + x = (x ^ y) * mult + mult += intmask(82520 + len + len) + return x + +# ---------- + +from pypy.rpython.extregistry import ExtRegistryEntry + +class Entry(ExtRegistryEntry): + _about_ = compute_hash + + def compute_result_annotation(self, s_x): + from pypy.annotation import model as annmodel + return annmodel.SomeInteger() + + def specialize_call(self, hop): + r_obj, = hop.args_r + v_obj, = hop.inputargs(r_obj) + ll_fn = r_obj.get_ll_hash_function() + return hop.gendirectcall(ll_fn, v_obj) + +class Entry(ExtRegistryEntry): + _about_ = compute_identity_hash + + def compute_result_annotation(self, s_x): + from pypy.annotation import model as annmodel + return annmodel.SomeInteger() + + def specialize_call(self, hop): + vobj, = hop.inputargs(hop.args_r[0]) + if hop.rtyper.type_system.name == 'lltypesystem': + from pypy.rpython.lltypesystem import lltype + ok = (isinstance(vobj.concretetype, lltype.Ptr) and + vobj.concretetype.TO._gckind == 'gc') + else: + from pypy.rpython.lltypesystem import ootype + ok = isinstance(vobj.concretetype, ootype.OOType) + if not ok: + from pypy.rpython.error import TyperError + raise TyperError("compute_identity_hash() cannot be applied to" + " %r" % (vobj.concretetype,)) + return hop.genop('identityhash', [vobj], resulttype=lltype.Signed) + class Entry(ExtRegistryEntry): _about_ = compute_unique_id def compute_result_annotation(self, s_x): from pypy.annotation import model as annmodel - assert isinstance(s_x, annmodel.SomeInstance) return annmodel.SomeInteger() def specialize_call(self, hop): @@ -173,7 +296,7 @@ # XXX wrong implementation for now, fix me from pypy.rpython.rmodel import warning warning("compute_unique_id() is not fully supported on ootype") - return hop.genop('ooidentityhash', [vobj], + return hop.genop('identityhash', [vobj], resulttype = ootype.Signed) from pypy.rpython.error import TyperError raise TyperError("compute_unique_id() cannot be applied to %r" % ( @@ -184,7 +307,6 @@ def compute_result_annotation(self, s_x): from pypy.annotation import model as annmodel - assert isinstance(s_x, annmodel.SomeInstance) return annmodel.SomeInteger() def specialize_call(self, hop): @@ -197,12 +319,14 @@ elif hop.rtyper.type_system.name == 'ootypesystem': from pypy.rpython.ootypesystem import ootype if isinstance(vobj.concretetype, ootype.Instance): - return hop.genop('ooidentityhash', [vobj], + return hop.genop('identityhash', [vobj], resulttype = ootype.Signed) from pypy.rpython.error import TyperError raise TyperError("current_object_addr_as_int() cannot be applied to" " %r" % (vobj.concretetype,)) +# ____________________________________________________________ + def hlinvoke(repr, llcallable, *args): raise TypeError, "hlinvoke is meant to be rtyped and not called direclty" From antocuni at codespeak.net Mon Oct 12 16:50:47 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 12 Oct 2009 16:50:47 +0200 (CEST) Subject: [pypy-svn] r68341 - in pypy/trunk/pypy: jit/metainterp/test translator/cli translator/oosupport Message-ID: <20091012145047.1A1E0168048@codespeak.net> Author: antocuni Date: Mon Oct 12 16:50:44 2009 New Revision: 68341 Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py pypy/trunk/pypy/translator/cli/constant.py pypy/trunk/pypy/translator/cli/cts.py pypy/trunk/pypy/translator/oosupport/constant.py pypy/trunk/pypy/translator/oosupport/metavm.py Log: add a jit test for math.modf, and fix the cli backend to run it Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Mon Oct 12 16:50:44 2009 @@ -945,6 +945,32 @@ res = self.interp_operations(f, [-10]) assert res == 456 * 2 + def test_residual_external_call(self): + class CustomPolicy(JitPolicy): + def look_inside_function(self, func): + mod = func.__module__ or '?' + if mod == 'pypy.rpython.lltypesystem.module.ll_math': + # XXX temporary, contains force_cast + return False + return super(CustomPolicy, self).look_inside_function(func) + + import math + myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'res']) + def f(x, y): + x = float(x) + res = 0 + while y > 0: + myjitdriver.can_enter_jit(x=x, y=y, res=res) + myjitdriver.jit_merge_point(x=x, y=y, res=res) + rpart, ipart = math.modf(x) + res += ipart + y -= 1 + return res + res = self.meta_interp(f, [6, 7], policy=CustomPolicy()) + assert res == 42 + self.check_loop_count(1) + + class TestOOtype(BasicTests, OOJitMixin): def test_oohash(self): @@ -1031,6 +1057,7 @@ + class BaseLLtypeTests(BasicTests): def test_oops_on_nongc(self): Modified: pypy/trunk/pypy/translator/cli/constant.py ============================================================================== --- pypy/trunk/pypy/translator/cli/constant.py (original) +++ pypy/trunk/pypy/translator/cli/constant.py Mon Oct 12 16:50:44 2009 @@ -288,7 +288,7 @@ class CLIStaticMethodConst(CLIBaseConstMixin, StaticMethodConst): def create_pointer(self, gen): assert not self.is_null() - signature = self.cts.graph_to_signature(self.value.graph) + signature = self.cts.static_meth_to_signature(self.value) gen.ilasm.opcode('ldnull') gen.ilasm.opcode('ldftn', signature) gen.ilasm.new('instance void class %s::.ctor(object, native int)' % self.delegate_type) Modified: pypy/trunk/pypy/translator/cli/cts.py ============================================================================== --- pypy/trunk/pypy/translator/cli/cts.py (original) +++ pypy/trunk/pypy/translator/cli/cts.py Mon Oct 12 16:50:44 2009 @@ -305,20 +305,33 @@ def ctor_name(self, t): return 'instance void %s::.ctor()' % self.lltype_to_cts(t) + def static_meth_to_signature(self, sm): + from pypy.translator.oosupport import metavm + graph = getattr(sm, 'graph', None) + if graph: + return self.graph_to_signature(graph) + module, name = metavm.get_primitive_name(sm) + func_name = '[pypylib]pypy.builtin.%s::%s' % (module, name) + T = ootype.typeOf(sm) + return self.format_signatue(func_name, T.ARGS, T.RESULT) + def graph_to_signature(self, graph, is_method = False, func_name = None): - ret_type, ret_var = self.llvar_to_cts(graph.getreturnvar()) func_name = func_name or graph.name func_name = self.escape_name(func_name) namespace = getattr(graph.func, '_namespace_', None) if namespace: func_name = '%s::%s' % (namespace, func_name) - args = [arg for arg in graph.getargs() if arg.concretetype is not ootype.Void] + ARGS = [arg.concretetype for arg in graph.getargs() if arg.concretetype is not ootype.Void] if is_method: - args = args[1:] + ARGS = ARGS[1:] + RESULT = graph.getreturnvar().concretetype + return self.format_signatue(func_name, ARGS, RESULT) - arg_types = [self.lltype_to_cts(arg.concretetype).typename() for arg in args] + def format_signatue(self, func_name, ARGS, RESULT): + arg_types = [self.lltype_to_cts(ARG).typename() for ARG in ARGS] arg_list = ', '.join(arg_types) + ret_type = self.lltype_to_cts(RESULT) return '%s %s(%s)' % (ret_type, func_name, arg_list) Modified: pypy/trunk/pypy/translator/oosupport/constant.py ============================================================================== --- pypy/trunk/pypy/translator/oosupport/constant.py (original) +++ pypy/trunk/pypy/translator/oosupport/constant.py Mon Oct 12 16:50:44 2009 @@ -748,7 +748,8 @@ def record_dependencies(self): if self.value is ootype.null(self.value._TYPE): return - self.db.pending_function(self.value.graph) + if hasattr(self.value, 'graph'): + self.db.pending_function(self.value.graph) self.delegate_type = self.db.record_delegate(self.value._TYPE) def initialize_data(self, constgen, gen): Modified: pypy/trunk/pypy/translator/oosupport/metavm.py ============================================================================== --- pypy/trunk/pypy/translator/oosupport/metavm.py (original) +++ pypy/trunk/pypy/translator/oosupport/metavm.py Mon Oct 12 16:50:44 2009 @@ -437,23 +437,23 @@ generator.branch_conditionally(False, self.label) -class _Call(MicroInstruction): +def get_primitive_name(sm): + try: + sm.graph + return None + except AttributeError: + pass + try: + return 'rffi', sm._obj.oo_primitive + except AttributeError: + pass + return sm._name.rsplit('.', 1) - def _get_primitive_name(self, callee): - try: - callee.graph - return None - except AttributeError: - pass - try: - return 'rffi', callee._obj.oo_primitive - except AttributeError: - pass - return callee._name.rsplit('.', 1) +class _Call(MicroInstruction): def render(self, generator, op): callee = op.args[0].value - is_primitive = self._get_primitive_name(callee) + is_primitive = get_primitive_name(callee) if is_primitive: module, name = is_primitive From arigo at codespeak.net Mon Oct 12 17:33:02 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Oct 2009 17:33:02 +0200 (CEST) Subject: [pypy-svn] r68342 - pypy/branch/gc-hash/pypy/rlib Message-ID: <20091012153302.366AD16804C@codespeak.net> Author: arigo Date: Mon Oct 12 17:33:00 2009 New Revision: 68342 Modified: pypy/branch/gc-hash/pypy/rlib/objectmodel.py Log: Improve comments. Simplify _hash_tuple() with appropriate comments. Modified: pypy/branch/gc-hash/pypy/rlib/objectmodel.py ============================================================================== --- pypy/branch/gc-hash/pypy/rlib/objectmodel.py (original) +++ pypy/branch/gc-hash/pypy/rlib/objectmodel.py Mon Oct 12 17:33:00 2009 @@ -136,9 +136,13 @@ def compute_hash(x): """RPython equivalent of hash(x), where 'x' is an immutable - RPython-level object. For strings or unicodes it computes the - hash as in Python. For tuples it calls compute_hash() + RPython-level or low-level object. For strings or unicodes it + computes the hash as in Python. For tuples it calls compute_hash() recursively. For instances it uses compute_identity_hash(). + For low-level objects it returns the hash of the original RPython- + level object, if any, or just compute_identity_hash() otherwise. + It cannot be used on llmemory.GCREF. + Note that this can return 0 or -1 too. Behavior across translation: @@ -164,11 +168,11 @@ def compute_identity_hash(x): """RPython equivalent of object.__hash__(x). This returns the - so-called 'identity hash', which is the non-overridable default - hash of Python. Can be called for any RPython-level object - that turns into a GC object, or for any low-level GC object. - The value is not guaranteed to be the same before and after - translation, except for RPython instances on the lltypesystem. + so-called 'identity hash', which is the non-overridable default hash + of Python. Can be called for any RPython-level object that turns + into a GC object, or for any low-level GC object, including + llmemory.GCREF. The value is not guaranteed to be the same before + and after translation, except for RPython instances on the lltypesystem. """ result = object.__hash__(x) try: @@ -227,14 +231,16 @@ TAKE_NEXT = float(2**31) def _hash_tuple(t): - """NOT_RPYTHON. The algorithm behind compute_hash() for a tuple.""" + """NOT_RPYTHON. The algorithm behind compute_hash() for a tuple. + It is modelled after the old algorithm of Python 2.3, which is + slightly faster than the one introduced by Python 2.4. We assume + that nested tuples are very uncommon in RPython, making the case + unlikely. + """ x = 0x345678 - mult = 1000003 - length = len(t) for item in t: y = compute_hash(item) - x = (x ^ y) * mult - mult += intmask(82520 + len + len) + x = (1000003 * x) ^ y return x # ---------- From fijal at codespeak.net Mon Oct 12 17:46:18 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 12 Oct 2009 17:46:18 +0200 (CEST) Subject: [pypy-svn] r68343 - in pypy/branch/inline-fastpath-malloc/pypy/rpython: . lltypesystem Message-ID: <20091012154618.889FC16804C@codespeak.net> Author: fijal Date: Mon Oct 12 17:46:17 2009 New Revision: 68343 Modified: pypy/branch/inline-fastpath-malloc/pypy/rpython/llinterp.py pypy/branch/inline-fastpath-malloc/pypy/rpython/lltypesystem/lloperation.py Log: I suppose these are operations that I need, not working so far Modified: pypy/branch/inline-fastpath-malloc/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/inline-fastpath-malloc/pypy/rpython/llinterp.py (original) +++ pypy/branch/inline-fastpath-malloc/pypy/rpython/llinterp.py Mon Oct 12 17:46:17 2009 @@ -848,6 +848,12 @@ def op_gc_restore_exception(self, exc): raise NotImplementedError("gc_restore_exception") + def op_gc_nursery_size(self): + raise NotImplementedError + + def op_gc_adr_of_nursery_pointer(self): + raise NotImplementedError + def op_gc_call_rtti_destructor(self, rtti, addr): if hasattr(rtti._obj, 'destructor_funcptr'): d = rtti._obj.destructor_funcptr Modified: pypy/branch/inline-fastpath-malloc/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/inline-fastpath-malloc/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/inline-fastpath-malloc/pypy/rpython/lltypesystem/lloperation.py Mon Oct 12 17:46:17 2009 @@ -439,6 +439,10 @@ 'gc_thread_run' : LLOp(), 'gc_thread_die' : LLOp(), 'gc_assume_young_pointers': LLOp(), + 'gc_adr_of_nursery_pointer' : LLOp(), + # ^^^ returns an address of nursery pointer, for later modifications + 'gc_nursery_size' : LLOp(), + # experimental operations in support of thread cloning, only # implemented by the Mark&Sweep GC 'gc_x_swap_pool': LLOp(canraise=(MemoryError,), canunwindgc=True), From fijal at codespeak.net Mon Oct 12 18:02:55 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 12 Oct 2009 18:02:55 +0200 (CEST) Subject: [pypy-svn] r68344 - pypy/branch/inline-fastpath-malloc/pypy/rpython/memory/gc Message-ID: <20091012160255.7AAE7168049@codespeak.net> Author: fijal Date: Mon Oct 12 18:02:55 2009 New Revision: 68344 Modified: pypy/branch/inline-fastpath-malloc/pypy/rpython/memory/gc/generation.py Log: Make get_young_* staticmethods as they don't depend on runtime values Modified: pypy/branch/inline-fastpath-malloc/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/inline-fastpath-malloc/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/inline-fastpath-malloc/pypy/rpython/memory/gc/generation.py Mon Oct 12 18:02:55 2009 @@ -122,10 +122,12 @@ # a new nursery (e.g. if it invokes finalizers). self.semispace_collect() - def get_young_fixedsize(self, nursery_size): + @staticmethod + def get_young_fixedsize(nursery_size): return nursery_size // 2 - 1 - def get_young_var_basesize(self, nursery_size): + @staticmethod + def get_young_var_basesize(nursery_size): return nursery_size // 4 - 1 def is_in_nursery(self, addr): From fijal at codespeak.net Mon Oct 12 18:03:23 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 12 Oct 2009 18:03:23 +0200 (CEST) Subject: [pypy-svn] r68345 - in pypy/branch/inline-fastpath-malloc/pypy/rpython: . lltypesystem Message-ID: <20091012160323.DD444168049@codespeak.net> Author: fijal Date: Mon Oct 12 18:03:21 2009 New Revision: 68345 Modified: pypy/branch/inline-fastpath-malloc/pypy/rpython/llinterp.py pypy/branch/inline-fastpath-malloc/pypy/rpython/lltypesystem/lloperation.py Log: Adjust names and meanings of lloperations introduced for GC & JIT interaction. Modified: pypy/branch/inline-fastpath-malloc/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/inline-fastpath-malloc/pypy/rpython/llinterp.py (original) +++ pypy/branch/inline-fastpath-malloc/pypy/rpython/llinterp.py Mon Oct 12 18:03:21 2009 @@ -848,7 +848,7 @@ def op_gc_restore_exception(self, exc): raise NotImplementedError("gc_restore_exception") - def op_gc_nursery_size(self): + def op_gc_adr_of_nursery_end(self): raise NotImplementedError def op_gc_adr_of_nursery_pointer(self): Modified: pypy/branch/inline-fastpath-malloc/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/inline-fastpath-malloc/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/inline-fastpath-malloc/pypy/rpython/lltypesystem/lloperation.py Mon Oct 12 18:03:21 2009 @@ -439,9 +439,13 @@ 'gc_thread_run' : LLOp(), 'gc_thread_die' : LLOp(), 'gc_assume_young_pointers': LLOp(), + + # ------- JIT & GC interaction, only for some GCs ---------- + 'gc_adr_of_nursery_pointer' : LLOp(), # ^^^ returns an address of nursery pointer, for later modifications - 'gc_nursery_size' : LLOp(), + 'gc_adr_of_nursery_end' : LLOp(), + # ^^^ returns an address of pointer, since it can change at runtime # experimental operations in support of thread cloning, only # implemented by the Mark&Sweep GC From afa at codespeak.net Mon Oct 12 18:04:11 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 12 Oct 2009 18:04:11 +0200 (CEST) Subject: [pypy-svn] r68346 - pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc Message-ID: <20091012160411.55445168003@codespeak.net> Author: afa Date: Mon Oct 12 18:04:10 2009 New Revision: 68346 Modified: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py Log: Refactor fixlocalvars(), later we will allow attributes which are lists of localvars. Modified: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py Mon Oct 12 18:04:10 2009 @@ -461,27 +461,32 @@ insn1.framesize = size_at_insn1 def fixlocalvars(self): + def fixvar(localvar): + match = r_localvar_esp.match(localvar) + if match: + if localvar == '0(%esp)': # for pushl and popl, by + hint = None # default ebp addressing is + else: # a bit nicer + hint = 'esp' + ofs_from_esp = int(match.group(1) or '0') + localvar = ofs_from_esp - insn.framesize + assert localvar != 0 # that's the return address + return LocalVar(localvar, hint=hint) + elif self.uses_frame_pointer: + match = r_localvar_ebp.match(localvar) + if match: + ofs_from_ebp = int(match.group(1) or '0') + localvar = ofs_from_ebp - 4 + assert localvar != 0 # that's the return address + return LocalVar(localvar, hint='ebp') + return localvar + for insn in self.insns: - if hasattr(insn, 'framesize'): - for name in insn._locals_: - localvar = getattr(insn, name) - match = r_localvar_esp.match(localvar) - if match: - if localvar == '0(%esp)': # for pushl and popl, by - hint = None # default ebp addressing is - else: # a bit nicer - hint = 'esp' - ofs_from_esp = int(match.group(1) or '0') - localvar = ofs_from_esp - insn.framesize - assert localvar != 0 # that's the return address - setattr(insn, name, LocalVar(localvar, hint=hint)) - elif self.uses_frame_pointer: - match = r_localvar_ebp.match(localvar) - if match: - ofs_from_ebp = int(match.group(1) or '0') - localvar = ofs_from_ebp - 4 - assert localvar != 0 # that's the return address - setattr(insn, name, LocalVar(localvar, hint='ebp')) + if not hasattr(insn, 'framesize'): + continue + for name in insn._locals_: + localvar = getattr(insn, name) + setattr(insn, name, fixvar(localvar)) def trackgcroots(self): From fijal at codespeak.net Mon Oct 12 18:06:29 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 12 Oct 2009 18:06:29 +0200 (CEST) Subject: [pypy-svn] r68347 - in pypy/branch/inline-fastpath-malloc/pypy/rpython: . lltypesystem Message-ID: <20091012160629.13299168049@codespeak.net> Author: fijal Date: Mon Oct 12 18:06:28 2009 New Revision: 68347 Modified: pypy/branch/inline-fastpath-malloc/pypy/rpython/llinterp.py pypy/branch/inline-fastpath-malloc/pypy/rpython/lltypesystem/lloperation.py Log: adjust names of operations, once again, so they're consistent with generation's gc fields Modified: pypy/branch/inline-fastpath-malloc/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/inline-fastpath-malloc/pypy/rpython/llinterp.py (original) +++ pypy/branch/inline-fastpath-malloc/pypy/rpython/llinterp.py Mon Oct 12 18:06:28 2009 @@ -848,10 +848,10 @@ def op_gc_restore_exception(self, exc): raise NotImplementedError("gc_restore_exception") - def op_gc_adr_of_nursery_end(self): + def op_gc_adr_of_nursery_top(self): raise NotImplementedError - def op_gc_adr_of_nursery_pointer(self): + def op_gc_adr_of_nursery_free(self): raise NotImplementedError def op_gc_call_rtti_destructor(self, rtti, addr): Modified: pypy/branch/inline-fastpath-malloc/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/inline-fastpath-malloc/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/inline-fastpath-malloc/pypy/rpython/lltypesystem/lloperation.py Mon Oct 12 18:06:28 2009 @@ -442,9 +442,9 @@ # ------- JIT & GC interaction, only for some GCs ---------- - 'gc_adr_of_nursery_pointer' : LLOp(), - # ^^^ returns an address of nursery pointer, for later modifications - 'gc_adr_of_nursery_end' : LLOp(), + 'gc_adr_of_nursery_free' : LLOp(), + # ^^^ returns an address of nursery free pointer, for later modifications + 'gc_adr_of_nursery_top' : LLOp(), # ^^^ returns an address of pointer, since it can change at runtime # experimental operations in support of thread cloning, only From afa at codespeak.net Mon Oct 12 18:18:26 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 12 Oct 2009 18:18:26 +0200 (CEST) Subject: [pypy-svn] r68348 - in pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc: . test/darwin Message-ID: <20091012161826.58CEF168049@codespeak.net> Author: afa Date: Mon Oct 12 18:18:24 2009 New Revision: 68348 Added: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/test/darwin/track_switch0.s Modified: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py Log: Handle table-based switches on Darwin: where the address is computed in several instructions, we walk backwards to find all possible inputs for the indirect jump, and use the first label we meet. Added: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/test/darwin/track_switch0.s ============================================================================== --- (empty file) +++ pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/test/darwin/track_switch0.s Mon Oct 12 18:18:24 2009 @@ -0,0 +1,94 @@ +_pypy_g_MetaInterp_prepare_resume_from_failure: + subl $28, %esp + movl %ebx, 20(%esp) + movl %esi, 24(%esp) + call L2245 +"L00000000130$pb": +L2245: + popl %ebx + movl 32(%esp), %edx +L2231: + movl 36(%esp), %eax + subl $6, %eax + cmpl $7, %eax + jbe L2244 +L2243: + movl 20(%esp), %ebx + movl 24(%esp), %esi + addl $28, %esp + ret + .align 4,0x90 +L2244: + movl L2237-"L00000000130$pb"(%ebx,%eax,4), %eax + addl %ebx, %eax + jmp *%eax + .align 2,0x90 +L2237: + .long L2233-"L00000000130$pb" + .long L2234-"L00000000130$pb" + .long L2243-"L00000000130$pb" + .long L2243-"L00000000130$pb" + .long L2243-"L00000000130$pb" + .long L2235-"L00000000130$pb" + .long L2235-"L00000000130$pb" + .long L2236-"L00000000130$pb" +L2236: + movl %edx, 32(%esp) + movl 20(%esp), %ebx + movl 24(%esp), %esi + addl $28, %esp + jmp L_pypy_g_MetaInterp_raise_overflow_error$stub +L2235: + movl %edx, 32(%esp) + movl 20(%esp), %ebx + movl 24(%esp), %esi + addl $28, %esp + jmp L_pypy_g_MetaInterp_handle_exception$stub +L2234: + movl 16(%edx), %eax + movl 8(%eax), %edx + movl 4(%eax), %eax + movl 4(%edx,%eax,4), %eax + movl %eax, 32(%esp) + movl 20(%esp), %ebx + movl 24(%esp), %esi + addl $28, %esp + jmp L_pypy_g_MIFrame_dont_follow_jump$stub +L2233: + movl 16(%edx), %eax + movl 8(%eax), %edx + movl 4(%eax), %eax + movl 4(%edx,%eax,4), %esi + movl 8(%esi), %ecx + movl 52(%esi), %eax + testl %eax, %eax + js L2238 + movl %eax, %edx +L2240: + cmpb $0, 12(%ecx,%edx) + je L2241 +L2242: + movl L_pypy_g_exceptions_AssertionError$non_lazy_ptr-"L00000000130$pb"(%ebx), %eax + movl %eax, 36(%esp) + movl L_pypy_g_exceptions_AssertionError_vtable$non_lazy_ptr-"L00000000130$pb"(%ebx), %eax + movl %eax, 32(%esp) + movl 20(%esp), %ebx + movl 24(%esp), %esi + addl $28, %esp + jmp L_pypy_g_RPyRaiseException$stub +L2241: + incl %eax + movl %eax, 52(%esi) + movl %esi, (%esp) + call L_pypy_g_MIFrame_load_3byte$stub + ;; expected {28(%esp) | 20(%esp), 24(%esp), %edi, %ebp | } + movl %eax, 52(%esi) + movl 20(%esp), %ebx + movl 24(%esp), %esi + addl $28, %esp + ret +L2238: + movl %eax, %edx + addl 8(%ecx), %edx + jmp L2240 + .align 4,0x90 Modified: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py Mon Oct 12 18:18:24 2009 @@ -37,6 +37,8 @@ r_jmp_switch = re.compile(r"\tjmp\t[*]"+LABEL+"[(]") r_jmptable_item = re.compile(r"\t.long\t"+LABEL+"\s*$") r_jmptable_end = re.compile(r"\t.text|\t.section\s+.text") +r_jmptable_item = re.compile(r"\t.long\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") +r_jmptable_end = re.compile(r"\t.text|\t.section\s+.text|"+LABEL) r_binaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+"),\s*("+OPERAND+")\s*$") LOCALVAR = r"%eax|%edx|%ecx|%ebx|%esi|%edi|%ebp|\d*[(]%esp[)]" LOCALVARFP = LOCALVAR + r"|-?\d*[(]%ebp[)]" @@ -462,6 +464,11 @@ def fixlocalvars(self): def fixvar(localvar): + if localvar is None: + return None + elif isinstance(localvar, (list, tuple)): + return [fixvar(var) for var in localvar] + match = r_localvar_esp.match(localvar) if match: if localvar == '0(%esp)': # for pushl and popl, by @@ -597,6 +604,7 @@ def visit_addl(self, line, sign=+1): match = r_binaryinsn.match(line) + source = match.group(1) target = match.group(2) if target == '%esp': count = match.group(1) @@ -605,7 +613,7 @@ return InsnCannotFollowEsp() return InsnStackAdjust(sign * int(count[1:])) elif self.r_localvar.match(target): - return InsnSetLocal(target) + return InsnSetLocal(target, [source, target]) else: return [] @@ -624,6 +632,7 @@ match = r_binaryinsn.match(line) if not match: raise UnrecognizedOperation(line) + source = match.group(1) target = match.group(2) if self.r_localvar.match(target): return InsnSetLocal(target) @@ -694,7 +703,7 @@ if self.r_localvar.match(source): return [InsnCopyLocal(source, target)] else: - return [InsnSetLocal(target)] + return [InsnSetLocal(target, [source])] else: return [] @@ -742,31 +751,38 @@ return InsnRet() def visit_jmp(self, line): - tablelabel = None + tablelabels = [] match = r_jmp_switch.match(line) if match: # this is a jmp *Label(%index), used for table-based switches. # Assume that the table is just a list of lines looking like # .long LABEL or .long 0, ending in a .text or .section .text.hot. - tablelabel = match.group(1) + tablelabels.append(match.group(1)) elif r_unaryinsn_star.match(line): # maybe a jmp similar to the above, but stored in a # registry: # movl L9341(%eax), %eax # jmp *%eax operand = r_unaryinsn_star.match(line).group(1)[1:] - prev_line = self.lines[self.currentlineno-1] - match = r_insn.match(prev_line) - binaryinsn = r_binaryinsn.match(prev_line) - if (match and binaryinsn and - match.group(1) == 'movl' and binaryinsn.group(2) == operand - and '(' in binaryinsn.group(1)): - tablelabel = binaryinsn.group(1).split('(')[0] - if tablelabel not in self.labels: - # Probably an indirect tail-call. - tablelabel = None - if tablelabel: - tablelin = self.labels[tablelabel].lineno + 1 + def walker(insn, locs): + sources = [] + for loc in locs: + sources.extend(insn.all_sources_of(loc)) + for source in sources: + label_match = re.compile(LABEL).match(source) + if label_match: + tablelabels.append(label_match.group(0)) + return + yield tuple(sources) + insn = InsnStop() + insn.previous_insns = [self.insns[-1]] + self.walk_instructions_backwards(walker, insn, (operand,)) + + # Probably an indirect tail-call. + tablelabels = [label for label in tablelabels if label in self.labels] + assert len(tablelabels) <= 1 + if tablelabels: + tablelin = self.labels[tablelabels[0]].lineno + 1 while not r_jmptable_end.match(self.lines[tablelin]): match = r_jmptable_item.match(self.lines[tablelin]) if not match: @@ -920,6 +936,13 @@ def source_of(self, localvar, tag): return localvar + def all_sources_of(self, localvar): + source = self.source_of(localvar, None) + if source is somenewvalue: + return [] + else: + return [source] + class Label(Insn): _args_ = ['label', 'lineno'] def __init__(self, label, lineno): @@ -952,14 +975,19 @@ return self.arguments[localvar] class InsnSetLocal(Insn): - _args_ = ['target'] - _locals_ = ['target'] - def __init__(self, target): + _args_ = ['target', 'sources'] + _locals_ = ['target', 'sources'] + def __init__(self, target, sources=()): self.target = target + self.sources = sources def source_of(self, localvar, tag): if localvar == self.target: return somenewvalue return localvar + def all_sources_of(self, localvar): + if localvar == self.target: + return self.sources + return [localvar] class InsnCopyLocal(Insn): _args_ = ['source', 'target'] From pedronis at codespeak.net Mon Oct 12 19:29:27 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 12 Oct 2009 19:29:27 +0200 (CEST) Subject: [pypy-svn] r68349 - in pypy/branch/improve-kwd-args/pypy: interpreter interpreter/test module/__builtin__ module/pypyjit objspace/flow Message-ID: <20091012172927.851EB16804B@codespeak.net> Author: pedronis Date: Mon Oct 12 19:29:24 2009 New Revision: 68349 Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py pypy/branch/improve-kwd-args/pypy/interpreter/pyframe.py pypy/branch/improve-kwd-args/pypy/interpreter/pyopcode.py pypy/branch/improve-kwd-args/pypy/interpreter/test/test_argument.py pypy/branch/improve-kwd-args/pypy/interpreter/test/test_function.py pypy/branch/improve-kwd-args/pypy/module/__builtin__/functional.py pypy/branch/improve-kwd-args/pypy/module/pypyjit/interp_jit.py pypy/branch/improve-kwd-args/pypy/objspace/flow/flowcontext.py pypy/branch/improve-kwd-args/pypy/objspace/flow/objspace.py Log: (pedronis, cfbolz): progress and regress all mixed up in a blob of confusion: - make the argument parsing not rely on dicts, which is JIT-friendlier - at the moment Arguments objects are not immutable any more, which is a bad idea Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/argument.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/argument.py Mon Oct 12 19:29:24 2009 @@ -3,6 +3,7 @@ """ from pypy.interpreter.error import OperationError +from pypy.rlib.debug import make_sure_not_resized class Arguments(object): """ @@ -13,16 +14,21 @@ ### Construction ### - def __init__(self, space, args_w, kwds_w=None, + def __init__(self, space, args_w, keywords=None, keywords_w=None, w_stararg=None, w_starstararg=None): self.space = space assert isinstance(args_w, list) self.arguments_w = args_w - from pypy.rlib.debug import make_sure_not_resized + self.keywords = keywords + self.keywords_w = keywords_w + if keywords is not None: + assert keywords_w is not None + assert len(keywords_w) == len(keywords) + make_sure_not_resized(self.keywords) + make_sure_not_resized(self.keywords_w) + make_sure_not_resized(self.arguments_w) - self.kwds_w = kwds_w - self.w_stararg = w_stararg - self.w_starstararg = w_starstararg + self._combine_wrapped(w_stararg, w_starstararg) @staticmethod def factory(space, args_w, kwds_w=None, @@ -31,65 +37,52 @@ w_stararg, w_starstararg) def num_args(self): # only used in module/__builtin__/interp_classobj.py - self._unpack() return len(self.arguments_w) def num_kwds(self): # only used in module/__builtin__/interp_classobj.py - self._unpack() - return len(self.kwds_w) + return len(self.keywords) def __repr__(self): """ NOT_RPYTHON """ - if self.w_starstararg is not None: - return '%s(%s, %s, %s, %s)' % (self.__class__, - self.arguments_w, - self.kwds_w, - self.w_stararg, - self.w_starstararg) - if self.w_stararg is None: - if not self.kwds_w: - return '%s(%s)' % (self.__class__, self.arguments_w,) - else: - return '%s(%s, %s)' % (self.__class__, self.arguments_w, - self.kwds_w) + if not self.keywords: + return '%s(%s)' % (self.__class__, self.arguments_w,) else: return '%s(%s, %s, %s)' % (self.__class__, self.arguments_w, - self.kwds_w, self.w_stararg) + self.keywords, self.keywords_w) ### Manipulation ### def unpack(self): # used often "Return a ([w1,w2...], {'kw':w3...}) pair." - self._unpack() - return self.arguments_w, self.kwds_w + # XXX may want to change that later + kwds_w = {} + if self.keywords: + for i in range(len(self.keywords)): + kwds_w[self.keywords[i]] = self.keywords_w[i] + return self.arguments_w, kwds_w def prepend(self, w_firstarg): # used often "Return a new Arguments with a new argument inserted first." return self.factory(self.space, [w_firstarg] + self.arguments_w, - self.kwds_w, self.w_stararg, self.w_starstararg) + self.keywords, self.keywords_w) - def _unpack(self): - "unpack the *arg and **kwd into w_arguments and kwds_w" - # --- unpack the * argument now --- - if self.w_stararg is not None: + def _combine_wrapped(self, w_stararg, w_starstararg): + "unpack the *arg and **kwd into arguments_w and keywords_w" + # unpack the * arguments + if w_stararg is not None: self.arguments_w = (self.arguments_w + - self.space.unpackiterable(self.w_stararg)) - self.w_stararg = None - # --- unpack the ** argument now --- - if self.kwds_w is None: - self.kwds_w = {} - if self.w_starstararg is not None: + self.space.viewiterable(w_stararg)) + # unpack the ** arguments + if w_starstararg is not None: space = self.space - w_starstararg = self.w_starstararg - # maybe we could allow general mappings? if not space.is_true(space.isinstance(w_starstararg, space.w_dict)): raise OperationError(space.w_TypeError, space.wrap("argument after ** must be " "a dictionary")) - # don't change the original yet, - # in case something goes wrong - d = self.kwds_w.copy() + keywords_w = [None] * space.int_w(space.len(w_starstararg)) + keywords = [None] * space.int_w(space.len(w_starstararg)) + i = 0 for w_key in space.unpackiterable(w_starstararg): try: key = space.str_w(w_key) @@ -98,14 +91,20 @@ raise raise OperationError(space.w_TypeError, space.wrap("keywords must be strings")) - if key in d: + if self.keywords and key in self.keywords: raise OperationError(self.space.w_TypeError, self.space.wrap("got multiple values " "for keyword argument " "'%s'" % key)) - d[key] = space.getitem(w_starstararg, w_key) - self.kwds_w = d - self.w_starstararg = None + keywords[i] = key + keywords_w[i] = space.getitem(w_starstararg, w_key) + i += 1 + if self.keywords is None: + self.keywords = keywords + self.keywords_w = keywords_w + else: + self.keywords += keywords + self.keywords_w += keywords_w def fixedunpack(self, argcount): # used by ./objspace/std/typeobject.py @@ -113,16 +112,10 @@ # ./annotation/description.py """The simplest argument parsing: get the 'argcount' arguments, or raise a real ValueError if the length is wrong.""" - if bool(self.kwds_w) or (self.w_starstararg is not None and - self.space.is_true(self.w_starstararg)): + if self.keywords: raise ValueError, "no keyword arguments expected" if len(self.arguments_w) > argcount: raise ValueError, "too many arguments (%d expected)" % argcount - if self.w_stararg is not None: - self.arguments_w = (self.arguments_w + - self.space.viewiterable(self.w_stararg, - argcount - len(self.arguments_w))) - self.w_stararg = None elif len(self.arguments_w) < argcount: raise ValueError, "not enough arguments (%d expected)" % argcount return self.arguments_w @@ -135,15 +128,7 @@ "Return the first argument for inspection." if self.arguments_w: return self.arguments_w[0] - if self.w_stararg is None: - return None - w_iter = self.space.iter(self.w_stararg) - try: - return self.space.next(w_iter) - except OperationError, e: - if not e.match(self.space, self.space.w_StopIteration): - raise - return None + return None ### Parsing for function calls ### @@ -173,27 +158,15 @@ else: upfront = 0 - if self.w_stararg is not None: - # There is a case where we don't have to unpack() a w_stararg: - # if it matches exactly a *arg in the signature. - if (len(self.arguments_w) + upfront == co_argcount and - has_vararg and - self.space.is_w(self.space.type(self.w_stararg), - self.space.w_tuple)): - pass - else: - self._unpack() # sets self.w_stararg to None - # always unpack the ** arguments - if self.w_starstararg is not None: - self._unpack() - args_w = self.arguments_w + assert args_w is not None, "tried to match arguments again" num_args = len(args_w) - kwds_w = self.kwds_w + keywords = self.keywords + keywords_w = self.keywords_w num_kwds = 0 - if kwds_w is not None: - num_kwds = len(kwds_w) + if keywords is not None: + num_kwds = len(keywords) avail = num_args + upfront @@ -208,29 +181,36 @@ scope_w[i + input_argcount] = args_w[i] input_argcount += take - # check that no keyword argument conflicts with these - # note that for this purpose we ignore the first blindargs, - # which were put into place by prepend(). This way, keywords do - # not conflict with the hidden extra argument bound by methods. - if kwds_w and input_argcount > blindargs: - for name in argnames[blindargs:input_argcount]: - if name in kwds_w: - raise ArgErrMultipleValues(name) - - remainingkwds_w = self.kwds_w + # the code assumes that keywords can potentially be large, but that + # argnames is typically not too large + num_remainingkwds = num_kwds + if keywords: + for i in range(num_kwds): + name = keywords[i] + try: + j = argnames.index(name) + except ValueError: + continue + if j < input_argcount: + # check that no keyword argument conflicts with these note + # that for this purpose we ignore the first blindargs, + # which were put into place by prepend(). This way, + # keywords do not conflict with the hidden extra argument + # bound by methods. + if blindargs <= j: + raise ArgErrMultipleValues(name) + else: + assert scope_w[j] is None + scope_w[j] = keywords_w[i] + keywords[i] = None # mark as used + num_remainingkwds -= 1 missing = 0 if input_argcount < co_argcount: - if remainingkwds_w is None: - remainingkwds_w = {} - else: - remainingkwds_w = remainingkwds_w.copy() - # not enough args, fill in kwargs or defaults if exists def_first = co_argcount - len(defaults_w) for i in range(input_argcount, co_argcount): name = argnames[i] - if name in remainingkwds_w: - scope_w[i] = remainingkwds_w[name] - del remainingkwds_w[name] + if scope_w[i] is not None: + pass elif i >= def_first: scope_w[i] = defaults_w[i-def_first] else: @@ -239,23 +219,19 @@ # keyword arguments, which will be checked for below. missing += 1 - # collect extra positional arguments into the *vararg if has_vararg: - if self.w_stararg is None: # common case - args_left = co_argcount - upfront - if args_left < 0: # check required by rpython - assert extravarargs is not None - starargs_w = extravarargs - if num_args: - starargs_w = starargs_w + args_w - elif num_args > args_left: - starargs_w = args_w[args_left:] - else: - starargs_w = [] - scope_w[co_argcount] = self.space.newtuple(starargs_w) - else: # shortcut for the non-unpack() case above - scope_w[co_argcount] = self.w_stararg + args_left = co_argcount - upfront + if args_left < 0: # check required by rpython + assert extravarargs is not None + starargs_w = extravarargs + if num_args: + starargs_w = starargs_w + args_w + elif num_args > args_left: + starargs_w = args_w[args_left:] + else: + starargs_w = [] + scope_w[co_argcount] = self.space.newtuple(starargs_w) elif avail > co_argcount: raise ArgErrCount(avail, num_kwds, (co_argcount, has_vararg, has_kwarg), @@ -264,18 +240,23 @@ # collect extra keyword arguments into the **kwarg if has_kwarg: w_kwds = self.space.newdict() - if remainingkwds_w: - for key, w_value in remainingkwds_w.items(): - self.space.setitem(w_kwds, self.space.wrap(key), w_value) + if num_remainingkwds: + for i in range(len(keywords)): + key = keywords[i] + if key is not None: + self.space.setitem(w_kwds, self.space.wrap(key), keywords_w[i]) scope_w[co_argcount + has_vararg] = w_kwds - elif remainingkwds_w: - raise ArgErrUnknownKwds(remainingkwds_w) + elif num_remainingkwds: + raise ArgErrUnknownKwds(num_remainingkwds, keywords) if missing: raise ArgErrCount(avail, num_kwds, (co_argcount, has_vararg, has_kwarg), defaults_w, missing) + self.arguments_w = None + self.keywords = None + self.keywords_w = None return co_argcount + has_vararg + has_kwarg @@ -358,14 +339,11 @@ space.setitem(w_kwds, space.wrap(key), w_value) return w_args, w_kwds + @staticmethod def fromshape(space, (shape_cnt,shape_keys,shape_star,shape_stst), data_w): # used by geninterped code args_w = data_w[:shape_cnt] - p = shape_cnt - kwds_w = {} - for i in range(len(shape_keys)): - kwds_w[shape_keys[i]] = data_w[p] - p += 1 + p = end_keys = shape_cnt + len(shape_keys) if shape_star: w_star = data_w[p] p += 1 @@ -376,27 +354,64 @@ p += 1 else: w_starstar = None - return Arguments(space, args_w, kwds_w, w_star, w_starstar) - fromshape = staticmethod(fromshape) + return Arguments(space, args_w, list(shape_keys), + data_w[shape_cnt:end_keys], w_star, + w_starstar) + + def copy(self): + if self.keywords is None: + keywords = None + keywords_w = None + else: + keywords = self.keywords[:] + keywords_w = self.keywords_w[:] + return Arguments(self.space, self.arguments_w[:], keywords, + keywords_w) - def normalize(self): - """Return an instance of the Arguments class. (Instances of other - classes may not be suitable for long-term storage or multiple - usage.) Also force the type and validity of the * and ** arguments - to be checked now. - """ - args_w, kwds_w = self.unpack() - return Arguments(self.space, args_w, kwds_w) +class ArgumentsForTranslation(Arguments): + def __init__(self, space, args_w, keywords=None, keywords_w=None, + w_stararg=None, w_starstararg=None): + self.w_stararg = w_stararg + self.w_starstararg = w_starstararg + self.combine_has_happened = False + Arguments.__init__(self, space, args_w, keywords, keywords_w) + def combine_if_necessary(self): + if self.combine_has_happened: + return + self._combine_wrapped(self.w_stararg, self.w_starstararg) + self.combine_has_happened = True -class ArgumentsForTranslation(Arguments): @staticmethod def factory(space, args_w, kwds_w=None, w_stararg=None, w_starstararg=None): return ArgumentsForTranslation(space, args_w, kwds_w, w_stararg, w_starstararg) + def copy(self): + if self.keywords is None: + keywords = None + keywords_w = None + else: + keywords = self.keywords[:] + keywords_w = self.keywords_w[:] + return ArgumentsForTranslation(self.space, self.arguments_w[:], keywords, + keywords_w, self.w_stararg, + self.w_starstararg) + + def _match_signature(self, w_firstarg, scope_w, argnames, has_vararg=False, + has_kwarg=False, defaults_w=[], blindargs=0): + self.combine_if_necessary() + # _match_signature is destructive + return Arguments._match_signature( + self, w_firstarg, scope_w, argnames, has_vararg, has_kwarg, + defaults_w, blindargs) + + def unpack(self): + self.combine_if_necessary() + return Arguments.unpack(self) + def match_signature(self, signature, defaults_w): """Parse args and kwargs according to the signature of a code object, or raise an ArgErr in case of failure. @@ -444,27 +459,26 @@ args_w[i + datalen] = stararg_w[i] assert len(args_w) == need_cnt - kwds_w = {} + keywords = [] + keywords_w = [] for key in need_kwds: - kwds_w[key] = unfiltered_kwds_w[key] + keywords.append(key) + keywords_w.append(unfiltered_kwds_w[key]) - return ArgumentsForTranslation(self.space, args_w, kwds_w) + return ArgumentsForTranslation(self.space, args_w, keywords, keywords_w) @staticmethod def frompacked(space, w_args=None, w_kwds=None): raise NotImplementedError("go away") + @staticmethod def fromshape(space, (shape_cnt,shape_keys,shape_star,shape_stst), data_w): # used by # ./rpython/callparse.py # ./rpython/rbuiltin.py # ./annotation/bookkeeper.py args_w = data_w[:shape_cnt] - p = shape_cnt - kwds_w = {} - for i in range(len(shape_keys)): - kwds_w[shape_keys[i]] = data_w[p] - p += 1 + p = end_keys = shape_cnt + len(shape_keys) if shape_star: w_star = data_w[p] p += 1 @@ -475,14 +489,16 @@ p += 1 else: w_starstar = None - return ArgumentsForTranslation(space, args_w, kwds_w, w_star, w_starstar) - fromshape = staticmethod(fromshape) + return ArgumentsForTranslation(space, args_w, list(shape_keys), + data_w[shape_cnt:end_keys], w_star, + w_starstar) def flatten(self): # used by ./objspace/flow/objspace.py """ Argument <-> list of w_objects together with "shape" information """ shape_cnt, shape_keys, shape_star, shape_stst = self._rawshape() - data_w = self.arguments_w + [self.kwds_w[key] for key in shape_keys] + data_w = self.arguments_w + [self.keywords_w[self.keywords.index(key)] + for key in shape_keys] if shape_star: data_w.append(self.w_stararg) if shape_stst: @@ -490,9 +506,10 @@ return (shape_cnt, shape_keys, shape_star, shape_stst), data_w def _rawshape(self, nextra=0): + assert not self.combine_has_happened shape_cnt = len(self.arguments_w)+nextra # Number of positional args - if self.kwds_w: - shape_keys = self.kwds_w.keys() # List of keywords (strings) + if self.keywords: + shape_keys = self.keywords[:] # List of keywords (strings) shape_keys.sort() else: shape_keys = [] @@ -581,11 +598,14 @@ class ArgErrUnknownKwds(ArgErr): - def __init__(self, kwds_w): + def __init__(self, num_remainingkwds, keywords): self.kwd_name = '' - self.num_kwds = len(kwds_w) - if self.num_kwds == 1: - self.kwd_name = kwds_w.keys()[0] + self.num_kwds = num_remainingkwds + if num_remainingkwds == 1: + for kwd_name in keywords: + if kwd_name is not None: + self.kwd_name = kwd_name + break def getmsg(self, fnname): if self.num_kwds == 1: Modified: pypy/branch/improve-kwd-args/pypy/interpreter/pyframe.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/pyframe.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/pyframe.py Mon Oct 12 19:29:24 2009 @@ -184,17 +184,6 @@ self.valuestackdepth = hint(depth, promote=True) return w_object - def popstrdictvalues(self, n): - dic_w = {} - while True: - n -= 1 - if n < 0: - break - w_value = self.popvalue() - w_key = self.popvalue() - key = self.space.str_w(w_key) - dic_w[key] = w_value - return dic_w # we need two popvalues that return different data types: # one in case we want list another in case of tuple @@ -282,8 +271,8 @@ def make_arguments(self, nargs): return Arguments(self.space, self.peekvalues(nargs)) - def argument_factory(self, arguments, keywords, w_star, w_starstar): - return Arguments(self.space, arguments, keywords, w_star, w_starstar) + def argument_factory(self, arguments, keywords, keywords_w, w_star, w_starstar): + return Arguments(self.space, arguments, keywords, keywords_w, w_star, w_starstar) @jit.dont_look_inside def descr__reduce__(self, space): Modified: pypy/branch/improve-kwd-args/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/pyopcode.py Mon Oct 12 19:29:24 2009 @@ -9,7 +9,6 @@ from pypy.interpreter.baseobjspace import UnpackValueError, Wrappable from pypy.interpreter import gateway, function, eval from pypy.interpreter import pyframe, pytraceback -from pypy.interpreter.argument import Arguments from pypy.interpreter.pycode import PyCode from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.objectmodel import we_are_translated @@ -896,11 +895,24 @@ n_arguments = oparg & 0xff n_keywords = (oparg>>8) & 0xff - keywords = None if n_keywords: - keywords = f.popstrdictvalues(n_keywords) + keywords = [None] * n_keywords + keywords_w = [None] * n_keywords + dic_w = {} + while True: + n_keywords -= 1 + if n_keywords < 0: + break + w_value = f.popvalue() + w_key = f.popvalue() + key = f.space.str_w(w_key) + keywords[n_keywords] = key + keywords_w[n_keywords] = w_value + else: + keywords = None + keywords_w = None arguments = f.popvalues(n_arguments) - args = f.argument_factory(arguments, keywords, w_star, w_starstar) + args = f.argument_factory(arguments, keywords, keywords_w, w_star, w_starstar) w_function = f.popvalue() if we_are_jitted(): w_result = f.space.call_args(w_function, args) Modified: pypy/branch/improve-kwd-args/pypy/interpreter/test/test_argument.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/test/test_argument.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/test/test_argument.py Mon Oct 12 19:29:24 2009 @@ -11,6 +11,9 @@ def is_true(self, obj): return bool(obj) + def viewiterable(self, it): + return list(it) + def unpackiterable(self, it): return list(it) @@ -29,6 +32,12 @@ def str_w(self, s): return str(s) + def len(self, x): + return len(x) + + def int_w(self, x): + return x + def isinstance(self, obj, cls): return isinstance(obj, cls) @@ -47,23 +56,31 @@ args._match_signature(None, l, []) assert len(l) == 0 l = [None, None] + args = Arguments(space, []) py.test.raises(ArgErr, args._match_signature, None, l, ["a"]) + args = Arguments(space, []) py.test.raises(ArgErr, args._match_signature, None, l, ["a"], has_vararg=True) + args = Arguments(space, []) l = [None] args._match_signature(None, l, ["a"], defaults_w=[1]) assert l == [1] + args = Arguments(space, []) l = [None] args._match_signature(None, l, [], has_vararg=True) assert l == [()] + args = Arguments(space, []) l = [None] args._match_signature(None, l, [], has_kwarg=True) assert l == [{}] + args = Arguments(space, []) l = [None, None] py.test.raises(ArgErr, args._match_signature, 41, l, []) + args = Arguments(space, []) l = [None] args._match_signature(1, l, ["a"]) assert l == [1] + args = Arguments(space, []) l = [None] args._match_signature(1, l, [], has_vararg=True) assert l == [(1,)] @@ -73,29 +90,37 @@ values = [4, 5, 6, 7] for havefirstarg in [0, 1]: for i in range(len(values)-havefirstarg): - args = values[havefirstarg:i+havefirstarg] + arglist = values[havefirstarg:i+havefirstarg] starargs = tuple(values[i+havefirstarg:]) if havefirstarg: firstarg = values[0] else: firstarg = None - args = Arguments(space, args, w_stararg=starargs) + args = Arguments(space, arglist, w_stararg=starargs) l = [None, None, None, None] args._match_signature(firstarg, l, ["a", "b", "c", "d"]) assert l == [4, 5, 6, 7] + args = Arguments(space, arglist, w_stararg=starargs) l = [None, None, None, None, None, None] py.test.raises(ArgErr, args._match_signature, firstarg, l, ["a"]) + args = Arguments(space, arglist, w_stararg=starargs) + l = [None, None, None, None, None, None] py.test.raises(ArgErr, args._match_signature, firstarg, l, ["a", "b", "c", "d", "e"]) + args = Arguments(space, arglist, w_stararg=starargs) + l = [None, None, None, None, None, None] py.test.raises(ArgErr, args._match_signature, firstarg, l, ["a", "b", "c", "d", "e"], has_vararg=True) l = [None, None, None, None, None] + args = Arguments(space, arglist, w_stararg=starargs) args._match_signature(firstarg, l, ["a", "b", "c", "d", "e"], defaults_w=[1]) assert l == [4, 5, 6, 7, 1] for j in range(len(values)): l = [None] * (j + 1) + args = Arguments(space, arglist, w_stararg=starargs) args._match_signature(firstarg, l, ["a", "b", "c", "d", "e"][:j], has_vararg=True) assert l == values[:j] + [tuple(values[j:])] l = [None, None, None, None, None] + args = Arguments(space, arglist, w_stararg=starargs) args._match_signature(firstarg, l, ["a", "b", "c", "d"], has_kwarg=True) assert l == [4, 5, 6, 7, {}] @@ -104,24 +129,33 @@ for i in range(3): kwds = [("c", 3)] kwds_w = dict(kwds[:i]) + keywords = kwds_w.keys() + keywords_w = kwds_w.values() w_kwds = dict(kwds[i:]) if i == 2: w_kwds = None - args = Arguments(space, [1, 2], kwds_w, w_starstararg=w_kwds) + assert len(keywords) == len(keywords_w) + args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds) l = [None, None, None] args._match_signature(None, l, ["a", "b", "c"], defaults_w=[4]) assert l == [1, 2, 3] + args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds) l = [None, None, None, None] args._match_signature(None, l, ["a", "b", "b1", "c"], defaults_w=[4, 5]) assert l == [1, 2, 4, 3] + args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds) l = [None, None, None, None] args._match_signature(None, l, ["a", "b", "c", "d"], defaults_w=[4, 5]) assert l == [1, 2, 3, 5] + args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds) l = [None, None, None, None] py.test.raises(ArgErr, args._match_signature, None, l, ["c", "b", "a", "d"], defaults_w=[4, 5]) + args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds) + l = [None, None, None, None] py.test.raises(ArgErr, args._match_signature, None, l, ["a", "b", "c1", "d"], defaults_w=[4, 5]) + args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds) l = [None, None, None] args._match_signature(None, l, ["a", "b"], has_kwarg=True) assert l == [1, 2, {'c': 3}] @@ -131,26 +165,26 @@ kwds = [("c", 3), ('d', 4)] for i in range(4): kwds_w = dict(kwds[:i]) + keywords = kwds_w.keys() + keywords_w = kwds_w.values() w_kwds = dict(kwds[i:]) if i == 3: w_kwds = None - args = Arguments(space, [1, 2], kwds_w, w_starstararg=w_kwds) + args = Arguments(space, [1, 2], keywords, keywords_w, w_starstararg=w_kwds) l = [None, None, None, None] args._match_signature(None, l, ["a", "b", "c"], has_kwarg=True) assert l == [1, 2, 3, {'d': 4}] def test_duplicate_kwds(self): space = DummySpace() - args = Arguments(space, [], {"a": 1}, w_starstararg={"a": 2}) - excinfo = py.test.raises(OperationError, args._match_signature, None, - [None], [], has_kwarg=True) + excinfo = py.test.raises(OperationError, Arguments, space, [], ["a"], + [1], w_starstararg={"a": 2}) assert excinfo.value.w_type is TypeError def test_starstararg_wrong_type(self): space = DummySpace() - args = Arguments(space, [], {"a": 1}, w_starstararg="hello") - excinfo = py.test.raises(OperationError, args._match_signature, None, - [None], [], has_kwarg=True) + excinfo = py.test.raises(OperationError, Arguments, space, [], ["a"], + [1], w_starstararg="hello") assert excinfo.value.w_type is TypeError def test_unwrap_error(self): @@ -163,14 +197,12 @@ raise OperationError(ValueError, None) return str(w) space.str_w = str_w - args = Arguments(space, [], {"a": 1}, w_starstararg={None: 1}) - excinfo = py.test.raises(OperationError, args._match_signature, None, - [None], [], has_kwarg=True) + excinfo = py.test.raises(OperationError, Arguments, space, [], + ["a"], [1], w_starstararg={None: 1}) assert excinfo.value.w_type is TypeError assert excinfo.value.w_value is not None - args = Arguments(space, [], {"a": 1}, w_starstararg={valuedummy: 1}) - excinfo = py.test.raises(OperationError, args._match_signature, None, - [None], [], has_kwarg=True) + excinfo = py.test.raises(OperationError, Arguments, space, [], + ["a"], [1], w_starstararg={valuedummy: 1}) assert excinfo.value.w_type is ValueError assert excinfo.value.w_value is None @@ -180,142 +212,153 @@ kwds = [("a", 3), ('b', 4)] for i in range(4): kwds_w = dict(kwds[:i]) + keywords = kwds_w.keys() + keywords_w = kwds_w.values() w_kwds = dict(kwds[i:]) if i == 3: w_kwds = None - args = Arguments(space, [1, 2], kwds_w, w_starstararg=w_kwds) + args = Arguments(space, [1, 2], keywords[:], keywords_w[:], + w_starstararg=w_kwds) l = [None, None, None] args._match_signature(None, l, ["a", "b"], has_kwarg=True, blindargs=2) assert l == [1, 2, {'a':3, 'b': 4}] + args = Arguments(space, [1, 2], keywords[:], keywords_w[:], + w_starstararg=w_kwds) + l = [None, None, None] py.test.raises(ArgErrUnknownKwds, args._match_signature, None, l, ["a", "b"], blindargs=2) +def make_arguments_for_translation(space, args_w, keywords_w={}, + w_stararg=None, w_starstararg=None): + return ArgumentsForTranslation(space, args_w, keywords_w.keys(), + keywords_w.values(), w_stararg, + w_starstararg) class TestArgumentsForTranslation(object): def test_unmatch_signature(self): space = DummySpace() - args = ArgumentsForTranslation(space, [1,2,3]) + args = make_arguments_for_translation(space, [1,2,3]) sig = (['a', 'b', 'c'], None, None) - data = args.match_signature(sig, []) + data = args.copy().match_signature(sig, []) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() - args = ArgumentsForTranslation(space, [1]) + args = make_arguments_for_translation(space, [1]) sig = (['a', 'b', 'c'], None, None) - data = args.match_signature(sig, [2, 3]) + data = args.copy().match_signature(sig, [2, 3]) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() - args = ArgumentsForTranslation(space, [1,2,3,4,5]) + args = make_arguments_for_translation(space, [1,2,3,4,5]) sig = (['a', 'b', 'c'], 'r', None) - data = args.match_signature(sig, []) + data = args.copy().match_signature(sig, []) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() - args = ArgumentsForTranslation(space, [1], {'c': 3, 'b': 2}) + args = make_arguments_for_translation(space, [1], {'c': 3, 'b': 2}) sig = (['a', 'b', 'c'], None, None) - data = args.match_signature(sig, []) + data = args.copy().match_signature(sig, []) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() - args = ArgumentsForTranslation(space, [1], {'c': 5}) + args = make_arguments_for_translation(space, [1], {'c': 5}) sig = (['a', 'b', 'c'], None, None) - data = args.match_signature(sig, [2, 3]) + data = args.copy().match_signature(sig, [2, 3]) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() - args = ArgumentsForTranslation(space, [1], {'c': 5, 'd': 7}) + args = make_arguments_for_translation(space, [1], {'c': 5, 'd': 7}) sig = (['a', 'b', 'c'], None, 'kw') - data = args.match_signature(sig, [2, 3]) + data = args.copy().match_signature(sig, [2, 3]) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() - args = ArgumentsForTranslation(space, [1,2,3,4,5], {'e': 5, 'd': 7}) + args = make_arguments_for_translation(space, [1,2,3,4,5], {'e': 5, 'd': 7}) sig = (['a', 'b', 'c'], 'r', 'kw') - data = args.match_signature(sig, [2, 3]) + data = args.copy().match_signature(sig, [2, 3]) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() - args = ArgumentsForTranslation(space, [], {}, + args = make_arguments_for_translation(space, [], {}, w_stararg=[1], w_starstararg={'c': 5, 'd': 7}) sig = (['a', 'b', 'c'], None, 'kw') - data = args.match_signature(sig, [2, 3]) + data = args.copy().match_signature(sig, [2, 3]) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() - args = ArgumentsForTranslation(space, [1,2], {'g': 9}, + args = make_arguments_for_translation(space, [1,2], {'g': 9}, w_stararg=[3,4,5], w_starstararg={'e': 5, 'd': 7}) sig = (['a', 'b', 'c'], 'r', 'kw') - data = args.match_signature(sig, [2, 3]) + data = args.copy().match_signature(sig, [2, 3]) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() def test_rawshape(self): space = DummySpace() - args = ArgumentsForTranslation(space, [1,2,3]) + args = make_arguments_for_translation(space, [1,2,3]) assert rawshape(args) == (3, (), False, False) - args = ArgumentsForTranslation(space, [1]) + args = make_arguments_for_translation(space, [1]) assert rawshape(args, 2) == (3, (), False, False) - args = ArgumentsForTranslation(space, [1,2,3,4,5]) + args = make_arguments_for_translation(space, [1,2,3,4,5]) assert rawshape(args) == (5, (), False, False) - args = ArgumentsForTranslation(space, [1], {'c': 3, 'b': 2}) + args = make_arguments_for_translation(space, [1], {'c': 3, 'b': 2}) assert rawshape(args) == (1, ('b', 'c'), False, False) - args = ArgumentsForTranslation(space, [1], {'c': 5}) + args = make_arguments_for_translation(space, [1], {'c': 5}) assert rawshape(args) == (1, ('c', ), False, False) - args = ArgumentsForTranslation(space, [1], {'c': 5, 'd': 7}) + args = make_arguments_for_translation(space, [1], {'c': 5, 'd': 7}) assert rawshape(args) == (1, ('c', 'd'), False, False) - args = ArgumentsForTranslation(space, [1,2,3,4,5], {'e': 5, 'd': 7}) + args = make_arguments_for_translation(space, [1,2,3,4,5], {'e': 5, 'd': 7}) assert rawshape(args) == (5, ('d', 'e'), False, False) - args = ArgumentsForTranslation(space, [], {}, + args = make_arguments_for_translation(space, [], {}, w_stararg=[1], w_starstararg={'c': 5, 'd': 7}) assert rawshape(args) == (0, (), True, True) - args = ArgumentsForTranslation(space, [1,2], {'g': 9}, + args = make_arguments_for_translation(space, [1,2], {'g': 9}, w_stararg=[3,4,5], w_starstararg={'e': 5, 'd': 7}) assert rawshape(args) == (2, ('g', ), True, True) def test_flatten(self): space = DummySpace() - args = ArgumentsForTranslation(space, [1,2,3]) + args = make_arguments_for_translation(space, [1,2,3]) assert args.flatten() == ((3, (), False, False), [1, 2, 3]) - args = ArgumentsForTranslation(space, [1]) + args = make_arguments_for_translation(space, [1]) assert args.flatten() == ((1, (), False, False), [1]) - args = ArgumentsForTranslation(space, [1,2,3,4,5]) + args = make_arguments_for_translation(space, [1,2,3,4,5]) assert args.flatten() == ((5, (), False, False), [1,2,3,4,5]) - args = ArgumentsForTranslation(space, [1], {'c': 3, 'b': 2}) + args = make_arguments_for_translation(space, [1], {'c': 3, 'b': 2}) assert args.flatten() == ((1, ('b', 'c'), False, False), [1, 2, 3]) - args = ArgumentsForTranslation(space, [1], {'c': 5}) + args = make_arguments_for_translation(space, [1], {'c': 5}) assert args.flatten() == ((1, ('c', ), False, False), [1, 5]) - args = ArgumentsForTranslation(space, [1], {'c': 5, 'd': 7}) + args = make_arguments_for_translation(space, [1], {'c': 5, 'd': 7}) assert args.flatten() == ((1, ('c', 'd'), False, False), [1, 5, 7]) - args = ArgumentsForTranslation(space, [1,2,3,4,5], {'e': 5, 'd': 7}) + args = make_arguments_for_translation(space, [1,2,3,4,5], {'e': 5, 'd': 7}) assert args.flatten() == ((5, ('d', 'e'), False, False), [1, 2, 3, 4, 5, 7, 5]) - args = ArgumentsForTranslation(space, [], {}, + args = make_arguments_for_translation(space, [], {}, w_stararg=[1], w_starstararg={'c': 5, 'd': 7}) assert args.flatten() == ((0, (), True, True), [[1], {'c': 5, 'd': 7}]) - args = ArgumentsForTranslation(space, [1,2], {'g': 9}, + args = make_arguments_for_translation(space, [1,2], {'g': 9}, w_stararg=[3,4,5], w_starstararg={'e': 5, 'd': 7}) assert args.flatten() == ((2, ('g', ), True, True), [1, 2, 9, [3, 4, 5], {'e': 5, 'd': 7}]) @@ -324,7 +367,7 @@ space = DummySpace() var = object() shape = ((2, ('g', ), True, False), [1, 2, 9, var]) - args = ArgumentsForTranslation(space, [1,2], {'g': 9}, + args = make_arguments_for_translation(space, [1,2], {'g': 9}, w_stararg=var) assert args.flatten() == shape Modified: pypy/branch/improve-kwd-args/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/test/test_function.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/test/test_function.py Mon Oct 12 19:29:24 2009 @@ -480,20 +480,20 @@ w_meth1 = descr_function_get(space, func, obj1, space.type(obj1)) meth1 = space.unwrap(w_meth1) assert isinstance(meth1, Method) - assert meth1.call_args(args) == obj1 + assert meth1.call_args(args.copy()) == obj1 # Check method returned from method.__get__() # --- meth1 is already bound so meth1.__get__(*) is meth1. w_meth2 = meth1.descr_method_get(obj2, space.type(obj2)) meth2 = space.unwrap(w_meth2) assert isinstance(meth2, Method) - assert meth2.call_args(args) == obj1 + assert meth2.call_args(args.copy()) == obj1 # Check method returned from unbound_method.__get__() w_meth3 = descr_function_get(space, func, None, space.type(obj2)) meth3 = space.unwrap(w_meth3) w_meth4 = meth3.descr_method_get(obj2, space.w_None) meth4 = space.unwrap(w_meth4) assert isinstance(meth4, Method) - assert meth4.call_args(args) == obj2 + assert meth4.call_args(args.copy()) == obj2 # Check method returned from unbound_method.__get__() # --- with an incompatible class w_meth5 = meth3.descr_method_get(space.wrap('hello'), space.w_str) Modified: pypy/branch/improve-kwd-args/pypy/module/__builtin__/functional.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/module/__builtin__/functional.py (original) +++ pypy/branch/improve-kwd-args/pypy/module/__builtin__/functional.py Mon Oct 12 19:29:24 2009 @@ -108,6 +108,7 @@ compare = space.gt else: compare = space.lt + # XXXXXXXXX args, kwargs = arguments.unpack() if len(args) > 1: w_sequence = space.newtuple(args) @@ -120,9 +121,7 @@ w_key = kwargs["key"] except KeyError: w_key = None - else: - del kwargs["key"] - if kwargs: + if len(kwargs) != 1: msg = "%s() got unexpected keyword argument" % (implementation_of,) raise OperationError(space.w_TypeError, space.wrap(msg)) w_iter = space.iter(w_sequence) Modified: pypy/branch/improve-kwd-args/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/improve-kwd-args/pypy/module/pypyjit/interp_jit.py Mon Oct 12 19:29:24 2009 @@ -103,6 +103,7 @@ * set_param(name=value, ...) # as keyword arguments * set_param("name=value,name=value") # as a user-supplied string ''' + # XXXXXXXXX args_w, kwds_w = args.unpack() if len(args_w) > 1: msg = ("set_param() takes at most 1 non-keyword argument, %d given" Modified: pypy/branch/improve-kwd-args/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/objspace/flow/flowcontext.py (original) +++ pypy/branch/improve-kwd-args/pypy/objspace/flow/flowcontext.py Mon Oct 12 19:29:24 2009 @@ -397,5 +397,5 @@ class FlowSpaceFrame(pyframe.PyFrame): def make_arguments(self, nargs): return ArgumentsForTranslation(self.space, self.peekvalues(nargs)) - def argument_factory(self, arguments, keywords, w_star, w_starstar): - return ArgumentsForTranslation(self.space, arguments, keywords, w_star, w_starstar) + def argument_factory(self, *args): + return ArgumentsForTranslation(self.space, *args) Modified: pypy/branch/improve-kwd-args/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/objspace/flow/objspace.py (original) +++ pypy/branch/improve-kwd-args/pypy/objspace/flow/objspace.py Mon Oct 12 19:29:24 2009 @@ -370,7 +370,7 @@ return sc(self, fn, args) try: - args_w, kwds_w = args.unpack() + args_w, kwds_w = args.copy().unpack() except UnwrapException: args_w, kwds_w = '?', '?' # NOTE: annrpython needs to know about the following two operations! From arigo at codespeak.net Mon Oct 12 19:31:49 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Oct 2009 19:31:49 +0200 (CEST) Subject: [pypy-svn] r68350 - pypy/trunk/pypy/rpython/test Message-ID: <20091012173149.3EC59168049@codespeak.net> Author: arigo Date: Mon Oct 12 19:31:48 2009 New Revision: 68350 Modified: pypy/trunk/pypy/rpython/test/test_rdict.py Log: Make this test harder. Still works as far as I can tell, also in the CLI backend. Modified: pypy/trunk/pypy/rpython/test/test_rdict.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_rdict.py (original) +++ pypy/trunk/pypy/rpython/test/test_rdict.py Mon Oct 12 19:31:48 2009 @@ -451,9 +451,9 @@ def test_tuple_dict(self): def f(i): d = {} - d[(1, 2)] = 4 - d[(1, 3)] = 6 - return d[(1, i)] + d[(1, 4.5, (str(i), 2), 2)] = 4 + d[(1, 4.5, (str(i), 2), 3)] = 6 + return d[(1, 4.5, (str(i), 2), i)] res = self.interpret(f, [2]) assert res == f(2) From arigo at codespeak.net Mon Oct 12 19:47:12 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 12 Oct 2009 19:47:12 +0200 (CEST) Subject: [pypy-svn] r68351 - in pypy/branch/gc-hash/pypy/rlib: . test Message-ID: <20091012174712.C5F93168049@codespeak.net> Author: arigo Date: Mon Oct 12 19:47:12 2009 New Revision: 68351 Modified: pypy/branch/gc-hash/pypy/rlib/objectmodel.py pypy/branch/gc-hash/pypy/rlib/test/test_objectmodel.py Log: Update the docstring of compute_hash() & friends, to only promize to work on RPython-level objects. Tests. Modified: pypy/branch/gc-hash/pypy/rlib/objectmodel.py ============================================================================== --- pypy/branch/gc-hash/pypy/rlib/objectmodel.py (original) +++ pypy/branch/gc-hash/pypy/rlib/objectmodel.py Mon Oct 12 19:47:12 2009 @@ -136,12 +136,9 @@ def compute_hash(x): """RPython equivalent of hash(x), where 'x' is an immutable - RPython-level or low-level object. For strings or unicodes it - computes the hash as in Python. For tuples it calls compute_hash() - recursively. For instances it uses compute_identity_hash(). - For low-level objects it returns the hash of the original RPython- - level object, if any, or just compute_identity_hash() otherwise. - It cannot be used on llmemory.GCREF. + RPython-level. For strings or unicodes it computes the hash as + in Python. For tuples it calls compute_hash() recursively. + For instances it uses compute_identity_hash(). Note that this can return 0 or -1 too. @@ -170,9 +167,9 @@ """RPython equivalent of object.__hash__(x). This returns the so-called 'identity hash', which is the non-overridable default hash of Python. Can be called for any RPython-level object that turns - into a GC object, or for any low-level GC object, including - llmemory.GCREF. The value is not guaranteed to be the same before - and after translation, except for RPython instances on the lltypesystem. + into a GC object, but not NULL. The value is not guaranteed to be the + same before and after translation, except for RPython instances on the + lltypesystem. """ result = object.__hash__(x) try: @@ -206,14 +203,13 @@ from pypy.rlib.rarithmetic import intmask length = len(s) if length == 0: - x = 0 - else: - x = ord(s[0]) << 7 - i = 0 - while i < length: - x = (1000003*x) ^ ord(s[i]) - i += 1 - x ^= length + return 0 + x = ord(s[0]) << 7 + i = 0 + while i < length: + x = (1000003*x) ^ ord(s[i]) + i += 1 + x ^= length return intmask(x) def _hash_float(f): @@ -222,12 +218,13 @@ except the fact that the integer case is not treated specially. In RPython, floats cannot be used with ints in dicts, anyway. """ + from pypy.rlib.rarithmetic import intmask v, expo = math.frexp(f) v *= TAKE_NEXT hipart = int(v) v = (v - float(hipart)) * TAKE_NEXT x = hipart + int(v) + (expo << 15) - return x + return intmask(x) TAKE_NEXT = float(2**31) def _hash_tuple(t): @@ -237,10 +234,11 @@ that nested tuples are very uncommon in RPython, making the case unlikely. """ + from pypy.rlib.rarithmetic import intmask x = 0x345678 for item in t: y = compute_hash(item) - x = (1000003 * x) ^ y + x = intmask((1000003 * x) ^ y) return x # ---------- Modified: pypy/branch/gc-hash/pypy/rlib/test/test_objectmodel.py ============================================================================== --- pypy/branch/gc-hash/pypy/rlib/test/test_objectmodel.py (original) +++ pypy/branch/gc-hash/pypy/rlib/test/test_objectmodel.py Mon Oct 12 19:47:12 2009 @@ -141,6 +141,45 @@ py.test.raises(TypeError, "s1 < s2") py.test.raises(TypeError, "hash(s1)") +def test_compute_hash(): + from pypy.rlib.objectmodel import _hash_string, _hash_float, _hash_tuple + assert compute_hash("Hello") == _hash_string("Hello") + assert compute_hash(7) == 7 + assert compute_hash(-3.5) == _hash_float(-3.5) + assert compute_hash(None) == 0 + assert compute_hash(("world", None, 7)) == _hash_tuple(("world", None, 7)) + # + class Foo(object): + def __hash__(self): + return 42 + foo = Foo() + h = compute_hash(foo) + assert h == object.__hash__(foo) + assert h == getattr(foo, '__precomputed_identity_hash') + assert compute_hash(None) == 0 + +def test_compute_identity_hash(): + class Foo(object): + def __hash__(self): + return 42 + foo = Foo() + h = compute_identity_hash(foo) + assert h == object.__hash__(foo) + assert h == getattr(foo, '__precomputed_identity_hash') + +def test_compute_unique_id(): + class Foo(object): + pass + foo = Foo() + assert compute_unique_id(foo) == id(foo) + +def test_current_object_addr_as_int(): + from pypy.rlib.rarithmetic import intmask + class Foo(object): + pass + foo = Foo() + assert current_object_addr_as_int(foo) == intmask(id(foo)) + class BaseTestObjectModel(BaseRtypingTest): def test_we_are_translated(self): @@ -270,6 +309,27 @@ res = self.interpret(g, [3]) assert res == 77 + def test_compute_hash(self): + class Foo(object): + pass + def f(i): + assert compute_hash(i) == compute_hash(42) + assert compute_hash(i+1.0) == compute_hash(43.0) + assert compute_hash("Hello" + str(i)) == compute_hash("Hello42") + if i == 42: + p = None + else: + p = Foo() + assert compute_hash(p) == compute_hash(None) + assert (compute_hash(("world", None, i, 7.5)) == + compute_hash(("world", None, 42, 7.5))) + q = Foo() + assert compute_hash(q) == compute_identity_hash(q) + return i*2 + res = self.interpret(f, [42]) + assert res == 84 + + class TestLLtype(BaseTestObjectModel, LLRtypeMixin): def test_rtype_keepalive(self): @@ -283,6 +343,33 @@ res = self.interpret(f, []) assert res == 1 + def test_compute_hash_across_translation(self): + class Foo(object): + pass + q = Foo() + + def f(i): + assert compute_hash(i) == h_42 + assert compute_hash(i+1.0) == h_43_dot_0 + assert compute_hash("Hello" + str(i)) == h_Hello42 + if i == 42: + p = None + else: + p = Foo() + assert compute_hash(p) == h_None + assert compute_hash(("world", None, i, 7.5)) == h_tuple + assert compute_hash(q) == h_q + return i*2 + h_42 = compute_hash(42) + h_43_dot_0 = compute_hash(43.0) + h_Hello42 = compute_hash("Hello42") + h_None = compute_hash(None) + h_tuple = compute_hash(("world", None, 42, 7.5)) + h_q = compute_hash(q) + + res = self.interpret(f, [42]) + assert res == 84 + class TestOOtype(BaseTestObjectModel, OORtypeMixin): pass From fijal at codespeak.net Mon Oct 12 20:24:14 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 12 Oct 2009 20:24:14 +0200 (CEST) Subject: [pypy-svn] r68352 - in pypy/branch/inline-fastpath-malloc/pypy/rpython: . lltypesystem Message-ID: <20091012182414.089BF16804C@codespeak.net> Author: fijal Date: Mon Oct 12 20:24:14 2009 New Revision: 68352 Modified: pypy/branch/inline-fastpath-malloc/pypy/rpython/llinterp.py pypy/branch/inline-fastpath-malloc/pypy/rpython/lltypesystem/lloperation.py Log: This is another needed operation Modified: pypy/branch/inline-fastpath-malloc/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/inline-fastpath-malloc/pypy/rpython/llinterp.py (original) +++ pypy/branch/inline-fastpath-malloc/pypy/rpython/llinterp.py Mon Oct 12 20:24:14 2009 @@ -854,6 +854,9 @@ def op_gc_adr_of_nursery_free(self): raise NotImplementedError + def op_gc_size_of_header(self): + raise NotImplementedError + def op_gc_call_rtti_destructor(self, rtti, addr): if hasattr(rtti._obj, 'destructor_funcptr'): d = rtti._obj.destructor_funcptr Modified: pypy/branch/inline-fastpath-malloc/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/inline-fastpath-malloc/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/inline-fastpath-malloc/pypy/rpython/lltypesystem/lloperation.py Mon Oct 12 20:24:14 2009 @@ -446,6 +446,7 @@ # ^^^ returns an address of nursery free pointer, for later modifications 'gc_adr_of_nursery_top' : LLOp(), # ^^^ returns an address of pointer, since it can change at runtime + 'gc_size_of_header' : LLOp(), # experimental operations in support of thread cloning, only # implemented by the Mark&Sweep GC From cfbolz at codespeak.net Mon Oct 12 22:23:57 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Oct 2009 22:23:57 +0200 (CEST) Subject: [pypy-svn] r68353 - in pypy/branch/improve-kwd-args/pypy: annotation interpreter interpreter/test module/__builtin__ objspace/flow rpython Message-ID: <20091012202357.C458016800B@codespeak.net> Author: cfbolz Date: Mon Oct 12 22:23:56 2009 New Revision: 68353 Modified: pypy/branch/improve-kwd-args/pypy/annotation/bookkeeper.py pypy/branch/improve-kwd-args/pypy/interpreter/argument.py pypy/branch/improve-kwd-args/pypy/interpreter/gateway.py pypy/branch/improve-kwd-args/pypy/interpreter/test/test_argument.py pypy/branch/improve-kwd-args/pypy/interpreter/test/test_compiler.py pypy/branch/improve-kwd-args/pypy/interpreter/test/test_function.py pypy/branch/improve-kwd-args/pypy/interpreter/test/test_gateway.py pypy/branch/improve-kwd-args/pypy/module/__builtin__/functional.py pypy/branch/improve-kwd-args/pypy/objspace/flow/objspace.py pypy/branch/improve-kwd-args/pypy/rpython/callparse.py pypy/branch/improve-kwd-args/pypy/rpython/rbuiltin.py Log: make Arguments immutable again and fix variaous breakages Modified: pypy/branch/improve-kwd-args/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/improve-kwd-args/pypy/annotation/bookkeeper.py Mon Oct 12 22:23:56 2009 @@ -762,6 +762,7 @@ getattr(s_obj, 'from_ellipsis', False)): # see newtuple() return [Ellipsis] raise CallPatternTooComplex, "'*' argument must be SomeTuple" + viewiterable = unpackiterable def is_w(self, one, other): return one is other Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/argument.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/argument.py Mon Oct 12 22:23:56 2009 @@ -30,12 +30,6 @@ make_sure_not_resized(self.arguments_w) self._combine_wrapped(w_stararg, w_starstararg) - @staticmethod - def factory(space, args_w, kwds_w=None, - w_stararg=None, w_starstararg=None): - return Arguments(space, args_w, kwds_w, - w_stararg, w_starstararg) - def num_args(self): # only used in module/__builtin__/interp_classobj.py return len(self.arguments_w) @@ -64,8 +58,8 @@ def prepend(self, w_firstarg): # used often "Return a new Arguments with a new argument inserted first." - return self.factory(self.space, [w_firstarg] + self.arguments_w, - self.keywords, self.keywords_w) + return Arguments(self.space, [w_firstarg] + self.arguments_w, + self.keywords, self.keywords_w) def _combine_wrapped(self, w_stararg, w_starstararg): "unpack the *arg and **kwd into arguments_w and keywords_w" @@ -159,7 +153,6 @@ upfront = 0 args_w = self.arguments_w - assert args_w is not None, "tried to match arguments again" num_args = len(args_w) keywords = self.keywords @@ -184,7 +177,9 @@ # the code assumes that keywords can potentially be large, but that # argnames is typically not too large num_remainingkwds = num_kwds + used_keywords = None if keywords: + used_keywords = [False] * num_kwds for i in range(num_kwds): name = keywords[i] try: @@ -202,7 +197,7 @@ else: assert scope_w[j] is None scope_w[j] = keywords_w[i] - keywords[i] = None # mark as used + used_keywords[i] = True # mark as used num_remainingkwds -= 1 missing = 0 if input_argcount < co_argcount: @@ -242,21 +237,18 @@ w_kwds = self.space.newdict() if num_remainingkwds: for i in range(len(keywords)): - key = keywords[i] - if key is not None: + if not used_keywords[i]: + key = keywords[i] self.space.setitem(w_kwds, self.space.wrap(key), keywords_w[i]) scope_w[co_argcount + has_vararg] = w_kwds elif num_remainingkwds: - raise ArgErrUnknownKwds(num_remainingkwds, keywords) + raise ArgErrUnknownKwds(num_remainingkwds, keywords, used_keywords) if missing: raise ArgErrCount(avail, num_kwds, (co_argcount, has_vararg, has_kwarg), defaults_w, missing) - self.arguments_w = None - self.keywords = None - self.keywords_w = None return co_argcount + has_vararg + has_kwarg @@ -358,17 +350,6 @@ data_w[shape_cnt:end_keys], w_star, w_starstar) - def copy(self): - if self.keywords is None: - keywords = None - keywords_w = None - else: - keywords = self.keywords[:] - keywords_w = self.keywords_w[:] - return Arguments(self.space, self.arguments_w[:], keywords, - keywords_w) - - class ArgumentsForTranslation(Arguments): def __init__(self, space, args_w, keywords=None, keywords_w=None, w_stararg=None, w_starstararg=None): @@ -383,23 +364,19 @@ self._combine_wrapped(self.w_stararg, self.w_starstararg) self.combine_has_happened = True - @staticmethod - def factory(space, args_w, kwds_w=None, - w_stararg=None, w_starstararg=None): - return ArgumentsForTranslation(space, args_w, kwds_w, - w_stararg, w_starstararg) + def prepend(self, w_firstarg): # used often + "Return a new Arguments with a new argument inserted first." + return ArgumentsForTranslation(self.space, [w_firstarg] + self.arguments_w, + self.keywords, self.keywords_w, self.w_stararg, + self.w_starstararg) def copy(self): - if self.keywords is None: - keywords = None - keywords_w = None - else: - keywords = self.keywords[:] - keywords_w = self.keywords_w[:] - return ArgumentsForTranslation(self.space, self.arguments_w[:], keywords, - keywords_w, self.w_stararg, + return ArgumentsForTranslation(self.space, self.arguments_w, + self.keywords, self.keywords_w, self.w_stararg, self.w_starstararg) + + def _match_signature(self, w_firstarg, scope_w, argnames, has_vararg=False, has_kwarg=False, defaults_w=[], blindargs=0): self.combine_if_necessary() @@ -598,13 +575,13 @@ class ArgErrUnknownKwds(ArgErr): - def __init__(self, num_remainingkwds, keywords): + def __init__(self, num_remainingkwds, keywords, used_keywords): self.kwd_name = '' self.num_kwds = num_remainingkwds if num_remainingkwds == 1: - for kwd_name in keywords: - if kwd_name is not None: - self.kwd_name = kwd_name + for i in range(len(keywords)): + if not used_keywords[i]: + self.kwd_name = keywords[i] break def getmsg(self, fnname): Modified: pypy/branch/improve-kwd-args/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/gateway.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/gateway.py Mon Oct 12 22:23:56 2009 @@ -870,10 +870,11 @@ else: # ...which is merged with the previous arguments, if any if len(args_w) > 1: + # XXXXXXXXX more_args_w, more_kwds_w = args.unpack() args = Arguments(space, list(args_w[:-1]) + more_args_w, - more_kwds_w) + more_kwds_w.keys(), more_kwds_w.values()) w_func = self.wget(space, name) return space.call_args(w_func, args) def get_function(space): Modified: pypy/branch/improve-kwd-args/pypy/interpreter/test/test_argument.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/test/test_argument.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/test/test_argument.py Mon Oct 12 22:23:56 2009 @@ -241,43 +241,43 @@ space = DummySpace() args = make_arguments_for_translation(space, [1,2,3]) sig = (['a', 'b', 'c'], None, None) - data = args.copy().match_signature(sig, []) + data = args.match_signature(sig, []) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() args = make_arguments_for_translation(space, [1]) sig = (['a', 'b', 'c'], None, None) - data = args.copy().match_signature(sig, [2, 3]) + data = args.match_signature(sig, [2, 3]) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() args = make_arguments_for_translation(space, [1,2,3,4,5]) sig = (['a', 'b', 'c'], 'r', None) - data = args.copy().match_signature(sig, []) + data = args.match_signature(sig, []) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() args = make_arguments_for_translation(space, [1], {'c': 3, 'b': 2}) sig = (['a', 'b', 'c'], None, None) - data = args.copy().match_signature(sig, []) + data = args.match_signature(sig, []) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() args = make_arguments_for_translation(space, [1], {'c': 5}) sig = (['a', 'b', 'c'], None, None) - data = args.copy().match_signature(sig, [2, 3]) + data = args.match_signature(sig, [2, 3]) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() args = make_arguments_for_translation(space, [1], {'c': 5, 'd': 7}) sig = (['a', 'b', 'c'], None, 'kw') - data = args.copy().match_signature(sig, [2, 3]) + data = args.match_signature(sig, [2, 3]) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() args = make_arguments_for_translation(space, [1,2,3,4,5], {'e': 5, 'd': 7}) sig = (['a', 'b', 'c'], 'r', 'kw') - data = args.copy().match_signature(sig, [2, 3]) + data = args.match_signature(sig, [2, 3]) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() @@ -285,7 +285,7 @@ w_stararg=[1], w_starstararg={'c': 5, 'd': 7}) sig = (['a', 'b', 'c'], None, 'kw') - data = args.copy().match_signature(sig, [2, 3]) + data = args.match_signature(sig, [2, 3]) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() @@ -293,7 +293,7 @@ w_stararg=[3,4,5], w_starstararg={'e': 5, 'd': 7}) sig = (['a', 'b', 'c'], 'r', 'kw') - data = args.copy().match_signature(sig, [2, 3]) + data = args.match_signature(sig, [2, 3]) new_args = args.unmatch_signature(sig, data) assert args.unpack() == new_args.unpack() Modified: pypy/branch/improve-kwd-args/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/test/test_compiler.py Mon Oct 12 22:23:56 2009 @@ -330,8 +330,8 @@ space = self.space w_mod = space.appexec((), '():\n import warnings\n return warnings\n') #sys.getmodule('warnings') w_filterwarnings = space.getattr(w_mod, space.wrap('filterwarnings')) - filter_arg = Arguments(space, [ space.wrap('error') ], - dict(module=space.wrap(''))) + filter_arg = Arguments(space, [ space.wrap('error') ], ["module"], + [space.wrap("")]) for code in (''' def wrong1(): Modified: pypy/branch/improve-kwd-args/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/test/test_function.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/test/test_function.py Mon Oct 12 22:23:56 2009 @@ -480,20 +480,20 @@ w_meth1 = descr_function_get(space, func, obj1, space.type(obj1)) meth1 = space.unwrap(w_meth1) assert isinstance(meth1, Method) - assert meth1.call_args(args.copy()) == obj1 + assert meth1.call_args(args) == obj1 # Check method returned from method.__get__() # --- meth1 is already bound so meth1.__get__(*) is meth1. w_meth2 = meth1.descr_method_get(obj2, space.type(obj2)) meth2 = space.unwrap(w_meth2) assert isinstance(meth2, Method) - assert meth2.call_args(args.copy()) == obj1 + assert meth2.call_args(args) == obj1 # Check method returned from unbound_method.__get__() w_meth3 = descr_function_get(space, func, None, space.type(obj2)) meth3 = space.unwrap(w_meth3) w_meth4 = meth3.descr_method_get(obj2, space.w_None) meth4 = space.unwrap(w_meth4) assert isinstance(meth4, Method) - assert meth4.call_args(args.copy()) == obj2 + assert meth4.call_args(args) == obj2 # Check method returned from unbound_method.__get__() # --- with an incompatible class w_meth5 = meth3.descr_method_get(space.wrap('hello'), space.w_str) Modified: pypy/branch/improve-kwd-args/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/test/test_gateway.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/test/test_gateway.py Mon Oct 12 22:23:56 2009 @@ -77,7 +77,7 @@ gateway.W_Root, gateway.Arguments]) w = self.space.wrap - args = argument.Arguments(self.space, [w(123), w(23)], {}, + args = argument.Arguments(self.space, [w(123), w(23)], [], [], w_stararg = w((0, True)), w_starstararg = w({'boo': 10})) w_result = code.funcrun(FakeFunc(self.space, "c"), args) @@ -116,8 +116,7 @@ gg = gateway.app2interp_temp(app_general) args = gateway.Arguments(self.space, [w(6), w(7)]) assert self.space.int_w(gg(self.space, w(3), args)) == 23 - args = gateway.Arguments(self.space, [w(6)], {'hello': w(7), - 'world': w(8)}) + args = gateway.Arguments(self.space, [w(6)], ['hello', 'world'], [w(7), w(8)]) assert self.space.int_w(gg(self.space, w(3), args)) == 213 def test_interp2app(self): Modified: pypy/branch/improve-kwd-args/pypy/module/__builtin__/functional.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/module/__builtin__/functional.py (original) +++ pypy/branch/improve-kwd-args/pypy/module/__builtin__/functional.py Mon Oct 12 22:23:56 2009 @@ -121,7 +121,7 @@ w_key = kwargs["key"] except KeyError: w_key = None - if len(kwargs) != 1: + if len(kwargs) > 1: msg = "%s() got unexpected keyword argument" % (implementation_of,) raise OperationError(space.w_TypeError, space.wrap(msg)) w_iter = space.iter(w_sequence) Modified: pypy/branch/improve-kwd-args/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/objspace/flow/objspace.py (original) +++ pypy/branch/improve-kwd-args/pypy/objspace/flow/objspace.py Mon Oct 12 22:23:56 2009 @@ -360,6 +360,12 @@ return self.do_operation_with_implicit_exceptions('setitem', w_obj, w_key, w_val) + def call_function(self, w_func, *args_w): + from pypy.interpreter.argument import ArgumentsForTranslation + nargs = len(args_w) + args = ArgumentsForTranslation(self, list(args_w)) + return self.call_args(w_func, args) + def call_args(self, w_callable, args): try: fn = self.unwrap(w_callable) Modified: pypy/branch/improve-kwd-args/pypy/rpython/callparse.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/rpython/callparse.py (original) +++ pypy/branch/improve-kwd-args/pypy/rpython/callparse.py Mon Oct 12 22:23:56 2009 @@ -178,6 +178,7 @@ raise ValueError return list(items) raise CallPatternTooComplex, "'*' argument must be a tuple" + viewiterable = unpackiterable def is_w(self, one, other): return one is other Modified: pypy/branch/improve-kwd-args/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/rpython/rbuiltin.py (original) +++ pypy/branch/improve-kwd-args/pypy/rpython/rbuiltin.py Mon Oct 12 22:23:56 2009 @@ -52,9 +52,10 @@ def call_args_expand(hop, takes_kwds = True): hop = hop.copy() - from pypy.interpreter.argument import Arguments - arguments = Arguments.fromshape(None, hop.args_s[1].const, # shape - range(hop.nb_args-2)) + from pypy.interpreter.argument import ArgumentsForTranslation + arguments = ArgumentsForTranslation.fromshape( + None, hop.args_s[1].const, # shape + range(hop.nb_args-2)) if arguments.w_starstararg is not None: raise TyperError("**kwds call not implemented") if arguments.w_stararg is not None: @@ -75,12 +76,13 @@ hop.args_s.append(s_tuple.items[i]) hop.args_r.append(r_tuple.items_r[i]) - kwds = arguments.kwds_w or {} - if not takes_kwds and kwds: + keywords = arguments.keywords + if not takes_kwds and keywords: raise TyperError("kwds args not supported") # prefix keyword arguments with 'i_' kwds_i = {} - for key, index in kwds.items(): + for i, key in enumerate(keywords): + index = arguments.keywords_w[i] kwds_i['i_'+key] = index return hop, kwds_i From cfbolz at codespeak.net Mon Oct 12 22:49:54 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 12 Oct 2009 22:49:54 +0200 (CEST) Subject: [pypy-svn] r68354 - in pypy/branch/improve-kwd-args/pypy: interpreter translator Message-ID: <20091012204954.65ADC168048@codespeak.net> Author: cfbolz Date: Mon Oct 12 22:49:53 2009 New Revision: 68354 Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py pypy/branch/improve-kwd-args/pypy/translator/geninterplevel.py Log: fix geninterp to no longer use fromshape Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/argument.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/argument.py Mon Oct 12 22:49:53 2009 @@ -331,25 +331,6 @@ space.setitem(w_kwds, space.wrap(key), w_value) return w_args, w_kwds - @staticmethod - def fromshape(space, (shape_cnt,shape_keys,shape_star,shape_stst), data_w): - # used by geninterped code - args_w = data_w[:shape_cnt] - p = end_keys = shape_cnt + len(shape_keys) - if shape_star: - w_star = data_w[p] - p += 1 - else: - w_star = None - if shape_stst: - w_starstar = data_w[p] - p += 1 - else: - w_starstar = None - return Arguments(space, args_w, list(shape_keys), - data_w[shape_cnt:end_keys], w_star, - w_starstar) - class ArgumentsForTranslation(Arguments): def __init__(self, space, args_w, keywords=None, keywords_w=None, w_stararg=None, w_starstararg=None): Modified: pypy/branch/improve-kwd-args/pypy/translator/geninterplevel.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/translator/geninterplevel.py (original) +++ pypy/branch/improve-kwd-args/pypy/translator/geninterplevel.py Mon Oct 12 22:49:53 2009 @@ -52,7 +52,7 @@ from pypy.interpreter.pycode import CO_VARARGS, CO_VARKEYWORDS from types import FunctionType, CodeType, ModuleType, MethodType from pypy.interpreter.error import OperationError -from pypy.interpreter.argument import Arguments +from pypy.interpreter.argument import ArgumentsForTranslation, Arguments from pypy.translator.backendopt.ssa import SSI_to_SSA from pypy.translator.translator import TranslationContext @@ -71,7 +71,7 @@ log = py.log.Producer("geninterp") py.log.setconsumer("geninterp", ansi_log) -GI_VERSION = '1.1.26' # bump this for substantial changes +GI_VERSION = '1.2.1' # bump this for substantial changes # ____________________________________________________________ try: @@ -264,16 +264,21 @@ v = op.args[0] exv = self.expr(v, localscope) fmt = ( - "_args = %(Arg)s.fromshape(space, %(shape)s, [%(data_w)s])\n" + "_args = %(Arg)s(space, [%(args_w)s], %(keywords)s, [%(keywords_w)s], %(w_stararg)s, %(w_starstararg)s)\n" "%(res)s = space.call_args(%(func)s, _args)") assert isinstance(op.args[1], Constant) shape = op.args[1].value # make a list out of the second shape elt. shape = shape[0], list(shape[1]), shape[2], shape[3] + arglist = [self.expr(arg, localscope) for arg in op.args[2:]] + args = ArgumentsForTranslation.fromshape(None, shape, arglist) return fmt % {"res": self.expr(op.result, localscope), "func": exv, - "shape": repr(shape), - "data_w": self.arglist(op.args[2:], localscope), + "args_w": ", ".join(args.arguments_w), + "keywords": args.keywords, + "keywords_w": ", ".join(args.keywords_w), + "w_stararg": args.w_stararg, + "w_starstararg": args.w_starstararg, 'Arg': self.nameof(Arguments) } if op.opname == "hint": return "%s = %s" % (self.expr(op.result, localscope), From afa at codespeak.net Mon Oct 12 23:05:18 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 12 Oct 2009 23:05:18 +0200 (CEST) Subject: [pypy-svn] r68355 - pypy/branch/asmgcc-mingw32-2/pypy/translator/platform Message-ID: <20091012210518.752B916804C@codespeak.net> Author: afa Date: Mon Oct 12 23:05:17 2009 New Revision: 68355 Modified: pypy/branch/asmgcc-mingw32-2/pypy/translator/platform/__init__.py Log: (probably temporarily) don't trim stderr output I want to investigate a failure in trackgcroot.py Modified: pypy/branch/asmgcc-mingw32-2/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/asmgcc-mingw32-2/pypy/translator/platform/__init__.py (original) +++ pypy/branch/asmgcc-mingw32-2/pypy/translator/platform/__init__.py Mon Oct 12 23:05:17 2009 @@ -110,10 +110,8 @@ errorfile = outname.new(ext='errors') errorfile.write(stderr) stderrlines = stderr.splitlines() - for line in stderrlines[:50]: + for line in stderrlines: log.ERROR(line) - if len(stderrlines) > 50: - log.ERROR('...') raise CompilationError(stdout, stderr) From afa at codespeak.net Tue Oct 13 00:10:34 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 13 Oct 2009 00:10:34 +0200 (CEST) Subject: [pypy-svn] r68356 - pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/test/darwin Message-ID: <20091012221034.187DB168015@codespeak.net> Author: afa Date: Tue Oct 13 00:10:33 2009 New Revision: 68356 Added: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/test/darwin/track_tailcall0.s Log: A failing testcase Added: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/test/darwin/track_tailcall0.s ============================================================================== --- (empty file) +++ pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/test/darwin/track_tailcall0.s Tue Oct 13 00:10:33 2009 @@ -0,0 +1,29 @@ +_pypy_g___mm_le_0_perform_call: + subl $12, %esp + movl %ebx, (%esp) + movl %esi, 4(%esp) + movl %edi, 8(%esp) + call L82 +"L00000000004$pb": +L82: + popl %ebx + movl 16(%esp), %edi + movl 20(%esp), %esi +L79: + movl 4(%edi), %eax + movl 20(%eax), %ecx + incl %ecx + movl 4(%esi), %eax + movl 20(%eax), %edx + movl L_pypy_g_array_45$non_lazy_ptr-"L00000000004$pb"(%ebx), %eax + addl 8(%eax,%ecx,4), %edx + andl $255, %edx + movl L_pypy_g_array_2193$non_lazy_ptr-"L00000000004$pb"(%ebx), %eax + movl 8(%eax,%edx,4), %ecx + movl (%esp), %ebx + movl 4(%esp), %esi + movl 8(%esp), %edi + addl $12, %esp + jmp *%ecx +L80: + nop From afa at codespeak.net Tue Oct 13 00:18:09 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 13 Oct 2009 00:18:09 +0200 (CEST) Subject: [pypy-svn] r68357 - pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc Message-ID: <20091012221809.3027816804E@codespeak.net> Author: afa Date: Tue Oct 13 00:18:08 2009 New Revision: 68357 Modified: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py Log: Fix previous test Modified: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py Tue Oct 13 00:18:08 2009 @@ -35,8 +35,7 @@ r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+")\s*$") r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+([*]"+OPERAND+")\s*$") r_jmp_switch = re.compile(r"\tjmp\t[*]"+LABEL+"[(]") -r_jmptable_item = re.compile(r"\t.long\t"+LABEL+"\s*$") -r_jmptable_end = re.compile(r"\t.text|\t.section\s+.text") +r_jmp_source = re.compile(r"\d+[(](%[\w]+),") r_jmptable_item = re.compile(r"\t.long\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") r_jmptable_end = re.compile(r"\t.text|\t.section\s+.text|"+LABEL) r_binaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+"),\s*("+OPERAND+")\s*$") @@ -767,7 +766,14 @@ def walker(insn, locs): sources = [] for loc in locs: - sources.extend(insn.all_sources_of(loc)) + for s in insn.all_sources_of(loc): + # if the source looks like 8(%eax,%edx,4) + # %eax is the real source, %edx is an offset + match = r_jmp_source.match(s) + if match: + sources.append(match.group(1)) + else: + sources.append(s) for source in sources: label_match = re.compile(LABEL).match(source) if label_match: @@ -778,8 +784,9 @@ insn.previous_insns = [self.insns[-1]] self.walk_instructions_backwards(walker, insn, (operand,)) - # Probably an indirect tail-call. - tablelabels = [label for label in tablelabels if label in self.labels] + # Remove probable tail-calls + tablelabels = [label for label in tablelabels + if label in self.labels] assert len(tablelabels) <= 1 if tablelabels: tablelin = self.labels[tablelabels[0]].lineno + 1 From afa at codespeak.net Tue Oct 13 00:28:41 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 13 Oct 2009 00:28:41 +0200 (CEST) Subject: [pypy-svn] r68358 - in pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc: . test/darwin Message-ID: <20091012222841.046C216804E@codespeak.net> Author: afa Date: Tue Oct 13 00:28:40 2009 New Revision: 68358 Added: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/test/darwin/track_switch1.s Modified: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py Log: A switch table may end with a ".align" directive. Test and fix. Added: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/test/darwin/track_switch1.s ============================================================================== --- (empty file) +++ pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/test/darwin/track_switch1.s Tue Oct 13 00:28:40 2009 @@ -0,0 +1,149 @@ +_pypy_g___mm_mul_W_TransparentList_W_Root: + pushl %esi + pushl %ebx + subl $20, %esp + call L6833 +"L00000000284$pb": +L6833: + popl %ebx + movl 32(%esp), %edx +L6820: + movl 4(%edx), %eax + cmpl $61, 20(%eax) + je L6821 +L6822: + movl L_pypy_g_pypy_rpython_memory_gc_hybrid_HybridGC$non_lazy_ptr-"L00000000284$pb"(%ebx), %esi + movl 112(%esi), %edx + movl 124(%esi), %eax + subl %edx, %eax + cmpl $15, %eax + jle L6823 +L6825: + movl $14, (%edx) + leal 16(%edx), %eax + movl %eax, 112(%esi) +L6826: + movl L_pypy_g_pypy_objspace_std_multimethod_FailedToImplement_$non_lazy_ptr-"L00000000284$pb"(%ebx), %eax + movl %eax, 4(%edx) + movl $0, 8(%edx) + movl $0, 12(%edx) + movl %edx, 4(%esp) + movl 4(%edx), %eax + movl %eax, (%esp) + call L_pypy_g_RPyRaiseException$stub + ;; expected {28(%esp) | 20(%esp), 24(%esp), %edi, %ebp | } + xorl %ecx, %ecx +L6827: + movl %ecx, %eax + addl $20, %esp + popl %ebx + popl %esi + ret + .align 4,0x90 +L6821: + movl 36(%esp), %ecx + movl 4(%ecx), %eax + cmpl $66, 20(%eax) + ja L6822 + movl 20(%eax), %eax + movl L6831-"L00000000284$pb"(%ebx,%eax,4), %eax + addl %ebx, %eax + jmp *%eax + .align 2,0x90 +L6831: + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6822-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6822-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6822-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6822-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .long L6830-"L00000000284$pb" + .align 4,0x90 +L6823: + movl %esi, (%esp) + call L_pypy_g_GenerationGC_collect_nursery$stub + ;; expected {28(%esp) | 20(%esp), 24(%esp), %edi, %ebp | } + xorl %ecx, %ecx + movl %eax, %edx + movl L_pypy_g_ExcData$non_lazy_ptr-"L00000000284$pb"(%ebx), %eax + movl (%eax), %eax + testl %eax, %eax + je L6825 + movl %ecx, %eax + addl $20, %esp + popl %ebx + popl %esi + ret +L6830: + /* keepalive 36(%esp) */ + movl 36(%esp), %eax + movl %eax, 8(%esp) + movl L_pypy_g_pypy_objspace_std_stringobject_W_StringObject_574$non_lazy_ptr-"L00000000284$pb"(%ebx), %eax + movl %eax, 4(%esp) + movl 8(%edx), %eax + movl %eax, (%esp) + call L_pypy_g_call_function__star_2$stub + ;; expected {28(%esp) | 20(%esp), 24(%esp), %edi, %ebp | } + movl %eax, %ecx + jmp L6827 + .align 4,0x90 Modified: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py Tue Oct 13 00:28:40 2009 @@ -37,7 +37,7 @@ r_jmp_switch = re.compile(r"\tjmp\t[*]"+LABEL+"[(]") r_jmp_source = re.compile(r"\d+[(](%[\w]+),") r_jmptable_item = re.compile(r"\t.long\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") -r_jmptable_end = re.compile(r"\t.text|\t.section\s+.text|"+LABEL) +r_jmptable_end = re.compile(r"\t.text|\t.section\s+.text|\t\.align|"+LABEL) r_binaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+"),\s*("+OPERAND+")\s*$") LOCALVAR = r"%eax|%edx|%ecx|%ebx|%esi|%edi|%ebp|\d*[(]%esp[)]" LOCALVARFP = LOCALVAR + r"|-?\d*[(]%ebp[)]" From afa at codespeak.net Tue Oct 13 00:53:27 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 13 Oct 2009 00:53:27 +0200 (CEST) Subject: [pypy-svn] r68359 - in pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc: . test/darwin Message-ID: <20091012225327.9E740168050@codespeak.net> Author: afa Date: Tue Oct 13 00:53:26 2009 New Revision: 68359 Added: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/test/darwin/track_tailcall1.s Modified: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py Log: Another failing test, and its fix. Added: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/test/darwin/track_tailcall1.s ============================================================================== --- (empty file) +++ pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/test/darwin/track_tailcall1.s Tue Oct 13 00:53:26 2009 @@ -0,0 +1,344 @@ +_pypy_g_call__Type: + subl $92, %esp + movl %ebx, 76(%esp) + movl %esi, 80(%esp) + movl %edi, 84(%esp) + movl %ebp, 88(%esp) + call L12381 +"L00000000489$pb": +L12381: + popl %ebx + movl 96(%esp), %esi + movl 100(%esp), %edi +L12333: + movl L_pypy_g_pypy_objspace_std_typeobject_W_TypeObject_25$non_lazy_ptr-"L00000000489$pb"(%ebx), %eax + movl %eax, 28(%esp) + cmpl %eax, %esi + je L12334 + movl %esi, %ebp +L12336: + movl 416(%ebp), %eax + testl %eax, %eax + je L12337 +L12338: + movl %edi, 8(%esp) + movl %ebp, 4(%esp) + movl %eax, (%esp) + call L_pypy_g_ObjSpace_call_obj_args$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | %edi} + movl %edi, %edx + movl L___gcmapend$non_lazy_ptr-"L00000000489$pb"(%ebx), %ecx + /* GCROOT %edx */ + movl L_pypy_g_ExcData$non_lazy_ptr-"L00000000489$pb"(%ebx), %esi + movl (%esi), %esi + testl %esi, %esi + je L12378 +L12339: + xorl %eax, %eax +L12346: + movl 76(%esp), %ebx + movl 80(%esp), %esi + movl 84(%esp), %edi + movl 88(%esp), %ebp + addl $92, %esp + ret + .align 4,0x90 +L12378: + movl %edx, %edi + movl %eax, %esi +L12341: + movl L_pypy_g_rpy_string_154$non_lazy_ptr-"L00000000489$pb"(%ebx), %eax + movl %eax, 4(%esp) + movl %esi, (%esp) + call L_pypy_g_lookup____init__$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | %esi, %edi} + movl %edi, %edx + movl L___gcmapend$non_lazy_ptr-"L00000000489$pb"(%ebx), %ecx + /* GCROOT %edx */ + /* GCROOT %esi */ + movl L_pypy_g_ExcData$non_lazy_ptr-"L00000000489$pb"(%ebx), %ecx + movl (%ecx), %ecx + testl %ecx, %ecx + jne L12339 +L12342: + movl %edx, 8(%esp) + movl %esi, 4(%esp) + movl %eax, (%esp) + call L_pypy_g_get_and_call_args$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | %esi} + movl %eax, %edx + movl %esi, %eax + movl L___gcmapend$non_lazy_ptr-"L00000000489$pb"(%ebx), %ecx + /* GCROOT %eax */ + movl L_pypy_g_ExcData$non_lazy_ptr-"L00000000489$pb"(%ebx), %esi + movl (%esi), %ebp + testl %ebp, %ebp + jne L12339 +L12343: + cmpl L_pypy_g_pypy_objspace_std_noneobject_W_NoneObject$non_lazy_ptr-"L00000000489$pb"(%ebx), %edx + je L12346 +L12344: + movl L_pypy_g_pypy_rpython_memory_gc_hybrid_HybridGC$non_lazy_ptr-"L00000000489$pb"(%ebx), %esi + movl 112(%esi), %edx + movl 124(%esi), %eax + subl %edx, %eax + cmpl $19, %eax + jle L12347 + movl %edx, %ecx +L12349: + movl $4, (%ecx) + leal 20(%ecx), %eax + movl %eax, 112(%esi) +L12350: + movl L_pypy_g_pypy_interpreter_error_OperationError_vtable$non_lazy_ptr-"L00000000489$pb"(%ebx), %edx + movl %edx, 4(%ecx) + movl L_pypy_g_pypy_objspace_std_typeobject_W_TypeObject$non_lazy_ptr-"L00000000489$pb"(%ebx), %eax + movl %eax, 12(%ecx) + movl L_pypy_g_pypy_objspace_std_stringobject_W_StringObject_775$non_lazy_ptr-"L00000000489$pb"(%ebx), %eax + movl %eax, 16(%ecx) + movl $0, 8(%ecx) +L12351: + movl %ecx, 4(%esp) + movl %edx, (%esp) + call L_pypy_g_RPyRaiseException$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | } + xorl %eax, %eax + jmp L12346 + .align 4,0x90 +L12337: + movl L_pypy_g_rpy_string_211$non_lazy_ptr-"L00000000489$pb"(%ebx), %eax + movl %eax, 4(%esp) + movl %ebp, (%esp) + call L_pypy_g_W_TypeObject_lookup_where_with_method_cache$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | %edi, %ebp} + movl L___gcmapend$non_lazy_ptr-"L00000000489$pb"(%ebx), %edx + /* GCROOT %ebp */ + /* GCROOT %edi */ + movl L_pypy_g_ExcData$non_lazy_ptr-"L00000000489$pb"(%ebx), %ecx + movl (%ecx), %esi + testl %esi, %esi + jne L12339 +L12353: + movl 4(%eax), %esi + movl $0, 8(%esp) + movl %ebp, 4(%esp) + movl 8(%eax), %eax + movl %eax, (%esp) + call L_pypy_g_get$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | %esi, %edi, %ebp} + movl %eax, 36(%esp) + movl L___gcmapend$non_lazy_ptr-"L00000000489$pb"(%ebx), %eax + /* GCROOT %ebp */ + /* GCROOT %edi */ + movl %edi, 40(%esp) + movl %esi, %eax + movl L___gcmapend$non_lazy_ptr-"L00000000489$pb"(%ebx), %edx + /* GCROOT %eax */ + movl %esi, %edx + movl L___gcmapend$non_lazy_ptr-"L00000000489$pb"(%ebx), %ecx + /* GCROOT %edx */ + movl L_pypy_g_ExcData$non_lazy_ptr-"L00000000489$pb"(%ebx), %esi + movl (%esi), %ecx + testl %ecx, %ecx + jne L12339 +L12354: + testl %eax, %eax + je L12356 +L12355: + movl 4(%eax), %eax + movl (%eax), %eax + cmpl $623, %eax + jle L12356 +L12367: + cmpl $629, %eax + jg L12356 +L12368: + testb $2, 9(%edx) + jne L12356 +L12369: + cmpl 28(%esp), %edx + je L12356 +L12370: + testb $8, 2(%ebp) + je L12372 +L12371: + movl 36(%esp), %eax + movl %eax, 4(%esp) + movl %ebp, (%esp) + call L_pypy_g_remember_young_pointer$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | %ebp, 40(%esp)} +L12372: + movl 36(%esp), %esi + movl %esi, 416(%ebp) +L12356: + movl 40(%esp), %eax + movl %eax, 8(%esp) + movl %ebp, 4(%esp) + movl 36(%esp), %edx + movl %edx, (%esp) + call L_pypy_g_ObjSpace_call_obj_args$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | %ebp, 40(%esp)} + movl %eax, %edi + movl %ebp, %esi + movl L___gcmapend$non_lazy_ptr-"L00000000489$pb"(%ebx), %ecx + /* GCROOT %esi */ + movl %esi, 44(%esp) + movl 40(%esp), %eax + /* GCROOT %eax */ + movl %eax, 48(%esp) + movl %ebp, %esi + /* GCROOT %esi */ + movl L_pypy_g_ExcData$non_lazy_ptr-"L00000000489$pb"(%ebx), %edx + movl (%edx), %eax + testl %eax, %eax + jne L12339 +L12357: + movl 4(%edi), %eax + movl %edi, (%esp) + call *32(%eax) + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | %edi, 48(%esp)} + /* keepalive %eax */ + /* keepalive 44(%esp) */ + movl %esi, 4(%esp) + movl %eax, (%esp) + call L_pypy_g___mm_issubtype_perform_call$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | %edi, 48(%esp)} + movl %eax, %esi + movl %edi, %ebp + movl L___gcmapend$non_lazy_ptr-"L00000000489$pb"(%ebx), %ecx + /* GCROOT %ebp */ + movl 48(%esp), %edi + /* GCROOT %edi */ + movl L_pypy_g_ExcData$non_lazy_ptr-"L00000000489$pb"(%ebx), %eax + movl (%eax), %eax + testl %eax, %eax + jne L12339 +L12358: + movl %esi, (%esp) + call L_pypy_g___mm_nonzero_0_perform_call$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | %esi, %edi, %ebp} + movl L___gcmapend$non_lazy_ptr-"L00000000489$pb"(%ebx), %edx + /* GCROOT %ebp */ + movl %ebp, 52(%esp) + movl %ebp, %ecx + /* GCROOT %edi */ + movl %edi, 56(%esp) + /* GCROOT %esi */ + movl %esi, 60(%esp) + movl L_pypy_g_ExcData$non_lazy_ptr-"L00000000489$pb"(%ebx), %esi + movl (%esi), %ebp + testl %ebp, %ebp + je L12379 +L12359: + movl L_pypy_g_ExcData$non_lazy_ptr-"L00000000489$pb"(%ebx), %eax + movl 4(%eax), %esi + movl $0, 4(%eax) + movl $0, (%eax) + /* keepalive %esi */ + /* keepalive 60(%esp) */ + /* keepalive 52(%esp) */ + /* keepalive 56(%esp) */ + movl L_pypy_g_pypy_objspace_std_multimethod_FailedToImplement_$non_lazy_ptr-"L00000000489$pb"(%ebx), %eax + movl %eax, 4(%esp) + movl %ebp, (%esp) + call L_pypy_g_ll_issubclass__object_vtablePtr_object_vtablePtr$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | 52(%esp), 56(%esp)} + movl %ebp, %edx + movl %esi, %ecx + testb %al, %al + je L12351 +L12364: + movl 60(%esp), %edx + movl %edx, (%esp) + call L_pypy_g_is_true$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | 52(%esp), 56(%esp)} + movl %eax, %esi + movl 52(%esp), %eax + movl L___gcmapend$non_lazy_ptr-"L00000000489$pb"(%ebx), %ecx + /* GCROOT %eax */ + movl 56(%esp), %edx + /* GCROOT %edx */ + movl L_pypy_g_ExcData$non_lazy_ptr-"L00000000489$pb"(%ebx), %ecx + movl (%ecx), %ecx + testl %ecx, %ecx + jne L12339 + movl %eax, %ecx + movl %edx, %edi + movl %esi, %edx +L12361: + movl %ecx, %eax + movl %ecx, %esi + testb %dl, %dl + jne L12341 + jmp L12346 + .align 4,0x90 +L12334: + movl $1, 8(%esp) + movl %edi, 4(%esp) + movl 4(%edi), %eax + movsbl 22(%eax),%eax + movl %eax, (%esp) + call L_pypy_g_dispatcher_60$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | %esi, %edi} + movl %esi, %ecx + movl L___gcmapend$non_lazy_ptr-"L00000000489$pb"(%ebx), %edx + /* GCROOT %ecx */ + movl %ecx, %ebp + movl %edi, %edx + movl L___gcmapend$non_lazy_ptr-"L00000000489$pb"(%ebx), %esi + /* GCROOT %edx */ + movl %edx, %edi + movl L_pypy_g_ExcData$non_lazy_ptr-"L00000000489$pb"(%ebx), %esi + movl (%esi), %esi + movl %esi, 32(%esp) + testl %esi, %esi + je L12380 +L12373: + movl L_pypy_g_ExcData$non_lazy_ptr-"L00000000489$pb"(%ebx), %eax + movl 4(%eax), %esi + movl $0, 4(%eax) + movl $0, (%eax) + /* keepalive %esi */ + /* keepalive %edx */ + /* keepalive %ecx */ + movl L_pypy_g_exceptions_ValueError_vtable$non_lazy_ptr-"L00000000489$pb"(%ebx), %eax + movl %eax, 4(%esp) + movl 32(%esp), %edx + movl %edx, (%esp) + call L_pypy_g_ll_issubclass__object_vtablePtr_object_vtablePtr$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | %edi, %ebp} + movl 32(%esp), %edx + movl %esi, %ecx + testb %al, %al + je L12351 + jmp L12336 +L12380: +L12374: + cmpl $1, 4(%eax) + jne L12336 +L12375: + movl 8(%eax), %eax + movl 4(%eax), %edx + movl %eax, 96(%esp) + movl 32(%edx), %ecx + movl 76(%esp), %ebx + movl 80(%esp), %esi + movl 84(%esp), %edi + movl 88(%esp), %ebp + addl $92, %esp + jmp *%ecx +L12347: + movl %esi, (%esp) + call L_pypy_g_GenerationGC_collect_nursery$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | } + movl %eax, %ecx + movl L_pypy_g_ExcData$non_lazy_ptr-"L00000000489$pb"(%ebx), %eax + movl (%eax), %edi + testl %edi, %edi + jne L12339 + jmp L12349 +L12379: +L12360: + movzbl 8(%eax), %edx + jmp L12361 + .align 4,0x90 Modified: pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/asmgcc-mingw32-2/pypy/translator/c/gcc/trackgcroot.py Tue Oct 13 00:53:26 2009 @@ -35,7 +35,7 @@ r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+")\s*$") r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+([*]"+OPERAND+")\s*$") r_jmp_switch = re.compile(r"\tjmp\t[*]"+LABEL+"[(]") -r_jmp_source = re.compile(r"\d+[(](%[\w]+),") +r_jmp_source = re.compile(r"\d+[(](%[\w]+)[,)]") r_jmptable_item = re.compile(r"\t.long\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") r_jmptable_end = re.compile(r"\t.text|\t.section\s+.text|\t\.align|"+LABEL) r_binaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+"),\s*("+OPERAND+")\s*$") From pedronis at codespeak.net Tue Oct 13 11:04:41 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 13 Oct 2009 11:04:41 +0200 (CEST) Subject: [pypy-svn] r68363 - pypy/branch/improve-kwd-args/pypy/objspace/std Message-ID: <20091013090441.BED2916802D@codespeak.net> Author: pedronis Date: Tue Oct 13 11:04:40 2009 New Revision: 68363 Modified: pypy/branch/improve-kwd-args/pypy/objspace/std/proxyobject.py Log: (cfbolz, pedronis) fix proxyobject.py, not in the nicest way Modified: pypy/branch/improve-kwd-args/pypy/objspace/std/proxyobject.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/objspace/std/proxyobject.py (original) +++ pypy/branch/improve-kwd-args/pypy/objspace/std/proxyobject.py Tue Oct 13 11:04:40 2009 @@ -22,10 +22,11 @@ self.space = space def descr_call_mismatch(self, space, name, reqcls, args): - args_w, kwds_w = args.unpack() - args_w = args_w[:] + # xxx fishing + args_w = args.arguments_w[:] args_w[0] = space.wrap(name) - args = argument.Arguments(space, args_w, kwds_w) + args = argument.Arguments(space, args_w, args.keywords, + args.keywords_w) return space.call_args(self.w_controller, args) def getclass(self, space): From cfbolz at codespeak.net Tue Oct 13 12:01:44 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 13 Oct 2009 12:01:44 +0200 (CEST) Subject: [pypy-svn] r68364 - pypy/branch/improve-kwd-args/pypy/interpreter Message-ID: <20091013100144.41927168014@codespeak.net> Author: cfbolz Date: Tue Oct 13 12:01:43 2009 New Revision: 68364 Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py Log: fix repr Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/argument.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/argument.py Tue Oct 13 12:01:43 2009 @@ -38,10 +38,11 @@ def __repr__(self): """ NOT_RPYTHON """ + name = self.__class__.__name__ if not self.keywords: - return '%s(%s)' % (self.__class__, self.arguments_w,) + return '%s(%s)' % (name, self.arguments_w,) else: - return '%s(%s, %s, %s)' % (self.__class__, self.arguments_w, + return '%s(%s, %s, %s)' % (name, self.arguments_w, self.keywords, self.keywords_w) From pedronis at codespeak.net Tue Oct 13 12:06:57 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 13 Oct 2009 12:06:57 +0200 (CEST) Subject: [pypy-svn] r68365 - in pypy/branch/improve-kwd-args/pypy: interpreter module/__builtin__ module/__builtin__/test module/thread objspace/std Message-ID: <20091013100657.9C101168014@codespeak.net> Author: pedronis Date: Tue Oct 13 12:06:57 2009 New Revision: 68365 Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py pypy/branch/improve-kwd-args/pypy/interpreter/gateway.py pypy/branch/improve-kwd-args/pypy/module/__builtin__/functional.py pypy/branch/improve-kwd-args/pypy/module/__builtin__/test/test_functional.py pypy/branch/improve-kwd-args/pypy/module/thread/os_local.py pypy/branch/improve-kwd-args/pypy/objspace/std/proxyobject.py Log: (cfbolz, pedronis) fixes and cleanups Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/argument.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/argument.py Tue Oct 13 12:06:57 2009 @@ -57,11 +57,14 @@ kwds_w[self.keywords[i]] = self.keywords_w[i] return self.arguments_w, kwds_w + def replace_arguments(self, args_w): + "Return a new Arguments with a args_w as positional arguments." + return Arguments(self.space, args_w, self.keywords, self.keywords_w) + def prepend(self, w_firstarg): # used often "Return a new Arguments with a new argument inserted first." - return Arguments(self.space, [w_firstarg] + self.arguments_w, - self.keywords, self.keywords_w) - + return self.replace_arguments([w_firstarg] + self.arguments_w) + def _combine_wrapped(self, w_stararg, w_starstararg): "unpack the *arg and **kwd into arguments_w and keywords_w" # unpack the * arguments Modified: pypy/branch/improve-kwd-args/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/gateway.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/gateway.py Tue Oct 13 12:06:57 2009 @@ -860,22 +860,22 @@ if ret_w is not None: # it was RPython return ret_w # the last argument can be an Arguments + w_func = self.wget(space, name) if not args_w: - args = Arguments(space, []) + return space.call_function(w_func) else: args = args_w[-1] assert args is not None if not isinstance(args, Arguments): - args = Arguments(space, list(args_w)) + return space.call_function(w_func, *args_w) else: - # ...which is merged with the previous arguments, if any - if len(args_w) > 1: - # XXXXXXXXX - more_args_w, more_kwds_w = args.unpack() - args = Arguments(space, - list(args_w[:-1]) + more_args_w, - more_kwds_w.keys(), more_kwds_w.values()) - w_func = self.wget(space, name) + if len(args_w) == 2: + return space.call_obj_args(w_func, args_w[0], args) + elif len(args_w) > 2: + # xxx is this used at all? + # ...which is merged with the previous arguments, if any + args = args.replace_arguments(list(args_w[:-1]) + + args.arguments_w) return space.call_args(w_func, args) def get_function(space): w_func = self.wget(space, name) Modified: pypy/branch/improve-kwd-args/pypy/module/__builtin__/functional.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/module/__builtin__/functional.py (original) +++ pypy/branch/improve-kwd-args/pypy/module/__builtin__/functional.py Tue Oct 13 12:06:57 2009 @@ -103,27 +103,28 @@ @specialize.arg(2) -def min_max(space, arguments, implementation_of): +def min_max(space, args, implementation_of): if implementation_of == "max": compare = space.gt else: compare = space.lt - # XXXXXXXXX - args, kwargs = arguments.unpack() - if len(args) > 1: - w_sequence = space.newtuple(args) - elif len(args): - w_sequence = args[0] + args_w = args.arguments_w + if len(args_w) > 1: + w_sequence = space.newtuple(args_w) + elif len(args_w): + w_sequence = args_w[0] else: msg = "%s() expects at least one argument" % (implementation_of,) raise OperationError(space.w_TypeError, space.wrap(msg)) - try: - w_key = kwargs["key"] - except KeyError: - w_key = None - if len(kwargs) > 1: - msg = "%s() got unexpected keyword argument" % (implementation_of,) - raise OperationError(space.w_TypeError, space.wrap(msg)) + w_key = None + kwds = args.keywords + if kwds: + if kwds[0] == "key" and len(kwds) == 1: + w_key = args.keywords_w[0] + else: + msg = "%s() got unexpected keyword argument" % (implementation_of,) + raise OperationError(space.w_TypeError, space.wrap(msg)) + w_iter = space.iter(w_sequence) w_max_item = None w_max_val = None Modified: pypy/branch/improve-kwd-args/pypy/module/__builtin__/test/test_functional.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/module/__builtin__/test/test_functional.py (original) +++ pypy/branch/improve-kwd-args/pypy/module/__builtin__/test/test_functional.py Tue Oct 13 12:06:57 2009 @@ -208,3 +208,17 @@ S = [10, 20, 30] assert any([x > 42 for x in S]) == False +class AppTestMinMax: + def test_min(self): + assert min(1, 2) == 1 + assert min(1, 2, key=lambda x: -x) == 2 + assert min([1, 2, 3]) == 1 + raises(TypeError, min, 1, 2, bar=2) + raises(TypeError, min, 1, 2, key=lambda x: x, bar=2) + + def test_max(self): + assert max(1, 2) == 2 + assert max(1, 2, key=lambda x: -x) == 1 + assert max([1, 2, 3]) == 3 + raises(TypeError, max, 1, 2, bar=2) + raises(TypeError, max, 1, 2, key=lambda x: x, bar=2) Modified: pypy/branch/improve-kwd-args/pypy/module/thread/os_local.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/module/thread/os_local.py (original) +++ pypy/branch/improve-kwd-args/pypy/module/thread/os_local.py Tue Oct 13 12:06:57 2009 @@ -12,7 +12,7 @@ def __init__(self, space, initargs): self.space = space - self.initargs = initargs.normalize() + self.initargs = initargs ident = thread.get_ident() self.dicts = {ident: space.newdict()} Modified: pypy/branch/improve-kwd-args/pypy/objspace/std/proxyobject.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/objspace/std/proxyobject.py (original) +++ pypy/branch/improve-kwd-args/pypy/objspace/std/proxyobject.py Tue Oct 13 12:06:57 2009 @@ -22,11 +22,9 @@ self.space = space def descr_call_mismatch(self, space, name, reqcls, args): - # xxx fishing args_w = args.arguments_w[:] args_w[0] = space.wrap(name) - args = argument.Arguments(space, args_w, args.keywords, - args.keywords_w) + args = args.replace_arguments(args_w) return space.call_args(self.w_controller, args) def getclass(self, space): From cfbolz at codespeak.net Tue Oct 13 12:19:11 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 13 Oct 2009 12:19:11 +0200 (CEST) Subject: [pypy-svn] r68366 - pypy/branch/improve-kwd-args/pypy/interpreter Message-ID: <20091013101911.0B91A168014@codespeak.net> Author: cfbolz Date: Tue Oct 13 12:19:10 2009 New Revision: 68366 Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py Log: fix translation Modified: pypy/branch/improve-kwd-args/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/improve-kwd-args/pypy/interpreter/argument.py (original) +++ pypy/branch/improve-kwd-args/pypy/interpreter/argument.py Tue Oct 13 12:19:10 2009 @@ -101,8 +101,8 @@ self.keywords = keywords self.keywords_w = keywords_w else: - self.keywords += keywords - self.keywords_w += keywords_w + self.keywords = self.keywords + keywords + self.keywords_w = self.keywords_w + keywords_w def fixedunpack(self, argcount): # used by ./objspace/std/typeobject.py From arigo at codespeak.net Tue Oct 13 13:15:26 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 13 Oct 2009 13:15:26 +0200 (CEST) Subject: [pypy-svn] r68367 - in pypy/branch/gc-hash/pypy/rlib: . test Message-ID: <20091013111526.804D6168013@codespeak.net> Author: arigo Date: Tue Oct 13 13:15:25 2009 New Revision: 68367 Modified: pypy/branch/gc-hash/pypy/rlib/jit.py pypy/branch/gc-hash/pypy/rlib/objectmodel.py pypy/branch/gc-hash/pypy/rlib/rarithmetic.py pypy/branch/gc-hash/pypy/rlib/rope.py pypy/branch/gc-hash/pypy/rlib/test/test_objectmodel.py Log: Finish the refactoring of rlib/objectmodel. Modified: pypy/branch/gc-hash/pypy/rlib/jit.py ============================================================================== --- pypy/branch/gc-hash/pypy/rlib/jit.py (original) +++ pypy/branch/gc-hash/pypy/rlib/jit.py Tue Oct 13 13:15:25 2009 @@ -206,9 +206,6 @@ raise JitHintError("%s expects the following keyword " "arguments: %s" % (self.instance, expected)) - for name in driver.greens: - s_green_key = kwds_s['s_' + name] - s_green_key.hash() # force the hash cache to appear if self.instance.__name__ == 'jit_merge_point': self.annotate_hooks(**kwds_s) Modified: pypy/branch/gc-hash/pypy/rlib/objectmodel.py ============================================================================== --- pypy/branch/gc-hash/pypy/rlib/objectmodel.py (original) +++ pypy/branch/gc-hash/pypy/rlib/objectmodel.py Tue Oct 13 13:15:25 2009 @@ -230,9 +230,9 @@ def _hash_tuple(t): """NOT_RPYTHON. The algorithm behind compute_hash() for a tuple. It is modelled after the old algorithm of Python 2.3, which is - slightly faster than the one introduced by Python 2.4. We assume - that nested tuples are very uncommon in RPython, making the case - unlikely. + a bit faster than the one introduced by Python 2.4. We assume + that nested tuples are very uncommon in RPython, making the bad + case unlikely. """ from pypy.rlib.rarithmetic import intmask x = 0x345678 @@ -266,13 +266,13 @@ return annmodel.SomeInteger() def specialize_call(self, hop): + from pypy.rpython.lltypesystem import lltype vobj, = hop.inputargs(hop.args_r[0]) if hop.rtyper.type_system.name == 'lltypesystem': - from pypy.rpython.lltypesystem import lltype ok = (isinstance(vobj.concretetype, lltype.Ptr) and vobj.concretetype.TO._gckind == 'gc') else: - from pypy.rpython.lltypesystem import ootype + from pypy.rpython.ootypesystem import ootype ok = isinstance(vobj.concretetype, ootype.OOType) if not ok: from pypy.rpython.error import TyperError @@ -288,23 +288,19 @@ return annmodel.SomeInteger() def specialize_call(self, hop): + from pypy.rpython.lltypesystem import lltype vobj, = hop.inputargs(hop.args_r[0]) if hop.rtyper.type_system.name == 'lltypesystem': - from pypy.rpython.lltypesystem import lltype - if isinstance(vobj.concretetype, lltype.Ptr): - return hop.genop('gc_id', [vobj], - resulttype = lltype.Signed) - elif hop.rtyper.type_system.name == 'ootypesystem': + ok = (isinstance(vobj.concretetype, lltype.Ptr) and + vobj.concretetype.TO._gckind == 'gc') + else: from pypy.rpython.ootypesystem import ootype - if isinstance(vobj.concretetype, ootype.Instance): - # XXX wrong implementation for now, fix me - from pypy.rpython.rmodel import warning - warning("compute_unique_id() is not fully supported on ootype") - return hop.genop('identityhash', [vobj], - resulttype = ootype.Signed) - from pypy.rpython.error import TyperError - raise TyperError("compute_unique_id() cannot be applied to %r" % ( - vobj.concretetype,)) + ok = isinstance(vobj.concretetype, ootype.Instance) + if not ok: + from pypy.rpython.error import TyperError + raise TyperError("compute_unique_id() cannot be applied to" + " %r" % (vobj.concretetype,)) + return hop.genop('gc_id', [vobj], resulttype=lltype.Signed) class Entry(ExtRegistryEntry): _about_ = current_object_addr_as_int Modified: pypy/branch/gc-hash/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/gc-hash/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/gc-hash/pypy/rlib/rarithmetic.py Tue Oct 13 13:15:25 2009 @@ -463,23 +463,6 @@ return formatd(fmt, x) -# a common string hash function - -def _hash_string(s): - length = len(s) - if length == 0: - x = -1 - else: - x = ord(s[0]) << 7 - i = 0 - while i < length: - x = (1000003*x) ^ ord(s[i]) - i += 1 - x ^= length - if x == 0: - x = -1 - return intmask(x) - # the 'float' C type class r_singlefloat(object): Modified: pypy/branch/gc-hash/pypy/rlib/rope.py ============================================================================== --- pypy/branch/gc-hash/pypy/rlib/rope.py (original) +++ pypy/branch/gc-hash/pypy/rlib/rope.py Tue Oct 13 13:15:25 2009 @@ -1,6 +1,6 @@ import py import sys -from pypy.rlib.rarithmetic import intmask, _hash_string, ovfcheck +from pypy.rlib.rarithmetic import intmask, ovfcheck from pypy.rlib.rarithmetic import r_uint, LONG_BIT from pypy.rlib.objectmodel import we_are_translated import math Modified: pypy/branch/gc-hash/pypy/rlib/test/test_objectmodel.py ============================================================================== --- pypy/branch/gc-hash/pypy/rlib/test/test_objectmodel.py (original) +++ pypy/branch/gc-hash/pypy/rlib/test/test_objectmodel.py Tue Oct 13 13:15:25 2009 @@ -349,8 +349,10 @@ q = Foo() def f(i): + assert compute_hash(None) == 0 assert compute_hash(i) == h_42 assert compute_hash(i+1.0) == h_43_dot_0 + assert compute_hash((i+3)/6.0) == h_7_dot_5 assert compute_hash("Hello" + str(i)) == h_Hello42 if i == 42: p = None @@ -362,6 +364,7 @@ return i*2 h_42 = compute_hash(42) h_43_dot_0 = compute_hash(43.0) + h_7_dot_5 = compute_hash(7.5) h_Hello42 = compute_hash("Hello42") h_None = compute_hash(None) h_tuple = compute_hash(("world", None, 42, 7.5)) From arigo at codespeak.net Tue Oct 13 13:15:38 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 13 Oct 2009 13:15:38 +0200 (CEST) Subject: [pypy-svn] r68368 - pypy/branch/gc-hash/pypy/objspace/std Message-ID: <20091013111538.2ECC5168013@codespeak.net> Author: arigo Date: Tue Oct 13 13:15:37 2009 New Revision: 68368 Modified: pypy/branch/gc-hash/pypy/objspace/std/stringobject.py pypy/branch/gc-hash/pypy/objspace/std/unicodeobject.py Log: Fix stringobject. Modified: pypy/branch/gc-hash/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/gc-hash/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/gc-hash/pypy/objspace/std/stringobject.py Tue Oct 13 13:15:37 2009 @@ -2,8 +2,8 @@ from pypy.objspace.std.objspace import * from pypy.interpreter import gateway -from pypy.rlib.rarithmetic import ovfcheck, _hash_string -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib.objectmodel import we_are_translated, compute_hash from pypy.objspace.std.inttype import wrapint from pypy.objspace.std.sliceobject import W_SliceObject, normalize_simple_slice from pypy.objspace.std import slicetype @@ -755,12 +755,7 @@ def hash__String(space, w_str): s = w_str._value - if we_are_translated(): - x = hash(s) # to use the hash cache in rpython strings - else: - x = _hash_string(s) # to make sure we get the same hash as rpython - # (otherwise translation will freeze W_DictObjects where we can't find - # the keys any more!) + x = compute_hash(s) return wrapint(space, x) def lt__String_String(space, w_str1, w_str2): Modified: pypy/branch/gc-hash/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/gc-hash/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/gc-hash/pypy/objspace/std/unicodeobject.py Tue Oct 13 13:15:37 2009 @@ -7,6 +7,7 @@ from pypy.objspace.std import slicetype from pypy.objspace.std.tupleobject import W_TupleObject from pypy.rlib.rarithmetic import intmask, ovfcheck +from pypy.rlib.objectmodel import compute_hash from pypy.module.unicodedata import unicodedb_4_1_0 as unicodedb from pypy.tool.sourcetools import func_with_new_name @@ -211,13 +212,7 @@ x ^= ord(s[0]) h = intmask(x) return space.wrap(h) - if we_are_translated(): - x = hash(s) # to use the hash cache in rpython strings - else: - from pypy.rlib.rarithmetic import _hash_string - x = _hash_string(s) # to make sure we get the same hash as rpython - # (otherwise translation will freeze W_DictObjects where we can't find - # the keys any more!) + x = compute_hash(s) return space.wrap(x) def len__Unicode(space, w_uni): From arigo at codespeak.net Tue Oct 13 13:17:53 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 13 Oct 2009 13:17:53 +0200 (CEST) Subject: [pypy-svn] r68369 - in pypy/branch/gc-hash/pypy/annotation: . test Message-ID: <20091013111753.01B71168013@codespeak.net> Author: arigo Date: Tue Oct 13 13:17:53 2009 New Revision: 68369 Modified: pypy/branch/gc-hash/pypy/annotation/bookkeeper.py pypy/branch/gc-hash/pypy/annotation/dictdef.py pypy/branch/gc-hash/pypy/annotation/test/test_annrpython.py pypy/branch/gc-hash/pypy/annotation/unaryop.py Log: Refactoring in the annotator, mainly to make hash() not RPython any more, but also to force compute_identity_hash() on all prebuilt dictionary keys. Modified: pypy/branch/gc-hash/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/gc-hash/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/gc-hash/pypy/annotation/bookkeeper.py Tue Oct 13 13:17:53 2009 @@ -169,7 +169,6 @@ self.pending_specializations = [] # list of callbacks self.external_class_cache = {} # cache of ExternalType classes - self.needs_hash_support = {} self.needs_generic_instantiate = {} self.stats = Stats(self) @@ -219,12 +218,6 @@ self.consider_call_site_for_pbc(pbc, 'simple_call', args_s, s_ImpossibleValue) self.emulated_pbc_calls = {} - - for clsdef in self.needs_hash_support.keys(): - for clsdef2 in self.needs_hash_support: - if clsdef.issubclass(clsdef2) and clsdef is not clsdef2: - del self.needs_hash_support[clsdef] - break finally: self.leave() @@ -399,6 +392,7 @@ for ek, ev in items: result.dictdef.generalize_key(self.immutablevalue(ek)) result.dictdef.generalize_value(self.immutablevalue(ev)) + result.dictdef.seen_prebuilt_key(ek) seen_elements = len(items) # if the dictionary grew during the iteration, # start over again @@ -417,6 +411,7 @@ for ek, ev in x.iteritems(): dictdef.generalize_key(self.immutablevalue(ek, False)) dictdef.generalize_value(self.immutablevalue(ev, False)) + dictdef.seen_prebuilt_key(ek) result = SomeDict(dictdef) elif tp is weakref.ReferenceType: x1 = x() Modified: pypy/branch/gc-hash/pypy/annotation/dictdef.py ============================================================================== --- pypy/branch/gc-hash/pypy/annotation/dictdef.py (original) +++ pypy/branch/gc-hash/pypy/annotation/dictdef.py Tue Oct 13 13:17:53 2009 @@ -2,6 +2,7 @@ from pypy.annotation.model import SomeInteger, s_Bool, unionof from pypy.annotation.model import SomeInstance from pypy.annotation.listdef import ListItem +from pypy.rlib.objectmodel import compute_identity_hash class DictKey(ListItem): @@ -10,7 +11,6 @@ def __init__(self, bookkeeper, s_value, is_r_dict=False): ListItem.__init__(self, bookkeeper, s_value) self.is_r_dict = is_r_dict - self.enable_hashing() def patch(self): for dictdef in self.itemof: @@ -26,15 +26,8 @@ other.s_rdict_hashfn, other=other) - def enable_hashing(self): - # r_dicts don't need the RPython hash of their keys - if isinstance(self.s_value, SomeInstance) and not self.is_r_dict: - self.bookkeeper.needs_hash_support[self.s_value.classdef] = True - def generalize(self, s_other_value): updated = ListItem.generalize(self, s_other_value) - if updated: - self.enable_hashing() if updated and self.custom_eq_hash: self.emulate_rdict_calls() return updated @@ -139,6 +132,12 @@ def generalize_value(self, s_value): self.dictvalue.generalize(s_value) + def seen_prebuilt_key(self, x): + try: + compute_identity_hash(x) + except TypeError: + pass # e.g. if x is an int + def __repr__(self): return '<{%r: %r}>' % (self.dictkey.s_value, self.dictvalue.s_value) Modified: pypy/branch/gc-hash/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/gc-hash/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/gc-hash/pypy/annotation/test/test_annrpython.py Tue Oct 13 13:17:53 2009 @@ -3187,15 +3187,6 @@ s = a.build_types(f, [int]) assert s.const == 0 - def test_hash(self): - class A(object): - pass - def f(): - return hash(A()) + hash(None) - a = self.RPythonAnnotator() - s = a.build_types(f, []) - assert s.knowntype == int - def test_contains_of_empty_dict(self): class A(object): def meth(self): Modified: pypy/branch/gc-hash/pypy/annotation/unaryop.py ============================================================================== --- pypy/branch/gc-hash/pypy/annotation/unaryop.py (original) +++ pypy/branch/gc-hash/pypy/annotation/unaryop.py Tue Oct 13 13:17:53 2009 @@ -20,11 +20,12 @@ def immutablevalue(x): return getbookkeeper().immutablevalue(x) -UNARY_OPERATIONS = set(['len', 'is_true', 'getattr', 'setattr', 'delattr', 'hash', +UNARY_OPERATIONS = set(['len', 'is_true', 'getattr', 'setattr', 'delattr', 'simple_call', 'call_args', 'str', 'repr', 'iter', 'next', 'invert', 'type', 'issubtype', 'pos', 'neg', 'nonzero', 'abs', 'hex', 'oct', - 'ord', 'int', 'float', 'long', 'id', + 'ord', 'int', 'float', 'long', + 'hash', 'id', # <== not supported any more 'getslice', 'setslice', 'delslice', 'neg_ovf', 'abs_ovf', 'hint', 'unicode', 'unichr']) @@ -98,7 +99,8 @@ return obj.is_true() def hash(obj): - raise TypeError, "hash() is not generally supported" + raise TypeError, ("cannot use hash() in RPython; " + "see objectmodel.compute_xxx()") def str(obj): getbookkeeper().count('str', obj) @@ -121,10 +123,8 @@ return SomeString() def id(obj): - raise Exception("cannot use id() in RPython; pick one of:\n" - "\t\t objectmodel.compute_unique_id()\n" - "\t\t hash()\n" - "\t\t objectmodel.current_object_addr_as_int()") + raise Exception("cannot use id() in RPython; " + "see objectmodel.compute_xxx()") def int(obj): return SomeInteger() @@ -203,9 +203,6 @@ return getbookkeeper().immutablevalue(bool(self.const)) return s_Bool - def hash(flt): - return SomeInteger() - class __extend__(SomeInteger): def invert(self): @@ -272,11 +269,6 @@ def getanyitem(tup): return unionof(*tup.items) - def hash(tup): - for s_item in tup.items: - s_item.hash() # record that we need the hash of each item - return SomeInteger() - def getslice(tup, s_start, s_stop): assert s_start.is_immutable_constant(),"tuple slicing: needs constants" assert s_stop.is_immutable_constant(), "tuple slicing: needs constants" @@ -501,9 +493,6 @@ def ord(str): return SomeInteger(nonneg=True) - def hash(str): - return SomeInteger() - def method_split(str, patt): # XXX getbookkeeper().count("str_split", str, patt) return getbookkeeper().newlist(str.basestringclass()) @@ -632,10 +621,6 @@ # create or update the attribute in clsdef clsdef.generalize_attr(attr, s_value) - def hash(ins): - getbookkeeper().needs_hash_support[ins.classdef] = True - return SomeInteger() - def is_true_behavior(ins, s): if not ins.can_be_None: s.const = True @@ -694,13 +679,6 @@ else: return SomeObject() # len() on a pbc? no chance - def hash(pbc): - if pbc.isNone(): - # only supports hash(None) as part of hash() - return SomeInteger() - else: - return SomeObject.hash(pbc) - class __extend__(SomeGenericCallable): def call(self, args): bookkeeper = getbookkeeper() From arigo at codespeak.net Tue Oct 13 13:28:18 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 13 Oct 2009 13:28:18 +0200 (CEST) Subject: [pypy-svn] r68370 - in pypy/branch/gc-hash/pypy: annotation rpython rpython/lltypesystem rpython/lltypesystem/test rpython/ootypesystem rpython/ootypesystem/test Message-ID: <20091013112818.4589D168013@codespeak.net> Author: arigo Date: Tue Oct 13 13:28:16 2009 New Revision: 68370 Modified: pypy/branch/gc-hash/pypy/annotation/builtin.py pypy/branch/gc-hash/pypy/rpython/llinterp.py pypy/branch/gc-hash/pypy/rpython/lltypesystem/lloperation.py pypy/branch/gc-hash/pypy/rpython/lltypesystem/lltype.py pypy/branch/gc-hash/pypy/rpython/lltypesystem/rclass.py pypy/branch/gc-hash/pypy/rpython/lltypesystem/rstr.py pypy/branch/gc-hash/pypy/rpython/lltypesystem/test/test_lltype.py pypy/branch/gc-hash/pypy/rpython/ootypesystem/ooregistry.py pypy/branch/gc-hash/pypy/rpython/ootypesystem/ootype.py pypy/branch/gc-hash/pypy/rpython/ootypesystem/rbuiltin.py pypy/branch/gc-hash/pypy/rpython/ootypesystem/rclass.py pypy/branch/gc-hash/pypy/rpython/ootypesystem/rstr.py pypy/branch/gc-hash/pypy/rpython/ootypesystem/rtupletype.py pypy/branch/gc-hash/pypy/rpython/ootypesystem/test/test_ootype.py pypy/branch/gc-hash/pypy/rpython/rbuiltin.py pypy/branch/gc-hash/pypy/rpython/rclass.py pypy/branch/gc-hash/pypy/rpython/rdict.py pypy/branch/gc-hash/pypy/rpython/rtuple.py pypy/branch/gc-hash/pypy/rpython/rtyper.py Log: Refactor lltype and ootype to have only one method in common, called identityhash(). Refactor rclass to no longer generate the hash_cache field. Modified: pypy/branch/gc-hash/pypy/annotation/builtin.py ============================================================================== --- pypy/branch/gc-hash/pypy/annotation/builtin.py (original) +++ pypy/branch/gc-hash/pypy/annotation/builtin.py Tue Oct 13 13:28:16 2009 @@ -493,6 +493,10 @@ assert PtrT.is_constant() return SomePtr(ll_ptrtype=PtrT.const) +def identityhash(s_obj): + assert isinstance(s_obj, (SomePtr, SomeOOObject, SomeOOInstance)) + return SomeInteger() + def getRuntimeTypeInfo(T): assert T.is_constant() return immutablevalue(lltype.getRuntimeTypeInfo(T.const)) @@ -517,6 +521,7 @@ BUILTIN_ANALYZERS[lltype.direct_ptradd] = direct_ptradd BUILTIN_ANALYZERS[lltype.cast_ptr_to_int] = cast_ptr_to_int BUILTIN_ANALYZERS[lltype.cast_int_to_ptr] = cast_int_to_ptr +BUILTIN_ANALYZERS[lltype.identityhash] = identityhash BUILTIN_ANALYZERS[lltype.getRuntimeTypeInfo] = getRuntimeTypeInfo BUILTIN_ANALYZERS[lltype.runtime_type_info] = runtime_type_info BUILTIN_ANALYZERS[lltype.Ptr] = constPtr @@ -562,10 +567,6 @@ else: return SomeOOInstance(c.ootype) -def ooidentityhash(i): - assert isinstance(i, (SomeOOInstance, SomeOOObject)) - return SomeInteger() - def ooupcast(I, i): assert isinstance(I.const, ootype.Instance) if ootype.isSubclass(i.ootype, I.const): @@ -606,7 +607,6 @@ BUILTIN_ANALYZERS[ootype.runtimenew] = runtimenew BUILTIN_ANALYZERS[ootype.classof] = classof BUILTIN_ANALYZERS[ootype.subclassof] = subclassof -BUILTIN_ANALYZERS[ootype.ooidentityhash] = ooidentityhash BUILTIN_ANALYZERS[ootype.ooupcast] = ooupcast BUILTIN_ANALYZERS[ootype.oodowncast] = oodowncast BUILTIN_ANALYZERS[ootype.cast_to_object] = cast_to_object Modified: pypy/branch/gc-hash/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/llinterp.py (original) +++ pypy/branch/gc-hash/pypy/rpython/llinterp.py Tue Oct 13 13:28:16 2009 @@ -801,6 +801,9 @@ checkadr(adr) return llmemory.cast_adr_to_int(adr) + def op_identityhash(self, obj): + return lltype.identityhash(obj) + def op_weakref_create(self, v_obj): def objgetter(): # special support for gcwrapper.py return self.getval(v_obj) @@ -1180,9 +1183,6 @@ raise RuntimeError("calling abstract method %r" % (m,)) return self.perform_call(m, (lltype.typeOf(inst),)+lltype.typeOf(m).ARGS, [inst]+args) - def op_ooidentityhash(self, inst): - return ootype.ooidentityhash(inst) - def op_oostring(self, obj, base): return ootype.oostring(obj, base) @@ -1204,9 +1204,6 @@ except ValueError: self.make_llexception() - def op_oohash(self, s): - return ootype.oohash(s) - class Tracer(object): Counter = 0 file = None Modified: pypy/branch/gc-hash/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/gc-hash/pypy/rpython/lltypesystem/lloperation.py Tue Oct 13 13:28:16 2009 @@ -378,6 +378,7 @@ 'direct_arrayitems': LLOp(canfold=True), 'direct_ptradd': LLOp(canfold=True), 'cast_opaque_ptr': LLOp(sideeffects=False), + 'identityhash': LLOp(sideeffects=False), # see rlib/objectmodel # __________ address operations __________ @@ -530,11 +531,9 @@ 'instanceof': LLOp(oo=True, canfold=True), 'classof': LLOp(oo=True, canfold=True), 'subclassof': LLOp(oo=True, canfold=True), - 'ooidentityhash': LLOp(oo=True, sideeffects=False), # not an id()! 'oostring': LLOp(oo=True, sideeffects=False), 'ooparse_int': LLOp(oo=True, canraise=(ValueError,)), 'ooparse_float': LLOp(oo=True, canraise=(ValueError,)), - 'oohash': LLOp(oo=True, sideeffects=False), 'oounicode': LLOp(oo=True, canraise=(UnicodeDecodeError,)), # _____ read frame var support ___ Modified: pypy/branch/gc-hash/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/gc-hash/pypy/rpython/lltypesystem/lltype.py Tue Oct 13 13:28:16 2009 @@ -1119,6 +1119,14 @@ return callb(*args) raise TypeError("%r instance is not a function" % (self._T,)) + def _identityhash(self): + p = normalizeptr(self) + try: + return p._obj._hash_cache_ + except AttributeError: + result = p._obj._hash_cache_ = hash(p._obj) + return result + class _ptr(_abstract_ptr): __slots__ = ('_TYPE', '_weak', '_solid', @@ -1840,19 +1848,18 @@ "should have been: %s" % (p, result2, result)) return result -def hash_gc_object(p): - """Returns the lltype-level hash of the given GcStruct.""" - p = normalizeptr(p) - if not p: - return 0 # hash(NULL) - try: - return p._obj._hash_cache_ - except AttributeError: - result = p._obj._hash_cache_ = intmask(id(p._obj)) - return result +def identityhash(p): + """Returns the lltype-level hash of the given GcStruct. + Also works with most ootype objects. Not for NULL. + See rlib.objectmodel.compute_identiy_hash() for more + information about the RPython-level meaning of this. + """ + assert p + return p._identityhash() -def init_hash_gc_object(p, value): +def init_identity_hash(p, value): """For a prebuilt object p, initialize its hash value to 'value'.""" + assert isinstance(typeOf(p), Ptr) p = normalizeptr(p) if not p: raise ValueError("cannot change hash(NULL)!") Modified: pypy/branch/gc-hash/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/gc-hash/pypy/rpython/lltypesystem/rclass.py Tue Oct 13 13:28:16 2009 @@ -20,6 +20,7 @@ from pypy.rpython.extregistry import ExtRegistryEntry from pypy.annotation import model as annmodel from pypy.rlib.rarithmetic import intmask +from pypy.rlib import objectmodel # # There is one "vtable" per user class, with the following structure: @@ -331,12 +332,6 @@ mangled_name = 'inst_' + name fields[name] = mangled_name, r llfields.append((mangled_name, r.lowleveltype)) - # - # hash() support - if self.rtyper.needs_hash_support(self.classdef): - from pypy.rpython import rint - fields['_hash_cache_'] = 'hash_cache', rint.signed_repr - llfields.append(('hash_cache', Signed)) self.rbase = getinstancerepr(self.rtyper, self.classdef.basedef, self.gcflavor) @@ -348,10 +343,6 @@ if hints is None: hints = {} hints = self._check_for_immutable_hints(hints) - if ('_hash_cache_' in fields or - '_hash_cache_' in self.rbase.allinstancefields): - adtmeths = adtmeths.copy() - adtmeths['gethash'] = self.get_ll_hash_function() object_type = MkStruct(self.classdef.name, ('super', self.rbase.object_type), hints=hints, @@ -406,21 +397,6 @@ def create_instance(self): return malloc(self.object_type, flavor=self.gcflavor) - def get_ll_hash_function(self): - if self.classdef is None: - raise TyperError, 'missing hash support flag in classdef' - if self.rtyper.needs_hash_support(self.classdef): - try: - return self._ll_hash_function - except AttributeError: - INSPTR = self.lowleveltype - def _ll_hash_function(ins): - return ll_inst_hash(cast_pointer(INSPTR, ins)) - self._ll_hash_function = _ll_hash_function - return _ll_hash_function - else: - return self.rbase.get_ll_hash_function() - def initialize_prebuilt_data(self, value, classdef, result): if self.classdef is not None: # recursively build the parent part of the instance @@ -429,8 +405,6 @@ for name, (mangled_name, r) in self.fields.items(): if r.lowleveltype is Void: llattrvalue = None - elif name == '_hash_cache_': # hash() support - continue # already done by initialize_prebuilt_hash() else: try: attrvalue = getattr(value, name) @@ -451,12 +425,9 @@ result.typeptr = rclass.getvtable() def initialize_prebuilt_hash(self, value, result): - if self.classdef is not None: - self.rbase.initialize_prebuilt_hash(value, result.super) - if '_hash_cache_' in self.fields: - mangled_name, r = self.fields['_hash_cache_'] - llattrvalue = hash(value) - setattr(result, mangled_name, llattrvalue) + llattrvalue = getattr(value, '__precomputed_identity_hash', None) + if llattrvalue is not None: + lltype.init_identity_hash(result, llattrvalue) def getfieldrepr(self, attr): """Return the repr used for the given attribute.""" @@ -523,10 +494,7 @@ mangled_name, r = self.allinstancefields[fldname] if r.lowleveltype is Void: continue - if fldname == '_hash_cache_': - value = Constant(0, Signed) - else: - value = self.classdef.classdesc.read_attribute(fldname, None) + value = self.classdef.classdesc.read_attribute(fldname, None) if value is not None: cvalue = inputconst(r.lowleveltype, r.convert_desc_or_const(value)) @@ -697,18 +665,6 @@ def ll_runtime_type_info(obj): return obj.typeptr.rtti -def ll_inst_hash(ins): - if not ins: - return 0 # for None - cached = ins.hash_cache - if cached == 0: - # XXX this should ideally be done in a GC-dependent way: we only - # need a hash_cache for moving GCs, and we only need the '~' to - # avoid Boehm keeping the object alive if the value is passed - # around - cached = ins.hash_cache = ~cast_ptr_to_int(ins) - return cached - def ll_inst_type(obj): if obj: return obj.typeptr Modified: pypy/branch/gc-hash/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/gc-hash/pypy/rpython/lltypesystem/rstr.py Tue Oct 13 13:28:16 2009 @@ -2,9 +2,9 @@ from pypy.tool.pairtype import pairtype from pypy.rpython.error import TyperError from pypy.rlib.objectmodel import malloc_zero_filled, we_are_translated +from pypy.rlib.objectmodel import _hash_string from pypy.rlib.debug import ll_assert from pypy.rpython.robject import PyObjRepr, pyobj_repr -from pypy.rlib.rarithmetic import _hash_string from pypy.rpython.rmodel import inputconst, IntegerRepr from pypy.rpython.rstr import AbstractStringRepr,AbstractCharRepr,\ AbstractUniCharRepr, AbstractStringIteratorRepr,\ Modified: pypy/branch/gc-hash/pypy/rpython/lltypesystem/test/test_lltype.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/lltypesystem/test/test_lltype.py (original) +++ pypy/branch/gc-hash/pypy/rpython/lltypesystem/test/test_lltype.py Tue Oct 13 13:28:16 2009 @@ -740,29 +740,31 @@ import gc; gc.collect(); gc.collect() ptr2[0] = 5 # crashes if the array was deallocated -def test_hash_gc_object(): +def test_identityhash(): S = GcStruct('S', ('x', Signed)) S2 = GcStruct('S2', ('super', S)) S3 = GcStruct('S3', ('super', S2)) + py.test.raises(AssertionError, identityhash, nullptr(S2)) + s3 = malloc(S3) - hash3 = hash_gc_object(s3.super) - assert hash3 == hash_gc_object(s3) - assert hash3 == hash_gc_object(s3.super) - assert hash3 == hash_gc_object(s3.super.super) - py.test.raises(ValueError, init_hash_gc_object, s3, hash3^1) - py.test.raises(ValueError, init_hash_gc_object, s3.super, hash3^4) - py.test.raises(ValueError, init_hash_gc_object, s3.super.super, hash3^9) + hash3 = identityhash(s3.super) + assert hash3 == identityhash(s3) + assert hash3 == identityhash(s3.super) + assert hash3 == identityhash(s3.super.super) + py.test.raises(ValueError, init_identity_hash, s3, hash3^1) + py.test.raises(ValueError, init_identity_hash, s3.super, hash3^4) + py.test.raises(ValueError, init_identity_hash, s3.super.super, hash3^9) s3 = malloc(S3) - init_hash_gc_object(s3.super, -123) - assert -123 == hash_gc_object(s3) - assert -123 == hash_gc_object(s3.super) - assert -123 == hash_gc_object(s3.super.super) - py.test.raises(ValueError, init_hash_gc_object, s3, 4313) - py.test.raises(ValueError, init_hash_gc_object, s3.super, 0) - py.test.raises(ValueError, init_hash_gc_object, s3.super.super, -124) + init_identity_hash(s3.super, -123) + assert -123 == identityhash(s3) + assert -123 == identityhash(s3.super) + assert -123 == identityhash(s3.super.super) + py.test.raises(ValueError, init_identity_hash, s3, 4313) + py.test.raises(ValueError, init_identity_hash, s3.super, 0) + py.test.raises(ValueError, init_identity_hash, s3.super.super, -124) from pypy.rpython.lltypesystem import llmemory p3 = cast_opaque_ptr(llmemory.GCREF, s3) - assert -123 == hash_gc_object(p3) + assert -123 == identityhash(p3) Modified: pypy/branch/gc-hash/pypy/rpython/ootypesystem/ooregistry.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/ootypesystem/ooregistry.py (original) +++ pypy/branch/gc-hash/pypy/rpython/ootypesystem/ooregistry.py Tue Oct 13 13:28:16 2009 @@ -81,21 +81,3 @@ hop.has_implicit_exception(ValueError) hop.exception_is_here() return hop.genop('ooparse_float', vlist, resulttype = ootype.Float) - - -class Entry_oohash(ExtRegistryEntry): - _about_ = ootype.oohash - - def compute_result_annotation(self, str_s): - if not (isinstance(str_s, annmodel.SomeOOInstance) - and (str_s.ootype is ootype.String or - str_s.ootype is ootype.Unicode)): - return annmodel.s_ImpossibleValue - return annmodel.SomeInteger() - - def specialize_call(self, hop): - assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)\ - and (hop.args_s[0].ootype is ootype.String or - hop.args_s[0].ootype is ootype.Unicode) - vlist = hop.inputargs(hop.args_r[0]) - return hop.genop('oohash', vlist, resulttype=ootype.Signed) Modified: pypy/branch/gc-hash/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/branch/gc-hash/pypy/rpython/ootypesystem/ootype.py Tue Oct 13 13:28:16 2009 @@ -4,6 +4,7 @@ from pypy.rpython.lltypesystem.lltype import Bool, Void, UniChar, typeOf, \ Primitive, isCompatibleType, enforce, saferecursive, SignedLongLong, UnsignedLongLong from pypy.rpython.lltypesystem.lltype import frozendict, isCompatibleType +from pypy.rpython.lltypesystem.lltype import identityhash from pypy.rlib.rarithmetic import intmask from pypy.rlib import objectmodel from pypy.tool.uid import uid @@ -331,9 +332,14 @@ # can treat them polymorphically, if they choose to do so. def __init__(self, fields, _hints={}): + if isinstance(fields, dict): + fields = fields.items() # random order in that case self._fields = frozendict() - for name, ITEMTYPE in fields.items(): + fields_in_order = [] + for name, ITEMTYPE in fields: self._fields[name] = ITEMTYPE, ITEMTYPE._defl() + fields_in_order.append(name) + self._fields_in_order = tuple(fields_in_order) self._null = _null_record(self) self._hints = frozendict(_hints) @@ -361,8 +367,8 @@ return self, None def __str__(self): - item_str = ["%s: %s" % (str(name), str(ITEMTYPE)) - for name, (ITEMTYPE, _) in self._fields.items()] + item_str = ["%s: %s" % (str(name), str(self._fields[name][0])) + for name in self._fields_in_order] return '%s(%s)' % (self.__class__.__name__, ", ".join(item_str)) class BuiltinADTType(BuiltinType): @@ -411,6 +417,7 @@ generic_types = { self.SELFTYPE_T: self } self._GENERIC_METHODS = frozendict({ + "ll_hash": Meth([], Signed), "ll_stritem_nonneg": Meth([Signed], self.CHAR), "ll_strlen": Meth([], Signed), "ll_strconcat": Meth([self.SELFTYPE_T], self.SELFTYPE_T), @@ -881,13 +888,10 @@ return hash(self.obj) def _identityhash(self): - if self: - try: - return self.obj._identityhash() - except AttributeError: - return intmask(id(self.obj)) - else: - return 0 # for all null objects + try: + return self.obj._identityhash() + except AttributeError: + return hash(self.obj) def _cast_to_object(self): return self @@ -986,10 +990,7 @@ return self def _identityhash(self): - if self: - return intmask(id(self)) - else: - return 0 # for all null instances + return hash(self) def _cast_to_object(self): return make_object(ooupcast(ROOT, self)) @@ -1386,6 +1387,12 @@ else: assert False, 'Unknown type %s' % self._TYPE + def ll_hash(self): + # NOT_RPYTHON + # hopefully, ll_hash() should not be called on NULL + assert self._str is not None + return objectmodel._hash_string(self._str) + def ll_stritem_nonneg(self, i): # NOT_RPYTHON s = self._str @@ -1622,10 +1629,7 @@ self._array[index] = item def _identityhash(self): - if self: - return intmask(id(self)) - else: - return 0 # for all null arrays + return hash(self) class _null_array(_null_mixin(_array), _array): @@ -1772,13 +1776,16 @@ self.__dict__[name] = value def _identityhash(self): - if self: - return intmask(id(self)) - else: - return 0 # for all null tuples + return hash(self) + + def _items_in_order(self): + return [self._items[name] for name in self._TYPE._fields_in_order] + + def _ll_hash(self): + return objectmodel._ll_hash_tuple(self._items_in_order()) def __hash__(self): - key = tuple(self._items.keys()), tuple(self._items.values()) + key = tuple(self._items_in_order()) return hash(key) def __eq__(self, other): @@ -1898,16 +1905,6 @@ assert typeOf(obj) is Object return obj._cast_to(EXPECTED_TYPE) -def ooidentityhash(inst): - T = typeOf(inst) - assert T is Object or isinstance(T, (Instance, Record, Array)) - return inst._identityhash() - -def oohash(inst): - assert typeOf(inst) is String or typeOf(inst) is Unicode - # for now only strings and unicode are supported - return hash(inst._str) - def oostring(obj, base): """ Convert char, int, float, instances and str to str. Modified: pypy/branch/gc-hash/pypy/rpython/ootypesystem/rbuiltin.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/ootypesystem/rbuiltin.py (original) +++ pypy/branch/gc-hash/pypy/rpython/ootypesystem/rbuiltin.py Tue Oct 13 13:28:16 2009 @@ -48,12 +48,6 @@ return hop.genop('runtimenew', vlist, resulttype = hop.r_result.lowleveltype) -def rtype_ooidentityhash(hop): - assert isinstance(hop.args_s[0], (annmodel.SomeOOInstance, annmodel.SomeOOObject)) - vlist = hop.inputargs(hop.args_r[0]) - return hop.genop('ooidentityhash', vlist, - resulttype = ootype.Signed) - def rtype_ooupcast(hop): assert isinstance(hop.args_s[0].const, ootype.Instance) assert isinstance(hop.args_s[1], annmodel.SomeOOInstance) @@ -132,7 +126,6 @@ BUILTIN_TYPER[ootype.subclassof] = rtype_subclassof BUILTIN_TYPER[ootype.instanceof] = rtype_instanceof BUILTIN_TYPER[ootype.runtimenew] = rtype_runtimenew -BUILTIN_TYPER[ootype.ooidentityhash] = rtype_ooidentityhash BUILTIN_TYPER[ootype.ooupcast] = rtype_ooupcast BUILTIN_TYPER[ootype.oodowncast] = rtype_oodowncast BUILTIN_TYPER[ootype.cast_from_object] = rtype_cast_from_object Modified: pypy/branch/gc-hash/pypy/rpython/ootypesystem/rclass.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/ootypesystem/rclass.py (original) +++ pypy/branch/gc-hash/pypy/rpython/ootypesystem/rclass.py Tue Oct 13 13:28:16 2009 @@ -152,7 +152,7 @@ if config.translation.ootype.mangle: return 'o' + name else: - not_allowed = ('_hash_cache_', 'meta', 'class_') + not_allowed = ('meta', 'class_') assert name not in not_allowed, "%s is a reserved name" % name return name @@ -253,13 +253,6 @@ allmethods[mangled] = meth_name, s_meth # else: it's the __init__ of a builtin exception - # - # hash() support - if self.rtyper.needs_hash_support(self.classdef): - from pypy.rpython import rint - allfields['_hash_cache_'] = rint.signed_repr - fields['_hash_cache_'] = ootype.Signed - ootype.addFields(self.lowleveltype, fields) self.rbase = getinstancerepr(self.rtyper, self.classdef.basedef) @@ -413,9 +406,6 @@ graph=graph) ootype.addMethods(self.lowleveltype, {mangled: m}) - def get_ll_hash_function(self): - return ll_inst_hash - def rtype_getattr(self, hop): if hop.s_result.is_constant(): return hop.inputconst(hop.r_result, hop.s_result.const) @@ -526,8 +516,6 @@ llattrvalue = None elif mangled == 'meta': llattrvalue = classrepr.get_meta_instance() - elif mangled == '_hash_cache_': # hash() support - continue # already done by initialize_prebuilt_hash() else: name = unmangle(mangled, self.rtyper.getconfig()) try: @@ -544,8 +532,7 @@ setattr(result, mangled, llattrvalue) def initialize_prebuilt_hash(self, value, result): - if '_hash_cache_' in self.lowleveltype._allfields(): - result._hash_cache_ = hash(value) + pass class __extend__(pairtype(InstanceRepr, InstanceRepr)): @@ -581,15 +568,6 @@ v = rpair.rtype_eq(hop) return hop.genop("bool_not", [v], resulttype=ootype.Bool) - -def ll_inst_hash(ins): - if not ins: - return 0 - cached = ins._hash_cache_ - if cached == 0: - cached = ins._hash_cache_ = ootype.ooidentityhash(ins) - return cached - def ll_inst_type(obj): if obj: return ootype.classof(obj) Modified: pypy/branch/gc-hash/pypy/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/ootypesystem/rstr.py (original) +++ pypy/branch/gc-hash/pypy/rpython/ootypesystem/rstr.py Tue Oct 13 13:28:16 2009 @@ -113,10 +113,10 @@ return ootype.oounicode(ch, -1) def ll_strhash(s): - return ootype.oohash(s) + return s.ll_hash() def ll_strfasthash(s): - return ootype.oohash(s) + return s.ll_hash() def ll_char_mul(ch, times): if times < 0: Modified: pypy/branch/gc-hash/pypy/rpython/ootypesystem/rtupletype.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/ootypesystem/rtupletype.py (original) +++ pypy/branch/gc-hash/pypy/rpython/ootypesystem/rtupletype.py Tue Oct 13 13:28:16 2009 @@ -12,4 +12,4 @@ fields = [('item%d' % i, TYPE) for i, TYPE in enumerate(field_lltypes)] hints = {'immutable': True, 'noidentity': True} - return ootype.Record(dict(fields), _hints=hints) + return ootype.Record(fields, _hints=hints) Modified: pypy/branch/gc-hash/pypy/rpython/ootypesystem/test/test_ootype.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/ootypesystem/test/test_ootype.py (original) +++ pypy/branch/gc-hash/pypy/rpython/ootypesystem/test/test_ootype.py Tue Oct 13 13:28:16 2009 @@ -622,27 +622,27 @@ cls2 = cast_from_object(Class, obj) assert cls is cls2 -def test_object_ooidentityhash(): +def test_object_identityhash(): A = Instance("Foo", ROOT) a = new(A) obj1 = cast_to_object(a) obj2 = cast_to_object(a) - assert ooidentityhash(obj1) == ooidentityhash(obj2) + assert identityhash(obj1) == identityhash(obj2) -def test_object_ooidentityhash_sm(): +def test_object_identityhash_sm(): M = StaticMethod([Signed], Signed) def m_(x): return x m = static_meth(M, "m", _callable=m_) obj1 = cast_to_object(m) obj2 = cast_to_object(m) - assert ooidentityhash(obj1) == ooidentityhash(obj2) + assert identityhash(obj1) == identityhash(obj2) -def test_ooidentityhash_array(): +def test_identityhash_array(): A = Array(Signed) a = oonewarray(A, 10) b = oonewarray(A, 10) - assert ooidentityhash(a) != ooidentityhash(b) + assert identityhash(a) != identityhash(b) # likely def test_bool_class(): A = Instance("Foo", ROOT) Modified: pypy/branch/gc-hash/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/rbuiltin.py (original) +++ pypy/branch/gc-hash/pypy/rpython/rbuiltin.py Tue Oct 13 13:28:16 2009 @@ -492,6 +492,10 @@ return hop.genop('cast_int_to_ptr', [v_input], resulttype = hop.r_result.lowleveltype) +def rtype_identity_hash(hop): + vlist = hop.inputargs(hop.args_r[0]) + return hop.genop('identityhash', vlist, resulttype=lltype.Signed) + def rtype_runtime_type_info(hop): assert isinstance(hop.args_r[0], rptr.PtrRepr) vlist = hop.inputargs(hop.args_r[0]) @@ -510,6 +514,7 @@ BUILTIN_TYPER[lltype.cast_int_to_ptr] = rtype_cast_int_to_ptr BUILTIN_TYPER[lltype.typeOf] = rtype_const_result BUILTIN_TYPER[lltype.nullptr] = rtype_const_result +BUILTIN_TYPER[lltype.identityhash] = rtype_identity_hash BUILTIN_TYPER[lltype.getRuntimeTypeInfo] = rtype_const_result BUILTIN_TYPER[lltype.Ptr] = rtype_const_result BUILTIN_TYPER[lltype.runtime_type_info] = rtype_runtime_type_info Modified: pypy/branch/gc-hash/pypy/rpython/rclass.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/rclass.py (original) +++ pypy/branch/gc-hash/pypy/rpython/rclass.py Tue Oct 13 13:28:16 2009 @@ -236,15 +236,18 @@ self.setup() result = self.create_instance() self._reusable_prebuilt_instance = result - self.initialize_prebuilt_instance(Ellipsis, self.classdef, result) + self.initialize_prebuilt_data(Ellipsis, self.classdef, result) return result def initialize_prebuilt_instance(self, value, classdef, result): - # must fill in the _hash_cache_ field before the other ones + # must fill in the hash cache before the other ones # (see test_circular_hash_initialization) self.initialize_prebuilt_hash(value, result) self.initialize_prebuilt_data(value, classdef, result) + def get_ll_hash_function(self): + return ll_inst_hash + def rtype_type(self, hop): raise NotImplementedError @@ -272,6 +275,13 @@ rinstance = getinstancerepr(rtyper, classdef) return rinstance.new_instance(llops, classcallhop) +def ll_inst_hash(ins): + if not ins: + return 0 # for None + else: + from pypy.rpython.lltypesystem import lltype + return lltype.identityhash(ins) # also works for ootype + _missing = object() Modified: pypy/branch/gc-hash/pypy/rpython/rdict.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/rdict.py (original) +++ pypy/branch/gc-hash/pypy/rpython/rdict.py Tue Oct 13 13:28:16 2009 @@ -46,14 +46,8 @@ else: return self._externalvsinternal(self.rtyper, item_repr) - def pickkeyrepr(self, key_repr): - external, internal = self.pickrepr(key_repr) - if external != internal: - internal = external - while not self.rtyper.needs_hash_support(internal.classdef): - internal = internal.rbase - return external, internal - + pickkeyrepr = pickrepr + def compact_repr(self): return 'DictR %s %s' % (self.key_repr.compact_repr(), self.value_repr.compact_repr()) Modified: pypy/branch/gc-hash/pypy/rpython/rtuple.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/rtuple.py (original) +++ pypy/branch/gc-hash/pypy/rpython/rtuple.py Tue Oct 13 13:28:16 2009 @@ -100,14 +100,14 @@ autounrolling_funclist = unrolling_iterable(enumerate(hash_funcs)) def ll_hash(t): - retval = 0x345678 - mult = 1000003 + """Must be kept in sync with rlib.objectmodel._hash_tuple().""" + x = 0x345678 for i, hash_func in autounrolling_funclist: attrname = 'item%d' % i item = getattr(t, attrname) - retval = intmask((retval ^ hash_func(item)) * intmask(mult)) - mult = mult + 82520 + 2*len(items_r) - return retval + y = hash_func(item) + x = intmask((1000003 * x) ^ y) + return x _gen_hash_function_cache[key] = ll_hash return ll_hash Modified: pypy/branch/gc-hash/pypy/rpython/rtyper.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/rtyper.py (original) +++ pypy/branch/gc-hash/pypy/rpython/rtyper.py Tue Oct 13 13:28:16 2009 @@ -651,9 +651,6 @@ # __________ utilities __________ - def needs_hash_support(self, clsdef): - return clsdef in self.annotator.bookkeeper.needs_hash_support - def needs_wrapper(self, cls): return cls in self.classes_with_wrapper From arigo at codespeak.net Tue Oct 13 13:29:00 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 13 Oct 2009 13:29:00 +0200 (CEST) Subject: [pypy-svn] r68371 - pypy/branch/gc-hash/pypy/rpython Message-ID: <20091013112900.748BE168013@codespeak.net> Author: arigo Date: Tue Oct 13 13:28:59 2009 New Revision: 68371 Modified: pypy/branch/gc-hash/pypy/rpython/llinterp.py Log: Control the generation of trace files from llinterp with a command-line setting: the PYPY_TRACE env var. Modified: pypy/branch/gc-hash/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/gc-hash/pypy/rpython/llinterp.py (original) +++ pypy/branch/gc-hash/pypy/rpython/llinterp.py Tue Oct 13 13:28:59 2009 @@ -1207,7 +1207,7 @@ class Tracer(object): Counter = 0 file = None - TRACE = False + TRACE = int(os.getenv('PYPY_TRACE') or '0') HEADER = """ + """ Modified: pypy/build/bot2/pypybuildbot/test/test_summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/test/test_summary.py (original) +++ pypy/build/bot2/pypybuildbot/test/test_summary.py Sat Oct 17 18:12:04 2009 @@ -491,7 +491,7 @@ assert 'TEST1' in out assert ':-)' in out - assert '\n - + success' in out + assert '\n - + success' in out def test_two_builds_samerev(self): builder = status_builder.BuilderStatus('builder0') From cfbolz at codespeak.net Sat Oct 17 18:30:19 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 17 Oct 2009 18:30:19 +0200 (CEST) Subject: [pypy-svn] r68591 - pypy/branch/pyjitpl5-c Message-ID: <20091017163019.DB20B16801F@codespeak.net> Author: cfbolz Date: Sat Oct 17 18:30:19 2009 New Revision: 68591 Removed: pypy/branch/pyjitpl5-c/ Log: the C backend is not going anywhere From cfbolz at codespeak.net Sat Oct 17 18:31:59 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 17 Oct 2009 18:31:59 +0200 (CEST) Subject: [pypy-svn] r68592 - pypy/branch/pyjitpl5-experiments Message-ID: <20091017163159.D86C516801F@codespeak.net> Author: cfbolz Date: Sat Oct 17 18:31:59 2009 New Revision: 68592 Removed: pypy/branch/pyjitpl5-experiments/ Log: This is a stale as well. From cfbolz at codespeak.net Sat Oct 17 18:32:24 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 17 Oct 2009 18:32:24 +0200 (CEST) Subject: [pypy-svn] r68593 - pypy/branch/gc-hash Message-ID: <20091017163224.1DE0716801F@codespeak.net> Author: cfbolz Date: Sat Oct 17 18:32:23 2009 New Revision: 68593 Removed: pypy/branch/gc-hash/ Log: merged From cfbolz at codespeak.net Sat Oct 17 18:32:42 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 17 Oct 2009 18:32:42 +0200 (CEST) Subject: [pypy-svn] r68594 - pypy/branch/gc-hash-merge Message-ID: <20091017163242.7449316801F@codespeak.net> Author: cfbolz Date: Sat Oct 17 18:32:41 2009 New Revision: 68594 Removed: pypy/branch/gc-hash-merge/ Log: merged From cfbolz at codespeak.net Sat Oct 17 18:34:00 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 17 Oct 2009 18:34:00 +0200 (CEST) Subject: [pypy-svn] r68595 - pypy/branch/gc-in-genc Message-ID: <20091017163400.E771F16801F@codespeak.net> Author: cfbolz Date: Sat Oct 17 18:34:00 2009 New Revision: 68595 Removed: pypy/branch/gc-in-genc/ Log: armin tells me I can kill this From cfbolz at codespeak.net Sat Oct 17 18:36:20 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 17 Oct 2009 18:36:20 +0200 (CEST) Subject: [pypy-svn] r68596 - pypy/branch/tagged-pointers-framework Message-ID: <20091017163620.17B4316801F@codespeak.net> Author: cfbolz Date: Sat Oct 17 18:36:19 2009 New Revision: 68596 Removed: pypy/branch/tagged-pointers-framework/ Log: this has been merged into tagged-pointers-framework2 From cfbolz at codespeak.net Sat Oct 17 18:40:42 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 17 Oct 2009 18:40:42 +0200 (CEST) Subject: [pypy-svn] r68597 - pypy/branch/unified-rtti Message-ID: <20091017164042.0DB98168014@codespeak.net> Author: cfbolz Date: Sat Oct 17 18:40:41 2009 New Revision: 68597 Removed: pypy/branch/unified-rtti/ Log: another one bites the dust From cfbolz at codespeak.net Sat Oct 17 18:46:45 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 17 Oct 2009 18:46:45 +0200 (CEST) Subject: [pypy-svn] r68598 - pypy/branch/shrink-multidict Message-ID: <20091017164645.33F10168014@codespeak.net> Author: cfbolz Date: Sat Oct 17 18:46:44 2009 New Revision: 68598 Removed: pypy/branch/shrink-multidict/ Log: kill this branch, I want what it contains, but the world has changed too much to make it mergeable. From cfbolz at codespeak.net Sat Oct 17 18:49:37 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 17 Oct 2009 18:49:37 +0200 (CEST) Subject: [pypy-svn] r68599 - pypy/branch/shrink-multidict Message-ID: <20091017164937.7CE40168014@codespeak.net> Author: cfbolz Date: Sat Oct 17 18:49:36 2009 New Revision: 68599 Added: pypy/branch/shrink-multidict/ - copied from r68598, pypy/trunk/ Log: resurrect the branch to try again From cfbolz at codespeak.net Sat Oct 17 19:29:30 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 17 Oct 2009 19:29:30 +0200 (CEST) Subject: [pypy-svn] r68604 - pypy/branch/tagged-pointers-framework2/pypy/config Message-ID: <20091017172930.97413168015@codespeak.net> Author: cfbolz Date: Sat Oct 17 19:29:29 2009 New Revision: 68604 Modified: pypy/branch/tagged-pointers-framework2/pypy/config/pypyoption.py Log: disable small ints again, now that the test run is finished Modified: pypy/branch/tagged-pointers-framework2/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/tagged-pointers-framework2/pypy/config/pypyoption.py (original) +++ pypy/branch/tagged-pointers-framework2/pypy/config/pypyoption.py Sat Oct 17 19:29:29 2009 @@ -357,7 +357,6 @@ if level in ['2', '3', 'jit']: config.objspace.opcodes.suggest(CALL_METHOD=True) config.objspace.std.suggest(withmultidict=True) - config.objspace.std.suggest(withsmallint=True) config.objspace.std.suggest(withshadowtracking=True) config.objspace.std.suggest(withrangelist=True) config.objspace.std.suggest(withmethodcache=True) From arigo at codespeak.net Sat Oct 17 19:34:05 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 17 Oct 2009 19:34:05 +0200 (CEST) Subject: [pypy-svn] r68605 - in pypy/build/bot2/pypybuildbot: . test Message-ID: <20091017173405.00EE9168016@codespeak.net> Author: arigo Date: Sat Oct 17 19:33:59 2009 New Revision: 68605 Modified: pypy/build/bot2/pypybuildbot/summary.py pypy/build/bot2/pypybuildbot/test/test_summary.py Log: Fix the color of the "-" signs. Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Sat Oct 17 19:33:59 2009 @@ -419,17 +419,15 @@ if self.SUCCESS_LINE: success = [] for label, outcome_set in by_label: - symbol = "+" - clazz = "success" if outcome_set.failed: - clazz = "failed" symbol = html.a("-", id="a%dc%d" % (a_num, 1<- + success' in out + assert '\n - + success' in out def test_two_builds_samerev(self): builder = status_builder.BuilderStatus('builder0') From cfbolz at codespeak.net Sat Oct 17 20:06:06 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 17 Oct 2009 20:06:06 +0200 (CEST) Subject: [pypy-svn] r68607 - pypy/trunk/pypy/config Message-ID: <20091017180606.7CB29168016@codespeak.net> Author: cfbolz Date: Sat Oct 17 20:06:05 2009 New Revision: 68607 Added: pypy/trunk/pypy/config/pypyoption.py.merge.tmp - copied, changed from r68605, pypy/trunk/pypy/config/pypyoption.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/tagged-pointers-framework2/pypy/config/pypyoption.py revisions 68538 to 68605: ------------------------------------------------------------------------ r68604 | cfbolz | 2009-10-17 19:29:29 +0200 (Sat, 17 Oct 2009) | 2 lines disable small ints again, now that the test run is finished ------------------------------------------------------------------------ r68589 | cfbolz | 2009-10-17 17:20:38 +0200 (Sat, 17 Oct 2009) | 2 lines enable this for a bit, to get it when doing a lib-python run ------------------------------------------------------------------------ r68545 | cfbolz | 2009-10-16 23:02:19 +0200 (Fri, 16 Oct 2009) | 230 lines merge in tagged-pointers-framework branch: ----------------------------------------------------------------------- r65789 | arigo | 2009-06-16 10:06:57 +0200 (Tue, 16 Jun 2009) | 3 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gc/generation.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gctransform/framework.py Missing a gc.points_to_valid_gc_object() in the thread-specific code. The rest is just clean-ups. ------------------------------------------------------------------------ r65784 | arigo | 2009-06-15 22:02:59 +0200 (Mon, 15 Jun 2009) | 4 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gctransform/asmgcroot.py M /pypy/branch/tagged-pointers-framework/pypy/translator/c/gcc/test/test_asmgcroot.py Revert r65770. It turns out that asmgcc is not compatible with non-moving GCs. It could be fixed but it's not worthwhile (and definitely the "fix" of r65770 was completely broken :-) ------------------------------------------------------------------------ r65770 | cfbolz | 2009-06-14 12:21:20 +0200 (Sun, 14 Jun 2009) | 3 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gctransform/asmgcroot.py M /pypy/branch/tagged-pointers-framework/pypy/translator/c/gcc/test/test_asmgcroot.py Make asmgcc work at least somewhat with non-moving GCs. In the current form, some of the tests segfault, no clue whose fault that is yet. ------------------------------------------------------------------------ r65751 | cfbolz | 2009-06-12 19:12:45 +0200 (Fri, 12 Jun 2009) | 3 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gc/base.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gc/hybrid.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/test/test_gc.py Fix id computation in the presence of tagged pointers: make sure that real objects have even ids, so that the tagged ints can have odd ones. ------------------------------------------------------------------------ r65456 | cfbolz | 2009-05-27 15:59:42 +0200 (Wed, 27 May 2009) | 3 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/rpython/lltypesystem/rtagged.py Be more precise about getting the type of tagged-or-normal-or-None. Gets rid of a lot of warnings when translating. ------------------------------------------------------------------------ r65454 | cfbolz | 2009-05-27 15:04:33 +0200 (Wed, 27 May 2009) | 2 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gctransform/asmgcroot.py M /pypy/branch/tagged-pointers-framework/pypy/translator/c/gcc/test/test_asmgcroot.py Fix asmgcroot stack walker in conjunction with tagged pointers. ------------------------------------------------------------------------ r65453 | cfbolz | 2009-05-27 15:03:14 +0200 (Wed, 27 May 2009) | 2 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/lang/smalltalk/model.py Adapt smalltalk vm to make the translation.taggedpointers option do something. ------------------------------------------------------------------------ r65452 | cfbolz | 2009-05-27 15:02:21 +0200 (Wed, 27 May 2009) | 2 lines Changed paths: A /pypy/branch/tagged-pointers-framework/pypy/doc/config/translation.taggedpointers.txt add documentation ------------------------------------------------------------------------ r65448 | cfbolz | 2009-05-27 14:18:39 +0200 (Wed, 27 May 2009) | 2 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/translator/c/test/test_rtagged.py fix another test ------------------------------------------------------------------------ r65447 | cfbolz | 2009-05-27 14:16:22 +0200 (Wed, 27 May 2009) | 2 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/rlib/test/test_objectmodel.py forgot to fix a test ------------------------------------------------------------------------ r65422 | cfbolz | 2009-05-26 17:40:45 +0200 (Tue, 26 May 2009) | 3 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/rpython/lltypesystem/rtagged.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/lltypesystem/test/test_rtagged.py waaaaaa, how could this ever have worked? type(x) where x=None but x could also be a tagged pointer segfaulted so far. ------------------------------------------------------------------------ r65420 | cfbolz | 2009-05-26 15:10:34 +0200 (Tue, 26 May 2009) | 2 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/lang/prolog/interpreter/term.py make it possible to use tagged pointers in the prolog implementation. ------------------------------------------------------------------------ r65415 | cfbolz | 2009-05-25 17:44:47 +0200 (Mon, 25 May 2009) | 3 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/rlib/objectmodel.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/lltypesystem/rtagged.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/lltypesystem/test/test_rtagged.py Rename the very generic method name "getvalue" to get_untagged_value. So far it's not really used much anyway, only in tests. ------------------------------------------------------------------------ r65414 | cfbolz | 2009-05-25 17:39:35 +0200 (Mon, 25 May 2009) | 2 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/rlib/objectmodel.py update comment ------------------------------------------------------------------------ r65413 | cfbolz | 2009-05-25 17:26:28 +0200 (Mon, 25 May 2009) | 2 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/translator/c/test/test_boehm.py M /pypy/branch/tagged-pointers-framework/pypy/translator/c/test/test_newgc.py Fix translating tagging tests to use the new option. ------------------------------------------------------------------------ r65412 | cfbolz | 2009-05-25 17:22:29 +0200 (Mon, 25 May 2009) | 5 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/config/pypyoption.py M /pypy/branch/tagged-pointers-framework/pypy/config/translationoption.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/lltypesystem/rclass.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/lltypesystem/test/test_rtagged.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/test/test_gc.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/test/test_transformed_gc.py Add a translation config option to use pointer tagging. This should make it possible for e.g. the Smalltalk VM to make pointer tagging a translate.py (as it is not really natural there to have to different classes as in the Python interpreter). ------------------------------------------------------------------------ r65409 | cfbolz | 2009-05-25 14:25:19 +0200 (Mon, 25 May 2009) | 5 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/rpython/lltypesystem/llmemory.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/lltypesystem/lltype.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gc/base.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gc/hybrid.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/test/test_gc.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/test/test_transformed_gc.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/test/test_rptr.py - fix casting of opaque pointers to ints. Needed for correct GCREF handing - allowing casting of addresses of dead objects to int. Fixes some test_transformed_gc tests - proper id handling of tagged pointers in moving GCs ------------------------------------------------------------------------ r65133 | cfbolz | 2009-05-07 14:00:26 +0200 (Thu, 07 May 2009) | 3 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gctransform/framework.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/test/test_gc.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/test/test_transformed_gc.py tests with prebuilt data structures. This finds a bug. The tests still fails on mark-n-sweep, no clue why yet. ------------------------------------------------------------------------ r65083 | cfbolz | 2009-05-06 12:26:50 +0200 (Wed, 06 May 2009) | 3 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gcwrapper.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/test/test_gc.py Fix gcwrapper to support prebuilt tagged pointers. Doesn't find the bug I am hunting though. ------------------------------------------------------------------------ r65079 | cfbolz | 2009-05-06 09:55:29 +0200 (Wed, 06 May 2009) | 2 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gc/base.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gc/generation.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/test/test_gc.py fix write barrier ------------------------------------------------------------------------ r65078 | cfbolz | 2009-05-06 09:39:43 +0200 (Wed, 06 May 2009) | 2 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gc/markcompact.py missed a copy of trace ------------------------------------------------------------------------ r65041 | cfbolz | 2009-05-04 23:35:44 +0200 (Mon, 04 May 2009) | 2 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/config/pypyoption.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gc/base.py In theory, this should be enough. In practise, it obviously segfaults. ------------------------------------------------------------------------ r65021 | cfbolz | 2009-05-04 13:59:29 +0200 (Mon, 04 May 2009) | 4 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gc/base.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gc/generation.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gc/markcompact.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gc/marksweep.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gc/semispace.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gctransform/framework.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/gcwrapper.py Refactor things so that there is only one place where it is checked whether a pointed-to object is NULL. This place can later be used for checking tagged pointers as well. ------------------------------------------------------------------------ r65020 | cfbolz | 2009-05-04 13:58:19 +0200 (Mon, 04 May 2009) | 2 lines Changed paths: M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/test/test_gc.py M /pypy/branch/tagged-pointers-framework/pypy/rpython/memory/test/test_transformed_gc.py (failing) tests that I would like to pass ------------------------------------------------------------------------ r65019 | cfbolz | 2009-05-04 13:54:06 +0200 (Mon, 04 May 2009) | 2 lines Changed paths: A /pypy/branch/tagged-pointers-framework (from /pypy/trunk:65018) A branch to play with supporting tagged pointers with the framework GCs ------------------------------------------------------------------------ r68540 | cfbolz | 2009-10-16 15:59:17 +0200 (Fri, 16 Oct 2009) | 2 lines a new copy of trunk to merge in the tagged-pointers-framework branch ------------------------------------------------------------------------ Copied: pypy/trunk/pypy/config/pypyoption.py.merge.tmp (from r68605, pypy/trunk/pypy/config/pypyoption.py) ============================================================================== --- pypy/trunk/pypy/config/pypyoption.py (original) +++ pypy/trunk/pypy/config/pypyoption.py.merge.tmp Sat Oct 17 20:06:05 2009 @@ -183,8 +183,8 @@ BoolOption("withsmallint", "use tagged integers", default=False, - requires=[("translation.gc", "boehm"), - ("objspace.std.withprebuiltint", False)]), + requires=[("objspace.std.withprebuiltint", False), + ("translation.taggedpointers", True)]), BoolOption("withprebuiltint", "prebuild commonly used int objects", default=False), From cfbolz at codespeak.net Sat Oct 17 20:06:51 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 17 Oct 2009 20:06:51 +0200 (CEST) Subject: [pypy-svn] r68608 - in pypy/trunk/pypy: config doc/config lang rlib rpython translator/c Message-ID: <20091017180651.4327C168016@codespeak.net> Author: cfbolz Date: Sat Oct 17 20:06:50 2009 New Revision: 68608 Added: pypy/trunk/pypy/config/pypyoption.py - copied unchanged from r68607, pypy/trunk/pypy/config/pypyoption.py.merge.tmp pypy/trunk/pypy/config/translationoption.py - copied unchanged from r68607, pypy/branch/tagged-pointers-framework2/pypy/config/translationoption.py pypy/trunk/pypy/doc/config/translation.taggedpointers.txt - copied unchanged from r68607, pypy/branch/tagged-pointers-framework2/pypy/doc/config/translation.taggedpointers.txt pypy/trunk/pypy/lang/ - copied from r68607, pypy/branch/tagged-pointers-framework2/pypy/lang/ pypy/trunk/pypy/rlib/ - copied from r68607, pypy/branch/tagged-pointers-framework2/pypy/rlib/ pypy/trunk/pypy/rpython/ - copied from r68607, pypy/branch/tagged-pointers-framework2/pypy/rpython/ pypy/trunk/pypy/translator/c/ - copied from r68607, pypy/branch/tagged-pointers-framework2/pypy/translator/c/ Removed: pypy/trunk/pypy/config/pypyoption.py.merge.tmp Log: merge tagged-pointers-framework2 branch From cfbolz at codespeak.net Sat Oct 17 20:07:12 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 17 Oct 2009 20:07:12 +0200 (CEST) Subject: [pypy-svn] r68609 - pypy/branch/tagged-pointers-framework2 Message-ID: <20091017180712.73F4A168016@codespeak.net> Author: cfbolz Date: Sat Oct 17 20:07:11 2009 New Revision: 68609 Removed: pypy/branch/tagged-pointers-framework2/ Log: kill merged branch From arigo at codespeak.net Sat Oct 17 21:30:27 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 17 Oct 2009 21:30:27 +0200 (CEST) Subject: [pypy-svn] r68612 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20091017193027.47B11168015@codespeak.net> Author: arigo Date: Sat Oct 17 21:30:24 2009 New Revision: 68612 Modified: pypy/trunk/pypy/rpython/lltypesystem/llarena.py Log: On Linux, to clear large regions of memory, the best way seems to be madvise(MADV_DONTNEED). Indeed, the previous trick of reading /dev/zero does not seem to have the correct effect of lazily-allocating pages on all Linux systems (e.g. bigdog-vm1). Note that this usage of madvise() is purely Linux-specific. Modified: pypy/trunk/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/llarena.py Sat Oct 17 21:30:24 2009 @@ -331,7 +331,46 @@ from pypy.rpython.extfunc import register_external from pypy.rlib.objectmodel import CDefinedIntSymbolic -if os.name == 'posix': +if sys.platform == 'linux2': + # This only works with linux's madvise(), which is really not a memory + # usage hint but a real command. It guarantees that after MADV_DONTNEED + # the pages are cleared again. + from pypy.rpython.tool import rffi_platform + MADV_DONTNEED = rffi_platform.getconstantinteger('MADV_DONTNEED', + '#include ') + linux_madvise = rffi.llexternal('madvise', + [llmemory.Address, rffi.SIZE_T, rffi.INT], + rffi.INT) + linux_getpagesize = rffi.llexternal('getpagesize', [], rffi.INT) + + class LinuxPageSize: + def __init__(self): + self.pagesize = 0 + _freeze_ = __init__ + linuxpagesize = LinuxPageSize() + + def clear_large_memory_chunk(baseaddr, size): + pagesize = linuxpagesize.pagesize + if pagesize == 0: + pagesize = rffi.cast(lltype.Signed, linux_getpagesize()) + linuxpagesize.pagesize = pagesize + if size > 2 * pagesize: + lowbits = rffi.cast(lltype.Signed, baseaddr) & (pagesize - 1) + if lowbits: # clear the initial misaligned part, if any + partpage = pagesize - lowbits + llmemory.raw_memclear(baseaddr, partpage) + baseaddr += partpage + size -= partpage + madv_length = size & -pagesize + madv_flags = rffi.cast(rffi.INT, MADV_DONTNEED) + err = linux_madvise(baseaddr, madv_length, madv_flags) + if rffi.cast(lltype.Signed, err) == 0: + baseaddr += madv_length # madvise() worked + size -= madv_length + if size > 0: # clear the final misaligned part, if any + llmemory.raw_memclear(baseaddr, size) + +elif os.name == 'posix': READ_MAX = (sys.maxint//4) + 1 # upper bound on reads to avoid surprises raw_os_open = rffi.llexternal('open', [rffi.CCHARP, rffi.INT, rffi.MODE_T], @@ -348,7 +387,7 @@ _dev_zero = rffi.str2charp('/dev/zero') # prebuilt def clear_large_memory_chunk(baseaddr, size): - # on Linux at least, reading from /dev/zero is the fastest way + # on some Unixy platforms, reading from /dev/zero is the fastest way # to clear arenas, because the kernel knows that it doesn't # need to even allocate the pages before they are used. @@ -373,6 +412,10 @@ else: # XXX any better implementation on Windows? + # Should use VirtualAlloc() to reserve the range of pages, + # and commit some pages gradually with support from the GC. + # Or it might be enough to decommit the pages and recommit + # them immediately. clear_large_memory_chunk = llmemory.raw_memclear From afa at codespeak.net Mon Oct 19 00:16:23 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 19 Oct 2009 00:16:23 +0200 (CEST) Subject: [pypy-svn] r68614 - in pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc: . test test/msvc Message-ID: <20091018221623.E598A168009@codespeak.net> Author: afa Date: Mon Oct 19 00:16:22 2009 New Revision: 68614 Added: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Log: Start parsing a msvc assembler Added: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s ============================================================================== --- (empty file) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s Mon Oct 19 00:16:22 2009 @@ -0,0 +1,275 @@ +PUBLIC _pypy_g_ll_join_strs__Signed_arrayPtr +; Function compile flags: /Ogtpy +; COMDAT _pypy_g_ll_join_strs__Signed_arrayPtr +_TEXT SEGMENT +_l_result_2$ = -8 ; size = 4 +_l_v405$ = -4 ; size = 4 +_l_num_items_0$ = 8 ; size = 4 +_l_items_2$ = 12 ; size = 4 +_pypy_g_ll_join_strs__Signed_arrayPtr PROC ; COMDAT + +; 1457 : struct pypy_rpy_string0 *pypy_g_ll_join_strs__Signed_arrayPtr(long l_num_items_0, struct pypy_array0 *l_items_2) { + + sub esp, 8 + push ebx + push ebp + push esi + +; 1458 : long l_i_22; long l_i_23; long l_res_index_0; +; 1459 : struct pypy_rpy_string0 *l_result_2; bool_t l_v403; bool_t l_v404; +; 1460 : bool_t l_v409; bool_t l_v410; bool_t l_v411; long l_v402; +; 1461 : long l_v414; long l_v417; long l_v418; long l_v421; long l_v422; +; 1462 : long l_v423; struct pypy_object_vtable0 *l_v408; +; 1463 : struct pypy_rpy_string0 *l_v412; struct pypy_rpy_string0 *l_v413; +; 1464 : struct pypy_rpy_string0 *l_v415; struct pypy_rpy_string0 *l_v419; +; 1465 : struct pypy_rpy_string0 *l_v420; struct pypy_rpy_string0 *l_v424; +; 1466 : void* l_v405; void* l_v406; +; 1467 : +; 1468 : block0: +; 1469 : l_i_23 = 0L; + + xor esi, esi + push edi + +; 1470 : l_v402 = 0L; + + xor edi, edi + +; 1471 : goto block1; +; 1472 : +; 1473 : block1: +; 1474 : OP_INT_LT(l_i_23, l_num_items_0, l_v403); + + cmp DWORD PTR _l_num_items_0$[esp+20], esi +$block0$40039: +$block1$40040: + +; 1475 : l_v404 = l_v403; +; 1476 : while (l_v404) { + + jle SHORT $block2$40046 + mov ebp, DWORD PTR _l_items_2$[esp+20] + add ebp, 8 +$LL5 at pypy_g_ll_@139: +$block6$40044: + +; 1525 : goto block3_back; +; 1526 : +; 1527 : block6: +; 1528 : l_v419 = RPyItem(l_items_2, l_i_23); + + test esi, esi + jl SHORT $LN14 at pypy_g_ll_@139 + mov eax, DWORD PTR _l_items_2$[esp+20] + cmp esi, DWORD PTR [eax+4] + jl SHORT $LN15 at pypy_g_ll_@139 +$LN14 at pypy_g_ll_@139: + call _RPyAbort +$LN15 at pypy_g_ll_@139: + +; 1529 : l_v420 = l_v419; + + mov ebx, DWORD PTR [ebp] + +; 1530 : l_v421 = RPyField(l_v420, rs_chars).length; + + test ebx, ebx + jne SHORT $LN16 at pypy_g_ll_@139 + call _RPyAbort +$LN16 at pypy_g_ll_@139: + +; 1531 : OP_INT_ADD(l_v402, l_v421, l_v422); +; 1532 : OP_INT_ADD(l_i_23, 1L, l_v423); +; 1533 : l_i_23 = l_v423; + + inc esi + add ebp, 4 + +; 1534 : l_v402 = l_v422; + + add edi, DWORD PTR [ebx+8] + cmp esi, DWORD PTR _l_num_items_0$[esp+20] +$block1_back$40045: + jl SHORT $LL5 at pypy_g_ll_@139 +$block2$40046: + +; 1477 : goto block6; +; 1478 : block1_back: ; +; 1479 : OP_INT_LT(l_i_23, l_num_items_0, l_v403); +; 1480 : l_v404 = l_v403; +; 1481 : } +; 1482 : goto block2; +; 1483 : +; 1484 : block2: +; 1485 : l_result_2 = pypy_g_mallocstr__Signed(l_v402); + + push edi + call _pypy_g_mallocstr__Signed + +; 1486 : l_v405 = (void*)l_items_2; + + mov ecx, DWORD PTR _l_items_2$[esp+24] + add esp, 4 + mov DWORD PTR _l_result_2$[esp+24], eax + mov DWORD PTR _l_v405$[esp+24], ecx + +; 1487 : l_v406 = pypy_asm_gcroot(l_v405); + + test DWORD PTR _l_v405$[esp+24], 0 + +; 1488 : l_items_2 = l_v406; /* for moving GCs */ +; 1489 : l_v408 = RPyField((&pypy_g_ExcData), ed_exc_type); +; 1490 : l_v409 = (l_v408 == NULL); + + cmp DWORD PTR _pypy_g_ExcData, 0 + +; 1491 : if (!l_v409) { + + je SHORT $LN3 at pypy_g_ll_@139 + pop edi + pop esi + pop ebp + +; 1492 : l_v424 = ((struct pypy_rpy_string0 *) NULL); + + xor eax, eax + pop ebx + +; 1535 : goto block1_back; +; 1536 : } + + add esp, 8 + ret 0 +$LN3 at pypy_g_ll_@139: + +; 1493 : goto block4; +; 1494 : } +; 1495 : l_i_22 = 0L; + + xor esi, esi + +; 1496 : l_res_index_0 = 0L; + + xor ebp, ebp + +; 1497 : goto block3; +; 1498 : +; 1499 : block3: +; 1500 : OP_INT_LT(l_i_22, l_num_items_0, l_v410); + + cmp DWORD PTR _l_num_items_0$[esp+20], ebp +$block3$40053: + +; 1501 : l_v411 = l_v410; +; 1502 : while (l_v411) { + + jle SHORT $LN1 at pypy_g_ll_@139 + mov ebx, ecx + add ebx, 8 +$LL2 at pypy_g_ll_@139: +$block5$40057: + +; 1514 : +; 1515 : block5: +; 1516 : l_v412 = RPyItem(l_items_2, l_i_22); + + test esi, esi + jl SHORT $LN9 at pypy_g_ll_@139 + mov edx, DWORD PTR _l_items_2$[esp+20] + cmp esi, DWORD PTR [edx+4] + jl SHORT $LN10 at pypy_g_ll_@139 +$LN9 at pypy_g_ll_@139: + call _RPyAbort +$LN10 at pypy_g_ll_@139: + +; 1517 : l_v413 = l_v412; + + mov edi, DWORD PTR [ebx] + +; 1518 : l_v414 = RPyField(l_v413, rs_chars).length; + + test edi, edi + jne SHORT $LN11 at pypy_g_ll_@139 + call _RPyAbort +$LN11 at pypy_g_ll_@139: + mov edi, DWORD PTR [edi+8] + +; 1519 : l_v415 = RPyItem(l_items_2, l_i_22); + + test esi, esi + jl SHORT $LN12 at pypy_g_ll_@139 + mov eax, DWORD PTR _l_items_2$[esp+20] + cmp esi, DWORD PTR [eax+4] + jl SHORT $LN13 at pypy_g_ll_@139 +$LN12 at pypy_g_ll_@139: + call _RPyAbort +$LN13 at pypy_g_ll_@139: + +; 1520 : pypy_g_copy_string_contents__rpy_stringPtr_rpy_stringPt(l_v415, l_result_2, 0L, l_res_index_0, l_v414); + + mov ecx, DWORD PTR [ebx] + mov edx, DWORD PTR _l_result_2$[esp+24] + push edi + add ecx, 12 ; 0000000cH + push ecx + lea eax, DWORD PTR [edx+ebp+12] + push eax +$block0$80664: +$block0$80634: +$block0$80639: +$block1$80640: +$block0$80644: +$block1$80645: +$block1$80635: +$block0$80659: +$block0$80667: +$block1$80668: +$block0$80673: +$block1$80674: +$block1$80661: +$block0$80678: +$block1$80679: + call _memcpy + add esp, 12 ; 0000000cH + +; 1521 : OP_INT_ADD(l_res_index_0, l_v414, l_v417); +; 1522 : OP_INT_ADD(l_i_22, 1L, l_v418); +; 1523 : l_i_22 = l_v418; + + inc esi + add ebx, 4 + +; 1524 : l_res_index_0 = l_v417; + + add ebp, edi + cmp esi, DWORD PTR _l_num_items_0$[esp+20] +$block1$80671: +$block3_back$40058: + jl SHORT $LL2 at pypy_g_ll_@139 +$LN1 at pypy_g_ll_@139: + +; 1503 : goto block5; +; 1504 : block3_back: ; +; 1505 : OP_INT_LT(l_i_22, l_num_items_0, l_v410); +; 1506 : l_v411 = l_v410; +; 1507 : } +; 1508 : l_v424 = l_result_2; +; 1509 : goto block4; +; 1510 : +; 1511 : block4: +; 1512 : RPY_DEBUG_RETURN(); +; 1513 : return l_v424; + + mov eax, DWORD PTR _l_result_2$[esp+24] + pop edi + pop esi + pop ebp +$block4$40052: + pop ebx + +; 1535 : goto block1_back; +; 1536 : } + + add esp, 8 + ret 0 +_pypy_g_ll_join_strs__Signed_arrayPtr ENDP Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py Mon Oct 19 00:16:22 2009 @@ -109,7 +109,7 @@ def test_computegcmaptable(): tests = [] - for format in ('elf', 'darwin'): + for format in ('elf', 'darwin', 'msvc'): for path in this_dir.join(format).listdir("track*.s"): n = path.purebasename[5:] try: Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Mon Oct 19 00:16:22 2009 @@ -33,7 +33,6 @@ r_globl = re.compile(r"\t[.]globl\t"+LABEL+"\s*$") r_globllabel = re.compile(LABEL+r"=[.][+]%d\s*$"%OFFSET_LABELS) r_insn = re.compile(r"\t([a-z]\w*)\s") -r_jump = re.compile(r"\tj\w+\s+"+LABEL+"\s*$") OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+")\s*$") r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+([*]"+OPERAND+")\s*$") @@ -53,6 +52,8 @@ class FunctionGcRootTracker(object): + r_jump = re.compile(r"\tj\w+\s+"+LABEL+"\s*$") + def __init__(self, funcname, lines, filetag=0): self.funcname = funcname self.lines = lines @@ -585,7 +586,7 @@ self.labels[label].previous_insns.append(self.insns[-1]) def conditional_jump(self, line): - match = r_jump.match(line) + match = self.r_jump.match(line) if not match: raise UnrecognizedOperation(line) label = match.group(1) @@ -677,6 +678,25 @@ class Mingw32FunctionGcRootTracker(DarwinFunctionGcRootTracker): format = 'mingw32' +class MsvcFunctionGcRootTracker(FunctionGcRootTracker): + format = 'msvc' + + r_functionstart = re.compile(r"PUBLIC\t"+LABEL+"$") + r_jump = re.compile(r"\tj\w+\s+(?:SHORT )?"+LABEL+"\s*$") + + def __init__(self, lines, filetag=0): + match = self.r_functionstart.match(lines[0]) + funcname = match.group(1) + super(MsvcFunctionGcRootTracker, self).__init__( + funcname, lines, filetag) + + for name in ''' + push mov + xor sub + '''.split(): + locals()['visit_' + name] = getattr(FunctionGcRootTracker, + 'visit_' + name + 'l') + class AssemblerParser(object): def __init__(self, verbose=0, shuffle=False): self.verbose = verbose @@ -795,10 +815,15 @@ if functionlines: yield in_function, functionlines +class MsvcAssemblerParser(AssemblerParser): + format = "msvc" + FunctionGcRootTracker = MsvcFunctionGcRootTracker + PARSERS = { 'elf': ElfAssemblerParser, 'darwin': DarwinAssemblerParser, 'mingw32': Mingw32AssemblerParser, + 'msvc': MsvcAssemblerParser, } class GcRootTracker(object): From afa at codespeak.net Mon Oct 19 00:58:34 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 19 Oct 2009 00:58:34 +0200 (CEST) Subject: [pypy-svn] r68615 - pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc Message-ID: <20091018225834.4F181168009@codespeak.net> Author: afa Date: Mon Oct 19 00:58:33 2009 New Revision: 68615 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Log: Little progress Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Mon Oct 19 00:58:33 2009 @@ -33,14 +33,10 @@ r_globl = re.compile(r"\t[.]globl\t"+LABEL+"\s*$") r_globllabel = re.compile(LABEL+r"=[.][+]%d\s*$"%OFFSET_LABELS) r_insn = re.compile(r"\t([a-z]\w*)\s") -OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' -r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+")\s*$") -r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+([*]"+OPERAND+")\s*$") r_jmp_switch = re.compile(r"\tjmp\t[*]"+LABEL+"[(]") r_jmp_source = re.compile(r"\d*[(](%[\w]+)[,)]") r_jmptable_item = re.compile(r"\t.long\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") r_jmptable_end = re.compile(r"\t.text|\t.section\s+.text|\t\.align|"+LABEL) -r_binaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+"),\s*("+OPERAND+")\s*$") LOCALVAR = r"%eax|%edx|%ecx|%ebx|%esi|%edi|%ebp|\d*[(]%esp[)]" LOCALVARFP = LOCALVAR + r"|-?\d*[(]%ebp[)]" r_gcroot_marker = re.compile(r"\t/[*] GCROOT ("+LOCALVARFP+") [*]/") @@ -52,6 +48,11 @@ class FunctionGcRootTracker(object): + OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' + r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+")\s*$") + r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+([*]"+OPERAND+")\s*$") + r_binaryinsn = re.compile(r"\t[a-z]\w*\s+(?P"+OPERAND+"),\s*(?P"+OPERAND+")\s*$") + r_jump = re.compile(r"\tj\w+\s+"+LABEL+"\s*$") def __init__(self, funcname, lines, filetag=0): @@ -370,9 +371,9 @@ visit_xorw = visit_nop def visit_addl(self, line, sign=+1): - match = r_binaryinsn.match(line) - source = match.group(1) - target = match.group(2) + match = self.r_binaryinsn.match(line) + source = match.group("source") + target = match.group("target") if target == '%esp': count = match.group(1) if not count.startswith('$'): @@ -388,7 +389,7 @@ return self.visit_addl(line, sign=-1) def unary_insn(self, line): - match = r_unaryinsn.match(line) + match = self.r_unaryinsn.match(line) target = match.group(1) if self.r_localvar.match(target): return InsnSetLocal(target) @@ -396,11 +397,11 @@ return [] def binary_insn(self, line): - match = r_binaryinsn.match(line) + match = self.r_binaryinsn.match(line) if not match: raise UnrecognizedOperation(line) - source = match.group(1) - target = match.group(2) + source = match.group("source") + target = match.group("target") if self.r_localvar.match(target): return InsnSetLocal(target, [source]) elif target == '%esp': @@ -428,8 +429,8 @@ visit_cmovno = binary_insn def visit_andl(self, line): - match = r_binaryinsn.match(line) - target = match.group(2) + match = self.r_binaryinsn.match(line) + target = match.group("target") if target == '%esp': # only for andl $-16, %esp used to align the stack in main(). # The exact amount of adjutment is not known yet, so we use @@ -440,11 +441,11 @@ return self.binary_insn(line) def visit_leal(self, line): - match = r_binaryinsn.match(line) - target = match.group(2) + match = self.r_binaryinsn.match(line) + target = match.group("target") if target == '%esp': # only for leal -12(%ebp), %esp in function epilogues - source = match.group(1) + source = match.group("source") match = r_localvar_ebp.match(source) if match: if not self.uses_frame_pointer: @@ -475,9 +476,9 @@ return [] def visit_movl(self, line): - match = r_binaryinsn.match(line) - source = match.group(1) - target = match.group(2) + match = self.r_binaryinsn.match(line) + source = match.group("source") + target = match.group("target") if source == '%esp' and target == '%ebp': return self._visit_prologue() elif source == '%ebp' and target == '%esp': @@ -485,7 +486,7 @@ return self.insns_for_copy(source, target) def visit_pushl(self, line): - match = r_unaryinsn.match(line) + match = self.r_unaryinsn.match(line) source = match.group(1) return [InsnStackAdjust(-4)] + self.insns_for_copy(source, '0(%esp)') @@ -496,7 +497,7 @@ return self.insns_for_copy('0(%esp)', target) + [InsnStackAdjust(+4)] def visit_popl(self, line): - match = r_unaryinsn.match(line) + match = self.r_unaryinsn.match(line) target = match.group(1) return self._visit_pop(target) @@ -525,12 +526,12 @@ # Assume that the table is just a list of lines looking like # .long LABEL or .long 0, ending in a .text or .section .text.hot. tablelabels.append(match.group(1)) - elif r_unaryinsn_star.match(line): + elif self.r_unaryinsn_star.match(line): # maybe a jmp similar to the above, but stored in a # registry: # movl L9341(%eax), %eax # jmp *%eax - operand = r_unaryinsn_star.match(line).group(1)[1:] + operand = self.r_unaryinsn_star.match(line).group(1)[1:] def walker(insn, locs): sources = [] for loc in locs: @@ -567,7 +568,7 @@ self.register_jump_to(label) tablelin += 1 return InsnStop() - if r_unaryinsn_star.match(line): + if self.r_unaryinsn_star.match(line): # that looks like an indirect tail-call. # tail-calls are equivalent to RET for us return InsnRet() @@ -575,7 +576,7 @@ self.conditional_jump(line) except KeyError: # label not found: check if it's a tail-call turned into a jump - match = r_unaryinsn.match(line) + match = self.r_unaryinsn.match(line) target = match.group(1) assert not target.startswith('.') # tail-calls are equivalent to RET for us @@ -615,17 +616,17 @@ def visit_xchgl(self, line): # only support the format used in VALGRIND_DISCARD_TRANSLATIONS # which is to use a marker no-op "xchgl %ebx, %ebx" - match = r_binaryinsn.match(line) - source = match.group(1) - target = match.group(2) + match = self.r_binaryinsn.match(line) + source = match.group("source") + target = match.group("target") if source == target: return [] raise UnrecognizedOperation(line) def visit_call(self, line): - match = r_unaryinsn.match(line) + match = self.r_unaryinsn.match(line) if match is None: - assert r_unaryinsn_star.match(line) # indirect call + assert self.r_unaryinsn_star.match(line) # indirect call else: target = match.group(1) if target in FUNCTIONS_NOT_RETURNING: @@ -682,6 +683,12 @@ format = 'msvc' r_functionstart = re.compile(r"PUBLIC\t"+LABEL+"$") + + OPERAND = r'(?:\w+|(?:DWORD PTR )?[_\w$]*\[[-+\w0-9]+\])' + r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+")\s*$") + r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+([*]"+OPERAND+")\s*$") + r_binaryinsn = re.compile(r"\t[a-z]\w*\s+(?P"+OPERAND+"),\s*(?P"+OPERAND+")\s*(?:;.+)?$") + r_jump = re.compile(r"\tj\w+\s+(?:SHORT )?"+LABEL+"\s*$") def __init__(self, lines, filetag=0): @@ -691,8 +698,8 @@ funcname, lines, filetag) for name in ''' - push mov - xor sub + push pop mov lea + xor sub add '''.split(): locals()['visit_' + name] = getattr(FunctionGcRootTracker, 'visit_' + name + 'l') From fijal at codespeak.net Mon Oct 19 11:01:54 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 19 Oct 2009 11:01:54 +0200 (CEST) Subject: [pypy-svn] r68617 - in pypy/trunk/pypy/jit/backend/x86: . test Message-ID: <20091019090154.6B26B168006@codespeak.net> Author: fijal Date: Mon Oct 19 11:01:53 2009 New Revision: 68617 Added: pypy/trunk/pypy/jit/backend/x86/support.py (contents, props changed) pypy/trunk/pypy/jit/backend/x86/test/test_support.py (contents, props changed) Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/runner.py pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py Log: Fix issue471 Don't have a hard limit on number of fail boxes, but grow them as needed Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Mon Oct 19 11:01:53 2009 @@ -13,15 +13,15 @@ from pypy.jit.backend.x86 import codebuf from pypy.jit.backend.x86.ri386 import * from pypy.jit.metainterp.resoperation import rop - - +from pypy.jit.backend.x86.support import NonmovableGrowableArrayFloat,\ + NonmovableGrowableArraySigned, NonmovableGrowableArrayGCREF,\ + CHUNK_SIZE # our calling convention - we pass first 6 args in registers # and the rest stays on the stack RET_BP = 5 # ret ip + bp + bx + esi + edi = 5 words -MAX_FAIL_BOXES = 1000 if sys.platform == 'darwin': # darwin requires the stack to be 16 bytes aligned on calls CALL_ALIGN = 4 @@ -76,30 +76,18 @@ self.malloc_array_func_addr = 0 self.malloc_str_func_addr = 0 self.malloc_unicode_func_addr = 0 - self.fail_boxes_int = lltype.malloc(lltype.GcArray(lltype.Signed), - MAX_FAIL_BOXES, zero=True) - self.fail_boxes_ptr = lltype.malloc(lltype.GcArray(llmemory.GCREF), - MAX_FAIL_BOXES, zero=True) - self.fail_boxes_float = lltype.malloc(lltype.GcArray(lltype.Float), - MAX_FAIL_BOXES, zero=True) + self.fail_boxes_int = NonmovableGrowableArraySigned() + self.fail_boxes_ptr = NonmovableGrowableArrayGCREF() + self.fail_boxes_float = NonmovableGrowableArrayFloat() def leave_jitted_hook(self): fail_boxes_ptr = self.fail_boxes_ptr - llop.gc_assume_young_pointers(lltype.Void, - llmemory.cast_ptr_to_adr(fail_boxes_ptr)) + for chunk in fail_boxes_ptr.chunks: + llop.gc_assume_young_pointers(lltype.Void, + llmemory.cast_ptr_to_adr(chunk)) def make_sure_mc_exists(self): if self.mc is None: - rffi.cast(lltype.Signed, self.fail_boxes_int) # workaround - rffi.cast(lltype.Signed, self.fail_boxes_ptr) # workaround - rffi.cast(lltype.Signed, self.fail_boxes_float) # workaround - self.fail_box_int_addr = rffi.cast(lltype.Signed, - lltype.direct_arrayitems(self.fail_boxes_int)) - self.fail_box_ptr_addr = rffi.cast(lltype.Signed, - lltype.direct_arrayitems(self.fail_boxes_ptr)) - self.fail_box_float_addr = rffi.cast(lltype.Signed, - lltype.direct_arrayitems(self.fail_boxes_float)) - # the address of the function called by 'new' gc_ll_descr = self.cpu.gc_ll_descr gc_ll_descr.initialize() @@ -213,23 +201,22 @@ # This uses XCHG to put zeroes in fail_boxes_ptr after # reading them self.mc.XOR(target, target) - self.mc.XCHG(target, addr_add(imm(self.fail_box_ptr_addr), - imm(i*WORD))) + adr = self.fail_boxes_ptr.get_addr_for_num(i) + self.mc.XCHG(target, heap(adr)) else: - self.mc.MOV(target, addr_add(imm(self.fail_box_int_addr), - imm(i*WORD))) + adr = self.fail_boxes_int.get_addr_for_num(i) + self.mc.MOV(target, heap(adr)) if target is not loc: self.mc.MOV(loc, target) for i in range(len(floatlocs)): loc = floatlocs[i] if loc is None: continue + adr = self.fail_boxes_float.get_addr_for_num(i) if isinstance(loc, REG): - self.mc.MOVSD(loc, addr64_add(imm(self.fail_box_float_addr), - imm(i*WORD*2))) + self.mc.MOVSD(loc, heap64(adr)) else: - self.mc.MOVSD(xmmtmp, addr64_add(imm(self.fail_box_float_addr), - imm(i*WORD*2))) + self.mc.MOVSD(xmmtmp, heap64(adr)) self.mc.MOVSD(loc, xmmtmp) return adr_stackadjust @@ -741,41 +728,38 @@ return addr def generate_failure(self, mc, faildescr, failargs, locs, exc): - assert len(failargs) < MAX_FAIL_BOXES pos = mc.tell() for i in range(len(failargs)): arg = failargs[i] loc = locs[i] if isinstance(loc, REG): if arg.type == FLOAT: - mc.MOVSD(addr64_add(imm(self.fail_box_float_addr), - imm(i*WORD*2)), loc) + adr = self.fail_boxes_float.get_addr_for_num(i) + mc.MOVSD(heap64(adr), loc) else: if arg.type == REF: - base = self.fail_box_ptr_addr + adr = self.fail_boxes_ptr.get_addr_for_num(i) else: - base = self.fail_box_int_addr - mc.MOV(addr_add(imm(base), imm(i*WORD)), loc) + adr = self.fail_boxes_int.get_addr_for_num(i) + mc.MOV(heap(adr), loc) for i in range(len(failargs)): arg = failargs[i] loc = locs[i] if not isinstance(loc, REG): if arg.type == FLOAT: mc.MOVSD(xmm0, loc) - mc.MOVSD(addr64_add(imm(self.fail_box_float_addr), - imm(i*WORD*2)), xmm0) + adr = self.fail_boxes_float.get_addr_for_num(i) + mc.MOVSD(heap64(adr), xmm0) else: if arg.type == REF: - base = self.fail_box_ptr_addr + adr = self.fail_boxes_ptr.get_addr_for_num(i) else: - base = self.fail_box_int_addr + adr = self.fail_boxes_int.get_addr_for_num(i) mc.MOV(eax, loc) - mc.MOV(addr_add(imm(base), imm(i*WORD)), eax) + mc.MOV(heap(adr), eax) if self.debug_markers: mc.MOV(eax, imm(pos)) - mc.MOV(addr_add(imm(self.fail_box_int_addr), - imm(len(locs) * WORD)), - eax) + mc.MOV(heap(self.fail_boxes_int.get_addr_for_num(len(locs))), eax) # we call a provided function that will # - call our on_leave_jitted_hook which will mark Modified: pypy/trunk/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/runner.py Mon Oct 19 11:01:53 2009 @@ -5,7 +5,7 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.rlib.objectmodel import we_are_translated from pypy.jit.metainterp import history -from pypy.jit.backend.x86.assembler import Assembler386, MAX_FAIL_BOXES +from pypy.jit.backend.x86.assembler import Assembler386 from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU @@ -38,28 +38,25 @@ self.assembler.assemble_bridge(faildescr, inputargs, operations) def set_future_value_int(self, index, intvalue): - assert index < MAX_FAIL_BOXES, "overflow!" - self.assembler.fail_boxes_int[index] = intvalue + self.assembler.fail_boxes_int.setitem(index, intvalue) def set_future_value_float(self, index, floatvalue): - assert index < MAX_FAIL_BOXES, "overflow!" - self.assembler.fail_boxes_float[index] = floatvalue + self.assembler.fail_boxes_float.setitem(index, floatvalue) def set_future_value_ref(self, index, ptrvalue): - assert index < MAX_FAIL_BOXES, "overflow!" - self.assembler.fail_boxes_ptr[index] = ptrvalue + self.assembler.fail_boxes_ptr.setitem(index, ptrvalue) def get_latest_value_int(self, index): - return self.assembler.fail_boxes_int[index] + return self.assembler.fail_boxes_int.getitem(index) def get_latest_value_float(self, index): - return self.assembler.fail_boxes_float[index] + return self.assembler.fail_boxes_float.getitem(index) def get_latest_value_ref(self, index): - ptrvalue = self.assembler.fail_boxes_ptr[index] + ptrvalue = self.assembler.fail_boxes_ptr.getitem(index) # clear after reading - self.assembler.fail_boxes_ptr[index] = lltype.nullptr( - llmemory.GCREF.TO) + self.assembler.fail_boxes_ptr.setitem(index, lltype.nullptr( + llmemory.GCREF.TO)) return ptrvalue def execute_token(self, executable_token): Added: pypy/trunk/pypy/jit/backend/x86/support.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/jit/backend/x86/support.py Mon Oct 19 11:01:53 2009 @@ -0,0 +1,42 @@ + +from pypy.rpython.lltypesystem import lltype, rffi, llmemory + +CHUNK_SIZE = 1000 + +def new_nonmovable_growable_array(TP): + ATP = lltype.GcArray(TP) + + class NonmovableGrowableArray(object): + def __init__(self): + self.chunks = [] + self._grow() + + def _grow(self): + self.chunks.append(lltype.malloc(ATP, CHUNK_SIZE, + zero=True)) + + def get_addr_for_num(self, i): + chunk_no, ofs = self._no_of(i) + chunk = self.chunks[chunk_no] + rffi.cast(lltype.Signed, chunk) + return rffi.cast(lltype.Signed, lltype.direct_ptradd( + lltype.direct_arrayitems(chunk), ofs)) + + def _no_of(self, i): + while i > len(self.chunks) * CHUNK_SIZE: + self._grow() + return i / CHUNK_SIZE, i % CHUNK_SIZE + + def setitem(self, i, v): + chunk_no, ofs = self._no_of(i) + self.chunks[chunk_no][ofs] = v + + def getitem(self, i): + chunk_no, ofs = self._no_of(i) + return self.chunks[chunk_no][ofs] + + return NonmovableGrowableArray + +NonmovableGrowableArrayFloat = new_nonmovable_growable_array(lltype.Float) +NonmovableGrowableArraySigned = new_nonmovable_growable_array(lltype.Signed) +NonmovableGrowableArrayGCREF = new_nonmovable_growable_array(llmemory.GCREF) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py Mon Oct 19 11:01:53 2009 @@ -198,8 +198,8 @@ ptr = lltype.malloc(S) self.interpret(ops, [0, ptr]) assert self.getptr(0, lltype.Ptr(S)) == ptr - assert not self.cpu.assembler.fail_boxes_ptr[0] - assert not self.cpu.assembler.fail_boxes_ptr[1] + assert not self.cpu.assembler.fail_boxes_ptr.getitem(0) + assert not self.cpu.assembler.fail_boxes_ptr.getitem(1) def test_exception_bridge_no_exception(self): ops = ''' Added: pypy/trunk/pypy/jit/backend/x86/test/test_support.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/jit/backend/x86/test/test_support.py Mon Oct 19 11:01:53 2009 @@ -0,0 +1,26 @@ + +from pypy.jit.backend.x86.support import NonmovableGrowableArraySigned, CHUNK_SIZE +from pypy.rpython.lltypesystem import lltype, llmemory, rffi + +def test_nonmovable_growable_array(): + ar = NonmovableGrowableArraySigned() + adr = ar.get_addr_for_num(10) + rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] = 42 + assert ar.getitem(10) == 42 + ar.setitem(42, 38) + adr = ar.get_addr_for_num(42) + assert rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] == 38 + adr = ar.get_addr_for_num(CHUNK_SIZE + 10) + rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] = 42 + assert ar.getitem(CHUNK_SIZE + 10) == 42 + ar.setitem(CHUNK_SIZE + 42, 38) + adr = ar.get_addr_for_num(CHUNK_SIZE + 42) + assert rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] == 38 + adr = ar.get_addr_for_num(3 * CHUNK_SIZE + 10) + rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] = 42 + assert ar.getitem(3 * CHUNK_SIZE + 10) == 42 + ar.setitem(3 * CHUNK_SIZE + 42, 38) + adr = ar.get_addr_for_num(3 * CHUNK_SIZE + 42) + assert rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] == 38 + + From pedronis at codespeak.net Mon Oct 19 11:18:33 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 19 Oct 2009 11:18:33 +0200 (CEST) Subject: [pypy-svn] r68618 - in pypy/trunk/pypy/jit: backend metainterp/test Message-ID: <20091019091833.A9A1E168006@codespeak.net> Author: pedronis Date: Mon Oct 19 11:18:33 2009 New Revision: 68618 Modified: pypy/trunk/pypy/jit/backend/loopviewer.py pypy/trunk/pypy/jit/backend/showstats.py pypy/trunk/pypy/jit/metainterp/test/oparser.py pypy/trunk/pypy/jit/metainterp/test/test_oparser.py Log: (pedronis, cfbolz): add a helper function that splits the content of the log file containing optimized loops into its parts (e.g. loops and bridges) Modified: pypy/trunk/pypy/jit/backend/loopviewer.py ============================================================================== --- pypy/trunk/pypy/jit/backend/loopviewer.py (original) +++ pypy/trunk/pypy/jit/backend/loopviewer.py Mon Oct 19 11:18:33 2009 @@ -2,9 +2,10 @@ """ Usage: loopviewer.py [loopnum] loopfile """ +import autopath import py import sys -from pypy.jit.metainterp.test.oparser import parse +from pypy.jit.metainterp.test.oparser import parse, split_logs_into_loops from pypy.jit.metainterp.history import ConstInt from pypy.rpython.lltypesystem import llmemory, lltype @@ -16,8 +17,8 @@ def main(loopnum, loopfile): data = py.path.local(loopfile).read() - loops = [i for i in data.split("[") if i] - inp = "[" + loops[loopnum] + loops = split_logs_into_loops(data) + inp = loops[loopnum] loop = parse(inp, namespace=alldict) loop.show() Modified: pypy/trunk/pypy/jit/backend/showstats.py ============================================================================== --- pypy/trunk/pypy/jit/backend/showstats.py (original) +++ pypy/trunk/pypy/jit/backend/showstats.py Mon Oct 19 11:18:33 2009 @@ -1,6 +1,7 @@ #!/usr/bin/env python +import autopath import sys, py -from pypy.jit.metainterp.test.oparser import parse +from pypy.jit.metainterp.test.oparser import parse, split_logs_into_loops from pypy.jit.metainterp.resoperation import rop from pypy.rpython.lltypesystem import lltype, llmemory @@ -11,9 +12,8 @@ alldict = AllDict() def main(argv): - lst = ("\n" + py.path.local(argv[0]).read()).split("\n[") - lst = ['[' + i for i in lst if i] - for oplist in lst: + parts = split_logs_into_loops(py.path.local(argv[0]).read()) + for oplist in parts: loop = parse(oplist, namespace=alldict) num_ops = 0 num_dmp = 0 Modified: pypy/trunk/pypy/jit/metainterp/test/oparser.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/oparser.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/oparser.py Mon Oct 19 11:18:33 2009 @@ -295,6 +295,24 @@ kwds['invent_fail_descr'] = None return parse(*args, **kwds) + +def split_logs_into_loops(text): + lines = text.splitlines() + parts = [] + last_with_hash = False + lines_of_part = [] + for i, line in enumerate(lines): + if (line.startswith("[") and last_with_hash + and len(lines_of_part) > 1): + parts.append("\n".join(lines_of_part[:-1])) + lines_of_part = [lines_of_part[-1], line] + else: + lines_of_part.append(line) + last_with_hash = line.startswith("#") + parts.append("\n".join(lines_of_part)) + return parts + + def _box_counter_more_than(s): if s.isdigit(): Box._counter = max(Box._counter, int(s)+1) Modified: pypy/trunk/pypy/jit/metainterp/test/test_oparser.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_oparser.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_oparser.py Mon Oct 19 11:18:33 2009 @@ -1,7 +1,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.jit.metainterp.test.oparser import parse +from pypy.jit.metainterp.test.oparser import parse, split_logs_into_loops from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.history import AbstractDescr, BoxInt, LoopToken,\ BoxFloat @@ -160,3 +160,72 @@ ''' loop = parse(x) # assert did not explode + +def test_split_logs_into_loops(): + text = '''\ +# Loop0 (loop), 12 ops +[i0, i1] +debug_merge_point('(no jitdriver.get_printable_location!)') +i3 = call(ConstClass(cls2), i0, descr=) +guard_no_exception(, descr=) [i0, i1, i3] +i5 = int_add(i1, 2) +i7 = call(ConstClass(cls6), i0, descr=) +p9 = guard_exception(4, descr=) [i5, i0, i7] +i11 = int_sub(i5, 1) +i12 = int_sub(i0, 1) +i14 = int_gt(i12, 3) +guard_true(i14, descr=) [i11, i12] +debug_merge_point('(no jitdriver.get_printable_location!)') +jump(i12, i11, descr=) +# Loop1 (entry bridge), 12 ops +[i0, i1] +debug_merge_point('(no jitdriver.get_printable_location!)') +i3 = call(ConstClass(cls2), i0, descr=) +p5 = guard_exception(4, descr=) [i0, i1, i3] +i7 = int_add(i1, 1) +i9 = call(ConstClass(cls8), i0, descr=) +p11 = guard_exception(4, descr=) [i7, i0, i9] +i12 = int_sub(i7, 1) +i13 = int_sub(i0, 1) +i15 = int_gt(i13, 3) +guard_true(i15, descr=) [i12, i13] +debug_merge_point('(no jitdriver.get_printable_location!)') +jump(i13, i12, descr=) +# bridge out of Guard5, 10 ops +[i0, i1, i2] +p4 = guard_exception(4, descr=) [i0, i1, i2] +i6 = int_add(i1, 1) +i8 = call(ConstClass(cls7), i0, descr=) +p10 = guard_exception(4, descr=) [i6, i0, i8] +i11 = int_sub(i6, 1) +i12 = int_sub(i0, 1) +i14 = int_gt(i12, 3) +guard_true(i14, descr=) [i11, i12] +debug_merge_point('(no jitdriver.get_printable_location!)') +jump(i12, i11, descr=) +# bridge out of Guard9, 6 ops +[i0, i1, i2] +i4 = int_add(i0, 2) +i6 = int_sub(i1, 1) +i8 = int_gt(i6, 3) +guard_true(i8, descr=) [i4, i6] +debug_merge_point('(no jitdriver.get_printable_location!)') +jump(i6, i4, descr=) +# bridge out of Guard12, 6 ops +[i0, i1, i2] +i4 = int_add(i0, 2) +i6 = int_sub(i1, 1) +i8 = int_gt(i6, 3) +guard_true(i8, descr=) [i4, i6] +debug_merge_point('(no jitdriver.get_printable_location!)') +jump(i6, i4, descr=) +''' + parts = split_logs_into_loops(text) + assert len(parts) == 5 + assert "\n".join(parts) == text.strip() + for part, typ in zip(parts, + ["Loop0", "Loop1", + "bridge out of Guard5", + "bridge out of Guard9", + "bridge out of Guard12"]): + assert part.startswith("# %s" % typ) From fijal at codespeak.net Mon Oct 19 11:41:14 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 19 Oct 2009 11:41:14 +0200 (CEST) Subject: [pypy-svn] r68619 - in pypy/trunk/pypy/jit/backend/x86: . test Message-ID: <20091019094114.EC9A7168006@codespeak.net> Author: fijal Date: Mon Oct 19 11:41:13 2009 New Revision: 68619 Modified: pypy/trunk/pypy/jit/backend/x86/support.py pypy/trunk/pypy/jit/backend/x86/test/test_support.py Log: oops Modified: pypy/trunk/pypy/jit/backend/x86/support.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/support.py (original) +++ pypy/trunk/pypy/jit/backend/x86/support.py Mon Oct 19 11:41:13 2009 @@ -23,7 +23,7 @@ lltype.direct_arrayitems(chunk), ofs)) def _no_of(self, i): - while i > len(self.chunks) * CHUNK_SIZE: + while i >= len(self.chunks) * CHUNK_SIZE: self._grow() return i / CHUNK_SIZE, i % CHUNK_SIZE Modified: pypy/trunk/pypy/jit/backend/x86/test/test_support.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_support.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_support.py Mon Oct 19 11:41:13 2009 @@ -22,5 +22,6 @@ ar.setitem(3 * CHUNK_SIZE + 42, 38) adr = ar.get_addr_for_num(3 * CHUNK_SIZE + 42) assert rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] == 38 - + ar.setitem(8*CHUNK_SIZE, 13) + assert ar.getitem(8*CHUNK_SIZE) == 13 From pedronis at codespeak.net Mon Oct 19 11:51:16 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 19 Oct 2009 11:51:16 +0200 (CEST) Subject: [pypy-svn] r68620 - in pypy/trunk/pypy/jit: backend metainterp/test Message-ID: <20091019095116.A701A168006@codespeak.net> Author: pedronis Date: Mon Oct 19 11:51:16 2009 New Revision: 68620 Modified: pypy/trunk/pypy/jit/backend/loopviewer.py pypy/trunk/pypy/jit/backend/showstats.py pypy/trunk/pypy/jit/metainterp/test/oparser.py pypy/trunk/pypy/jit/metainterp/test/test_oparser.py Log: (cfbolz, pedronis): refactor the oparser a bit to make it possible to not give it a namespace. Modified: pypy/trunk/pypy/jit/backend/loopviewer.py ============================================================================== --- pypy/trunk/pypy/jit/backend/loopviewer.py (original) +++ pypy/trunk/pypy/jit/backend/loopviewer.py Mon Oct 19 11:51:16 2009 @@ -9,17 +9,11 @@ from pypy.jit.metainterp.history import ConstInt from pypy.rpython.lltypesystem import llmemory, lltype -class AllDict(dict): - def __getitem__(self, item): - return lltype.nullptr(llmemory.GCREF.TO) - -alldict = AllDict() - def main(loopnum, loopfile): data = py.path.local(loopfile).read() loops = split_logs_into_loops(data) inp = loops[loopnum] - loop = parse(inp, namespace=alldict) + loop = parse(inp, no_namespace=True) loop.show() if __name__ == '__main__': Modified: pypy/trunk/pypy/jit/backend/showstats.py ============================================================================== --- pypy/trunk/pypy/jit/backend/showstats.py (original) +++ pypy/trunk/pypy/jit/backend/showstats.py Mon Oct 19 11:51:16 2009 @@ -5,16 +5,10 @@ from pypy.jit.metainterp.resoperation import rop from pypy.rpython.lltypesystem import lltype, llmemory -class AllDict(dict): - def __getitem__(self, item): - return lltype.nullptr(llmemory.GCREF.TO) - -alldict = AllDict() - def main(argv): parts = split_logs_into_loops(py.path.local(argv[0]).read()) for oplist in parts: - loop = parse(oplist, namespace=alldict) + loop = parse(oplist, no_namespace=True) num_ops = 0 num_dmp = 0 num_guards = 0 Modified: pypy/trunk/pypy/jit/metainterp/test/oparser.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/oparser.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/oparser.py Mon Oct 19 11:51:16 2009 @@ -50,18 +50,45 @@ class OpParser(object): - def __init__(self, descr, cpu, namespace, type_system, boxkinds, + def __init__(self, input, cpu, namespace, type_system, boxkinds, invent_fail_descr=default_fail_descr): - self.descr = descr + self.input = input self.vars = {} self.cpu = cpu - self.consts = namespace + self._consts = namespace self.type_system = type_system self.boxkinds = boxkinds or {} - self._cache = namespace.setdefault('_CACHE_', {}) + if namespace is not None: + self._cache = namespace.setdefault('_CACHE_', {}) + else: + self._cache = {} self.invent_fail_descr = invent_fail_descr self.looptoken = LoopToken() + def get_const(self, name, typ): + if self._consts is None: + return name + obj = self._consts[name] + if self.type_system == 'lltype': + if typ == 'ptr': + return ConstPtr(obj) + else: + assert typ == 'class' + return ConstAddr(llmemory.cast_ptr_to_adr(obj), + self.cpu) + else: + if typ == 'ptr': + return ConstObj(obj) + else: + assert typ == 'class' + return ConstObj(ootype.cast_to_object(obj)) + + def get_descr(self, poss_descr): + if poss_descr.startswith('<'): + return None + else: + return self._consts[poss_descr] + def box_for_var(self, elem): try: return self._cache[self.type_system, elem] @@ -122,11 +149,7 @@ llstr(info))) if arg.startswith('ConstClass('): name = arg[len('ConstClass('):-1] - if self.type_system == 'lltype': - return ConstAddr(llmemory.cast_ptr_to_adr(self.consts[name]), - self.cpu) - else: - return ConstObj(ootype.cast_to_object(self.consts[name])) + return self.get_const(name, 'class') elif arg == 'None': return None elif arg == 'NULL': @@ -136,10 +159,7 @@ return ConstObj(ConstObj.value) elif arg.startswith('ConstPtr('): name = arg[len('ConstPtr('):-1] - if self.type_system == 'lltype': - return ConstPtr(self.consts[name]) - else: - return ConstObj(self.consts[name]) + return self.get_const(name, 'ptr') return self.vars[arg] def parse_op(self, line): @@ -168,10 +188,7 @@ poss_descr = allargs[-1].strip() if poss_descr.startswith('descr='): - if poss_descr.startswith('descr=<'): - descr = None - else: - descr = self.consts[poss_descr[len('descr='):]] + descr = self.get_descr(poss_descr[len('descr='):]) allargs = allargs[:-1] for arg in allargs: arg = arg.strip() @@ -232,7 +249,7 @@ return self.parse_op_no_result(line) def parse(self): - lines = self.descr.splitlines() + lines = self.input.splitlines() ops = [] newlines = [] for line in lines: @@ -284,11 +301,12 @@ inpargs = self.parse_header_line(line[1:-1]) return base_indent, inpargs -def parse(descr, cpu=None, namespace=None, type_system='lltype', - boxkinds=None, invent_fail_descr=default_fail_descr): - if namespace is None: +def parse(input, cpu=None, namespace=None, type_system='lltype', + boxkinds=None, invent_fail_descr=default_fail_descr, + no_namespace=False): + if namespace is None and not no_namespace: namespace = {} - return OpParser(descr, cpu, namespace, type_system, boxkinds, + return OpParser(input, cpu, namespace, type_system, boxkinds, invent_fail_descr).parse() def pure_parse(*args, **kwds): Modified: pypy/trunk/pypy/jit/metainterp/test/test_oparser.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_oparser.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_oparser.py Mon Oct 19 11:51:16 2009 @@ -161,8 +161,7 @@ loop = parse(x) # assert did not explode -def test_split_logs_into_loops(): - text = '''\ +examplelog = '''\ # Loop0 (loop), 12 ops [i0, i1] debug_merge_point('(no jitdriver.get_printable_location!)') @@ -220,12 +219,19 @@ debug_merge_point('(no jitdriver.get_printable_location!)') jump(i6, i4, descr=) ''' - parts = split_logs_into_loops(text) + +def test_split_logs_into_loops(): + parts = split_logs_into_loops(examplelog) assert len(parts) == 5 - assert "\n".join(parts) == text.strip() + assert "\n".join(parts) == examplelog.strip() for part, typ in zip(parts, ["Loop0", "Loop1", "bridge out of Guard5", "bridge out of Guard9", "bridge out of Guard12"]): assert part.startswith("# %s" % typ) + +def test_parse_no_namespace(): + parts = split_logs_into_loops(examplelog) + for part in parts: + loop = parse(part, no_namespace=True) From afa at codespeak.net Mon Oct 19 12:01:29 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 19 Oct 2009 12:01:29 +0200 (CEST) Subject: [pypy-svn] r68621 - pypy/trunk/pypy/translator/c/gcc/test Message-ID: <20091019100129.C3CEC168006@codespeak.net> Author: afa Date: Mon Oct 19 12:01:29 2009 New Revision: 68621 Modified: pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py Log: Fix test_asmgccroot on win32: os.popen closes the stderr of the spawned shell, which caused the subprocess to fail. Modified: pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py Mon Oct 19 12:01:29 2009 @@ -55,7 +55,11 @@ def run(arg0, arg1): lines = [] print >> sys.stderr, 'RUN: starting', exe_name - g = os.popen('"%s" %d %d' % (exe_name, arg0, arg1), 'r') + if sys.platform == 'win32': + redirect = ' 2> NUL' + else: + redirect = '' + g = os.popen('"%s" %d %d%s' % (exe_name, arg0, arg1, redirect), 'r') for line in g: print >> sys.stderr, 'RUN:', line.rstrip() lines.append(line) @@ -92,6 +96,13 @@ f.writelines(lines) f.close() + if sys.platform == 'win32': + def test_callback_with_collect(self): + py.test.skip("No libffi yet with mingw32") + + def define_callback_with_collect(cls): + return lambda: 0 + class TestAsmGCRootWithSemiSpaceGC(AbstractTestAsmGCRoot, test_newgc.TestSemiSpaceGC): # for the individual tests see @@ -153,13 +164,6 @@ res = self.run('callback_simple') assert res == 4900 - if sys.platform == 'win32': - def test_callback_with_collect(self): - py.test.skip("No libffi yet with mingw32") - - def define_callback_with_collect(cls): - return lambda: 0 - class TestAsmGCRootWithHybridTagged(AbstractTestAsmGCRoot, test_newgc.TestHybridTaggedPointers): From pedronis at codespeak.net Mon Oct 19 12:54:28 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 19 Oct 2009 12:54:28 +0200 (CEST) Subject: [pypy-svn] r68623 - in pypy/trunk/pypy/jit/backend/x86: . test Message-ID: <20091019105428.067DE168006@codespeak.net> Author: pedronis Date: Mon Oct 19 12:54:28 2009 New Revision: 68623 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/runner.py pypy/trunk/pypy/jit/backend/x86/support.py (contents, props changed) pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py pypy/trunk/pypy/jit/backend/x86/test/test_support.py (contents, props changed) Log: reverting 68619, 68617 " I broke stuff" Also not clear that we need this fix at this point, if there are tons of variable around we have a problem and possibly the frontend should have given up on the loop earlier Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Mon Oct 19 12:54:28 2009 @@ -13,15 +13,15 @@ from pypy.jit.backend.x86 import codebuf from pypy.jit.backend.x86.ri386 import * from pypy.jit.metainterp.resoperation import rop -from pypy.jit.backend.x86.support import NonmovableGrowableArrayFloat,\ - NonmovableGrowableArraySigned, NonmovableGrowableArrayGCREF,\ - CHUNK_SIZE + + # our calling convention - we pass first 6 args in registers # and the rest stays on the stack RET_BP = 5 # ret ip + bp + bx + esi + edi = 5 words +MAX_FAIL_BOXES = 1000 if sys.platform == 'darwin': # darwin requires the stack to be 16 bytes aligned on calls CALL_ALIGN = 4 @@ -76,18 +76,30 @@ self.malloc_array_func_addr = 0 self.malloc_str_func_addr = 0 self.malloc_unicode_func_addr = 0 - self.fail_boxes_int = NonmovableGrowableArraySigned() - self.fail_boxes_ptr = NonmovableGrowableArrayGCREF() - self.fail_boxes_float = NonmovableGrowableArrayFloat() + self.fail_boxes_int = lltype.malloc(lltype.GcArray(lltype.Signed), + MAX_FAIL_BOXES, zero=True) + self.fail_boxes_ptr = lltype.malloc(lltype.GcArray(llmemory.GCREF), + MAX_FAIL_BOXES, zero=True) + self.fail_boxes_float = lltype.malloc(lltype.GcArray(lltype.Float), + MAX_FAIL_BOXES, zero=True) def leave_jitted_hook(self): fail_boxes_ptr = self.fail_boxes_ptr - for chunk in fail_boxes_ptr.chunks: - llop.gc_assume_young_pointers(lltype.Void, - llmemory.cast_ptr_to_adr(chunk)) + llop.gc_assume_young_pointers(lltype.Void, + llmemory.cast_ptr_to_adr(fail_boxes_ptr)) def make_sure_mc_exists(self): if self.mc is None: + rffi.cast(lltype.Signed, self.fail_boxes_int) # workaround + rffi.cast(lltype.Signed, self.fail_boxes_ptr) # workaround + rffi.cast(lltype.Signed, self.fail_boxes_float) # workaround + self.fail_box_int_addr = rffi.cast(lltype.Signed, + lltype.direct_arrayitems(self.fail_boxes_int)) + self.fail_box_ptr_addr = rffi.cast(lltype.Signed, + lltype.direct_arrayitems(self.fail_boxes_ptr)) + self.fail_box_float_addr = rffi.cast(lltype.Signed, + lltype.direct_arrayitems(self.fail_boxes_float)) + # the address of the function called by 'new' gc_ll_descr = self.cpu.gc_ll_descr gc_ll_descr.initialize() @@ -201,22 +213,23 @@ # This uses XCHG to put zeroes in fail_boxes_ptr after # reading them self.mc.XOR(target, target) - adr = self.fail_boxes_ptr.get_addr_for_num(i) - self.mc.XCHG(target, heap(adr)) + self.mc.XCHG(target, addr_add(imm(self.fail_box_ptr_addr), + imm(i*WORD))) else: - adr = self.fail_boxes_int.get_addr_for_num(i) - self.mc.MOV(target, heap(adr)) + self.mc.MOV(target, addr_add(imm(self.fail_box_int_addr), + imm(i*WORD))) if target is not loc: self.mc.MOV(loc, target) for i in range(len(floatlocs)): loc = floatlocs[i] if loc is None: continue - adr = self.fail_boxes_float.get_addr_for_num(i) if isinstance(loc, REG): - self.mc.MOVSD(loc, heap64(adr)) + self.mc.MOVSD(loc, addr64_add(imm(self.fail_box_float_addr), + imm(i*WORD*2))) else: - self.mc.MOVSD(xmmtmp, heap64(adr)) + self.mc.MOVSD(xmmtmp, addr64_add(imm(self.fail_box_float_addr), + imm(i*WORD*2))) self.mc.MOVSD(loc, xmmtmp) return adr_stackadjust @@ -728,38 +741,41 @@ return addr def generate_failure(self, mc, faildescr, failargs, locs, exc): + assert len(failargs) < MAX_FAIL_BOXES pos = mc.tell() for i in range(len(failargs)): arg = failargs[i] loc = locs[i] if isinstance(loc, REG): if arg.type == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(i) - mc.MOVSD(heap64(adr), loc) + mc.MOVSD(addr64_add(imm(self.fail_box_float_addr), + imm(i*WORD*2)), loc) else: if arg.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) + base = self.fail_box_ptr_addr else: - adr = self.fail_boxes_int.get_addr_for_num(i) - mc.MOV(heap(adr), loc) + base = self.fail_box_int_addr + mc.MOV(addr_add(imm(base), imm(i*WORD)), loc) for i in range(len(failargs)): arg = failargs[i] loc = locs[i] if not isinstance(loc, REG): if arg.type == FLOAT: mc.MOVSD(xmm0, loc) - adr = self.fail_boxes_float.get_addr_for_num(i) - mc.MOVSD(heap64(adr), xmm0) + mc.MOVSD(addr64_add(imm(self.fail_box_float_addr), + imm(i*WORD*2)), xmm0) else: if arg.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) + base = self.fail_box_ptr_addr else: - adr = self.fail_boxes_int.get_addr_for_num(i) + base = self.fail_box_int_addr mc.MOV(eax, loc) - mc.MOV(heap(adr), eax) + mc.MOV(addr_add(imm(base), imm(i*WORD)), eax) if self.debug_markers: mc.MOV(eax, imm(pos)) - mc.MOV(heap(self.fail_boxes_int.get_addr_for_num(len(locs))), eax) + mc.MOV(addr_add(imm(self.fail_box_int_addr), + imm(len(locs) * WORD)), + eax) # we call a provided function that will # - call our on_leave_jitted_hook which will mark Modified: pypy/trunk/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/runner.py Mon Oct 19 12:54:28 2009 @@ -5,7 +5,7 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.rlib.objectmodel import we_are_translated from pypy.jit.metainterp import history -from pypy.jit.backend.x86.assembler import Assembler386 +from pypy.jit.backend.x86.assembler import Assembler386, MAX_FAIL_BOXES from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU @@ -38,25 +38,28 @@ self.assembler.assemble_bridge(faildescr, inputargs, operations) def set_future_value_int(self, index, intvalue): - self.assembler.fail_boxes_int.setitem(index, intvalue) + assert index < MAX_FAIL_BOXES, "overflow!" + self.assembler.fail_boxes_int[index] = intvalue def set_future_value_float(self, index, floatvalue): - self.assembler.fail_boxes_float.setitem(index, floatvalue) + assert index < MAX_FAIL_BOXES, "overflow!" + self.assembler.fail_boxes_float[index] = floatvalue def set_future_value_ref(self, index, ptrvalue): - self.assembler.fail_boxes_ptr.setitem(index, ptrvalue) + assert index < MAX_FAIL_BOXES, "overflow!" + self.assembler.fail_boxes_ptr[index] = ptrvalue def get_latest_value_int(self, index): - return self.assembler.fail_boxes_int.getitem(index) + return self.assembler.fail_boxes_int[index] def get_latest_value_float(self, index): - return self.assembler.fail_boxes_float.getitem(index) + return self.assembler.fail_boxes_float[index] def get_latest_value_ref(self, index): - ptrvalue = self.assembler.fail_boxes_ptr.getitem(index) + ptrvalue = self.assembler.fail_boxes_ptr[index] # clear after reading - self.assembler.fail_boxes_ptr.setitem(index, lltype.nullptr( - llmemory.GCREF.TO)) + self.assembler.fail_boxes_ptr[index] = lltype.nullptr( + llmemory.GCREF.TO) return ptrvalue def execute_token(self, executable_token): Modified: pypy/trunk/pypy/jit/backend/x86/support.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/support.py (original) +++ pypy/trunk/pypy/jit/backend/x86/support.py Mon Oct 19 12:54:28 2009 @@ -23,7 +23,7 @@ lltype.direct_arrayitems(chunk), ofs)) def _no_of(self, i): - while i >= len(self.chunks) * CHUNK_SIZE: + while i > len(self.chunks) * CHUNK_SIZE: self._grow() return i / CHUNK_SIZE, i % CHUNK_SIZE Modified: pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py Mon Oct 19 12:54:28 2009 @@ -198,8 +198,8 @@ ptr = lltype.malloc(S) self.interpret(ops, [0, ptr]) assert self.getptr(0, lltype.Ptr(S)) == ptr - assert not self.cpu.assembler.fail_boxes_ptr.getitem(0) - assert not self.cpu.assembler.fail_boxes_ptr.getitem(1) + assert not self.cpu.assembler.fail_boxes_ptr[0] + assert not self.cpu.assembler.fail_boxes_ptr[1] def test_exception_bridge_no_exception(self): ops = ''' Modified: pypy/trunk/pypy/jit/backend/x86/test/test_support.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_support.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_support.py Mon Oct 19 12:54:28 2009 @@ -22,6 +22,5 @@ ar.setitem(3 * CHUNK_SIZE + 42, 38) adr = ar.get_addr_for_num(3 * CHUNK_SIZE + 42) assert rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] == 38 - ar.setitem(8*CHUNK_SIZE, 13) - assert ar.getitem(8*CHUNK_SIZE) == 13 + From afa at codespeak.net Mon Oct 19 13:49:44 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 19 Oct 2009 13:49:44 +0200 (CEST) Subject: [pypy-svn] r68624 - in pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc: . test test/msvc Message-ID: <20091019114944.1BFC1168006@codespeak.net> Author: afa Date: Mon Oct 19 13:49:43 2009 New Revision: 68624 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Log: The first msvc test passes! Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s Mon Oct 19 13:49:43 2009 @@ -65,6 +65,7 @@ jl SHORT $LN15 at pypy_g_ll_@139 $LN14 at pypy_g_ll_@139: call _RPyAbort + ;; expected {24(%esp) | %ebx, %esi, %edi, %ebp | } $LN15 at pypy_g_ll_@139: ; 1529 : l_v420 = l_v419; @@ -76,6 +77,7 @@ test ebx, ebx jne SHORT $LN16 at pypy_g_ll_@139 call _RPyAbort + ;; expected {24(%esp) | %ebx, %esi, %edi, %ebp | } $LN16 at pypy_g_ll_@139: ; 1531 : OP_INT_ADD(l_v402, l_v421, l_v422); @@ -105,6 +107,7 @@ push edi call _pypy_g_mallocstr__Signed + ;; expected {28(%esp) | %ebx, %esi, %edi, %ebp | } ; 1486 : l_v405 = (void*)l_items_2; @@ -180,6 +183,7 @@ jl SHORT $LN10 at pypy_g_ll_@139 $LN9 at pypy_g_ll_@139: call _RPyAbort + ;; expected {24(%esp) | %ebx, %esi, %edi, %ebp | } $LN10 at pypy_g_ll_@139: ; 1517 : l_v413 = l_v412; @@ -191,6 +195,7 @@ test edi, edi jne SHORT $LN11 at pypy_g_ll_@139 call _RPyAbort + ;; expected {24(%esp) | %ebx, %esi, %edi, %ebp | } $LN11 at pypy_g_ll_@139: mov edi, DWORD PTR [edi+8] @@ -203,6 +208,7 @@ jl SHORT $LN13 at pypy_g_ll_@139 $LN12 at pypy_g_ll_@139: call _RPyAbort + ;; expected {24(%esp) | %ebx, %esi, %edi, %ebp | } $LN13 at pypy_g_ll_@139: ; 1520 : pypy_g_copy_string_contents__rpy_stringPtr_rpy_stringPt(l_v415, l_result_2, 0L, l_res_index_0, l_v414); @@ -230,6 +236,7 @@ $block0$80678: $block1$80679: call _memcpy + ;; expected {36(%esp) | %ebx, %esi, %edi, %ebp | } add esp, 12 ; 0000000cH ; 1521 : OP_INT_ADD(l_res_index_0, l_v414, l_v417); Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py Mon Oct 19 13:49:43 2009 @@ -126,7 +126,7 @@ def check_computegcmaptable(format, path): print - print path.basename + print path.dirpath().basename + '/' + path.basename lines = path.readlines() expectedlines = lines[:] tracker = PARSERS[format].FunctionGcRootTracker(lines) Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Mon Oct 19 13:49:43 2009 @@ -48,12 +48,13 @@ class FunctionGcRootTracker(object): - OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' - r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+")\s*$") - r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+([*]"+OPERAND+")\s*$") - r_binaryinsn = re.compile(r"\t[a-z]\w*\s+(?P"+OPERAND+"),\s*(?P"+OPERAND+")\s*$") + @classmethod + def init_regexp(cls): + cls.r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+cls.OPERAND+")\s*$") + cls.r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+([*]"+cls.OPERAND+")\s*$") + cls.r_binaryinsn = re.compile(r"\t[a-z]\w*\s+(?P"+cls.OPERAND+"),\s*(?P"+cls.OPERAND+")\s*$") - r_jump = re.compile(r"\tj\w+\s+"+LABEL+"\s*$") + cls.r_jump = re.compile(r"\tj\w+\s+"+cls.LABEL+"\s*$") def __init__(self, funcname, lines, filetag=0): self.funcname = funcname @@ -158,6 +159,7 @@ except AttributeError: self.find_missing_visit_method(opname) meth = getattr(self, 'visit_' + opname) + line = line.rsplit(';', 1)[0] insn = meth(line) elif r_gcroot_marker.match(line): insn = self._visit_gcroot_marker(line) @@ -374,12 +376,12 @@ match = self.r_binaryinsn.match(line) source = match.group("source") target = match.group("target") - if target == '%esp': - count = match.group(1) - if not count.startswith('$'): + if target == self.ESP: + count = self.extract_immediate(source) + if count is None: # strange instruction - I've seen 'subl %eax, %esp' return InsnCannotFollowEsp() - return InsnStackAdjust(sign * int(count[1:])) + return InsnStackAdjust(sign * count) elif self.r_localvar.match(target): return InsnSetLocal(target, [source, target]) else: @@ -404,7 +406,7 @@ target = match.group("target") if self.r_localvar.match(target): return InsnSetLocal(target, [source]) - elif target == '%esp': + elif target == self.ESP: raise UnrecognizedOperation(line) else: return [] @@ -431,7 +433,7 @@ def visit_andl(self, line): match = self.r_binaryinsn.match(line) target = match.group("target") - if target == '%esp': + if target == self.ESP: # only for andl $-16, %esp used to align the stack in main(). # The exact amount of adjutment is not known yet, so we use # an odd-valued estimate to make sure the real value is not used @@ -443,7 +445,7 @@ def visit_leal(self, line): match = self.r_binaryinsn.match(line) target = match.group("target") - if target == '%esp': + if target == self.ESP: # only for leal -12(%ebp), %esp in function epilogues source = match.group("source") match = r_localvar_ebp.match(source) @@ -465,7 +467,7 @@ return self.binary_insn(line) def insns_for_copy(self, source, target): - if source == '%esp' or target == '%esp': + if source == self.ESP or target == self.ESP: raise UnrecognizedOperation('%s -> %s' % (source, target)) elif self.r_localvar.match(target): if self.r_localvar.match(source): @@ -479,9 +481,9 @@ match = self.r_binaryinsn.match(line) source = match.group("source") target = match.group("target") - if source == '%esp' and target == '%ebp': + if source == self.ESP and target == '%ebp': return self._visit_prologue() - elif source == '%ebp' and target == '%esp': + elif source == '%ebp' and target == self.ESP: return self._visit_epilogue() return self.insns_for_copy(source, target) @@ -658,6 +660,10 @@ class ElfFunctionGcRootTracker(FunctionGcRootTracker): format = 'elf' + ESP = '%esp' + OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' + LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' + def __init__(self, lines, filetag=0): match = r_functionstart_elf.match(lines[0]) funcname = match.group(1) @@ -667,14 +673,20 @@ super(ElfFunctionGcRootTracker, self).__init__( funcname, lines, filetag) -class DarwinFunctionGcRootTracker(FunctionGcRootTracker): + def extract_immediate(self, value): + if not value.startswith('$'): + return None + return int(value[1:]) + +ElfFunctionGcRootTracker.init_regexp() + +class DarwinFunctionGcRootTracker(ElfFunctionGcRootTracker): format = 'darwin' def __init__(self, lines, filetag=0): match = r_functionstart_darwin.match(lines[0]) funcname = '_'+match.group(1) - super(DarwinFunctionGcRootTracker, self).__init__( - funcname, lines, filetag) + FunctionGcRootTracker.__init__(self, funcname, lines, filetag) class Mingw32FunctionGcRootTracker(DarwinFunctionGcRootTracker): format = 'mingw32' @@ -684,12 +696,16 @@ r_functionstart = re.compile(r"PUBLIC\t"+LABEL+"$") - OPERAND = r'(?:\w+|(?:DWORD PTR )?[_\w$]*\[[-+\w0-9]+\])' - r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+")\s*$") - r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+([*]"+OPERAND+")\s*$") - r_binaryinsn = re.compile(r"\t[a-z]\w*\s+(?P"+OPERAND+"),\s*(?P"+OPERAND+")\s*(?:;.+)?$") + ESP = 'esp' - r_jump = re.compile(r"\tj\w+\s+(?:SHORT )?"+LABEL+"\s*$") + OPERAND = r'(?:\w+|(?:DWORD PTR )?[_\w$]*\[[-+\w0-9]+\])' + LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' + + @classmethod + def init_regexp(cls): + super(MsvcFunctionGcRootTracker, cls).init_regexp() + cls.r_binaryinsn = re.compile(r"\t[a-z]\w*\s+(?P"+cls.OPERAND+"),\s*(?P"+cls.OPERAND+")\s*(?:;.+)?$") + cls.r_jump = re.compile(r"\tj\w+\s+(?:SHORT )?"+LABEL+"\s*$") def __init__(self, lines, filetag=0): match = self.r_functionstart.match(lines[0]) @@ -704,6 +720,14 @@ locals()['visit_' + name] = getattr(FunctionGcRootTracker, 'visit_' + name + 'l') + def extract_immediate(self, value): + try: + return int(value) + except ValueError: + return None + +MsvcFunctionGcRootTracker.init_regexp() + class AssemblerParser(object): def __init__(self, verbose=0, shuffle=False): self.verbose = verbose From fijal at codespeak.net Mon Oct 19 14:49:42 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 19 Oct 2009 14:49:42 +0200 (CEST) Subject: [pypy-svn] r68625 - in pypy/trunk/pypy: rlib rpython/memory/test Message-ID: <20091019124942.0226016800D@codespeak.net> Author: fijal Date: Mon Oct 19 14:49:42 2009 New Revision: 68625 Modified: pypy/trunk/pypy/rlib/rgc.py pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py Log: support for malloc_nonmovable with zero=True Modified: pypy/trunk/pypy/rlib/rgc.py ============================================================================== --- pypy/trunk/pypy/rlib/rgc.py (original) +++ pypy/trunk/pypy/rlib/rgc.py Mon Oct 19 14:49:42 2009 @@ -184,7 +184,7 @@ hop.exception_cannot_occur() return hop.genop('gc_can_move', hop.args_v, resulttype=hop.r_result) -def malloc_nonmovable(TP, n=None): +def malloc_nonmovable(TP, n=None, zero=False): """ Allocate a non-moving buffer or return nullptr. When running directly, will pretend that gc is always moving (might be configurable in a future) @@ -195,21 +195,26 @@ class MallocNonMovingEntry(ExtRegistryEntry): _about_ = malloc_nonmovable - def compute_result_annotation(self, s_TP, s_n=None): + def compute_result_annotation(self, s_TP, s_n=None, s_zero=None): # basically return the same as malloc from pypy.annotation.builtin import malloc - return malloc(s_TP, s_n) + return malloc(s_TP, s_n, s_zero=s_zero) - def specialize_call(self, hop): + def specialize_call(self, hop, i_zero=None): from pypy.rpython.lltypesystem import lltype # XXX assume flavor and zero to be None by now assert hop.args_s[0].is_constant() vlist = [hop.inputarg(lltype.Void, arg=0)] opname = 'malloc_nonmovable' flags = {'flavor': 'gc'} + if i_zero is not None: + flags['zero'] = hop.args_s[i_zero].const + nb_args = hop.nb_args - 1 + else: + nb_args = hop.nb_args vlist.append(hop.inputconst(lltype.Void, flags)) - if hop.nb_args == 2: + if nb_args == 2: vlist.append(hop.inputarg(lltype.Signed, arg=1)) opname += '_varsize' Modified: pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py Mon Oct 19 14:49:42 2009 @@ -480,7 +480,7 @@ TP = lltype.GcArray(lltype.Char) def func(): #try: - a = rgc.malloc_nonmovable(TP, 3) + a = rgc.malloc_nonmovable(TP, 3, zero=True) rgc.collect() if a: assert not rgc.can_move(a) From fijal at codespeak.net Mon Oct 19 14:52:22 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 19 Oct 2009 14:52:22 +0200 (CEST) Subject: [pypy-svn] r68626 - in pypy/trunk/pypy/jit/backend/x86: . test Message-ID: <20091019125222.452DB16800D@codespeak.net> Author: fijal Date: Mon Oct 19 14:52:21 2009 New Revision: 68626 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/runner.py pypy/trunk/pypy/jit/backend/x86/support.py (contents, props changed) pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py pypy/trunk/pypy/jit/backend/x86/test/test_support.py (contents, props changed) Log: Re-revert 68623, I have a proper fix Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Mon Oct 19 14:52:21 2009 @@ -13,15 +13,15 @@ from pypy.jit.backend.x86 import codebuf from pypy.jit.backend.x86.ri386 import * from pypy.jit.metainterp.resoperation import rop - - +from pypy.jit.backend.x86.support import NonmovableGrowableArrayFloat,\ + NonmovableGrowableArraySigned, NonmovableGrowableArrayGCREF,\ + CHUNK_SIZE # our calling convention - we pass first 6 args in registers # and the rest stays on the stack RET_BP = 5 # ret ip + bp + bx + esi + edi = 5 words -MAX_FAIL_BOXES = 1000 if sys.platform == 'darwin': # darwin requires the stack to be 16 bytes aligned on calls CALL_ALIGN = 4 @@ -76,30 +76,18 @@ self.malloc_array_func_addr = 0 self.malloc_str_func_addr = 0 self.malloc_unicode_func_addr = 0 - self.fail_boxes_int = lltype.malloc(lltype.GcArray(lltype.Signed), - MAX_FAIL_BOXES, zero=True) - self.fail_boxes_ptr = lltype.malloc(lltype.GcArray(llmemory.GCREF), - MAX_FAIL_BOXES, zero=True) - self.fail_boxes_float = lltype.malloc(lltype.GcArray(lltype.Float), - MAX_FAIL_BOXES, zero=True) + self.fail_boxes_int = NonmovableGrowableArraySigned() + self.fail_boxes_ptr = NonmovableGrowableArrayGCREF() + self.fail_boxes_float = NonmovableGrowableArrayFloat() def leave_jitted_hook(self): fail_boxes_ptr = self.fail_boxes_ptr - llop.gc_assume_young_pointers(lltype.Void, - llmemory.cast_ptr_to_adr(fail_boxes_ptr)) + for chunk in fail_boxes_ptr.chunks: + llop.gc_assume_young_pointers(lltype.Void, + llmemory.cast_ptr_to_adr(chunk)) def make_sure_mc_exists(self): if self.mc is None: - rffi.cast(lltype.Signed, self.fail_boxes_int) # workaround - rffi.cast(lltype.Signed, self.fail_boxes_ptr) # workaround - rffi.cast(lltype.Signed, self.fail_boxes_float) # workaround - self.fail_box_int_addr = rffi.cast(lltype.Signed, - lltype.direct_arrayitems(self.fail_boxes_int)) - self.fail_box_ptr_addr = rffi.cast(lltype.Signed, - lltype.direct_arrayitems(self.fail_boxes_ptr)) - self.fail_box_float_addr = rffi.cast(lltype.Signed, - lltype.direct_arrayitems(self.fail_boxes_float)) - # the address of the function called by 'new' gc_ll_descr = self.cpu.gc_ll_descr gc_ll_descr.initialize() @@ -213,23 +201,22 @@ # This uses XCHG to put zeroes in fail_boxes_ptr after # reading them self.mc.XOR(target, target) - self.mc.XCHG(target, addr_add(imm(self.fail_box_ptr_addr), - imm(i*WORD))) + adr = self.fail_boxes_ptr.get_addr_for_num(i) + self.mc.XCHG(target, heap(adr)) else: - self.mc.MOV(target, addr_add(imm(self.fail_box_int_addr), - imm(i*WORD))) + adr = self.fail_boxes_int.get_addr_for_num(i) + self.mc.MOV(target, heap(adr)) if target is not loc: self.mc.MOV(loc, target) for i in range(len(floatlocs)): loc = floatlocs[i] if loc is None: continue + adr = self.fail_boxes_float.get_addr_for_num(i) if isinstance(loc, REG): - self.mc.MOVSD(loc, addr64_add(imm(self.fail_box_float_addr), - imm(i*WORD*2))) + self.mc.MOVSD(loc, heap64(adr)) else: - self.mc.MOVSD(xmmtmp, addr64_add(imm(self.fail_box_float_addr), - imm(i*WORD*2))) + self.mc.MOVSD(xmmtmp, heap64(adr)) self.mc.MOVSD(loc, xmmtmp) return adr_stackadjust @@ -741,41 +728,38 @@ return addr def generate_failure(self, mc, faildescr, failargs, locs, exc): - assert len(failargs) < MAX_FAIL_BOXES pos = mc.tell() for i in range(len(failargs)): arg = failargs[i] loc = locs[i] if isinstance(loc, REG): if arg.type == FLOAT: - mc.MOVSD(addr64_add(imm(self.fail_box_float_addr), - imm(i*WORD*2)), loc) + adr = self.fail_boxes_float.get_addr_for_num(i) + mc.MOVSD(heap64(adr), loc) else: if arg.type == REF: - base = self.fail_box_ptr_addr + adr = self.fail_boxes_ptr.get_addr_for_num(i) else: - base = self.fail_box_int_addr - mc.MOV(addr_add(imm(base), imm(i*WORD)), loc) + adr = self.fail_boxes_int.get_addr_for_num(i) + mc.MOV(heap(adr), loc) for i in range(len(failargs)): arg = failargs[i] loc = locs[i] if not isinstance(loc, REG): if arg.type == FLOAT: mc.MOVSD(xmm0, loc) - mc.MOVSD(addr64_add(imm(self.fail_box_float_addr), - imm(i*WORD*2)), xmm0) + adr = self.fail_boxes_float.get_addr_for_num(i) + mc.MOVSD(heap64(adr), xmm0) else: if arg.type == REF: - base = self.fail_box_ptr_addr + adr = self.fail_boxes_ptr.get_addr_for_num(i) else: - base = self.fail_box_int_addr + adr = self.fail_boxes_int.get_addr_for_num(i) mc.MOV(eax, loc) - mc.MOV(addr_add(imm(base), imm(i*WORD)), eax) + mc.MOV(heap(adr), eax) if self.debug_markers: mc.MOV(eax, imm(pos)) - mc.MOV(addr_add(imm(self.fail_box_int_addr), - imm(len(locs) * WORD)), - eax) + mc.MOV(heap(self.fail_boxes_int.get_addr_for_num(len(locs))), eax) # we call a provided function that will # - call our on_leave_jitted_hook which will mark Modified: pypy/trunk/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/runner.py Mon Oct 19 14:52:21 2009 @@ -5,7 +5,7 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.rlib.objectmodel import we_are_translated from pypy.jit.metainterp import history -from pypy.jit.backend.x86.assembler import Assembler386, MAX_FAIL_BOXES +from pypy.jit.backend.x86.assembler import Assembler386 from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU @@ -38,28 +38,25 @@ self.assembler.assemble_bridge(faildescr, inputargs, operations) def set_future_value_int(self, index, intvalue): - assert index < MAX_FAIL_BOXES, "overflow!" - self.assembler.fail_boxes_int[index] = intvalue + self.assembler.fail_boxes_int.setitem(index, intvalue) def set_future_value_float(self, index, floatvalue): - assert index < MAX_FAIL_BOXES, "overflow!" - self.assembler.fail_boxes_float[index] = floatvalue + self.assembler.fail_boxes_float.setitem(index, floatvalue) def set_future_value_ref(self, index, ptrvalue): - assert index < MAX_FAIL_BOXES, "overflow!" - self.assembler.fail_boxes_ptr[index] = ptrvalue + self.assembler.fail_boxes_ptr.setitem(index, ptrvalue) def get_latest_value_int(self, index): - return self.assembler.fail_boxes_int[index] + return self.assembler.fail_boxes_int.getitem(index) def get_latest_value_float(self, index): - return self.assembler.fail_boxes_float[index] + return self.assembler.fail_boxes_float.getitem(index) def get_latest_value_ref(self, index): - ptrvalue = self.assembler.fail_boxes_ptr[index] + ptrvalue = self.assembler.fail_boxes_ptr.getitem(index) # clear after reading - self.assembler.fail_boxes_ptr[index] = lltype.nullptr( - llmemory.GCREF.TO) + self.assembler.fail_boxes_ptr.setitem(index, lltype.nullptr( + llmemory.GCREF.TO)) return ptrvalue def execute_token(self, executable_token): Modified: pypy/trunk/pypy/jit/backend/x86/support.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/support.py (original) +++ pypy/trunk/pypy/jit/backend/x86/support.py Mon Oct 19 14:52:21 2009 @@ -23,7 +23,7 @@ lltype.direct_arrayitems(chunk), ofs)) def _no_of(self, i): - while i > len(self.chunks) * CHUNK_SIZE: + while i >= len(self.chunks) * CHUNK_SIZE: self._grow() return i / CHUNK_SIZE, i % CHUNK_SIZE Modified: pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py Mon Oct 19 14:52:21 2009 @@ -198,8 +198,8 @@ ptr = lltype.malloc(S) self.interpret(ops, [0, ptr]) assert self.getptr(0, lltype.Ptr(S)) == ptr - assert not self.cpu.assembler.fail_boxes_ptr[0] - assert not self.cpu.assembler.fail_boxes_ptr[1] + assert not self.cpu.assembler.fail_boxes_ptr.getitem(0) + assert not self.cpu.assembler.fail_boxes_ptr.getitem(1) def test_exception_bridge_no_exception(self): ops = ''' Modified: pypy/trunk/pypy/jit/backend/x86/test/test_support.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_support.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_support.py Mon Oct 19 14:52:21 2009 @@ -22,5 +22,6 @@ ar.setitem(3 * CHUNK_SIZE + 42, 38) adr = ar.get_addr_for_num(3 * CHUNK_SIZE + 42) assert rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] == 38 - + ar.setitem(8*CHUNK_SIZE, 13) + assert ar.getitem(8*CHUNK_SIZE) == 13 From fijal at codespeak.net Mon Oct 19 14:53:14 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 19 Oct 2009 14:53:14 +0200 (CEST) Subject: [pypy-svn] r68627 - pypy/trunk/pypy/jit/backend/x86 Message-ID: <20091019125314.D1F6516800D@codespeak.net> Author: fijal Date: Mon Oct 19 14:53:14 2009 New Revision: 68627 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/support.py Log: Correct fix. This is a bit hairy, leave at least a comment. I suppose one more test would not hurt a lot Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Mon Oct 19 14:53:14 2009 @@ -81,10 +81,15 @@ self.fail_boxes_float = NonmovableGrowableArrayFloat() def leave_jitted_hook(self): - fail_boxes_ptr = self.fail_boxes_ptr - for chunk in fail_boxes_ptr.chunks: + # XXX BIG FAT WARNING XXX + # At this point, we should not call anyone here, because + # RPython-level exception might be set. Here be dragons + i = 0 + while i < self.fail_boxes_ptr.lgt: + chunk = self.fail_boxes_ptr.chunks[i] llop.gc_assume_young_pointers(lltype.Void, llmemory.cast_ptr_to_adr(chunk)) + i += 1 def make_sure_mc_exists(self): if self.mc is None: Modified: pypy/trunk/pypy/jit/backend/x86/support.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/support.py (original) +++ pypy/trunk/pypy/jit/backend/x86/support.py Mon Oct 19 14:53:14 2009 @@ -1,5 +1,7 @@ from pypy.rpython.lltypesystem import lltype, rffi, llmemory +from pypy.rlib import rgc +from pypy.rlib.objectmodel import we_are_translated CHUNK_SIZE = 1000 @@ -9,11 +11,17 @@ class NonmovableGrowableArray(object): def __init__(self): self.chunks = [] - self._grow() + self.lgt = 0 def _grow(self): - self.chunks.append(lltype.malloc(ATP, CHUNK_SIZE, - zero=True)) + # XXX workaround for a fact that rgc.malloc_nonmovable always + # returns nullptr when run on top of python + if we_are_translated(): + new_item = rgc.malloc_nonmovable(ATP, CHUNK_SIZE, zero=True) + else: + new_item = lltype.malloc(ATP, CHUNK_SIZE, zero=True) + self.chunks.append(new_item) + self.lgt += 1 def get_addr_for_num(self, i): chunk_no, ofs = self._no_of(i) From pedronis at codespeak.net Mon Oct 19 15:18:22 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 19 Oct 2009 15:18:22 +0200 (CEST) Subject: [pypy-svn] r68628 - pypy/trunk/pypy/module/pypyjit/test Message-ID: <20091019131822.74C6B16801B@codespeak.net> Author: pedronis Date: Mon Oct 19 15:18:21 2009 New Revision: 68628 Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Log: (cfbolz, pedronis): improve the tests to parse the produced traces. So far we only check that a number of opcodes produce nothing in the trace. Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Mon Oct 19 15:18:21 2009 @@ -1,14 +1,34 @@ from pypy.conftest import gettestobjspace, option from pypy.tool.udir import udir import py +from py.test import skip import sys, os +class BytecodeTrace(list): + pass + + +ZERO_OP_BYTECODES = [ + 'POP_TOP', + 'ROT_TWO', + 'ROT_THREE', + 'DUP_TOP', + 'ROT_FOUR', + 'NOP', + 'DUP_TOPX', + 'LOAD_CONST', + 'JUMP_FORWARD', + #'JUMP_ABSOLUTE' in theory, but contains signals stuff + #'LOAD_FAST' should be here, but currently needs a guard for nonzeroness + 'STORE_FAST', + ] + class PyPyCJITTests(object): - def run_source(self, source, testcases): + def run_source(self, source, *testcases): source = py.code.Source(source) filepath = self.tmpdir.join('case%d.py' % self.counter) logfilepath = filepath.new(ext='.log') - self.counter += 1 + self.__class__.counter += 1 f = filepath.open('w') print >> f, source # some support code... @@ -40,18 +60,44 @@ assert result assert result.splitlines()[-1].strip() == 'OK :-)' assert logfilepath.check() + opslogfile = logfilepath.new(ext='.log.ops') + self.parse_loops(opslogfile) + + def parse_loops(self, opslogfile): + from pypy.jit.metainterp.test.oparser import parse, split_logs_into_loops + assert opslogfile.check() + logs = opslogfile.read() + parts = split_logs_into_loops(logs) + # skip entry bridges, they can contain random things + self.loops = [parse(part, no_namespace=True) for part in parts + if "entry bridge" not in part] + self.sliced_loops = [] # contains all bytecodes of all loops + for loop in self.loops: + for op in loop.operations: + if op.getopname() == "debug_merge_point": + sliced_loop = BytecodeTrace() + sliced_loop.bytecode = op.args[0]._get_str().rsplit(" ", 1)[1] + self.sliced_loops.append(sliced_loop) + else: + sliced_loop.append(op) + self.check_0_op_bytecodes() + def check_0_op_bytecodes(self): + for bytecodetrace in self.sliced_loops: + if bytecodetrace.bytecode not in ZERO_OP_BYTECODES: + continue + assert not bytecodetrace def test_f(self): self.run_source(""" def main(n): return (n+5)+6 """, - [([100], 111), + ([100], 111), ([-5], 6), ([sys.maxint], sys.maxint+11), ([-sys.maxint-5], long(-sys.maxint+6)), - ]) + ) def test_f1(self): self.run_source(''' @@ -67,7 +113,7 @@ i = i + 1 return x ''', - [([2117], 1083876708)]) + ([2117], 1083876708)) def test_factorial(self): self.run_source(''' @@ -78,10 +124,11 @@ n -= 1 return r ''', - [([5], 120), - ([20], 2432902008176640000L)]) + ([5], 120), + ([20], 2432902008176640000L)) def test_factorialrec(self): + skip("does not make sense yet") self.run_source(''' def main(n): if n > 1: @@ -89,8 +136,8 @@ else: return 1 ''', - [([5], 120), - ([20], 2432902008176640000L)]) + ([5], 120), + ([20], 2432902008176640000L)) def test_richards(self): self.run_source(''' @@ -100,24 +147,13 @@ def main(): return richards.main(iterations = 1) ''' % (sys.path,), - [([], 42)]) - - def test_inplace_op(self): - self.run_source(''' - def main(x, y): - r = 5 - r += x - r += -y - return r - ''', [([17, 3], 19), - ([sys.maxint-3, 5], long(sys.maxint - 3)), - ([17, -sys.maxint - 1], sys.maxint + 23) - ]) + ([], 42)) class AppTestJIT(PyPyCJITTests): def setup_class(cls): if not option.runappdirect: py.test.skip("meant only for pypy-c") + # the next line skips stuff if the pypy-c is not a jit build cls.space = gettestobjspace(usemodules=['pypyjit']) cls.tmpdir = udir.join('pypy-jit') cls.tmpdir.ensure(dir=1) From arigo at codespeak.net Mon Oct 19 15:28:12 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 19 Oct 2009 15:28:12 +0200 (CEST) Subject: [pypy-svn] r68630 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20091019132812.D9333168021@codespeak.net> Author: arigo Date: Mon Oct 19 15:28:12 2009 New Revision: 68630 Modified: pypy/trunk/pypy/rpython/lltypesystem/llarena.py Log: Fix. It fixes notably translator/sandbox. Modified: pypy/trunk/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/llarena.py Mon Oct 19 15:28:12 2009 @@ -340,8 +340,10 @@ '#include ') linux_madvise = rffi.llexternal('madvise', [llmemory.Address, rffi.SIZE_T, rffi.INT], - rffi.INT) - linux_getpagesize = rffi.llexternal('getpagesize', [], rffi.INT) + rffi.INT, + sandboxsafe=True, _nowrapper=True) + linux_getpagesize = rffi.llexternal('getpagesize', [], rffi.INT, + sandboxsafe=True, _nowrapper=True) class LinuxPageSize: def __init__(self): @@ -361,12 +363,13 @@ llmemory.raw_memclear(baseaddr, partpage) baseaddr += partpage size -= partpage - madv_length = size & -pagesize + length = size & -pagesize + madv_length = rffi.cast(rffi.SIZE_T, length) madv_flags = rffi.cast(rffi.INT, MADV_DONTNEED) err = linux_madvise(baseaddr, madv_length, madv_flags) if rffi.cast(lltype.Signed, err) == 0: - baseaddr += madv_length # madvise() worked - size -= madv_length + baseaddr += length # madvise() worked + size -= length if size > 0: # clear the final misaligned part, if any llmemory.raw_memclear(baseaddr, size) From pedronis at codespeak.net Mon Oct 19 15:35:18 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 19 Oct 2009 15:35:18 +0200 (CEST) Subject: [pypy-svn] r68631 - pypy/trunk/pypy/module/pypyjit/test Message-ID: <20091019133518.C791D16802F@codespeak.net> Author: pedronis Date: Mon Oct 19 15:35:18 2009 New Revision: 68631 Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Log: (cfbolz, pedronis): a test that checks LOAD_GLOBAL and an easy call Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Mon Oct 19 15:35:18 2009 @@ -5,8 +5,9 @@ import sys, os class BytecodeTrace(list): - pass - + def get_opnames(self, prefix=""): + return [op.getopname() for op in self + if op.getopname().startswith(prefix)] ZERO_OP_BYTECODES = [ 'POP_TOP', @@ -37,12 +38,11 @@ pypyjit.set_param(threshold=3) def check(args, expected): - for i in range(3): - print >> sys.stderr, 'trying:', args - result = main(*args) - print >> sys.stderr, 'got:', repr(result) - assert result == expected - assert type(result) is type(expected) + print >> sys.stderr, 'trying:', args + result = main(*args) + print >> sys.stderr, 'got:', repr(result) + assert result == expected + assert type(result) is type(expected) """) for testcase in testcases * 2: print >> f, "check(%r, %r)" % testcase @@ -88,10 +88,14 @@ continue assert not bytecodetrace + def get_by_bytecode(self, name): + return [ops for ops in self.sliced_loops if ops.bytecode == name] + def test_f(self): self.run_source(""" def main(n): - return (n+5)+6 + for i in range(3): + return (n+5)+6 """, ([100], 111), ([-5], 6), @@ -149,6 +153,33 @@ ''' % (sys.path,), ([], 42)) + def test_simple_call(self): + self.run_source(''' + def f(i): + return i + 1 + def main(n): + i = 0 + while i < n: + i = f(f(i)) + return i + ''', + ([20], 20), + ([31], 32)) + ops = self.get_by_bytecode("LOAD_GLOBAL") + assert len(ops) == 2 + assert ops[0].get_opnames() == ["getfield_gc", "getarrayitem_gc", + "getfield_gc", "ooisnull", + "guard_false"] + assert not ops[1] # second LOAD_GLOBAL folded away + ops = self.get_by_bytecode("CALL_FUNCTION") + assert len(ops) == 2 + for bytecode in ops: + assert not bytecode.get_opnames("call") + assert not bytecode.get_opnames("new") + assert len(bytecode.get_opnames("guard")) <= 10 + + + class AppTestJIT(PyPyCJITTests): def setup_class(cls): if not option.runappdirect: From pedronis at codespeak.net Mon Oct 19 15:56:00 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 19 Oct 2009 15:56:00 +0200 (CEST) Subject: [pypy-svn] r68632 - pypy/trunk/pypy/module/pypyjit/test Message-ID: <20091019135600.B223A168012@codespeak.net> Author: pedronis Date: Mon Oct 19 15:56:00 2009 New Revision: 68632 Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Log: (cfbolz, pedronis): check method lookups, attribute lookups, calls with defaults, calls with a keyword Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Mon Oct 19 15:56:00 2009 @@ -9,6 +9,9 @@ return [op.getopname() for op in self if op.getopname().startswith(prefix)] + def __repr__(self): + return "%s%s" % (self.bytecode, list.__repr__(self)) + ZERO_OP_BYTECODES = [ 'POP_TOP', 'ROT_TWO', @@ -177,7 +180,65 @@ assert not bytecode.get_opnames("call") assert not bytecode.get_opnames("new") assert len(bytecode.get_opnames("guard")) <= 10 - + + def test_method_call(self): + self.run_source(''' + class A(object): + def __init__(self, a): + self.a = a + def f(self, i): + return self.a + i + def main(n): + i = 0 + a = A(1) + while i < n: + x = a.f(i) + i = a.f(x) + return i + ''', + ([20], 20), + ([31], 32)) + ops = self.get_by_bytecode("LOOKUP_METHOD") + assert len(ops) == 2 + assert not ops[0].get_opnames("call") + assert not ops[0].get_opnames("new") + assert len(ops[0].get_opnames("guard")) <= 8 + assert not ops[1] # second LOOKUP_METHOD folded away + + ops = self.get_by_bytecode("CALL_METHOD") + assert len(ops) == 2 + for bytecode in ops: + assert not ops[0].get_opnames("call") + assert not ops[0].get_opnames("new") + assert len(ops[0].get_opnames("guard")) <= 9 + assert len(ops[1]) < len(ops[0]) + + ops = self.get_by_bytecode("LOAD_ATTR") + assert len(ops) == 2 + assert ops[0].get_opnames() == ["getfield_gc", "getarrayitem_gc", + "ooisnull", "guard_false"] + assert not ops[1] # second LOAD_ATTR folded away + + def test_default_and_kw(self): + self.run_source(''' + def f(i, j=1): + return i + j + def main(n): + i = 0 + while i < n: + i = f(f(i), j=1) + return i + ''', + ([20], 20), + ([31], 32)) + ops = self.get_by_bytecode("CALL_FUNCTION") + assert len(ops) == 2 + for bytecode in ops: + assert not bytecode.get_opnames("call") + assert not bytecode.get_opnames("new") + assert len(ops[0].get_opnames("guard")) <= 14 + assert len(ops[1].get_opnames("guard")) <= 3 + class AppTestJIT(PyPyCJITTests): From pedronis at codespeak.net Mon Oct 19 16:15:28 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 19 Oct 2009 16:15:28 +0200 (CEST) Subject: [pypy-svn] r68633 - pypy/trunk/pypy/module/pypyjit/test Message-ID: <20091019141528.63068168005@codespeak.net> Author: pedronis Date: Mon Oct 19 16:15:27 2009 New Revision: 68633 Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Log: (cfbolz, pedronis): a test for a virtual instance and for adding int and float Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Mon Oct 19 16:15:27 2009 @@ -94,18 +94,6 @@ def get_by_bytecode(self, name): return [ops for ops in self.sliced_loops if ops.bytecode == name] - def test_f(self): - self.run_source(""" - def main(n): - for i in range(3): - return (n+5)+6 - """, - ([100], 111), - ([-5], 6), - ([sys.maxint], sys.maxint+11), - ([-sys.maxint-5], long(-sys.maxint+6)), - ) - def test_f1(self): self.run_source(''' def main(n): @@ -208,9 +196,9 @@ ops = self.get_by_bytecode("CALL_METHOD") assert len(ops) == 2 for bytecode in ops: - assert not ops[0].get_opnames("call") - assert not ops[0].get_opnames("new") - assert len(ops[0].get_opnames("guard")) <= 9 + assert not bytecode.get_opnames("call") + assert not bytecode.get_opnames("new") + assert len(bytecode.get_opnames("guard")) <= 9 assert len(ops[1]) < len(ops[0]) ops = self.get_by_bytecode("LOAD_ATTR") @@ -230,7 +218,7 @@ return i ''', ([20], 20), - ([31], 32)) + ([31], 32)) ops = self.get_by_bytecode("CALL_FUNCTION") assert len(ops) == 2 for bytecode in ops: @@ -239,7 +227,48 @@ assert len(ops[0].get_opnames("guard")) <= 14 assert len(ops[1].get_opnames("guard")) <= 3 + def test_virtual_instance(self): + self.run_source(''' + class A(object): + pass + def main(n): + i = 0 + while i < n: + a = A() + a.x = 2 + i = i + a.x + return i + ''', + ([20], 20), + ([31], 32)) + + bytecode, = self.get_by_bytecode("CALL_FUNCTION") + assert not bytecode.get_opnames("call") + assert not bytecode.get_opnames("new") + assert len(bytecode.get_opnames("guard")) <= 9 + + bytecode, = self.get_by_bytecode("STORE_ATTR") + # XXX where does that come from? + assert bytecode.get_opnames() == ["getfield_gc", "guard_value"] + + def test_mixed_type_loop(self): + self.run_source(''' + class A(object): + pass + def main(n): + i = 0.0 + j = 2 + while i < n: + i = j + i + return i, type(i) is float + ''', + ([20], (20, True)), + ([31], (32, True))) + bytecode, = self.get_by_bytecode("BINARY_ADD") + assert not bytecode.get_opnames("call") + assert not bytecode.get_opnames("new") + assert len(bytecode.get_opnames("guard")) <= 3 class AppTestJIT(PyPyCJITTests): def setup_class(cls): From arigo at codespeak.net Mon Oct 19 16:45:05 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 19 Oct 2009 16:45:05 +0200 (CEST) Subject: [pypy-svn] r68634 - pypy/branch/gc-arena Message-ID: <20091019144505.CFF3E168005@codespeak.net> Author: arigo Date: Mon Oct 19 16:45:05 2009 New Revision: 68634 Added: pypy/branch/gc-arena/ - copied from r68633, pypy/trunk/ Log: A branch in which to add more memory-management options to llarena.py. From arigo at codespeak.net Mon Oct 19 16:49:27 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 19 Oct 2009 16:49:27 +0200 (CEST) Subject: [pypy-svn] r68635 - in pypy/branch/gc-arena/pypy/rpython/lltypesystem: . test Message-ID: <20091019144927.395E2168005@codespeak.net> Author: arigo Date: Mon Oct 19 16:49:26 2009 New Revision: 68635 Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena.py pypy/branch/gc-arena/pypy/rpython/lltypesystem/test/test_llarena.py Log: Add the Z_xxx constants describing how to clear the memory obtained or reseted. Test them. Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena.py Mon Oct 19 16:49:26 2009 @@ -13,6 +13,9 @@ class ArenaError(Exception): pass +class InaccessibleArenaError(ArenaError): + pass + class Arena(object): object_arena_location = {} # {container: (arena, offset)} old_object_arena_location = weakref.WeakKeyDictionary() @@ -44,10 +47,24 @@ del self.objectptrs[offset] del self.objectsizes[offset] obj._free() - if zero: + if zero == Z_DONT_CLEAR: + initialbyte = "#" + elif zero in (Z_CLEAR_LARGE_AREA, Z_CLEAR_SMALL_AREA): + initialbyte = "0" + elif zero == Z_INACCESSIBLE: + prev = self.usagemap[start:stop].tostring() + assert '!' not in prev, ( + "Z_INACCESSIBLE must be called only on a " + "previously-accessible memory range") + initialbyte = "!" + elif zero == Z_ACCESSIBLE: + prev = self.usagemap[start:stop].tostring() + assert prev == '!'*len(prev), ( + "Z_ACCESSIBLE must be called only on a " + "previously-inaccessible memory range") initialbyte = "0" else: - initialbyte = "#" + raise ValueError("argument 'zero' got bogus value %r" % (zero,)) self.usagemap[start:stop] = array.array('c', initialbyte*(stop-start)) def check(self): @@ -74,6 +91,8 @@ pass elif c == '#': zero = False + elif c == '!': + raise InaccessibleArenaError else: raise ArenaError("new object overlaps a previous object") assert offset not in self.objectptrs @@ -269,27 +288,39 @@ # work with fakearenaaddresses on which arbitrary arithmetic is # possible even on top of the llinterpreter. -# arena_new_view(ptr) is a no-op when translated, returns fresh view -# on previous arena when run on top of llinterp +# arena_malloc() and arena_reset() take as argument one of the +# following values: + +Z_DONT_CLEAR = 0 # it's ok to keep random bytes in the area +Z_CLEAR_LARGE_AREA = 1 # clear, optimized for a large area of memory +Z_CLEAR_SMALL_AREA = 2 # clear, optimized for a small or medium area of mem +Z_INACCESSIBLE = 3 # make the memory inaccessible (not reserved) +Z_ACCESSIBLE = 4 # make the memory accessible again + +# Note that Z_INACCESSIBLE and Z_ACCESSIBLE are restricted to whole +# pages, and you must not try to make inaccessible pages that are already +# inaccessible, nor make accessible pages that are already accessible. +# When they go through the Z_INACCESSIBLE-Z_ACCESSIBLE trip, pages are +# cleared. def arena_malloc(nbytes, zero): - """Allocate and return a new arena, optionally zero-initialized.""" + """Allocate and return a new arena, optionally zero-initialized. + The value of 'zero' is one the Z_xxx values. + """ return Arena(nbytes, zero).getaddr(0) -def arena_free(arena_addr): +def arena_free(arena_addr, nbytes): """Release an arena.""" assert isinstance(arena_addr, fakearenaaddress) assert arena_addr.offset == 0 - arena_addr.arena.reset(False) + assert nbytes == arena_addr.arena.nbytes + arena_addr.arena.reset(Z_DONT_CLEAR) arena_addr.arena.freed = True def arena_reset(arena_addr, size, zero): """Free all objects in the arena, which can then be reused. This can also be used on a subrange of the arena. - The value of 'zero' is: - * 0: don't fill the area with zeroes - * 1: clear, optimized for a very large area of memory - * 2: clear, optimized for a small or medium area of memory + The value of 'zero' is one of the Z_xxx values. """ arena_addr = _getfakearenaaddress(arena_addr) arena_addr.arena.reset(zero, arena_addr.offset, size) @@ -316,7 +347,8 @@ return RoundedUpForAllocation(size, minsize) def arena_new_view(ptr): - """Return a fresh memory view on an arena + """This is a no-op when translated, returns fresh view + on previous arena when run on top of llinterp. """ return Arena(ptr.arena.nbytes, False).getaddr(0) @@ -331,6 +363,31 @@ from pypy.rpython.extfunc import register_external from pypy.rlib.objectmodel import CDefinedIntSymbolic +# ---------- getting the page size ---------- + +if os.name == 'posix': + posix_getpagesize = rffi.llexternal('getpagesize', [], rffi.INT, + sandboxsafe=True, _nowrapper=True) + class PosixPageSize: + def __init__(self): + self.pagesize = 0 + _freeze_ = __init__ + posix_pagesize = PosixPageSize() + + def getpagesize(): + pagesize = posix_pagesize.pagesize + if pagesize == 0: + pagesize = rffi.cast(lltype.Signed, posix_getpagesize()) + posix_pagesize.pagesize = pagesize + return pagesize + +else: + # XXX a random value, but nothing really depends on it + def getpagesize(): + return 4096 + +# ---------- clearing a large range of memory ---------- + if sys.platform == 'linux2': # This only works with linux's madvise(), which is really not a memory # usage hint but a real command. It guarantees that after MADV_DONTNEED @@ -342,20 +399,9 @@ [llmemory.Address, rffi.SIZE_T, rffi.INT], rffi.INT, sandboxsafe=True, _nowrapper=True) - linux_getpagesize = rffi.llexternal('getpagesize', [], rffi.INT, - sandboxsafe=True, _nowrapper=True) - - class LinuxPageSize: - def __init__(self): - self.pagesize = 0 - _freeze_ = __init__ - linuxpagesize = LinuxPageSize() def clear_large_memory_chunk(baseaddr, size): - pagesize = linuxpagesize.pagesize - if pagesize == 0: - pagesize = rffi.cast(lltype.Signed, linux_getpagesize()) - linuxpagesize.pagesize = pagesize + pagesize = getpagesize() if size > 2 * pagesize: lowbits = rffi.cast(lltype.Signed, baseaddr) & (pagesize - 1) if lowbits: # clear the initial misaligned part, if any @@ -421,31 +467,119 @@ # them immediately. clear_large_memory_chunk = llmemory.raw_memclear +# ---------- platform-specific version of llimpl_arena_* ---------- -def llimpl_arena_malloc(nbytes, zero): - addr = llmemory.raw_malloc(nbytes) - if zero and bool(addr): - clear_large_memory_chunk(addr, nbytes) - return addr -register_external(arena_malloc, [int, bool], llmemory.Address, +if os.name == 'posix': + # llimpl_arena_*() functions based on mmap + from pypy.rpython.tool import rffi_platform + from pypy.translator.tool.cbuild import ExternalCompilationInfo + class CConfig: + _compilation_info_ = ExternalCompilationInfo( + includes=['sys/mman.h']) + off_t = rffi_platform.SimpleType('off_t') + PROT_NONE = rffi_platform.ConstantInteger('PROT_NONE') + PROT_READ = rffi_platform.ConstantInteger('PROT_READ') + PROT_WRITE = rffi_platform.ConstantInteger('PROT_WRITE') + MAP_PRIVATE = rffi_platform.ConstantInteger('MAP_PRIVATE') + MAP_ANON = rffi_platform.DefinedConstantInteger('MAP_ANON') + MAP_ANONYMOUS = rffi_platform.DefinedConstantInteger('MAP_ANONYMOUS') + MAP_NORESERVE = rffi_platform.DefinedConstantInteger('MAP_NORESERVE') + globals().update(rffi_platform.configure(CConfig)) + if MAP_ANONYMOUS is None: + MAP_ANONYMOUS = MAP_ANON + assert MAP_ANONYMOUS is not None + del MAP_ANON + + posix_mmap = rffi.llexternal('mmap', + [llmemory.Address, rffi.SIZE_T, rffi.INT, + rffi.INT, rffi.INT, off_t], + llmemory.Address, + sandboxsafe=True, _nowrapper=True) + posix_munmap = rffi.llexternal('munmap', + [llmemory.Address, rffi.SIZE_T], + rffi.INT, + sandboxsafe=True, _nowrapper=True) + posix_mprotect = rffi.llexternal('mprotect', + [llmemory.Address, rffi.SIZE_T, + rffi.INT], + rffi.INT, + sandboxsafe=True, _nowrapper=True) + + class MMapMemoryError(Exception): + pass + + def llimpl_arena_malloc(nbytes, zero): + flags = MAP_PRIVATE | MAP_ANONYMOUS + if zero == Z_INACCESSIBLE: + prot = PROT_NONE + if MAP_NORESERVE is not None: + flags |= MAP_NORESERVE + else: + prot = PROT_READ | PROT_WRITE + result = posix_mmap(llmemory.NULL, + rffi.cast(rffi.SIZE_T, nbytes), + rffi.cast(rffi.INT, prot), + rffi.cast(rffi.INT, flags), + rffi.cast(rffi.INT, -1), + rffi.cast(off_t, 0)) + if rffi.cast(lltype.Signed, result) == -1: + raise MMapMemoryError + return result + + def llimpl_arena_free(arena_addr, nbytes): + result = posix_munmap(arena_addr, rffi.cast(rffi.SIZE_T, nbytes)) + if rffi.cast(lltype.Signed, result) == -1: + raise MMapMemoryError + + def _arena_protect(arena_addr, size, flags): + res = posix_mprotect(arena_addr, + rffi.cast(rffi.SIZE_T, size), + rffi.cast(rffi.INT, flags)) + if rffi.cast(lltype.Signed, res) != 0: + raise MMapMemoryError + + def llimpl_arena_reset(arena_addr, size, zero): + if zero == Z_CLEAR_LARGE_AREA: + clear_large_memory_chunk(arena_addr, size) + elif zero == Z_CLEAR_SMALL_AREA: + llmemory.raw_memclear(arena_addr, size) + elif zero == Z_ACCESSIBLE: + _arena_protect(arena_addr, size, PROT_READ | PROT_WRITE) + elif zero == Z_INACCESSIBLE: + clear_large_memory_chunk(arena_addr, size) + _arena_protect(arena_addr, size, PROT_NONE) + +else: + # llimpl_arena_*() functions based on raw_malloc + def llimpl_arena_malloc(nbytes, zero): + addr = llmemory.raw_malloc(nbytes) + if zero in (Z_CLEAR_LARGE_AREA, Z_CLEAR_SMALL_AREA) and bool(addr): + clear_large_memory_chunk(addr, nbytes) + return addr + + def llimpl_arena_free(arena_addr, nbytes): + llmemory.raw_free(arena_addr) + + def llimpl_arena_reset(arena_addr, size, zero): + if zero in (Z_CLEAR_LARGE_AREA, Z_INACCESSIBLE): + clear_large_memory_chunk(arena_addr, size) + elif zero == Z_CLEAR_SMALL_AREA: + llmemory.raw_memclear(arena_addr, size) + +# ---------- + +register_external(arena_malloc, [int, int], llmemory.Address, 'll_arena.arena_malloc', llimpl=llimpl_arena_malloc, llfakeimpl=arena_malloc, sandboxsafe=True) -def llimpl_arena_free(arena_addr): - llmemory.raw_free(arena_addr) -register_external(arena_free, [llmemory.Address], None, 'll_arena.arena_free', +register_external(arena_free, [llmemory.Address, int], None, + 'll_arena.arena_free', llimpl=llimpl_arena_free, llfakeimpl=arena_free, sandboxsafe=True) -def llimpl_arena_reset(arena_addr, size, zero): - if zero: - if zero == 1: - clear_large_memory_chunk(arena_addr, size) - else: - llmemory.raw_memclear(arena_addr, size) register_external(arena_reset, [llmemory.Address, int, int], None, 'll_arena.arena_reset', llimpl=llimpl_arena_reset, Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/test/test_llarena.py Mon Oct 19 16:49:26 2009 @@ -5,13 +5,19 @@ from pypy.rpython.lltypesystem.llarena import arena_reserve, arena_free from pypy.rpython.lltypesystem.llarena import round_up_for_allocation from pypy.rpython.lltypesystem.llarena import ArenaError, arena_new_view +from pypy.rpython.lltypesystem.llarena import Z_DONT_CLEAR +from pypy.rpython.lltypesystem.llarena import Z_CLEAR_LARGE_AREA +from pypy.rpython.lltypesystem.llarena import Z_CLEAR_SMALL_AREA +from pypy.rpython.lltypesystem.llarena import Z_INACCESSIBLE +from pypy.rpython.lltypesystem.llarena import Z_ACCESSIBLE +from pypy.rpython.lltypesystem.llarena import InaccessibleArenaError def test_arena(): S = lltype.Struct('S', ('x',lltype.Signed)) SPTR = lltype.Ptr(S) ssize = llmemory.raw_malloc_usage(llmemory.sizeof(S)) myarenasize = 2*ssize+1 - a = arena_malloc(myarenasize, False) + a = arena_malloc(myarenasize, Z_DONT_CLEAR) assert a != llmemory.NULL assert a + 3 != llmemory.NULL @@ -51,7 +57,7 @@ py.test.raises(ArenaError, arena_reserve, a+2*ssize+1, llmemory.sizeof(S), False) - arena_reset(a, myarenasize, True) + arena_reset(a, myarenasize, Z_CLEAR_LARGE_AREA) py.test.raises(ArenaError, cast_adr_to_ptr, a, SPTR) arena_reserve(a, llmemory.sizeof(S)) s1_ptr1 = cast_adr_to_ptr(a, SPTR) @@ -100,12 +106,12 @@ def test_address_order(): - a = arena_malloc(24, False) + a = arena_malloc(24, Z_DONT_CLEAR) assert eq(a, a) assert lt(a, a+1) assert lt(a+5, a+20) - b = arena_malloc(24, False) + b = arena_malloc(24, Z_DONT_CLEAR) if a > b: a, b = b, a assert lt(a, b) @@ -137,28 +143,28 @@ def test_look_inside_object(): # this code is also used in translation tests below myarenasize = 50 - a = arena_malloc(myarenasize, False) + a = arena_malloc(myarenasize, Z_DONT_CLEAR) b = a + round_up_for_allocation(llmemory.sizeof(lltype.Char)) arena_reserve(b, precomputed_size) (b + llmemory.offsetof(SX, 'x')).signed[0] = 123 assert llmemory.cast_adr_to_ptr(b, SPTR).x == 123 llmemory.cast_adr_to_ptr(b, SPTR).x += 1 assert (b + llmemory.offsetof(SX, 'x')).signed[0] == 124 - arena_reset(a, myarenasize, True) + arena_reset(a, myarenasize, Z_CLEAR_LARGE_AREA) arena_reserve(b, round_up_for_allocation(llmemory.sizeof(SX))) assert llmemory.cast_adr_to_ptr(b, SPTR).x == 0 - arena_free(a) + arena_free(a, myarenasize) return 42 def test_arena_new_view(): - a = arena_malloc(50, False) + a = arena_malloc(50, Z_DONT_CLEAR) arena_reserve(a, precomputed_size) # we can now allocate the same space in new view b = arena_new_view(a) arena_reserve(b, precomputed_size) def test_partial_arena_reset(): - a = arena_malloc(72, False) + a = arena_malloc(72, Z_DONT_CLEAR) def reserve(i): b = a + i * llmemory.raw_malloc_usage(precomputed_size) arena_reserve(b, precomputed_size) @@ -171,7 +177,8 @@ blist.append(b) plist.append(llmemory.cast_adr_to_ptr(b, SPTR)) # clear blist[1] and blist[2] but not blist[0] nor blist[3] - arena_reset(blist[1], llmemory.raw_malloc_usage(precomputed_size)*2, False) + arena_reset(blist[1], llmemory.raw_malloc_usage(precomputed_size)*2, + Z_DONT_CLEAR) py.test.raises(RuntimeError, "plist[1].x") # marked as freed py.test.raises(RuntimeError, "plist[2].x") # marked as freed # re-reserve object at index 1 and 2 @@ -194,7 +201,8 @@ py.test.raises(lltype.UninitializedMemoryAccess, "(blist[2] + llmemory.offsetof(SX, 'x')).signed[0]") # clear and zero-fill the area over blist[0] and blist[1] - arena_reset(blist[0], llmemory.raw_malloc_usage(precomputed_size)*2, True) + arena_reset(blist[0], llmemory.raw_malloc_usage(precomputed_size)*2, + Z_CLEAR_LARGE_AREA) # re-reserve and check it's zero blist[0] = reserve(0) blist[1] = reserve(1) @@ -205,7 +213,7 @@ "(blist[2] + llmemory.offsetof(SX, 'x')).signed[0]") def test_address_eq_as_int(): - a = arena_malloc(50, False) + a = arena_malloc(50, Z_DONT_CLEAR) arena_reserve(a, precomputed_size) p = llmemory.cast_adr_to_ptr(a, SPTR) a1 = llmemory.cast_ptr_to_adr(p) @@ -226,7 +234,7 @@ size_gc_header = gcheaderbuilder.size_gc_header ssize = llmemory.raw_malloc_usage(llmemory.sizeof(S)) - a = arena_malloc(13*ssize, True) + a = arena_malloc(13*ssize, Z_CLEAR_LARGE_AREA) hdraddr = a + 3*ssize arena_reserve(hdraddr, size_gc_header + llmemory.sizeof(S)) hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR)) @@ -236,7 +244,7 @@ obj.z = -6 hdraddr = llmemory.cast_ptr_to_adr(obj) - size_gc_header - arena_reset(hdraddr, size_gc_header + llmemory.sizeof(S), False) + arena_reset(hdraddr, size_gc_header + llmemory.sizeof(S), Z_DONT_CLEAR) arena_reserve(hdraddr, size_gc_header + llmemory.sizeof(STUB)) # check that it possible to reach the newly reserved HDR+STUB @@ -268,3 +276,78 @@ fn = compile(test_look_inside_object, []) res = fn() assert res == 42 + +def fn_protection(i): + # + a = arena_malloc(32768, Z_INACCESSIBLE) + if i == -1: + arena_reserve(a, llmemory.sizeof(SX)) + s1 = cast_adr_to_ptr(a, SPTR) + s1.x = 5 # crash: memory is inaccessible + return 1 + # + arena_reset(a, 16384, Z_ACCESSIBLE) + arena_reserve(a, llmemory.sizeof(SX)) + s1 = cast_adr_to_ptr(a, SPTR) + assert s1.x == 0 + s1.x = 42 + # + arena_reset(a, 16384, Z_CLEAR_LARGE_AREA) + arena_reserve(a, llmemory.sizeof(SX)) + s1 = cast_adr_to_ptr(a, SPTR) + assert s1.x == 0 + s1.x = 43 + # + arena_reset(a, 16384, Z_CLEAR_SMALL_AREA) + arena_reserve(a, llmemory.sizeof(SX)) + s1 = cast_adr_to_ptr(a, SPTR) + assert s1.x == 0 + s1.x = 44 + # + if i == -2: + arena_reserve(a + 16384, llmemory.sizeof(SX)) + s1 = cast_adr_to_ptr(a + 16384, SPTR) + s1.x = 5 # crash: memory is inaccessible + return 1 + # + if i == -4: + arena_reset(a, 16384, Z_INACCESSIBLE) + arena_reserve(a, llmemory.sizeof(SX)) + s1 = cast_adr_to_ptr(a, SPTR) + s1.x = 5 # crash if Z_INACCESSIBLE is correctly implemented + return 1 + # + if i == -3: + arena_reset(a, 16384, Z_DONT_CLEAR) + arena_reserve(a, llmemory.sizeof(SX)) + s1 = cast_adr_to_ptr(a, SPTR) + # crash when not translated: memory was reset but not cleared + print s1.x + # + arena_free(a, 32768) + return 0 + +def test_protection(): + res = fn_protection(0) + assert res == 0 + py.test.raises(InaccessibleArenaError, fn_protection, -1) + py.test.raises(InaccessibleArenaError, fn_protection, -2) + py.test.raises(lltype.UninitializedMemoryAccess, fn_protection, -3) + py.test.raises(InaccessibleArenaError, fn_protection, -4) + +def test_compiled_protection(): + from pypy.translator.interactive import Translation + def fn_wrap(argv): + return fn_protection(int(argv[1])) + t = Translation(fn_wrap, standalone=True) + t.disable(['backendopt']) + path = t.compile_c() + def run(n): + return py.process.cmdexec('%s %d' % (path, n)) + run(0) + py.test.raises(py.error.Error, run, -1) # segfault + py.test.raises(py.error.Error, run, -2) # segfault + py.test.raises(py.error.Error, run, -4) # segfault + res = run(-3) + # good enough, although it should ideally crash: + assert '44\n' in res From pedronis at codespeak.net Mon Oct 19 17:03:09 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 19 Oct 2009 17:03:09 +0200 (CEST) Subject: [pypy-svn] r68636 - in pypy/trunk/pypy/module/pypyjit/test: . loops Message-ID: <20091019150309.EA56616800D@codespeak.net> Author: pedronis Date: Mon Oct 19 17:03:09 2009 New Revision: 68636 Removed: pypy/trunk/pypy/module/pypyjit/test/loops/ Modified: pypy/trunk/pypy/module/pypyjit/test/conftest.py pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Log: (cfbolz, pedronis) some more tests, switch the option to --pypy, kill unused directory Modified: pypy/trunk/pypy/module/pypyjit/test/conftest.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/conftest.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/conftest.py Mon Oct 19 17:03:09 2009 @@ -1,5 +1,5 @@ def pytest_addoption(parser): group = parser.addgroup("pypyjit options") - group.addoption("--pypy-c", action="store", default=None, dest="pypy_c", + group.addoption("--pypy", action="store", default=None, dest="pypy_c", help="the location of the JIT enabled pypy-c") Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Mon Oct 19 17:03:09 2009 @@ -235,6 +235,7 @@ i = 0 while i < n: a = A() + assert isinstance(a, A) a.x = 2 i = i + a.x return i @@ -242,10 +243,13 @@ ([20], 20), ([31], 32)) - bytecode, = self.get_by_bytecode("CALL_FUNCTION") - assert not bytecode.get_opnames("call") - assert not bytecode.get_opnames("new") - assert len(bytecode.get_opnames("guard")) <= 9 + callA, callisinstance = self.get_by_bytecode("CALL_FUNCTION") + assert not callA.get_opnames("call") + assert not callA.get_opnames("new") + assert len(callA.get_opnames("guard")) <= 9 + assert not callisinstance.get_opnames("call") + assert not callisinstance.get_opnames("new") + assert len(callisinstance.get_opnames("guard")) <= 2 bytecode, = self.get_by_bytecode("STORE_ATTR") # XXX where does that come from? From pedronis at codespeak.net Mon Oct 19 17:05:05 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 19 Oct 2009 17:05:05 +0200 (CEST) Subject: [pypy-svn] r68637 - pypy/build/bot2/pypybuildbot Message-ID: <20091019150505.11087168014@codespeak.net> Author: pedronis Date: Mon Oct 19 17:05:04 2009 New Revision: 68637 Modified: pypy/build/bot2/pypybuildbot/builds.py pypy/build/bot2/pypybuildbot/master.py Log: (cfbolz, pedronis) tentative for running the test_pypy_c.py jit tests Modified: pypy/build/bot2/pypybuildbot/builds.py ============================================================================== --- pypy/build/bot2/pypybuildbot/builds.py (original) +++ pypy/build/bot2/pypybuildbot/builds.py Mon Oct 19 17:05:04 2009 @@ -195,7 +195,7 @@ description="copy build", command=["scp", "pypy-c", "fijal at codespeak.net:builds/pypy-c-scratchbox"], workdir = workdir)) -class PyPyJITTranslatedLibPythonTestFactory(factory.BuildFactory): +class PyPyJITTranslatedTestFactory(factory.BuildFactory): def __init__(self, *a, **kw): platform = kw.pop('platform', 'linux') factory.BuildFactory.__init__(self, *a, **kw) @@ -214,6 +214,14 @@ "--resultlog=cpython.log", "lib-python"], logfiles={'pytestLog': 'cpython.log'})) + self.addStep(ShellCmd( + description="pypyjit tests", + command=["python", "pypy/test_all.py", + "--pypy=pypy/translator/goal/pypy-c", + "--resultlog=pypyjit.log", + "pypy/module/pypyjit/test/test_pypy_c.py"], + logfiles={'pytestLog': 'pypyjit.log'})) + class PyPyJITBenchmarkFactory(factory.BuildFactory): def __init__(self, *a, **kw): platform = kw.pop('platform', 'linux') Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Mon Oct 19 17:05:04 2009 @@ -53,7 +53,7 @@ pypyTranslatedAppLevelTestFactory = pypybuilds.PyPyTranslatedAppLevelTestFactory() pypyStacklessTranslatedAppLevelTestFactory = pypybuilds.PyPyStacklessTranslatedAppLevelTestFactory() -pypyJITTranslatedTestFactory = pypybuilds.PyPyJITTranslatedLibPythonTestFactory() +pypyJITTranslatedTestFactory = pypybuilds.PyPyJITTranslatedTestFactory() pypyJITBenchmarkFactory = pypybuilds.PyPyJITBenchmarkFactory() LINUX32 = "own-linux-x86-32" @@ -138,7 +138,7 @@ "slavenames": ["bigdogvm1"], 'builddir' : JITCPYLINUX32, 'factory' : pypyJITTranslatedTestFactory, - 'category' : 'lib-python', + 'category' : 'jit', }, {"name": JITONLYLINUX32, "slavenames": ["wyvern"], From pedronis at codespeak.net Mon Oct 19 17:07:56 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 19 Oct 2009 17:07:56 +0200 (CEST) Subject: [pypy-svn] r68638 - pypy/build/bot2/pypybuildbot Message-ID: <20091019150756.7CD6216800D@codespeak.net> Author: pedronis Date: Mon Oct 19 17:07:56 2009 New Revision: 68638 Modified: pypy/build/bot2/pypybuildbot/master.py Log: adjust indent Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Mon Oct 19 17:07:56 2009 @@ -76,8 +76,8 @@ Nightly("nightly", [LINUX32, CPYLINUX32, APPLVLLINUX32, CPYWIN32, STACKLESSAPPLVLLINUX32, JITCPYLINUX32], hour=4, minute=45), - Nightly("nightly-benchmark", [JITBENCH], - hour=2, minute=25), + Nightly("nightly-benchmark", [JITBENCH], + hour=2, minute=25), ], 'status': [status], From arigo at codespeak.net Mon Oct 19 17:16:17 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 19 Oct 2009 17:16:17 +0200 (CEST) Subject: [pypy-svn] r68639 - in pypy/branch/gc-arena/pypy/rpython/lltypesystem: . test Message-ID: <20091019151617.CFE6B16800D@codespeak.net> Author: arigo Date: Mon Oct 19 17:16:17 2009 New Revision: 68639 Added: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_generic.py (contents, props changed) pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_llinterp.py - copied, changed from r68635, pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena.py pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_posix.py (contents, props changed) Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena.py pypy/branch/gc-arena/pypy/rpython/lltypesystem/llmemory.py pypy/branch/gc-arena/pypy/rpython/lltypesystem/lltype.py pypy/branch/gc-arena/pypy/rpython/lltypesystem/test/test_llarena.py Log: Split up llarena.py in platform-specific parts. Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena.py Mon Oct 19 17:16:17 2009 @@ -1,356 +1,11 @@ -import array, weakref -from pypy.rpython.lltypesystem import lltype, llmemory # An "arena" is a large area of memory which can hold a number of # objects, not necessarily all of the same type or size. It's used by -# some of our framework GCs. Addresses that point inside arenas support -# direct arithmetic: adding and subtracting integers, and taking the -# difference of two addresses. When not translated to C, the arena -# keeps track of which bytes are used by what object to detect GC bugs; -# it internally uses raw_malloc_usage() to estimate the number of bytes -# it needs to reserve. - -class ArenaError(Exception): - pass - -class InaccessibleArenaError(ArenaError): - pass - -class Arena(object): - object_arena_location = {} # {container: (arena, offset)} - old_object_arena_location = weakref.WeakKeyDictionary() - - def __init__(self, nbytes, zero): - self.nbytes = nbytes - self.usagemap = array.array('c') - self.objectptrs = {} # {offset: ptr-to-container} - self.objectsizes = {} # {offset: size} - self.freed = False - self.reset(zero) - - def reset(self, zero, start=0, size=None): - self.check() - if size is None: - stop = self.nbytes - else: - stop = start + llmemory.raw_malloc_usage(size) - assert 0 <= start <= stop <= self.nbytes - for offset, ptr in self.objectptrs.items(): - size = self.objectsizes[offset] - if offset < start: # object is before the cleared area - assert offset + size <= start, "object overlaps cleared area" - elif offset + size > stop: # object is after the cleared area - assert offset >= stop, "object overlaps cleared area" - else: - obj = ptr._obj - del Arena.object_arena_location[obj] - del self.objectptrs[offset] - del self.objectsizes[offset] - obj._free() - if zero == Z_DONT_CLEAR: - initialbyte = "#" - elif zero in (Z_CLEAR_LARGE_AREA, Z_CLEAR_SMALL_AREA): - initialbyte = "0" - elif zero == Z_INACCESSIBLE: - prev = self.usagemap[start:stop].tostring() - assert '!' not in prev, ( - "Z_INACCESSIBLE must be called only on a " - "previously-accessible memory range") - initialbyte = "!" - elif zero == Z_ACCESSIBLE: - prev = self.usagemap[start:stop].tostring() - assert prev == '!'*len(prev), ( - "Z_ACCESSIBLE must be called only on a " - "previously-inaccessible memory range") - initialbyte = "0" - else: - raise ValueError("argument 'zero' got bogus value %r" % (zero,)) - self.usagemap[start:stop] = array.array('c', initialbyte*(stop-start)) - - def check(self): - if self.freed: - raise ArenaError("arena was already freed") - - def _getid(self): - address, length = self.usagemap.buffer_info() - return address - - def getaddr(self, offset): - if not (0 <= offset <= self.nbytes): - raise ArenaError("Address offset is outside the arena") - return fakearenaaddress(self, offset) - - def allocate_object(self, offset, size): - self.check() - bytes = llmemory.raw_malloc_usage(size) - if offset + bytes > self.nbytes: - raise ArenaError("object overflows beyond the end of the arena") - zero = True - for c in self.usagemap[offset:offset+bytes]: - if c == '0': - pass - elif c == '#': - zero = False - elif c == '!': - raise InaccessibleArenaError - else: - raise ArenaError("new object overlaps a previous object") - assert offset not in self.objectptrs - addr2 = size._raw_malloc([], zero=zero) - pattern = 'X' + 'x'*(bytes-1) - self.usagemap[offset:offset+bytes] = array.array('c', pattern) - self.setobject(addr2, offset, bytes) - # common case: 'size' starts with a GCHeaderOffset. In this case - # we can also remember that the real object starts after the header. - while isinstance(size, RoundedUpForAllocation): - size = size.basesize - if (isinstance(size, llmemory.CompositeOffset) and - isinstance(size.offsets[0], llmemory.GCHeaderOffset)): - objaddr = addr2 + size.offsets[0] - hdrbytes = llmemory.raw_malloc_usage(size.offsets[0]) - objoffset = offset + hdrbytes - self.setobject(objaddr, objoffset, bytes - hdrbytes) - return addr2 - - def setobject(self, objaddr, offset, bytes): - assert bytes > 0, ("llarena does not support GcStructs with no field" - " or empty arrays") - assert offset not in self.objectptrs - self.objectptrs[offset] = objaddr.ptr - self.objectsizes[offset] = bytes - container = objaddr.ptr._obj - Arena.object_arena_location[container] = self, offset - Arena.old_object_arena_location[container] = self, offset - -class fakearenaaddress(llmemory.fakeaddress): - - def __init__(self, arena, offset): - self.arena = arena - self.offset = offset - - def _getptr(self): - try: - return self.arena.objectptrs[self.offset] - except KeyError: - self.arena.check() - raise ArenaError("don't know yet what type of object " - "is at offset %d" % (self.offset,)) - ptr = property(_getptr) - - def __repr__(self): - return '' % (self.arena, self.offset) - - def __add__(self, other): - if isinstance(other, (int, long)): - position = self.offset + other - elif isinstance(other, llmemory.AddressOffset): - # this is really some Do What I Mean logic. There are two - # possible meanings: either we want to go past the current - # object in the arena, or we want to take the address inside - # the current object. Try to guess... - bytes = llmemory.raw_malloc_usage(other) - if (self.offset in self.arena.objectsizes and - bytes < self.arena.objectsizes[self.offset]): - # looks like we mean "inside the object" - return llmemory.fakeaddress.__add__(self, other) - position = self.offset + bytes - else: - return NotImplemented - return self.arena.getaddr(position) - - def __sub__(self, other): - if isinstance(other, llmemory.AddressOffset): - other = llmemory.raw_malloc_usage(other) - if isinstance(other, (int, long)): - return self.arena.getaddr(self.offset - other) - if isinstance(other, fakearenaaddress): - if self.arena is not other.arena: - raise ArenaError("The two addresses are from different arenas") - return self.offset - other.offset - return NotImplemented - - def __nonzero__(self): - return True - - def compare_with_fakeaddr(self, other): - other = other._fixup() - if not other: - return None, None - obj = other.ptr._obj - innerobject = False - while obj not in Arena.object_arena_location: - obj = obj._parentstructure() - if obj is None: - return None, None # not found in the arena - innerobject = True - arena, offset = Arena.object_arena_location[obj] - if innerobject: - # 'obj' is really inside the object allocated from the arena, - # so it's likely that its address "should be" a bit larger than - # what 'offset' says. - # We could estimate the correct offset but it's a bit messy; - # instead, let's check the answer doesn't depend on it - if self.arena is arena: - objectsize = arena.objectsizes[offset] - if offset < self.offset < offset+objectsize: - raise AssertionError( - "comparing an inner address with a " - "fakearenaaddress that points in the " - "middle of the same object") - offset += objectsize // 2 # arbitrary - return arena, offset - - def __eq__(self, other): - if isinstance(other, fakearenaaddress): - arena = other.arena - offset = other.offset - elif isinstance(other, llmemory.fakeaddress): - arena, offset = self.compare_with_fakeaddr(other) - else: - return llmemory.fakeaddress.__eq__(self, other) - return self.arena is arena and self.offset == offset - - def __lt__(self, other): - if isinstance(other, fakearenaaddress): - arena = other.arena - offset = other.offset - elif isinstance(other, llmemory.fakeaddress): - arena, offset = self.compare_with_fakeaddr(other) - if arena is None: - return False # self < other-not-in-any-arena => False - # (arbitrarily) - else: - raise TypeError("comparing a %s and a %s" % ( - self.__class__.__name__, other.__class__.__name__)) - if self.arena is arena: - return self.offset < offset - else: - return self.arena._getid() < arena._getid() - - def _cast_to_int(self): - return self.arena._getid() + self.offset - - -def _getfakearenaaddress(addr): - """Logic to handle test_replace_object_with_stub().""" - if isinstance(addr, fakearenaaddress): - return addr - else: - assert isinstance(addr, llmemory.fakeaddress) - assert addr, "NULL address" - # it must be possible to use the address of an already-freed - # arena object - obj = addr.ptr._getobj(check=False) - return _oldobj_to_address(obj) - -def _oldobj_to_address(obj): - obj = obj._normalizedcontainer(check=False) - try: - arena, offset = Arena.old_object_arena_location[obj] - except KeyError: - if obj._was_freed(): - msg = "taking address of %r, but it was freed" - else: - msg = "taking address of %r, but it is not in an arena" - raise RuntimeError(msg % (obj,)) - return arena.getaddr(offset) - -class RoundedUpForAllocation(llmemory.AddressOffset): - """A size that is rounded up in order to preserve alignment of objects - following it. For arenas containing heterogenous objects. - """ - def __init__(self, basesize, minsize): - assert isinstance(basesize, llmemory.AddressOffset) - assert isinstance(minsize, llmemory.AddressOffset) or minsize == 0 - self.basesize = basesize - self.minsize = minsize - - def __repr__(self): - return '< RoundedUpForAllocation %r %r >' % (self.basesize, - self.minsize) - - def known_nonneg(self): - return self.basesize.known_nonneg() - - def ref(self, ptr): - return self.basesize.ref(ptr) - - def _raw_malloc(self, rest, zero): - return self.basesize._raw_malloc(rest, zero=zero) - - def raw_memcopy(self, srcadr, dstadr): - self.basesize.raw_memcopy(srcadr, dstadr) - -# ____________________________________________________________ +# some of our framework GCs. # -# Public interface: arena_malloc(), arena_free(), arena_reset() -# are similar to raw_malloc(), raw_free() and raw_memclear(), but -# work with fakearenaaddresses on which arbitrary arithmetic is -# possible even on top of the llinterpreter. - -# arena_malloc() and arena_reset() take as argument one of the -# following values: - -Z_DONT_CLEAR = 0 # it's ok to keep random bytes in the area -Z_CLEAR_LARGE_AREA = 1 # clear, optimized for a large area of memory -Z_CLEAR_SMALL_AREA = 2 # clear, optimized for a small or medium area of mem -Z_INACCESSIBLE = 3 # make the memory inaccessible (not reserved) -Z_ACCESSIBLE = 4 # make the memory accessible again - -# Note that Z_INACCESSIBLE and Z_ACCESSIBLE are restricted to whole -# pages, and you must not try to make inaccessible pages that are already -# inaccessible, nor make accessible pages that are already accessible. -# When they go through the Z_INACCESSIBLE-Z_ACCESSIBLE trip, pages are -# cleared. - -def arena_malloc(nbytes, zero): - """Allocate and return a new arena, optionally zero-initialized. - The value of 'zero' is one the Z_xxx values. - """ - return Arena(nbytes, zero).getaddr(0) - -def arena_free(arena_addr, nbytes): - """Release an arena.""" - assert isinstance(arena_addr, fakearenaaddress) - assert arena_addr.offset == 0 - assert nbytes == arena_addr.arena.nbytes - arena_addr.arena.reset(Z_DONT_CLEAR) - arena_addr.arena.freed = True - -def arena_reset(arena_addr, size, zero): - """Free all objects in the arena, which can then be reused. - This can also be used on a subrange of the arena. - The value of 'zero' is one of the Z_xxx values. - """ - arena_addr = _getfakearenaaddress(arena_addr) - arena_addr.arena.reset(zero, arena_addr.offset, size) - -def arena_reserve(addr, size, check_alignment=True): - """Mark some bytes in an arena as reserved, and returns addr. - For debugging this can check that reserved ranges of bytes don't - overlap. The size must be symbolic; in non-translated version - this is used to know what type of lltype object to allocate.""" - from pypy.rpython.memory.lltypelayout import memory_alignment - addr = _getfakearenaaddress(addr) - if check_alignment and (addr.offset & (memory_alignment-1)) != 0: - raise ArenaError("object at offset %d would not be correctly aligned" - % (addr.offset,)) - addr.arena.allocate_object(addr.offset, size) - -def round_up_for_allocation(size, minsize=0): - """Round up the size in order to preserve alignment of objects - following an object. For arenas containing heterogenous objects. - If minsize is specified, it gives a minimum on the resulting size.""" - return _round_up_for_allocation(size, minsize) - -def _round_up_for_allocation(size, minsize): # internal - return RoundedUpForAllocation(size, minsize) - -def arena_new_view(ptr): - """This is a no-op when translated, returns fresh view - on previous arena when run on top of llinterp. - """ - return Arena(ptr.arena.nbytes, False).getaddr(0) +# See further comments in llarena_llinterp.py. + +from pypy.rpython.lltypesystem.llarena_llinterp import * # ____________________________________________________________ # @@ -358,231 +13,36 @@ # We can tweak these implementations to be more suited to very large # chunks of memory. -import os, sys -from pypy.rpython.lltypesystem import rffi, lltype +import os from pypy.rpython.extfunc import register_external -from pypy.rlib.objectmodel import CDefinedIntSymbolic - -# ---------- getting the page size ---------- +from pypy.rpython.lltypesystem import rffi if os.name == 'posix': - posix_getpagesize = rffi.llexternal('getpagesize', [], rffi.INT, - sandboxsafe=True, _nowrapper=True) - class PosixPageSize: - def __init__(self): - self.pagesize = 0 - _freeze_ = __init__ - posix_pagesize = PosixPageSize() - - def getpagesize(): - pagesize = posix_pagesize.pagesize - if pagesize == 0: - pagesize = rffi.cast(lltype.Signed, posix_getpagesize()) - posix_pagesize.pagesize = pagesize - return pagesize - -else: - # XXX a random value, but nothing really depends on it - def getpagesize(): - return 4096 - -# ---------- clearing a large range of memory ---------- - -if sys.platform == 'linux2': - # This only works with linux's madvise(), which is really not a memory - # usage hint but a real command. It guarantees that after MADV_DONTNEED - # the pages are cleared again. - from pypy.rpython.tool import rffi_platform - MADV_DONTNEED = rffi_platform.getconstantinteger('MADV_DONTNEED', - '#include ') - linux_madvise = rffi.llexternal('madvise', - [llmemory.Address, rffi.SIZE_T, rffi.INT], - rffi.INT, - sandboxsafe=True, _nowrapper=True) - - def clear_large_memory_chunk(baseaddr, size): - pagesize = getpagesize() - if size > 2 * pagesize: - lowbits = rffi.cast(lltype.Signed, baseaddr) & (pagesize - 1) - if lowbits: # clear the initial misaligned part, if any - partpage = pagesize - lowbits - llmemory.raw_memclear(baseaddr, partpage) - baseaddr += partpage - size -= partpage - length = size & -pagesize - madv_length = rffi.cast(rffi.SIZE_T, length) - madv_flags = rffi.cast(rffi.INT, MADV_DONTNEED) - err = linux_madvise(baseaddr, madv_length, madv_flags) - if rffi.cast(lltype.Signed, err) == 0: - baseaddr += length # madvise() worked - size -= length - if size > 0: # clear the final misaligned part, if any - llmemory.raw_memclear(baseaddr, size) - -elif os.name == 'posix': - READ_MAX = (sys.maxint//4) + 1 # upper bound on reads to avoid surprises - raw_os_open = rffi.llexternal('open', - [rffi.CCHARP, rffi.INT, rffi.MODE_T], - rffi.INT, - sandboxsafe=True, _nowrapper=True) - raw_os_read = rffi.llexternal('read', - [rffi.INT, llmemory.Address, rffi.SIZE_T], - rffi.SIZE_T, - sandboxsafe=True, _nowrapper=True) - raw_os_close = rffi.llexternal('close', - [rffi.INT], - rffi.INT, - sandboxsafe=True, _nowrapper=True) - _dev_zero = rffi.str2charp('/dev/zero') # prebuilt - - def clear_large_memory_chunk(baseaddr, size): - # on some Unixy platforms, reading from /dev/zero is the fastest way - # to clear arenas, because the kernel knows that it doesn't - # need to even allocate the pages before they are used. - - # NB.: careful, don't do anything that could malloc here! - # this code is called during GC initialization. - fd = raw_os_open(_dev_zero, - rffi.cast(rffi.INT, os.O_RDONLY), - rffi.cast(rffi.MODE_T, 0644)) - if rffi.cast(lltype.Signed, fd) != -1: - while size > 0: - size1 = rffi.cast(rffi.SIZE_T, min(READ_MAX, size)) - count = raw_os_read(fd, baseaddr, size1) - count = rffi.cast(lltype.Signed, count) - if count <= 0: - break - size -= count - baseaddr += count - raw_os_close(fd) - - if size > 0: # reading from /dev/zero failed, fallback - llmemory.raw_memclear(baseaddr, size) - + from pypy.rpython.lltypesystem import llarena_posix as llarena_impl +#elif os.name == 'win32': +# ... else: - # XXX any better implementation on Windows? - # Should use VirtualAlloc() to reserve the range of pages, - # and commit some pages gradually with support from the GC. - # Or it might be enough to decommit the pages and recommit - # them immediately. - clear_large_memory_chunk = llmemory.raw_memclear - -# ---------- platform-specific version of llimpl_arena_* ---------- + from pypy.rpython.lltypesystem import llarena_generic as llarena_impl -if os.name == 'posix': - # llimpl_arena_*() functions based on mmap - from pypy.rpython.tool import rffi_platform - from pypy.translator.tool.cbuild import ExternalCompilationInfo - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes=['sys/mman.h']) - off_t = rffi_platform.SimpleType('off_t') - PROT_NONE = rffi_platform.ConstantInteger('PROT_NONE') - PROT_READ = rffi_platform.ConstantInteger('PROT_READ') - PROT_WRITE = rffi_platform.ConstantInteger('PROT_WRITE') - MAP_PRIVATE = rffi_platform.ConstantInteger('MAP_PRIVATE') - MAP_ANON = rffi_platform.DefinedConstantInteger('MAP_ANON') - MAP_ANONYMOUS = rffi_platform.DefinedConstantInteger('MAP_ANONYMOUS') - MAP_NORESERVE = rffi_platform.DefinedConstantInteger('MAP_NORESERVE') - globals().update(rffi_platform.configure(CConfig)) - if MAP_ANONYMOUS is None: - MAP_ANONYMOUS = MAP_ANON - assert MAP_ANONYMOUS is not None - del MAP_ANON - - posix_mmap = rffi.llexternal('mmap', - [llmemory.Address, rffi.SIZE_T, rffi.INT, - rffi.INT, rffi.INT, off_t], - llmemory.Address, - sandboxsafe=True, _nowrapper=True) - posix_munmap = rffi.llexternal('munmap', - [llmemory.Address, rffi.SIZE_T], - rffi.INT, - sandboxsafe=True, _nowrapper=True) - posix_mprotect = rffi.llexternal('mprotect', - [llmemory.Address, rffi.SIZE_T, - rffi.INT], - rffi.INT, - sandboxsafe=True, _nowrapper=True) - - class MMapMemoryError(Exception): - pass - - def llimpl_arena_malloc(nbytes, zero): - flags = MAP_PRIVATE | MAP_ANONYMOUS - if zero == Z_INACCESSIBLE: - prot = PROT_NONE - if MAP_NORESERVE is not None: - flags |= MAP_NORESERVE - else: - prot = PROT_READ | PROT_WRITE - result = posix_mmap(llmemory.NULL, - rffi.cast(rffi.SIZE_T, nbytes), - rffi.cast(rffi.INT, prot), - rffi.cast(rffi.INT, flags), - rffi.cast(rffi.INT, -1), - rffi.cast(off_t, 0)) - if rffi.cast(lltype.Signed, result) == -1: - raise MMapMemoryError - return result - - def llimpl_arena_free(arena_addr, nbytes): - result = posix_munmap(arena_addr, rffi.cast(rffi.SIZE_T, nbytes)) - if rffi.cast(lltype.Signed, result) == -1: - raise MMapMemoryError - - def _arena_protect(arena_addr, size, flags): - res = posix_mprotect(arena_addr, - rffi.cast(rffi.SIZE_T, size), - rffi.cast(rffi.INT, flags)) - if rffi.cast(lltype.Signed, res) != 0: - raise MMapMemoryError - - def llimpl_arena_reset(arena_addr, size, zero): - if zero == Z_CLEAR_LARGE_AREA: - clear_large_memory_chunk(arena_addr, size) - elif zero == Z_CLEAR_SMALL_AREA: - llmemory.raw_memclear(arena_addr, size) - elif zero == Z_ACCESSIBLE: - _arena_protect(arena_addr, size, PROT_READ | PROT_WRITE) - elif zero == Z_INACCESSIBLE: - clear_large_memory_chunk(arena_addr, size) - _arena_protect(arena_addr, size, PROT_NONE) - -else: - # llimpl_arena_*() functions based on raw_malloc - def llimpl_arena_malloc(nbytes, zero): - addr = llmemory.raw_malloc(nbytes) - if zero in (Z_CLEAR_LARGE_AREA, Z_CLEAR_SMALL_AREA) and bool(addr): - clear_large_memory_chunk(addr, nbytes) - return addr - - def llimpl_arena_free(arena_addr, nbytes): - llmemory.raw_free(arena_addr) - - def llimpl_arena_reset(arena_addr, size, zero): - if zero in (Z_CLEAR_LARGE_AREA, Z_INACCESSIBLE): - clear_large_memory_chunk(arena_addr, size) - elif zero == Z_CLEAR_SMALL_AREA: - llmemory.raw_memclear(arena_addr, size) -# ---------- +implements_inaccessible = llarena_impl.implements_inaccessible +getpagesize = llarena_impl.getpagesize register_external(arena_malloc, [int, int], llmemory.Address, 'll_arena.arena_malloc', - llimpl=llimpl_arena_malloc, + llimpl=llarena_impl.llimpl_arena_malloc, llfakeimpl=arena_malloc, sandboxsafe=True) register_external(arena_free, [llmemory.Address, int], None, 'll_arena.arena_free', - llimpl=llimpl_arena_free, + llimpl=llarena_impl.llimpl_arena_free, llfakeimpl=arena_free, sandboxsafe=True) register_external(arena_reset, [llmemory.Address, int, int], None, 'll_arena.arena_reset', - llimpl=llimpl_arena_reset, + llimpl=llarena_impl.llimpl_arena_reset, llfakeimpl=arena_reset, sandboxsafe=True) @@ -599,7 +59,7 @@ lltype.Signed, sandboxsafe=True, _nowrapper=True) -register_external(_round_up_for_allocation, [int, int], int, +register_external(internal_round_up_for_allocation, [int, int], int, 'll_arena.round_up_for_allocation', llimpl=llimpl_round_up_for_allocation, llfakeimpl=round_up_for_allocation, Added: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_generic.py ============================================================================== --- (empty file) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_generic.py Mon Oct 19 17:16:17 2009 @@ -0,0 +1,24 @@ +from pypy.rpython.lltypesystem import llmemory +from pypy.rpython.lltypesystem.llarena_llinterp import Z_CLEAR_LARGE_AREA +from pypy.rpython.lltypesystem.llarena_llinterp import Z_CLEAR_SMALL_AREA +from pypy.rpython.lltypesystem.llarena_llinterp import Z_INACCESSIBLE + +implements_inaccessible = False + +# a random value, but nothing really depends on it +def getpagesize(): + return 4096 + +# llimpl_arena_*() functions based on raw_malloc +def llimpl_arena_malloc(nbytes, zero): + addr = llmemory.raw_malloc(nbytes) + if bool(addr): + llimpl_arena_reset(addr, nbytes, zero) + return addr + +def llimpl_arena_free(arena_addr, nbytes): + llmemory.raw_free(arena_addr) + +def llimpl_arena_reset(arena_addr, size, zero): + if zero in (Z_CLEAR_LARGE_AREA, Z_CLEAR_SMALL_AREA, Z_INACCESSIBLE): + llmemory.raw_memclear(arena_addr, size) Copied: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_llinterp.py (from r68635, pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena.py) ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_llinterp.py Mon Oct 19 17:16:17 2009 @@ -297,9 +297,10 @@ Z_INACCESSIBLE = 3 # make the memory inaccessible (not reserved) Z_ACCESSIBLE = 4 # make the memory accessible again -# Note that Z_INACCESSIBLE and Z_ACCESSIBLE are restricted to whole -# pages, and you must not try to make inaccessible pages that are already -# inaccessible, nor make accessible pages that are already accessible. +# Note that Z_CLEAR_LARGE_AREA, Z_INACCESSIBLE and Z_ACCESSIBLE are +# restricted to whole pages (at least one), and you must not try to make +# inaccessible pages that are already inaccessible, nor make accessible +# pages that are already accessible. # When they go through the Z_INACCESSIBLE-Z_ACCESSIBLE trip, pages are # cleared. @@ -341,9 +342,9 @@ """Round up the size in order to preserve alignment of objects following an object. For arenas containing heterogenous objects. If minsize is specified, it gives a minimum on the resulting size.""" - return _round_up_for_allocation(size, minsize) + return internal_round_up_for_allocation(size, minsize) -def _round_up_for_allocation(size, minsize): # internal +def internal_round_up_for_allocation(size, minsize): # internal return RoundedUpForAllocation(size, minsize) def arena_new_view(ptr): @@ -351,262 +352,3 @@ on previous arena when run on top of llinterp. """ return Arena(ptr.arena.nbytes, False).getaddr(0) - -# ____________________________________________________________ -# -# Translation support: the functions above turn into the code below. -# We can tweak these implementations to be more suited to very large -# chunks of memory. - -import os, sys -from pypy.rpython.lltypesystem import rffi, lltype -from pypy.rpython.extfunc import register_external -from pypy.rlib.objectmodel import CDefinedIntSymbolic - -# ---------- getting the page size ---------- - -if os.name == 'posix': - posix_getpagesize = rffi.llexternal('getpagesize', [], rffi.INT, - sandboxsafe=True, _nowrapper=True) - class PosixPageSize: - def __init__(self): - self.pagesize = 0 - _freeze_ = __init__ - posix_pagesize = PosixPageSize() - - def getpagesize(): - pagesize = posix_pagesize.pagesize - if pagesize == 0: - pagesize = rffi.cast(lltype.Signed, posix_getpagesize()) - posix_pagesize.pagesize = pagesize - return pagesize - -else: - # XXX a random value, but nothing really depends on it - def getpagesize(): - return 4096 - -# ---------- clearing a large range of memory ---------- - -if sys.platform == 'linux2': - # This only works with linux's madvise(), which is really not a memory - # usage hint but a real command. It guarantees that after MADV_DONTNEED - # the pages are cleared again. - from pypy.rpython.tool import rffi_platform - MADV_DONTNEED = rffi_platform.getconstantinteger('MADV_DONTNEED', - '#include ') - linux_madvise = rffi.llexternal('madvise', - [llmemory.Address, rffi.SIZE_T, rffi.INT], - rffi.INT, - sandboxsafe=True, _nowrapper=True) - - def clear_large_memory_chunk(baseaddr, size): - pagesize = getpagesize() - if size > 2 * pagesize: - lowbits = rffi.cast(lltype.Signed, baseaddr) & (pagesize - 1) - if lowbits: # clear the initial misaligned part, if any - partpage = pagesize - lowbits - llmemory.raw_memclear(baseaddr, partpage) - baseaddr += partpage - size -= partpage - length = size & -pagesize - madv_length = rffi.cast(rffi.SIZE_T, length) - madv_flags = rffi.cast(rffi.INT, MADV_DONTNEED) - err = linux_madvise(baseaddr, madv_length, madv_flags) - if rffi.cast(lltype.Signed, err) == 0: - baseaddr += length # madvise() worked - size -= length - if size > 0: # clear the final misaligned part, if any - llmemory.raw_memclear(baseaddr, size) - -elif os.name == 'posix': - READ_MAX = (sys.maxint//4) + 1 # upper bound on reads to avoid surprises - raw_os_open = rffi.llexternal('open', - [rffi.CCHARP, rffi.INT, rffi.MODE_T], - rffi.INT, - sandboxsafe=True, _nowrapper=True) - raw_os_read = rffi.llexternal('read', - [rffi.INT, llmemory.Address, rffi.SIZE_T], - rffi.SIZE_T, - sandboxsafe=True, _nowrapper=True) - raw_os_close = rffi.llexternal('close', - [rffi.INT], - rffi.INT, - sandboxsafe=True, _nowrapper=True) - _dev_zero = rffi.str2charp('/dev/zero') # prebuilt - - def clear_large_memory_chunk(baseaddr, size): - # on some Unixy platforms, reading from /dev/zero is the fastest way - # to clear arenas, because the kernel knows that it doesn't - # need to even allocate the pages before they are used. - - # NB.: careful, don't do anything that could malloc here! - # this code is called during GC initialization. - fd = raw_os_open(_dev_zero, - rffi.cast(rffi.INT, os.O_RDONLY), - rffi.cast(rffi.MODE_T, 0644)) - if rffi.cast(lltype.Signed, fd) != -1: - while size > 0: - size1 = rffi.cast(rffi.SIZE_T, min(READ_MAX, size)) - count = raw_os_read(fd, baseaddr, size1) - count = rffi.cast(lltype.Signed, count) - if count <= 0: - break - size -= count - baseaddr += count - raw_os_close(fd) - - if size > 0: # reading from /dev/zero failed, fallback - llmemory.raw_memclear(baseaddr, size) - -else: - # XXX any better implementation on Windows? - # Should use VirtualAlloc() to reserve the range of pages, - # and commit some pages gradually with support from the GC. - # Or it might be enough to decommit the pages and recommit - # them immediately. - clear_large_memory_chunk = llmemory.raw_memclear - -# ---------- platform-specific version of llimpl_arena_* ---------- - -if os.name == 'posix': - # llimpl_arena_*() functions based on mmap - from pypy.rpython.tool import rffi_platform - from pypy.translator.tool.cbuild import ExternalCompilationInfo - class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes=['sys/mman.h']) - off_t = rffi_platform.SimpleType('off_t') - PROT_NONE = rffi_platform.ConstantInteger('PROT_NONE') - PROT_READ = rffi_platform.ConstantInteger('PROT_READ') - PROT_WRITE = rffi_platform.ConstantInteger('PROT_WRITE') - MAP_PRIVATE = rffi_platform.ConstantInteger('MAP_PRIVATE') - MAP_ANON = rffi_platform.DefinedConstantInteger('MAP_ANON') - MAP_ANONYMOUS = rffi_platform.DefinedConstantInteger('MAP_ANONYMOUS') - MAP_NORESERVE = rffi_platform.DefinedConstantInteger('MAP_NORESERVE') - globals().update(rffi_platform.configure(CConfig)) - if MAP_ANONYMOUS is None: - MAP_ANONYMOUS = MAP_ANON - assert MAP_ANONYMOUS is not None - del MAP_ANON - - posix_mmap = rffi.llexternal('mmap', - [llmemory.Address, rffi.SIZE_T, rffi.INT, - rffi.INT, rffi.INT, off_t], - llmemory.Address, - sandboxsafe=True, _nowrapper=True) - posix_munmap = rffi.llexternal('munmap', - [llmemory.Address, rffi.SIZE_T], - rffi.INT, - sandboxsafe=True, _nowrapper=True) - posix_mprotect = rffi.llexternal('mprotect', - [llmemory.Address, rffi.SIZE_T, - rffi.INT], - rffi.INT, - sandboxsafe=True, _nowrapper=True) - - class MMapMemoryError(Exception): - pass - - def llimpl_arena_malloc(nbytes, zero): - flags = MAP_PRIVATE | MAP_ANONYMOUS - if zero == Z_INACCESSIBLE: - prot = PROT_NONE - if MAP_NORESERVE is not None: - flags |= MAP_NORESERVE - else: - prot = PROT_READ | PROT_WRITE - result = posix_mmap(llmemory.NULL, - rffi.cast(rffi.SIZE_T, nbytes), - rffi.cast(rffi.INT, prot), - rffi.cast(rffi.INT, flags), - rffi.cast(rffi.INT, -1), - rffi.cast(off_t, 0)) - if rffi.cast(lltype.Signed, result) == -1: - raise MMapMemoryError - return result - - def llimpl_arena_free(arena_addr, nbytes): - result = posix_munmap(arena_addr, rffi.cast(rffi.SIZE_T, nbytes)) - if rffi.cast(lltype.Signed, result) == -1: - raise MMapMemoryError - - def _arena_protect(arena_addr, size, flags): - res = posix_mprotect(arena_addr, - rffi.cast(rffi.SIZE_T, size), - rffi.cast(rffi.INT, flags)) - if rffi.cast(lltype.Signed, res) != 0: - raise MMapMemoryError - - def llimpl_arena_reset(arena_addr, size, zero): - if zero == Z_CLEAR_LARGE_AREA: - clear_large_memory_chunk(arena_addr, size) - elif zero == Z_CLEAR_SMALL_AREA: - llmemory.raw_memclear(arena_addr, size) - elif zero == Z_ACCESSIBLE: - _arena_protect(arena_addr, size, PROT_READ | PROT_WRITE) - elif zero == Z_INACCESSIBLE: - clear_large_memory_chunk(arena_addr, size) - _arena_protect(arena_addr, size, PROT_NONE) - -else: - # llimpl_arena_*() functions based on raw_malloc - def llimpl_arena_malloc(nbytes, zero): - addr = llmemory.raw_malloc(nbytes) - if zero in (Z_CLEAR_LARGE_AREA, Z_CLEAR_SMALL_AREA) and bool(addr): - clear_large_memory_chunk(addr, nbytes) - return addr - - def llimpl_arena_free(arena_addr, nbytes): - llmemory.raw_free(arena_addr) - - def llimpl_arena_reset(arena_addr, size, zero): - if zero in (Z_CLEAR_LARGE_AREA, Z_INACCESSIBLE): - clear_large_memory_chunk(arena_addr, size) - elif zero == Z_CLEAR_SMALL_AREA: - llmemory.raw_memclear(arena_addr, size) - -# ---------- - -register_external(arena_malloc, [int, int], llmemory.Address, - 'll_arena.arena_malloc', - llimpl=llimpl_arena_malloc, - llfakeimpl=arena_malloc, - sandboxsafe=True) - -register_external(arena_free, [llmemory.Address, int], None, - 'll_arena.arena_free', - llimpl=llimpl_arena_free, - llfakeimpl=arena_free, - sandboxsafe=True) - -register_external(arena_reset, [llmemory.Address, int, int], None, - 'll_arena.arena_reset', - llimpl=llimpl_arena_reset, - llfakeimpl=arena_reset, - sandboxsafe=True) - -def llimpl_arena_reserve(addr, size): - pass -register_external(arena_reserve, [llmemory.Address, int], None, - 'll_arena.arena_reserve', - llimpl=llimpl_arena_reserve, - llfakeimpl=arena_reserve, - sandboxsafe=True) - -llimpl_round_up_for_allocation = rffi.llexternal('ROUND_UP_FOR_ALLOCATION', - [lltype.Signed, lltype.Signed], - lltype.Signed, - sandboxsafe=True, - _nowrapper=True) -register_external(_round_up_for_allocation, [int, int], int, - 'll_arena.round_up_for_allocation', - llimpl=llimpl_round_up_for_allocation, - llfakeimpl=round_up_for_allocation, - sandboxsafe=True) - -def llimpl_arena_new_view(addr): - return addr -register_external(arena_new_view, [llmemory.Address], llmemory.Address, - 'll_arena.arena_new_view', llimpl=llimpl_arena_new_view, - llfakeimpl=arena_new_view, sandboxsafe=True) Added: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_posix.py ============================================================================== --- (empty file) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_posix.py Mon Oct 19 17:16:17 2009 @@ -0,0 +1,167 @@ +import os, sys +from pypy.rpython.lltypesystem import rffi, lltype, llmemory +from pypy.rpython.lltypesystem.llarena_llinterp import Z_CLEAR_LARGE_AREA +from pypy.rpython.lltypesystem.llarena_llinterp import Z_CLEAR_SMALL_AREA +from pypy.rpython.lltypesystem.llarena_llinterp import Z_INACCESSIBLE +from pypy.rpython.lltypesystem.llarena_llinterp import Z_ACCESSIBLE + +implements_inaccessible = True + +# ____________________________________________________________ + +posix_getpagesize = rffi.llexternal('getpagesize', [], rffi.INT, + sandboxsafe=True, _nowrapper=True) +class PosixPageSize: + def __init__(self): + self.pagesize = 0 + _freeze_ = __init__ +posix_pagesize = PosixPageSize() + +def getpagesize(): + pagesize = posix_pagesize.pagesize + if pagesize == 0: + pagesize = rffi.cast(lltype.Signed, posix_getpagesize()) + posix_pagesize.pagesize = pagesize + return pagesize + +# ____________________________________________________________ + +if sys.platform == 'linux2': + # This only works with linux's madvise(), which is really not a memory + # usage hint but a real command. It guarantees that after MADV_DONTNEED + # the pages are cleared again. + from pypy.rpython.tool import rffi_platform + MADV_DONTNEED = rffi_platform.getconstantinteger('MADV_DONTNEED', + '#include ') + linux_madvise = rffi.llexternal('madvise', + [llmemory.Address, rffi.SIZE_T, rffi.INT], + rffi.INT, + sandboxsafe=True, _nowrapper=True) + + def clear_large_memory_chunk(baseaddr, size): + madv_length = rffi.cast(rffi.SIZE_T, size) + madv_flags = rffi.cast(rffi.INT, MADV_DONTNEED) + err = linux_madvise(baseaddr, madv_length, madv_flags) + if rffi.cast(lltype.Signed, err) != 0: # did not work! + llmemory.raw_memclear(baseaddr, size) # clear manually... + +else: + READ_MAX = (sys.maxint//4) + 1 # upper bound on reads to avoid surprises + raw_os_open = rffi.llexternal('open', + [rffi.CCHARP, rffi.INT, rffi.MODE_T], + rffi.INT, + sandboxsafe=True, _nowrapper=True) + raw_os_read = rffi.llexternal('read', + [rffi.INT, llmemory.Address, rffi.SIZE_T], + rffi.SIZE_T, + sandboxsafe=True, _nowrapper=True) + raw_os_close = rffi.llexternal('close', + [rffi.INT], + rffi.INT, + sandboxsafe=True, _nowrapper=True) + _dev_zero = rffi.str2charp('/dev/zero') # prebuilt + + def clear_large_memory_chunk(baseaddr, size): + # on some Unixy platforms, reading from /dev/zero is the fastest way + # to clear arenas, because the kernel knows that it doesn't + # need to even allocate the pages before they are used. + + # NB.: careful, don't do anything that could malloc here! + # this code is called during GC initialization. + fd = raw_os_open(_dev_zero, + rffi.cast(rffi.INT, os.O_RDONLY), + rffi.cast(rffi.MODE_T, 0644)) + if rffi.cast(lltype.Signed, fd) != -1: + while size > 0: + size1 = rffi.cast(rffi.SIZE_T, min(READ_MAX, size)) + count = raw_os_read(fd, baseaddr, size1) + count = rffi.cast(lltype.Signed, count) + if count <= 0: + break + size -= count + baseaddr += count + raw_os_close(fd) + + if size > 0: # reading from /dev/zero failed, fallback + llmemory.raw_memclear(baseaddr, size) + +# ____________________________________________________________ + +# llimpl_arena_*() functions based on mmap +from pypy.rpython.tool import rffi_platform +from pypy.translator.tool.cbuild import ExternalCompilationInfo +class CConfig: + _compilation_info_ = ExternalCompilationInfo( + includes=['sys/mman.h']) + off_t = rffi_platform.SimpleType('off_t') + PROT_NONE = rffi_platform.ConstantInteger('PROT_NONE') + PROT_READ = rffi_platform.ConstantInteger('PROT_READ') + PROT_WRITE = rffi_platform.ConstantInteger('PROT_WRITE') + MAP_PRIVATE = rffi_platform.ConstantInteger('MAP_PRIVATE') + MAP_ANON = rffi_platform.DefinedConstantInteger('MAP_ANON') + MAP_ANONYMOUS = rffi_platform.DefinedConstantInteger('MAP_ANONYMOUS') + MAP_NORESERVE = rffi_platform.DefinedConstantInteger('MAP_NORESERVE') +globals().update(rffi_platform.configure(CConfig)) +if MAP_ANONYMOUS is None: + MAP_ANONYMOUS = MAP_ANON + assert MAP_ANONYMOUS is not None +del MAP_ANON + +posix_mmap = rffi.llexternal('mmap', + [llmemory.Address, rffi.SIZE_T, rffi.INT, + rffi.INT, rffi.INT, off_t], + llmemory.Address, + sandboxsafe=True, _nowrapper=True) +posix_munmap = rffi.llexternal('munmap', + [llmemory.Address, rffi.SIZE_T], + rffi.INT, + sandboxsafe=True, _nowrapper=True) +posix_mprotect = rffi.llexternal('mprotect', + [llmemory.Address, rffi.SIZE_T, + rffi.INT], + rffi.INT, + sandboxsafe=True, _nowrapper=True) + +class MMapMemoryError(Exception): + pass + +def llimpl_arena_malloc(nbytes, zero): + flags = MAP_PRIVATE | MAP_ANONYMOUS + if zero == Z_INACCESSIBLE: + prot = PROT_NONE + if MAP_NORESERVE is not None: + flags |= MAP_NORESERVE + else: + prot = PROT_READ | PROT_WRITE + result = posix_mmap(llmemory.NULL, + rffi.cast(rffi.SIZE_T, nbytes), + rffi.cast(rffi.INT, prot), + rffi.cast(rffi.INT, flags), + rffi.cast(rffi.INT, -1), + rffi.cast(off_t, 0)) + if rffi.cast(lltype.Signed, result) == -1: + raise MMapMemoryError + return result + +def llimpl_arena_free(arena_addr, nbytes): + result = posix_munmap(arena_addr, rffi.cast(rffi.SIZE_T, nbytes)) + if rffi.cast(lltype.Signed, result) == -1: + raise MMapMemoryError + +def _arena_protect(arena_addr, size, flags): + res = posix_mprotect(arena_addr, + rffi.cast(rffi.SIZE_T, size), + rffi.cast(rffi.INT, flags)) + if rffi.cast(lltype.Signed, res) != 0: + raise MMapMemoryError + +def llimpl_arena_reset(arena_addr, size, zero): + if zero == Z_CLEAR_LARGE_AREA: + clear_large_memory_chunk(arena_addr, size) + elif zero == Z_CLEAR_SMALL_AREA: + llmemory.raw_memclear(arena_addr, size) + elif zero == Z_ACCESSIBLE: + _arena_protect(arena_addr, size, PROT_READ | PROT_WRITE) + elif zero == Z_INACCESSIBLE: + clear_large_memory_chunk(arena_addr, size) + _arena_protect(arena_addr, size, PROT_NONE) Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/llmemory.py Mon Oct 19 17:16:17 2009 @@ -474,8 +474,8 @@ def _fixup(self): if self.ptr is not None and self.ptr._was_freed(): # hack to support llarena.test_replace_object_with_stub() - from pypy.rpython.lltypesystem import llarena - return llarena._getfakearenaaddress(self) + from pypy.rpython.lltypesystem import llarena_llinterp + return llarena_llinterp._getfakearenaaddress(self) else: return self Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/lltype.py Mon Oct 19 17:16:17 2009 @@ -1216,9 +1216,10 @@ if isinstance(self._T, FuncType): return llmemory.fakeaddress(self) elif self._was_freed(): - # hack to support llarena.test_replace_object_with_stub() - from pypy.rpython.lltypesystem import llarena - return llarena._oldobj_to_address(self._getobj(check=False)) + # hack to support test_llarena.test_replace_object_with_stub() + from pypy.rpython.lltypesystem import llarena_llinterp + return llarena_llinterp._oldobj_to_address( + self._getobj(check=False)) elif isinstance(self._obj, _subarray): return llmemory.fakeaddress(self) ## # return an address built as an offset in the whole array Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/test/test_llarena.py Mon Oct 19 17:16:17 2009 @@ -11,6 +11,7 @@ from pypy.rpython.lltypesystem.llarena import Z_INACCESSIBLE from pypy.rpython.lltypesystem.llarena import Z_ACCESSIBLE from pypy.rpython.lltypesystem.llarena import InaccessibleArenaError +from pypy.rpython.lltypesystem.llarena import implements_inaccessible def test_arena(): S = lltype.Struct('S', ('x',lltype.Signed)) @@ -345,9 +346,10 @@ def run(n): return py.process.cmdexec('%s %d' % (path, n)) run(0) - py.test.raises(py.error.Error, run, -1) # segfault - py.test.raises(py.error.Error, run, -2) # segfault - py.test.raises(py.error.Error, run, -4) # segfault + if implements_inaccessible: + py.test.raises(py.error.Error, run, -1) # segfault + py.test.raises(py.error.Error, run, -2) # segfault + py.test.raises(py.error.Error, run, -4) # segfault res = run(-3) # good enough, although it should ideally crash: assert '44\n' in res From arigo at codespeak.net Mon Oct 19 17:24:25 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 19 Oct 2009 17:24:25 +0200 (CEST) Subject: [pypy-svn] r68640 - pypy/branch/gc-arena/pypy/rpython/memory/gc Message-ID: <20091019152425.AA717168005@codespeak.net> Author: arigo Date: Mon Oct 19 17:24:24 2009 New Revision: 68640 Modified: pypy/branch/gc-arena/pypy/rpython/memory/gc/generation.py pypy/branch/gc-arena/pypy/rpython/memory/gc/semispace.py Log: Use the new interface. Modified: pypy/branch/gc-arena/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/gc-arena/pypy/rpython/memory/gc/generation.py Mon Oct 19 17:24:24 2009 @@ -337,7 +337,8 @@ if self.young_objects_with_id.length() > 0: self.update_young_objects_with_id() # mark the nursery as free and fill it with zeroes again - llarena.arena_reset(self.nursery, self.nursery_size, 2) + llarena.arena_reset(self.nursery, self.nursery_size, + llarena.Z_CLEAR_SMALL_AREA) if self.config.gcconfig.debugprint: llop.debug_print(lltype.Void, "survived (fraction of the size):", Modified: pypy/branch/gc-arena/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-arena/pypy/rpython/memory/gc/semispace.py Mon Oct 19 17:24:24 2009 @@ -59,10 +59,12 @@ def setup(self): if self.config.gcconfig.debugprint: self.program_start_time = time.time() - self.tospace = llarena.arena_malloc(self.space_size, True) + self.tospace = llarena.arena_malloc(self.space_size, + llarena.Z_CLEAR_LARGE_AREA) ll_assert(bool(self.tospace), "couldn't allocate tospace") self.top_of_space = self.tospace + self.space_size - self.fromspace = llarena.arena_malloc(self.space_size, True) + self.fromspace = llarena.arena_malloc(self.space_size, + llarena.Z_CLEAR_LARGE_AREA) ll_assert(bool(self.fromspace), "couldn't allocate fromspace") self.free = self.tospace MovingGCBase.setup(self) @@ -151,11 +153,12 @@ def double_space_size(self): self.red_zone = 0 old_fromspace = self.fromspace + oldsize = self.space_size newsize = self.space_size * 2 - newspace = llarena.arena_malloc(newsize, True) + newspace = llarena.arena_malloc(newsize, llarena.Z_CLEAR_LARGE_AREA) if not newspace: return False # out of memory - llarena.arena_free(old_fromspace) + llarena.arena_free(old_fromspace, oldsize) self.fromspace = newspace # now self.tospace contains the existing objects and # self.fromspace is the freshly allocated bigger space @@ -164,9 +167,9 @@ self.top_of_space = self.tospace + newsize # now self.tospace is the freshly allocated bigger space, # and self.fromspace is the old smaller space, now empty - llarena.arena_free(self.fromspace) + llarena.arena_free(self.fromspace, oldsize) - newspace = llarena.arena_malloc(newsize, True) + newspace = llarena.arena_malloc(newsize, llarena.Z_CLEAR_LARGE_AREA) if not newspace: # Complex failure case: we have in self.tospace a big chunk # of memory, and the two smaller original spaces are already gone. @@ -241,7 +244,8 @@ self.finished_full_collect() self.debug_check_consistency() if not size_changing: - llarena.arena_reset(fromspace, self.space_size, True) + llarena.arena_reset(fromspace, self.space_size, + llarena.Z_CLEAR_LARGE_AREA) self.record_red_zone() self.execute_finalizers() #llop.debug_print(lltype.Void, 'collected', self.space_size, size_changing, self.top_of_space - self.free) @@ -409,7 +413,8 @@ ll_assert(tid & GCFLAG_FORWARDED == 0, "unexpected GCFLAG_FORWARDED") # replace the object at 'obj' with a FORWARDSTUB. hdraddr = obj - size_gc_header - llarena.arena_reset(hdraddr, size_gc_header + objsize, False) + llarena.arena_reset(hdraddr, size_gc_header + objsize, + llarena.Z_DONT_CLEAR) llarena.arena_reserve(hdraddr, size_gc_header + stubsize) hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(self.HDR)) hdr.tid = tid | GCFLAG_FORWARDED From arigo at codespeak.net Mon Oct 19 17:26:03 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 19 Oct 2009 17:26:03 +0200 (CEST) Subject: [pypy-svn] r68641 - pypy/branch/gc-arena/pypy/rpython/lltypesystem Message-ID: <20091019152603.92E11168005@codespeak.net> Author: arigo Date: Mon Oct 19 17:26:03 2009 New Revision: 68641 Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_llinterp.py Log: Fix comment. Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_llinterp.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_llinterp.py (original) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_llinterp.py Mon Oct 19 17:26:03 2009 @@ -294,8 +294,8 @@ Z_DONT_CLEAR = 0 # it's ok to keep random bytes in the area Z_CLEAR_LARGE_AREA = 1 # clear, optimized for a large area of memory Z_CLEAR_SMALL_AREA = 2 # clear, optimized for a small or medium area of mem -Z_INACCESSIBLE = 3 # make the memory inaccessible (not reserved) -Z_ACCESSIBLE = 4 # make the memory accessible again +Z_INACCESSIBLE = 3 # make the pages inaccessible (i.e. not reserved) +Z_ACCESSIBLE = 4 # make the pages accessible again # Note that Z_CLEAR_LARGE_AREA, Z_INACCESSIBLE and Z_ACCESSIBLE are # restricted to whole pages (at least one), and you must not try to make From arigo at codespeak.net Mon Oct 19 17:59:58 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 19 Oct 2009 17:59:58 +0200 (CEST) Subject: [pypy-svn] r68642 - pypy/branch/gc-arena/pypy/rpython/lltypesystem Message-ID: <20091019155958.EC4D2168005@codespeak.net> Author: arigo Date: Mon Oct 19 17:59:58 2009 New Revision: 68642 Added: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_nt.py (contents, props changed) Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena.py Log: Attempt to write a Win32 version. Untested, probably contains at least typos. Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena.py Mon Oct 19 17:59:58 2009 @@ -19,8 +19,8 @@ if os.name == 'posix': from pypy.rpython.lltypesystem import llarena_posix as llarena_impl -#elif os.name == 'win32': -# ... +elif os.name == 'nt': + from pypy.rpython.lltypesystem import llarena_nt as llarena_impl else: from pypy.rpython.lltypesystem import llarena_generic as llarena_impl Added: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_nt.py ============================================================================== --- (empty file) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_nt.py Mon Oct 19 17:59:58 2009 @@ -0,0 +1,108 @@ +import os, sys +from pypy.rpython.tool import rffi_platform +from pypy.rpython.lltypesystem import rffi, lltype, llmemory +from pypy.rpython.lltypesystem.llarena_llinterp import Z_CLEAR_LARGE_AREA +from pypy.rpython.lltypesystem.llarena_llinterp import Z_CLEAR_SMALL_AREA +from pypy.rpython.lltypesystem.llarena_llinterp import Z_INACCESSIBLE +from pypy.rpython.lltypesystem.llarena_llinterp import Z_ACCESSIBLE +from pypy.rlib.rwin32 import DWORD, BOOL + +implements_inaccessible = True + +# ____________________________________________________________ + +class CConfig: + _compilation_info_ = ExternalCompilationInfo( + includes=['windows.h']) + + SYSTEM_INFO = rffi_platform.Struct( + 'SYSTEM_INFO', [ + ("dwPageSize", DWORD), + ]) + +config = rffi_platform.configure(CConfig) + +SYSTEM_INFO = config['SYSTEM_INFO'] +SYSTEM_INFO_P = lltype.Ptr(SYSTEM_INFO) + +def winexternal(name, args, result): + return rffi.llexternal(name, args, result, + compilation_info=CConfig._compilation_info_, + calling_conv='win') + +# ____________________________________________________________ + +GetSystemInfo = winexternal('GetSystemInfo', [SYSTEM_INFO_P], lltype.Void) + +class WinPageSize: + def __init__(self): + self.pagesize = 0 + _freeze_ = __init__ +win_pagesize = WinPageSize() + +def getpagesize(): + pagesize = win_pagesize.pagesize + if pagesize == 0: + si = rffi.make(SYSTEM_INFO) + try: + GetSystemInfo(si) + pagesize = rffi.cast(lltype.Signed, si.c_dwPageSize) + finally: + lltype.free(si, flavor="raw") + win_pagesize.pagesize = pagesize + return pagesize + +# ____________________________________________________________ + +VirtualAlloc = winexternal('VirtualAlloc', [llmemory.Address, + rffi.SIZE_T, DWORD, DWORD], + llmemory.Address) +VirtualFree = winexternal('VirtualFree', [llmemory.Address, + rffi.SIZE_T, DWORD], + BOOL) + +MEM_COMMIT = 0x1000 +MEM_RESERVE = 0x2000 +MEM_DECOMMIT = 0x4000 +MEM_RELEASE = 0x8000 +PAGE_READWRITE = 0x04 + +class VirtualAllocMemoryError(Exception): + pass + +def _virtual_alloc(arena_addr, nbytes, flags, protect): + result = VirtualFree(arena_addr, + rffi.cast(rffi.SIZE_T, nbytes), + rffi.cast(DWORD, flags), + rffi.cast(DWORD, protect)) + if result == llmemory.NULL: + raise VirtualAllocMemoryError + return result + +def _virtual_free(arena_addr, nbytes, flags): + result = VirtualFree(arena_addr, + rffi.cast(rffi.SIZE_T, nbytes), + rffi.cast(DWORD, flags)) + if rffi.cast(lltype.Signed, result) == 0: + raise VirtualAllocMemoryError + +def llimpl_arena_malloc(nbytes, zero): + flAllocationType = MEM_RESERVE + if zero != Z_INACCESSIBLE: + flAllocationType |= MEM_COMMIT + return _virtual_alloc(llmemory.NULL, nbytes, + flAllocationType, PAGE_READWRITE) + +def llimpl_arena_free(arena_addr, nbytes): + _virtual_free(arena_addr, 0, MEM_RELEASE) + +def llimpl_arena_reset(arena_addr, size, zero): + if zero == Z_CLEAR_LARGE_AREA: + _virtual_free(arena_addr, size, MEM_DECOMMIT) + _virtual_alloc(arena_addr, size, MEM_COMMIT, PAGE_READWRITE) + elif zero == Z_CLEAR_SMALL_AREA: + llmemory.raw_memclear(arena_addr, size) + elif zero == Z_ACCESSIBLE: + _virtual_alloc(arena_addr, size, MEM_COMMIT, PAGE_READWRITE) + elif zero == Z_INACCESSIBLE: + _virtual_free(arena_addr, size, MEM_DECOMMIT) From afa at codespeak.net Mon Oct 19 18:14:39 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 19 Oct 2009 18:14:39 +0200 (CEST) Subject: [pypy-svn] r68643 - pypy/branch/gc-arena/pypy/rpython/lltypesystem Message-ID: <20091019161439.D0BB0168006@codespeak.net> Author: afa Date: Mon Oct 19 18:14:39 2009 New Revision: 68643 Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_nt.py Log: Fix typo in llarena_nt Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_nt.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_nt.py (original) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_nt.py Mon Oct 19 18:14:39 2009 @@ -12,12 +12,12 @@ # ____________________________________________________________ class CConfig: - _compilation_info_ = ExternalCompilationInfo( + _compilation_info_ = rffi_platform.ExternalCompilationInfo( includes=['windows.h']) - SYSTEM_INFO = rffi_platform.Struct( - 'SYSTEM_INFO', [ - ("dwPageSize", DWORD), + SYSTEM_INFO = rffi_platform.Struct( + 'SYSTEM_INFO', [ + ("dwPageSize", DWORD), ]) config = rffi_platform.configure(CConfig) From pedronis at codespeak.net Mon Oct 19 18:20:42 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 19 Oct 2009 18:20:42 +0200 (CEST) Subject: [pypy-svn] r68644 - pypy/build/bot2/pypybuildbot Message-ID: <20091019162042.254DD168009@codespeak.net> Author: pedronis Date: Mon Oct 19 18:20:41 2009 New Revision: 68644 Modified: pypy/build/bot2/pypybuildbot/master.py Log: reorder categories Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Mon Oct 19 18:20:41 2009 @@ -34,7 +34,8 @@ # pypy test summary page summary = load('pypybuildbot.summary') -status.putChild('summary', summary.Summary(['own', 'applevel', 'lib-python', +status.putChild('summary', summary.Summary(['own', 'applevel', + 'lib-python', 'jit', 'stackless', 'windows', 'maemo', 'other'])) From afa at codespeak.net Mon Oct 19 18:21:11 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 19 Oct 2009 18:21:11 +0200 (CEST) Subject: [pypy-svn] r68645 - pypy/branch/gc-arena/pypy/rpython/lltypesystem Message-ID: <20091019162111.BA8A4168009@codespeak.net> Author: afa Date: Mon Oct 19 18:21:11 2009 New Revision: 68645 Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_nt.py Log: Really call VirtualAlloc when we mean to Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_nt.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_nt.py (original) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_nt.py Mon Oct 19 18:21:11 2009 @@ -71,10 +71,10 @@ pass def _virtual_alloc(arena_addr, nbytes, flags, protect): - result = VirtualFree(arena_addr, - rffi.cast(rffi.SIZE_T, nbytes), - rffi.cast(DWORD, flags), - rffi.cast(DWORD, protect)) + result = VirtualAlloc(arena_addr, + rffi.cast(rffi.SIZE_T, nbytes), + rffi.cast(DWORD, flags), + rffi.cast(DWORD, protect)) if result == llmemory.NULL: raise VirtualAllocMemoryError return result From arigo at codespeak.net Mon Oct 19 18:26:18 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 19 Oct 2009 18:26:18 +0200 (CEST) Subject: [pypy-svn] r68646 - in pypy/branch/gc-arena/pypy/rpython: lltypesystem memory/gc Message-ID: <20091019162618.E2C87168015@codespeak.net> Author: arigo Date: Mon Oct 19 18:26:18 2009 New Revision: 68646 Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena.py pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_generic.py pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_nt.py pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_posix.py pypy/branch/gc-arena/pypy/rpython/memory/gc/semispace.py Log: Clarify the out-of-memory behavior of the llarena functions. For now it seems saner to say that they all raise OutOfMemoryError instead of doing something like returning NULL, because arena_reset(Z_ACCESSIBLE) can also raise OutOfMemoryError. Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena.py Mon Oct 19 18:26:18 2009 @@ -27,6 +27,7 @@ implements_inaccessible = llarena_impl.implements_inaccessible getpagesize = llarena_impl.getpagesize +OutOfMemoryError = llarena_impl.OutOfMemoryError register_external(arena_malloc, [int, int], llmemory.Address, 'll_arena.arena_malloc', Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_generic.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_generic.py (original) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_generic.py Mon Oct 19 18:26:18 2009 @@ -5,6 +5,9 @@ implements_inaccessible = False +class OutOfMemoryError(Exception): + pass + # a random value, but nothing really depends on it def getpagesize(): return 4096 @@ -12,8 +15,9 @@ # llimpl_arena_*() functions based on raw_malloc def llimpl_arena_malloc(nbytes, zero): addr = llmemory.raw_malloc(nbytes) - if bool(addr): - llimpl_arena_reset(addr, nbytes, zero) + if not addr: + raise OutOfMemoryError + llimpl_arena_reset(addr, nbytes, zero) return addr def llimpl_arena_free(arena_addr, nbytes): Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_nt.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_nt.py (original) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_nt.py Mon Oct 19 18:26:18 2009 @@ -9,6 +9,12 @@ implements_inaccessible = True +class OutOfMemoryError(Exception): + pass + +class VirtualFreeInternalError(Exception): + pass + # ____________________________________________________________ class CConfig: @@ -76,7 +82,7 @@ rffi.cast(DWORD, flags), rffi.cast(DWORD, protect)) if result == llmemory.NULL: - raise VirtualAllocMemoryError + raise OutOfMemoryError return result def _virtual_free(arena_addr, nbytes, flags): @@ -84,7 +90,7 @@ rffi.cast(rffi.SIZE_T, nbytes), rffi.cast(DWORD, flags)) if rffi.cast(lltype.Signed, result) == 0: - raise VirtualAllocMemoryError + raise VirtualFreeInternalError def llimpl_arena_malloc(nbytes, zero): flAllocationType = MEM_RESERVE Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_posix.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_posix.py (original) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/llarena_posix.py Mon Oct 19 18:26:18 2009 @@ -7,6 +7,12 @@ implements_inaccessible = True +class OutOfMemoryError(Exception): + pass + +class MMapInternalError(Exception): + pass + # ____________________________________________________________ posix_getpagesize = rffi.llexternal('getpagesize', [], rffi.INT, @@ -122,9 +128,6 @@ rffi.INT, sandboxsafe=True, _nowrapper=True) -class MMapMemoryError(Exception): - pass - def llimpl_arena_malloc(nbytes, zero): flags = MAP_PRIVATE | MAP_ANONYMOUS if zero == Z_INACCESSIBLE: @@ -140,20 +143,20 @@ rffi.cast(rffi.INT, -1), rffi.cast(off_t, 0)) if rffi.cast(lltype.Signed, result) == -1: - raise MMapMemoryError + raise OutOfMemoryError return result def llimpl_arena_free(arena_addr, nbytes): result = posix_munmap(arena_addr, rffi.cast(rffi.SIZE_T, nbytes)) if rffi.cast(lltype.Signed, result) == -1: - raise MMapMemoryError + raise MMapInternalError def _arena_protect(arena_addr, size, flags): res = posix_mprotect(arena_addr, rffi.cast(rffi.SIZE_T, size), rffi.cast(rffi.INT, flags)) if rffi.cast(lltype.Signed, res) != 0: - raise MMapMemoryError + raise OutOfMemoryError def llimpl_arena_reset(arena_addr, size, zero): if zero == Z_CLEAR_LARGE_AREA: Modified: pypy/branch/gc-arena/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-arena/pypy/rpython/memory/gc/semispace.py Mon Oct 19 18:26:18 2009 @@ -61,11 +61,9 @@ self.program_start_time = time.time() self.tospace = llarena.arena_malloc(self.space_size, llarena.Z_CLEAR_LARGE_AREA) - ll_assert(bool(self.tospace), "couldn't allocate tospace") self.top_of_space = self.tospace + self.space_size self.fromspace = llarena.arena_malloc(self.space_size, llarena.Z_CLEAR_LARGE_AREA) - ll_assert(bool(self.fromspace), "couldn't allocate fromspace") self.free = self.tospace MovingGCBase.setup(self) self.objects_with_finalizers = self.AddressDeque() @@ -155,8 +153,10 @@ old_fromspace = self.fromspace oldsize = self.space_size newsize = self.space_size * 2 - newspace = llarena.arena_malloc(newsize, llarena.Z_CLEAR_LARGE_AREA) - if not newspace: + try: + newspace = llarena.arena_malloc(newsize, + llarena.Z_CLEAR_LARGE_AREA) + except llarena.OutOfMemoryError: return False # out of memory llarena.arena_free(old_fromspace, oldsize) self.fromspace = newspace @@ -169,8 +169,10 @@ # and self.fromspace is the old smaller space, now empty llarena.arena_free(self.fromspace, oldsize) - newspace = llarena.arena_malloc(newsize, llarena.Z_CLEAR_LARGE_AREA) - if not newspace: + try: + newspace = llarena.arena_malloc(newsize, + llarena.Z_CLEAR_LARGE_AREA) + except llarena.OutOfMemoryError: # Complex failure case: we have in self.tospace a big chunk # of memory, and the two smaller original spaces are already gone. # Unsure if it's worth these efforts, but we can artificially From afa at codespeak.net Mon Oct 19 19:36:00 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 19 Oct 2009 19:36:00 +0200 (CEST) Subject: [pypy-svn] r68647 - in pypy/branch/msvc-asmgcroot/pypy/translator/c: . gcc Message-ID: <20091019173600.2C67116800D@codespeak.net> Author: afa Date: Mon Oct 19 19:35:59 2009 New Revision: 68647 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py Log: More progress towards generating a correct gcmaptable.s file Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Mon Oct 19 19:35:59 2009 @@ -6,8 +6,6 @@ from pypy.translator.c.gcc.instruction import * LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' -r_functionstart_elf = re.compile(r"\t.type\s+"+LABEL+",\s*[@]function\s*$") -r_functionend_elf = re.compile(r"\t.size\s+"+LABEL+",\s*[.]-"+LABEL+"\s*$") # darwin r_textstart = re.compile(r"\t.text\s*$") @@ -24,7 +22,6 @@ 'const_data' ] r_sectionstart = re.compile(r"\t\.("+'|'.join(OTHERSECTIONS)+").*$") -r_functionstart_darwin = re.compile(r"_(\w+):\s*$") OFFSET_LABELS = 2**30 @@ -660,14 +657,17 @@ class ElfFunctionGcRootTracker(FunctionGcRootTracker): format = 'elf' - ESP = '%esp' - OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' - LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' + ESP = '%esp' + OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' + LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' + + r_functionstart = re.compile(r"\t.type\s+"+LABEL+",\s*[@]function\s*$") + r_functionend = re.compile(r"\t.size\s+"+LABEL+",\s*[.]-"+LABEL+"\s*$") def __init__(self, lines, filetag=0): - match = r_functionstart_elf.match(lines[0]) + match = self.r_functionstart.match(lines[0]) funcname = match.group(1) - match = r_functionend_elf.match(lines[-1]) + match = self.r_functionend.match(lines[-1]) assert funcname == match.group(1) assert funcname == match.group(2) super(ElfFunctionGcRootTracker, self).__init__( @@ -683,9 +683,11 @@ class DarwinFunctionGcRootTracker(ElfFunctionGcRootTracker): format = 'darwin' + r_functionstart = re.compile(r"_(\w+):\s*$") + def __init__(self, lines, filetag=0): - match = r_functionstart_darwin.match(lines[0]) - funcname = '_'+match.group(1) + match = self.r_functionstart.match(lines[0]) + funcname = '_' + match.group(1) FunctionGcRootTracker.__init__(self, funcname, lines, filetag) class Mingw32FunctionGcRootTracker(DarwinFunctionGcRootTracker): @@ -694,18 +696,19 @@ class MsvcFunctionGcRootTracker(FunctionGcRootTracker): format = 'msvc' - r_functionstart = re.compile(r"PUBLIC\t"+LABEL+"$") + r_functionstart = re.compile(LABEL+r"\s+PROC\s*(:?;.+)\n$") + r_functionend = re.compile(LABEL+r"\s+ENDP\s*$") ESP = 'esp' - OPERAND = r'(?:\w+|(?:DWORD PTR )?[_\w$]*\[[-+\w0-9]+\])' - LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' + OPERAND = r'(?:(:?WORD|DWORD|BYTE) PTR |OFFSET )?[_\w?@$]*(?:[-+0-9]+)?(:?\[[-+*\w0-9]+\])?' + LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' @classmethod def init_regexp(cls): super(MsvcFunctionGcRootTracker, cls).init_regexp() - cls.r_binaryinsn = re.compile(r"\t[a-z]\w*\s+(?P"+cls.OPERAND+"),\s*(?P"+cls.OPERAND+")\s*(?:;.+)?$") - cls.r_jump = re.compile(r"\tj\w+\s+(?:SHORT )?"+LABEL+"\s*$") + cls.r_binaryinsn = re.compile(r"\t[a-z]\w*\s+(?P"+cls.OPERAND+r"),\s*(?P"+cls.OPERAND+r")\s*(?:;.+)?$") + cls.r_jump = re.compile(r"\tj\w+\s+(?:SHORT |DWORD PTR )?"+LABEL+"\s*$") def __init__(self, lines, filetag=0): match = self.r_functionstart.match(lines[0]) @@ -720,6 +723,10 @@ locals()['visit_' + name] = getattr(FunctionGcRootTracker, 'visit_' + name + 'l') + visit_npad = FunctionGcRootTracker.visit_nop + # probably not GC pointers + visit_cdq = FunctionGcRootTracker.visit_nop + def extract_immediate(self, value): try: return int(value) @@ -774,14 +781,14 @@ functionlines = [] in_function = False for line in iterlines: - if r_functionstart_elf.match(line): + if cls.FunctionGcRootTracker.r_functionstart.match(line): assert not in_function, ( "missed the end of the previous function") yield False, functionlines in_function = True functionlines = [] functionlines.append(line) - if r_functionend_elf.match(line): + if cls.FunctionGcRootTracker.r_functionend.match(line): assert in_function, ( "missed the start of the current function") yield True, functionlines @@ -810,7 +817,7 @@ functionlines = [] in_text = False in_function = False - elif in_text and r_functionstart_darwin.match(line): + elif in_text and cls.FunctionGcRootTracker.r_functionstart.match(line): yield in_function, functionlines functionlines = [] in_function = True @@ -838,7 +845,7 @@ in_text = True elif r_sectionstart.match(line): in_text = False - elif in_text and r_functionstart_darwin.match(line): + elif in_text and cls.FunctionGcRootTracker.r_functionstart.match(line): yield in_function, functionlines functionlines = [] in_function = True @@ -850,6 +857,33 @@ format = "msvc" FunctionGcRootTracker = MsvcFunctionGcRootTracker + @classmethod + def find_functions(cls, iterlines): + functionlines = [] + in_function = False + for line in iterlines: + if cls.FunctionGcRootTracker.r_functionstart.match(line): + assert not in_function, ( + "missed the end of the previous function") + yield False, functionlines + in_function = True + functionlines = [] + functionlines.append(line) + if cls.FunctionGcRootTracker.r_functionend.match(line): + assert in_function, ( + "missed the start of the current function") + yield True, functionlines + in_function = False + functionlines = [] + assert not in_function, ( + "missed the end of the previous function") + yield False, functionlines + + def process_function(self, lines, entrypoint, filename): + entrypoint = '_' + entrypoint + return super(MsvcAssemblerParser, self).process_function( + lines, entrypoint, filename) + PARSERS = { 'elf': ElfAssemblerParser, 'darwin': DarwinAssemblerParser, @@ -885,98 +919,214 @@ shapes = {} shapelines = [] shapeofs = 0 - def _globalname(name): - if self.format in ('darwin', 'mingw32'): - return '_' + name - return name + def _globalname(name, disp=""): + if self.format in ('darwin', 'mingw32', 'msvc'): + name = '_' + name + + if disp: + return "%s + %s" % (name, disp) + else: + return name + def _globl(name): - print >> output, "\t.globl %s" % _globalname(name) + if self.format == 'msvc': + print >> output, "PUBLIC %s" % _globalname(name) + else: + print >> output, "\t.globl %s" % _globalname(name) def _label(name): print >> output, "%s:" % _globalname(name) def _variant(**kwargs): txt = kwargs[self.format] print >> output, "\t%s" % txt - print >> output, "\t.text" + def _comment(comment): + if self.format == 'msvc': + print >> output, "; %s" % comment + else: + print >> output, "/* %s */" % comment + + def _movl(source, target, comment): + if self.format == 'msvc': + print >> output, "\tmov\t%s, %s\t\t; %s" % (target, source, comment) + else: + print >> output, "\tmovl\t%s, %s\t\t/* %s */ " % (source, target, comment) + + def _pushl(source, comment): + if self.format == 'msvc': + print >> output, "\tpush\t%s\t\t; %s" % (source, comment) + else: + print >> output, "\tpushl\t%s\t\t/* %s */ " % (target, comment) + + def _popl(source, comment): + if self.format == 'msvc': + print >> output, "\tpop\t%s\t\t; %s" % (source, comment) + else: + print >> output, "\tpopl\t%s\t\t/* %s */ " % (target, comment) + + + def _register(name, disp=""): + if self.format == 'msvc': + if disp: + return "DWORD PTR [%s+%s]" % (name, disp) + else: + return name + else: + if disp: + return "%s(%%%s)" % (disp, name) + else: + return '%' + name + + def _offset(name): + if self.format == 'msvc': + return "OFFSET %s" % _globalname(name) + else: + return "$%s" % _globalname(name) + + def _call(arg, comment): + return "call\t%s\t\t;%s" % (arg, comment) + + def _indirectjmp(arg): + if self.format == 'msvc': + return arg + else: + return "*%" + arg + + if self.format == 'msvc': + print >> output, """\ + TITLE gcmaptable.s + .686P + .XMM + .model flat + """ + + _variant(elf='\t.text', + darwin='\t.text', + mingw32='\t.text', + msvc='_TEXT\tSEGMENT') + + _globl('__gcrootanchor') + _globl('pypy_asm_stackwalk') _variant(elf='.type pypy_asm_stackwalk, @function', darwin='', - mingw32='') + mingw32='', + msvc='') _label('pypy_asm_stackwalk') - print >> output, """\ - /* See description in asmgcroot.py */ - movl 4(%esp), %edx /* my argument, which is the callback */ - movl %esp, %eax /* my frame top address */ - pushl %eax /* ASM_FRAMEDATA[6] */ - pushl %ebp /* ASM_FRAMEDATA[5] */ - pushl %edi /* ASM_FRAMEDATA[4] */ - pushl %esi /* ASM_FRAMEDATA[3] */ - pushl %ebx /* ASM_FRAMEDATA[2] */ - - /* Add this ASM_FRAMEDATA to the front of the circular linked */ - /* list. Let's call it 'self'. */ - movl __gcrootanchor+4, %eax /* next = gcrootanchor->next */ - pushl %eax /* self->next = next */ - pushl $__gcrootanchor /* self->prev = gcrootanchor */ - movl %esp, __gcrootanchor+4 /* gcrootanchor->next = self */ - movl %esp, (%eax) /* next->prev = self */ - - /* note: the Mac OS X 16 bytes aligment must be respected. */ - call *%edx /* invoke the callback */ - - /* Detach this ASM_FRAMEDATA from the circular linked list */ - popl %esi /* prev = self->prev */ - popl %edi /* next = self->next */ - movl %edi, 4(%esi) /* prev->next = next */ - movl %esi, (%edi) /* next->prev = prev */ - - popl %ebx /* restore from ASM_FRAMEDATA[2] */ - popl %esi /* restore from ASM_FRAMEDATA[3] */ - popl %edi /* restore from ASM_FRAMEDATA[4] */ - popl %ebp /* restore from ASM_FRAMEDATA[5] */ - popl %ecx /* ignored ASM_FRAMEDATA[6] */ - /* the return value is the one of the 'call' above, */ - /* because %eax (and possibly %edx) are unmodified */ - ret -""".replace("__gcrootanchor", _globalname("__gcrootanchor")) + _comment("See description in asmgcroot.py") + _movl(_register("esp", disp="4"), _register("edx"), "my argument, which is the callback") + _movl(_register("esp"), _register("eax"), "my frame top address") + _pushl(_register("eax"), "ASM_FRAMEDATA[6]") + _pushl(_register("ebp"), "ASM_FRAMEDATA[5]") + _pushl(_register("edi"), "ASM_FRAMEDATA[4]") + _pushl(_register("esi"), "ASM_FRAMEDATA[3]") + _pushl(_register("ebx"), "ASM_FRAMEDATA[2]") + + print >> output + _comment("Add this ASM_FRAMEDATA to the front of the circular linked") + _comment("list. Let's call it 'self'.") + print >> output + _movl(_globalname("__gcrootanchor", disp=4), _register("eax"), "next = gcrootanchor->next") + _pushl(_register("eax"), "self->next = next") + _pushl(_offset("__gcrootanchor"), "self->prev = gcrootanchor") + _movl(_register("esp"), _globalname("__gcrootanchor", disp=4), "gcrootanchor->next = self") + _movl(_register("esp"), _register("eax", "0"), "next->prev = self") + print >> output + + _comment("note: the Mac OS X 16 bytes aligment must be respected.") + _call(_indirectjmp("edx"), "invoke the callback") + print >> output + + _comment("Detach this ASM_FRAMEDATA from the circular linked list") + _popl(_register("esi"), "prev = self->prev") + _popl(_register("edi"), "next = self->next") + _movl(_register("edi"), _register("esi", disp="4"), "prev->next = next") + _movl(_register("esi"), _register("edi", disp="0"), "next->prev = prev") + print >> output + + _popl(_register("ebx"), "restore from ASM_FRAMEDATA[2]") + _popl(_register("esi"), "restore from ASM_FRAMEDATA[3]") + _popl(_register("edi"), "restore from ASM_FRAMEDATA[4]") + _popl(_register("ebp"), "restore from ASM_FRAMEDATA[5]") + _popl(_register("ecx"), "restore from ASM_FRAMEDATA[6]") + _comment("the return value is the one of the 'call' above,") + _comment("because %eax (and possibly %edx) are unmodified") + + print >> output, "\tret" + _variant(elf='.size pypy_asm_stackwalk, .-pypy_asm_stackwalk', darwin='', - mingw32='') - print >> output, '\t.data' - print >> output, '\t.align\t4' - _globl('__gcrootanchor') - _label('__gcrootanchor') - print >> output, """\ - /* A circular doubly-linked list of all */ - /* the ASM_FRAMEDATAs currently alive */ + mingw32='', + msvc='') + + if self.format == 'msvc': + for label, state, is_range in self.gcmaptable: + print >> output, "EXTERN %s:NEAR" % label + + if self.format == 'msvc': + print >> output, '_DATA SEGMENT' + + _comment("A circular doubly-linked list of all") + _comment("the ASM_FRAMEDATAs currently alive") + if self.format == 'msvc': + print >> output, '%s DD ?' % _globalname("__gcrootanchor") + print >> output, '\tDD FLAT:___gcrootanchor ; prev' + print >> output, '\tDD FLAT:___gcrootanchor ; next' + else: + print >> output, '\t.data' + print >> output, '\t.align\t4' + _label('__gcrootanchor') + print >> output, """\ .long\t__gcrootanchor /* prev */ .long\t__gcrootanchor /* next */ """.replace("__gcrootanchor", _globalname("__gcrootanchor")) + _globl('__gcmapstart') - _label('__gcmapstart') + if self.format == 'msvc': + print >> output, '%s DD ?' % _globalname('__gcmapstart') + else: + _label('__gcmapstart') for label, state, is_range in self.gcmaptable: try: n = shapes[state] except KeyError: n = shapes[state] = shapeofs bytes = [str(b) for b in compress_callshape(state)] - shapelines.append('\t/*%d*/\t.byte\t%s\n' % ( - shapeofs, - ', '.join(bytes))) + if self.format == 'msvc': + shapelines.append('\tDD\t%s\t;%s\n' % ( + ', '.join(bytes), + shapeofs)) + else: + shapelines.append('\t/*%d*/\t.byte\t%s\n' % ( + shapeofs, + ', '.join(bytes))) shapeofs += len(bytes) if is_range: n = ~ n - print >> output, '\t.long\t%s-%d' % (label, OFFSET_LABELS) - print >> output, '\t.long\t%d' % (n,) + if self.format == 'msvc': + print >> output, '\tDD\t%s-%d' % (label, OFFSET_LABELS) + print >> output, '\tDD\t%d' % (n,) + else: + print >> output, '\t.long\t%s-%d' % (label, OFFSET_LABELS) + print >> output, '\t.long\t%d' % (n,) + _globl('__gcmapend') _label('__gcmapend') _variant(elf='.section\t.rodata', darwin='.const', - mingw32='') + mingw32='', + msvc='') + _globl('__gccallshapes') - _label('__gccallshapes') + if self.format == 'msvc': + print >> output, '%s DD ?' % _globalname('__gccallshapes') + else: + _label('__gccallshapes') output.writelines(shapelines) + if self.format == 'msvc': + print >> output, "_DATA ENDS" + print >> output, "END" + def process(self, iterlines, newfile, entrypoint='main', filename='?'): parser = PARSERS[format](verbose=self.verbose, shuffle=self.shuffle) for in_function, lines in parser.find_functions(iterlines): @@ -1010,6 +1160,7 @@ else: FUNCTIONS_NOT_RETURNING = { '_abort': None, + '__imp__abort': None, '__exit': None, '__assert': None, '__wassert': None, @@ -1169,7 +1320,7 @@ if sys.platform == 'darwin': format = 'darwin' elif sys.platform == 'win32': - format = 'mingw32' + format = 'msvc' else: format = 'elf' tracker = GcRootTracker(verbose=verbose, shuffle=shuffle, format=format) Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py Mon Oct 19 19:35:59 2009 @@ -493,7 +493,6 @@ mk.definition('ASMLBLFILES', lblsfiles) mk.definition('GCMAPFILES', gcmapfiles) mk.definition('DEBUGFLAGS', '-O2 -fomit-frame-pointer -g') - mk.definition('OBJECTS', '$(ASMLBLFILES) gcmaptable.s') if sys.platform == 'win32': python = sys.executable.replace('\\', '/') + ' ' @@ -501,13 +500,20 @@ python = '' if self.translator.platform.name == 'msvc': + lblofiles = ['%s.lbl.obj' % (cfile[:-2],) for cfile in mk.cfiles] + mk.definition('ASMLBLOBJFILES', lblofiles) + mk.definition('OBJECTS', '$(ASMLBLOBJFILES) gcmaptable.obj') + mk.rule('.SUFFIXES', '.s') + mk.rule('.s.obj', '', + 'c:\masm32\bin\ml $(CFLAGS) /Zm /coff /Fo$@ /c $< $(INCLUDEDIRS)') mk.rule('.c.gcmap', '', ['$(CC) $(CFLAGS) /c /FAs /Fa$*.s $< $(INCLUDEDIRS)', - python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -t $*.s > $@'] + 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -t $*.s > $@'] ) mk.rule('gcmaptable.s', '$(GCMAPFILES)', - python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py $(GCMAPFILES) > $@') + 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py $(GCMAPFILES) > $@') else: + mk.definition('OBJECTS', '$(ASMLBLFILES) gcmaptable.s') mk.rule('%.s', '%.c', '$(CC) $(CFLAGS) -frandom-seed=$< -o $@ -S $< $(INCLUDEDIRS)') mk.rule('%.lbl.s %.gcmap', '%.s', python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -t $< > $*.gcmap') From arigo at codespeak.net Mon Oct 19 20:51:07 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 19 Oct 2009 20:51:07 +0200 (CEST) Subject: [pypy-svn] r68648 - pypy/branch/gc-arena/pypy/translator/c/test Message-ID: <20091019185107.959B516800D@codespeak.net> Author: arigo Date: Mon Oct 19 20:51:07 2009 New Revision: 68648 Modified: pypy/branch/gc-arena/pypy/translator/c/test/test_newgc.py Log: Improve the test. Modified: pypy/branch/gc-arena/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/gc-arena/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/gc-arena/pypy/translator/c/test/test_newgc.py Mon Oct 19 20:51:07 2009 @@ -902,9 +902,9 @@ s.append("abcd") s.append("defg") s.append("rty") - s.append_multiple_char('y', 1000) + s.append_multiple_char('z', 1000) gc.collect() - s.append_multiple_char('y', 1000) + s.append_multiple_char('u', 1000) res = s.build() gc.collect() return res @@ -912,7 +912,7 @@ def test_string_builder_over_allocation(self): res = self.run('string_builder_over_allocation') - assert res[1000] == 'y' + assert res == 'abcddefgrty' * 1000*'z' + 1000*'u' class TestGenerationalGC(TestSemiSpaceGC): gcpolicy = "generation" From benjamin at codespeak.net Mon Oct 19 23:02:07 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 19 Oct 2009 23:02:07 +0200 (CEST) Subject: [pypy-svn] r68652 - pypy/trunk/pypy/interpreter/astcompiler Message-ID: <20091019210207.771AE168006@codespeak.net> Author: benjamin Date: Mon Oct 19 23:02:06 2009 New Revision: 68652 Modified: pypy/trunk/pypy/interpreter/astcompiler/consts.py Log: remove unused constants Modified: pypy/trunk/pypy/interpreter/astcompiler/consts.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/consts.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/consts.py Mon Oct 19 23:02:06 2009 @@ -1,15 +1,6 @@ -# operation flags -OP_ASSIGN = 0 # 'OP_ASSIGN' -OP_DELETE = 1 # 'OP_DELETE' -OP_APPLY = 2 # 'OP_APPLY' -OP_NONE = 3 - -SC_LOCAL = 1 -SC_GLOBAL = 2 -SC_FREE = 3 -SC_CELL = 4 -SC_UNKNOWN = 5 -SC_DEFAULT = 6 +""" +Various flags used during the compilation process. +""" CO_OPTIMIZED = 0x0001 CO_NEWLOCALS = 0x0002 From afa at codespeak.net Mon Oct 19 23:48:32 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 19 Oct 2009 23:48:32 +0200 (CEST) Subject: [pypy-svn] r68653 - in pypy/branch/msvc-asmgcroot/pypy/translator/c: . gcc gcc/test/msvc Message-ID: <20091019214832.9FFB4168006@codespeak.net> Author: afa Date: Mon Oct 19 23:48:31 2009 New Revision: 68653 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py Log: more fixes Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s Mon Oct 19 23:48:31 2009 @@ -1,11 +1,3 @@ -PUBLIC _pypy_g_ll_join_strs__Signed_arrayPtr -; Function compile flags: /Ogtpy -; COMDAT _pypy_g_ll_join_strs__Signed_arrayPtr -_TEXT SEGMENT -_l_result_2$ = -8 ; size = 4 -_l_v405$ = -4 ; size = 4 -_l_num_items_0$ = 8 ; size = 4 -_l_items_2$ = 12 ; size = 4 _pypy_g_ll_join_strs__Signed_arrayPtr PROC ; COMDAT ; 1457 : struct pypy_rpy_string0 *pypy_g_ll_join_strs__Signed_arrayPtr(long l_num_items_0, struct pypy_array0 *l_items_2) { Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Mon Oct 19 23:48:31 2009 @@ -324,14 +324,18 @@ break k += 1 self.labels[label] = None - # These global symbols are not directly labels pointing to the - # code location because such global labels in the middle of - # functions confuse gdb. Instead, we add to the global symbol's - # value a big constant, which is subtracted again when we need - # the original value for gcmaptable.s. That's a hack. - self.lines.insert(call.lineno+1, '%s=.+%d\n' % (label, - OFFSET_LABELS)) - self.lines.insert(call.lineno+1, '\t.globl\t%s\n' % (label,)) + if self.format == 'msvc': + self.lines.insert(call.lineno+1, '%s::\n' % (label,)) + #self.lines.insert(call.lineno+1, 'PUBLIC\t%s\n' % (label,)) + else: + # These global symbols are not directly labels pointing to the + # code location because such global labels in the middle of + # functions confuse gdb. Instead, we add to the global symbol's + # value a big constant, which is subtracted again when we need + # the original value for gcmaptable.s. That's a hack. + self.lines.insert(call.lineno+1, '%s=.+%d\n' % (label, + OFFSET_LABELS)) + self.lines.insert(call.lineno+1, '\t.globl\t%s\n' % (label,)) call.global_label = label # ____________________________________________________________ Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py Mon Oct 19 23:48:31 2009 @@ -503,9 +503,9 @@ lblofiles = ['%s.lbl.obj' % (cfile[:-2],) for cfile in mk.cfiles] mk.definition('ASMLBLOBJFILES', lblofiles) mk.definition('OBJECTS', '$(ASMLBLOBJFILES) gcmaptable.obj') - mk.rule('.SUFFIXES', '.s') + mk.rule('.SUFFIXES', '.s', []) mk.rule('.s.obj', '', - 'c:\masm32\bin\ml $(CFLAGS) /Zm /coff /Fo$@ /c $< $(INCLUDEDIRS)') + 'ml $(CFLAGS) /Zm /coff /Fo$@ /c $< $(INCLUDEDIRS)') mk.rule('.c.gcmap', '', ['$(CC) $(CFLAGS) /c /FAs /Fa$*.s $< $(INCLUDEDIRS)', 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -t $*.s > $@'] From afa at codespeak.net Tue Oct 20 01:17:02 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 20 Oct 2009 01:17:02 +0200 (CEST) Subject: [pypy-svn] r68654 - in pypy/branch/msvc-asmgcroot/pypy/translator/c: . gcc Message-ID: <20091019231702.1D577168006@codespeak.net> Author: afa Date: Tue Oct 20 01:17:01 2009 New Revision: 68654 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py Log: More progress, test program is almost compiling, two remaining problems: - nmake does not delay macro expansion, ASMLBLOBJFILES has to be defined before OBJECTS - the insertion of labels causes some "jmp SHORT" to become LONG jumps And of course, even after this the resulting executable does not work at all. Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Tue Oct 20 01:17:01 2009 @@ -326,7 +326,7 @@ self.labels[label] = None if self.format == 'msvc': self.lines.insert(call.lineno+1, '%s::\n' % (label,)) - #self.lines.insert(call.lineno+1, 'PUBLIC\t%s\n' % (label,)) + self.lines.insert(call.lineno+1, 'PUBLIC\t%s\n' % (label,)) else: # These global symbols are not directly labels pointing to the # code location because such global labels in the middle of @@ -750,10 +750,24 @@ for in_function, lines in self.find_functions(iterlines): if in_function: lines = self.process_function(lines, entrypoint, filename) - newfile.writelines(lines) + self.write_newfile(newfile, lines, filename.split('.')[0]) if self.verbose == 1: sys.stderr.write('\n') + def write_newfile(self, newfile, lines, grist): + # Workaround a bug in the .s files generated by msvc compiler: every + # float constant is exported with a name correcsponding to its value, + # and will conflict with other modules. + if self.format == 'msvc': + newlines = [] + for line in lines: + newlines.append( + line.replace('__real@', + '__%s__real@' % grist)) + lines = newlines + + newfile.writelines(lines) + def process_function(self, lines, entrypoint, filename): tracker = self.FunctionGcRootTracker( lines, filetag=getidentifier(filename)) @@ -1136,7 +1150,7 @@ for in_function, lines in parser.find_functions(iterlines): if in_function: lines = parser.process_function(lines, entrypoint, filename) - newfile.writelines(lines) + parser.write_newfile(newfile, lines, filename.split('.')[0]) if self.verbose == 1: sys.stderr.write('\n') if self.shuffle and random.random() < 0.5: @@ -1336,7 +1350,7 @@ tracker.reload_raw_table(f) f.close() else: - assert fn.endswith('.s') + assert fn.endswith('.s'), fn lblfn = fn[:-2] + '.lbl.s' g = open(lblfn, 'w') try: Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py Tue Oct 20 01:17:01 2009 @@ -505,7 +505,7 @@ mk.definition('OBJECTS', '$(ASMLBLOBJFILES) gcmaptable.obj') mk.rule('.SUFFIXES', '.s', []) mk.rule('.s.obj', '', - 'ml $(CFLAGS) /Zm /coff /Fo$@ /c $< $(INCLUDEDIRS)') + 'ml /Cx /Zm /coff /Fo$@ /c $< $(INCLUDEDIRS)') mk.rule('.c.gcmap', '', ['$(CC) $(CFLAGS) /c /FAs /Fa$*.s $< $(INCLUDEDIRS)', 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -t $*.s > $@'] From afa at codespeak.net Tue Oct 20 10:17:47 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 20 Oct 2009 10:17:47 +0200 (CEST) Subject: [pypy-svn] r68655 - in pypy/branch/msvc-asmgcroot/pypy/translator: c c/gcc/test platform Message-ID: <20091020081747.C352416800D@codespeak.net> Author: afa Date: Tue Oct 20 10:17:45 2009 New Revision: 68655 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py pypy/branch/msvc-asmgcroot/pypy/translator/platform/windows.py Log: More progress: fix the Makefile, which works now. One last compilation issue: the invalid "jmp NEAR" Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py Tue Oct 20 10:17:45 2009 @@ -121,10 +121,14 @@ for format, _, path in tests: yield check_computegcmaptable, format, path -r_globallabel = re.compile(r"([\w]+)=[.]+") + r_expected = re.compile(r"\s*;;\s*expected\s+([{].+[}])") def check_computegcmaptable(format, path): + if format == 'msvc': + r_globallabel = re.compile(r"([\w]+)::") + else: + r_globallabel = re.compile(r"([\w]+)=[.]+") print print path.dirpath().basename + '/' + path.basename lines = path.readlines() @@ -149,10 +153,16 @@ got = tabledict[label] assert format_callshape(got) == expected seen[label] = True - expectedlines.insert(i-2, '\t.globl\t%s\n' % (label,)) - expectedlines.insert(i-1, '%s=.+%d\n' % (label, OFFSET_LABELS)) + if format == 'msvc': + expectedlines.insert(i-2, 'PUBLIC\t%s\n' % (label,)) + expectedlines.insert(i-1, '%s::\n' % (label,)) + else: + expectedlines.insert(i-2, '\t.globl\t%s\n' % (label,)) + expectedlines.insert(i-1, '%s=.+%d\n' % (label, OFFSET_LABELS)) prevline = line assert len(seen) == len(tabledict), ( "computed table contains unexpected entries:\n%r" % [key for key in tabledict if key not in seen]) + print lines + print expectedlines assert lines == expectedlines Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py Tue Oct 20 10:17:45 2009 @@ -502,12 +502,12 @@ if self.translator.platform.name == 'msvc': lblofiles = ['%s.lbl.obj' % (cfile[:-2],) for cfile in mk.cfiles] mk.definition('ASMLBLOBJFILES', lblofiles) - mk.definition('OBJECTS', '$(ASMLBLOBJFILES) gcmaptable.obj') + mk.definition('OBJECTS', 'gcmaptable.obj $(ASMLBLOBJFILES)') mk.rule('.SUFFIXES', '.s', []) mk.rule('.s.obj', '', - 'ml /Cx /Zm /coff /Fo$@ /c $< $(INCLUDEDIRS)') + 'cmd /c $(MASM) /nologo /Cx /Zm /coff /Fo$@ /c $< $(INCLUDEDIRS)') mk.rule('.c.gcmap', '', - ['$(CC) $(CFLAGS) /c /FAs /Fa$*.s $< $(INCLUDEDIRS)', + ['$(CC) /nologo $(CFLAGS) /c /FAs /Fa$*.s $< $(INCLUDEDIRS)', 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -t $*.s > $@'] ) mk.rule('gcmaptable.s', '$(GCMAPFILES)', Modified: pypy/branch/msvc-asmgcroot/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/platform/windows.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/platform/windows.py Tue Oct 20 10:17:45 2009 @@ -96,6 +96,15 @@ # Probably not a msvc compiler... self.version = 0 + # Try to find a masm assembler + returncode, stdout, stderr = _run_subprocess('ml.exe', '', + env=self.c_environ) + r = re.search('Macro Assembler', stderr) + if r is None and os.path.exists('c:/masm32/bin/ml.exe'): + self.masm = 'c:/masm32/bin/ml.exe' + else: + self.masm = 'ml.exe' + # Install debug options only when interpreter is in debug mode if sys.executable.lower().endswith('_d.exe'): self.cflags = ['/MDd', '/Z7', '/Od'] @@ -177,10 +186,8 @@ errorfile = outname.new(ext='errors') errorfile.write(stderr) stderrlines = stderr.splitlines() - for line in stderrlines[:5]: + for line in stderrlines: log.ERROR(line) - if len(stderrlines) > 5: - log.ERROR('...') raise CompilationError(stdout, stderr) @@ -225,15 +232,17 @@ ('INCLUDEDIRS', self._includedirs(rel_includedirs)), ('CFLAGS', self.cflags + list(eci.compile_extra)), ('LDFLAGS', self.link_flags + list(eci.link_extra)), - ('CC', self.cc) + ('CC', self.cc), + ('CC_LINK', self.link), + ('MASM', self.masm), ] for args in definitions: m.definition(*args) rules = [ ('all', '$(DEFAULT_TARGET)', []), - ('$(TARGET)', '$(OBJECTS)', '$(CC) $(LDFLAGS) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS)'), - ('%.obj', '%.c', '$(CC) $(CFLAGS) -o $@ -c $< $(INCLUDEDIRS)'), + ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) /nologo $(LDFLAGS) $(OBJECTS) /out:$@ $(LIBDIRS) $(LIBS)'), + ('.c.o', '', '$(CC) $(CFLAGS) /Fo$@ /c $< $(INCLUDEDIRS)'), ] for rule in rules: @@ -251,14 +260,29 @@ try: returncode, stdout, stderr = _run_subprocess( 'nmake', - ['/f', str(path.join('Makefile'))]) + ['/nologo', '/f', str(path.join('Makefile'))]) finally: oldcwd.chdir() self._handle_error(returncode, stdout, stderr, path.join('make')) class NMakefile(posix.GnuMakefile): - pass # for the moment + def write(self, out=None): + # nmake expands macros when it parses rules. + # Write all macros before the rules. + if out is None: + f = self.makefile_dir.join('Makefile').open('w') + else: + f = out + for line in self.lines: + if not isinstance(line, posix.Rule): + line.write(f) + for line in self.lines: + if isinstance(line, posix.Rule): + line.write(f) + f.flush() + if out is None: + f.close() class MingwPlatform(posix.BasePosix): From afa at codespeak.net Tue Oct 20 10:30:12 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 20 Oct 2009 10:30:12 +0200 (CEST) Subject: [pypy-svn] r68656 - pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc Message-ID: <20091020083012.305C8168016@codespeak.net> Author: afa Date: Tue Oct 20 10:30:11 2009 New Revision: 68656 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Log: Now the test program compiles! Next step: make it work Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Tue Oct 20 10:30:11 2009 @@ -755,17 +755,6 @@ sys.stderr.write('\n') def write_newfile(self, newfile, lines, grist): - # Workaround a bug in the .s files generated by msvc compiler: every - # float constant is exported with a name correcsponding to its value, - # and will conflict with other modules. - if self.format == 'msvc': - newlines = [] - for line in lines: - newlines.append( - line.replace('__real@', - '__%s__real@' % grist)) - lines = newlines - newfile.writelines(lines) def process_function(self, lines, entrypoint, filename): @@ -902,6 +891,26 @@ return super(MsvcAssemblerParser, self).process_function( lines, entrypoint, filename) + def write_newfile(self, newfile, lines, grist): + newlines = [] + for line in lines: + # Workaround a bug in the .s files generated by msvc + # compiler: every float constant is exported with a name + # correcponding to its value, and will conflict with other + # modules. + line = line.replace('__real@', + '__%s__real@' % grist) + # Because we insert labels in the code, some "SHORT" jumps + # are now longer than 127 bytes. We turn them all into + # "NEAR" jumps. Note that the assembler allocates space + # for a near jump, but still generates a short jump when + # it can. + line = line.replace('\tjmp\tSHORT ', '\tjmp\t') + + newlines.append(line) + + newfile.writelines(newlines) + PARSERS = { 'elf': ElfAssemblerParser, 'darwin': DarwinAssemblerParser, From arigo at codespeak.net Tue Oct 20 11:53:32 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Oct 2009 11:53:32 +0200 (CEST) Subject: [pypy-svn] r68657 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091020095332.C99FC168005@codespeak.net> Author: arigo Date: Tue Oct 20 11:53:32 2009 New Revision: 68657 Modified: pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py pypy/trunk/pypy/jit/metainterp/warmspot.py Log: I would like to write "test and fix", but instead the fix is not necessary for the test to pass... This is due to the fact that equal strings are considered as identical both when run directly and on top of llinterp. I am not going to change that now... Modified: pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py Tue Oct 20 11:53:32 2009 @@ -1,21 +1,38 @@ import py from pypy.jit.metainterp.warmspot import ll_meta_interp, hash_whatever -from pypy.jit.metainterp.warmspot import get_stats +from pypy.jit.metainterp.warmspot import get_stats, equal_whatever from pypy.rlib.jit import JitDriver, OPTIMIZER_FULL, OPTIMIZER_SIMPLE from pypy.rlib.jit import unroll_safe from pypy.jit.backend.llgraph import runner +from pypy.rpython.test.test_llinterp import interpret from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin -def test_translate_hash_whatever(): - from pypy.rpython.test.test_llinterp import interpret - from pypy.rpython.lltypesystem import lltype +def test_hash_equal_whatever_lltype(): + from pypy.rpython.lltypesystem import lltype, rstr + s1 = rstr.mallocstr(2) + s2 = rstr.mallocstr(2) + s1.chars[0] = 'x'; s1.chars[1] = 'y' + s2.chars[0] = 'x'; s2.chars[1] = 'y' def fn(x): - return hash_whatever(lltype.typeOf(x), x) - for type_system in ('lltype', 'ootype'): - res = interpret(fn, [42], type_system=type_system) - assert res == 42 + assert hash_whatever(lltype.typeOf(x), x) == 42 + assert (hash_whatever(lltype.typeOf(s1), s1) == + hash_whatever(lltype.typeOf(s2), s2)) + assert equal_whatever(lltype.typeOf(s1), s1, s2) + fn(42) + interpret(fn, [42], type_system='lltype') + +def test_hash_equal_whatever_ootype(): + from pypy.rpython.ootypesystem import ootype + def fn(c): + s1 = ootype.oostring("xy", -1) + s2 = ootype.oostring("x" + chr(c), -1) + assert (hash_whatever(ootype.typeOf(s1), s1) == + hash_whatever(ootype.typeOf(s2), s2)) + assert equal_whatever(ootype.typeOf(s1), s1, s2) + fn(ord('y')) + interpret(fn, [ord('y')], type_system='ootype') class Exit(Exception): def __init__(self, result): Modified: pypy/trunk/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmspot.py Tue Oct 20 11:53:32 2009 @@ -634,6 +634,8 @@ if isinstance(TYPE, lltype.Ptr): if TYPE.TO is rstr.STR or TYPE.TO is rstr.UNICODE: return rstr.LLHelpers.ll_streq(x, y) + if TYPE is ootype.String or TYPE is ootype.Unicode: + return x.ll_streq(y) return x == y equal_whatever._annspecialcase_ = 'specialize:arg(0)' From arigo at codespeak.net Tue Oct 20 11:55:57 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Oct 2009 11:55:57 +0200 (CEST) Subject: [pypy-svn] r68658 - pypy/branch/warmspot-jitinfo Message-ID: <20091020095557.E2535168014@codespeak.net> Author: arigo Date: Tue Oct 20 11:55:52 2009 New Revision: 68658 Added: pypy/branch/warmspot-jitinfo/ - copied from r68657, pypy/trunk/ Log: A branch in which we remove the quasi-hash-table code of warmspot.py and replace it by calls to jitdriver. From arigo at codespeak.net Tue Oct 20 12:39:09 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Oct 2009 12:39:09 +0200 (CEST) Subject: [pypy-svn] r68659 - in pypy/branch/warmspot-jitinfo/pypy/jit/metainterp: . test Message-ID: <20091020103909.70226168005@codespeak.net> Author: arigo Date: Tue Oct 20 12:39:09 2009 New Revision: 68659 Added: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmstate.py (contents, props changed) pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmstate.py - copied, changed from r68658, pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmspot.py Log: Start to move the WarmEnterState class to its own file. Added: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmstate.py ============================================================================== --- (empty file) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmstate.py Tue Oct 20 12:39:09 2009 @@ -0,0 +1,54 @@ +from pypy.rpython.test.test_llinterp import interpret +from pypy.rpython.lltypesystem import lltype, llmemory, rstr +from pypy.rpython.ootypesystem import ootype +from pypy.jit.metainterp.warmstate import wrap, unwrap +from pypy.jit.metainterp.warmstate import equal_whatever, hash_whatever +from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr +from pypy.jit.metainterp.history import ConstInt, ConstFloat, ConstPtr + + +def test_unwrap(): + S = lltype.GcStruct('S') + p = lltype.malloc(S) + po = lltype.cast_opaque_ptr(llmemory.GCREF, p) + assert unwrap(lltype.Void, BoxInt(42)) is None + assert unwrap(lltype.Signed, BoxInt(42)) == 42 + assert unwrap(lltype.Char, BoxInt(42)) == chr(42) + assert unwrap(lltype.Float, BoxFloat(42.5)) == 42.5 + assert unwrap(lltype.Ptr(S), BoxPtr(po)) == p + +def test_wrap(): + def _is(box1, box2): + return (box1.__class__ == box2.__class__ and + box1.value == box2.value) + p = lltype.malloc(lltype.GcStruct('S')) + po = lltype.cast_opaque_ptr(llmemory.GCREF, p) + assert _is(wrap(None, 42), BoxInt(42)) + assert _is(wrap(None, 42.5), BoxFloat(42.5)) + assert _is(wrap(None, p), BoxPtr(po)) + assert _is(wrap(None, 42, in_const_box=True), ConstInt(42)) + assert _is(wrap(None, 42.5, in_const_box=True), ConstFloat(42.5)) + assert _is(wrap(None, p, in_const_box=True), ConstPtr(po)) + +def test_hash_equal_whatever_lltype(): + s1 = rstr.mallocstr(2) + s2 = rstr.mallocstr(2) + s1.chars[0] = 'x'; s1.chars[1] = 'y' + s2.chars[0] = 'x'; s2.chars[1] = 'y' + def fn(x): + assert hash_whatever(lltype.typeOf(x), x) == 42 + assert (hash_whatever(lltype.typeOf(s1), s1) == + hash_whatever(lltype.typeOf(s2), s2)) + assert equal_whatever(lltype.typeOf(s1), s1, s2) + fn(42) + interpret(fn, [42], type_system='lltype') + +def test_hash_equal_whatever_ootype(): + def fn(c): + s1 = ootype.oostring("xy", -1) + s2 = ootype.oostring("x" + chr(c), -1) + assert (hash_whatever(ootype.typeOf(s1), s1) == + hash_whatever(ootype.typeOf(s2), s2)) + assert equal_whatever(ootype.typeOf(s1), s1, s2) + fn(ord('y')) + interpret(fn, [ord('y')], type_system='ootype') Copied: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmstate.py (from r68658, pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmspot.py) ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmstate.py Tue Oct 20 12:39:09 2009 @@ -1,591 +1,13 @@ -import sys -from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr -from pypy.rpython.ootypesystem import ootype -from pypy.rpython.annlowlevel import llhelper, MixLevelHelperAnnotator,\ - cast_base_ptr_to_instance, hlstr -from pypy.annotation import model as annmodel -from pypy.rpython.llinterp import LLException -from pypy.rpython.test.test_llinterp import get_interpreter, clear_tcache -from pypy.objspace.flow.model import SpaceOperation, Variable, Constant -from pypy.objspace.flow.model import checkgraph, Link, copygraph -from pypy.rlib.objectmodel import we_are_translated -from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib.jit import PARAMETERS, OPTIMIZER_SIMPLE, OPTIMIZER_FULL -from pypy.rlib.rarithmetic import r_uint, intmask -from pypy.rlib.debug import debug_print -from pypy.rpython.lltypesystem.lloperation import llop -from pypy.translator.simplify import get_funcobj, get_functype -from pypy.translator.unsimplify import call_final_function - -from pypy.jit.metainterp import codewriter -from pypy.jit.metainterp import support, history, pyjitpl, gc -from pypy.jit.metainterp.pyjitpl import MetaInterpStaticData, MetaInterp -from pypy.jit.metainterp.policy import JitPolicy -from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper -from pypy.jit.metainterp.jitprof import Profiler, EmptyProfiler -from pypy.rlib.jit import DEBUG_STEPS, DEBUG_DETAILED, DEBUG_OFF, DEBUG_PROFILE -from pypy.rlib.nonconst import NonConstant - -# ____________________________________________________________ -# Bootstrapping - -def apply_jit(translator, backend_name="auto", debug_level=DEBUG_STEPS, - inline=False, - **kwds): - if 'CPUClass' not in kwds: - from pypy.jit.backend.detect_cpu import getcpuclass - kwds['CPUClass'] = getcpuclass(backend_name) - if debug_level > DEBUG_OFF: - ProfilerClass = Profiler - else: - ProfilerClass = EmptyProfiler - warmrunnerdesc = WarmRunnerDesc(translator, - translate_support_code=True, - listops=True, - no_stats = True, - ProfilerClass = ProfilerClass, - **kwds) - warmrunnerdesc.state.set_param_inlining(inline) - warmrunnerdesc.state.set_param_debug(debug_level) - warmrunnerdesc.finish() - translator.warmrunnerdesc = warmrunnerdesc # for later debugging - -def ll_meta_interp(function, args, backendopt=False, type_system='lltype', - listcomp=False, **kwds): - if listcomp: - extraconfigopts = {'translation.list_comprehension_operations': True} - else: - extraconfigopts = {} - interp, graph = get_interpreter(function, args, - backendopt=False, # will be done below - type_system=type_system, - **extraconfigopts) - clear_tcache() - return jittify_and_run(interp, graph, args, backendopt=backendopt, **kwds) - -def jittify_and_run(interp, graph, args, repeat=1, hash_bits=None, - backendopt=False, trace_limit=sys.maxint, - debug_level=DEBUG_STEPS, inline=False, **kwds): - translator = interp.typer.annotator.translator - translator.config.translation.gc = "boehm" - warmrunnerdesc = WarmRunnerDesc(translator, backendopt=backendopt, **kwds) - warmrunnerdesc.state.set_param_threshold(3) # for tests - warmrunnerdesc.state.set_param_trace_eagerness(2) # for tests - warmrunnerdesc.state.set_param_trace_limit(trace_limit) - warmrunnerdesc.state.set_param_inlining(inline) - warmrunnerdesc.state.set_param_debug(debug_level) - warmrunnerdesc.state.create_tables_now() # for tests - if hash_bits: - warmrunnerdesc.state.set_param_hash_bits(hash_bits) - warmrunnerdesc.finish() - res = interp.eval_graph(graph, args) - if not kwds.get('translate_support_code', False): - warmrunnerdesc.metainterp_sd.profiler.finish() - print '~~~ return value:', res - while repeat > 1: - print '~' * 79 - res1 = interp.eval_graph(graph, args) - if isinstance(res, int): - assert res1 == res - repeat -= 1 - return res - -def rpython_ll_meta_interp(function, args, backendopt=True, - loops='not used right now', **kwds): - return ll_meta_interp(function, args, backendopt=backendopt, - translate_support_code=True, **kwds) - -def _find_jit_marker(graphs, marker_name): - results = [] - for graph in graphs: - for block in graph.iterblocks(): - for i in range(len(block.operations)): - op = block.operations[i] - if (op.opname == 'jit_marker' and - op.args[0].value == marker_name): - results.append((graph, block, i)) - return results - -def find_can_enter_jit(graphs): - results = _find_jit_marker(graphs, 'can_enter_jit') - if not results: - raise Exception("no can_enter_jit found!") - return results - -def find_jit_merge_point(graphs): - results = _find_jit_marker(graphs, 'jit_merge_point') - if len(results) != 1: - raise Exception("found %d jit_merge_points, need exactly one!" % - (len(results),)) - return results[0] -def find_set_param(graphs): - return _find_jit_marker(graphs, 'set_param') - -def get_stats(): - return pyjitpl._warmrunnerdesc.stats - -def get_translator(): - return pyjitpl._warmrunnerdesc.translator - -def debug_checks(): - stats = get_stats() - stats.maybe_view() - stats.check_consistency() - -class JitException(Exception): - _go_through_llinterp_uncaught_ = True # ugh - -class ContinueRunningNormallyBase(JitException): - pass - -class CannotInlineCanEnterJit(JitException): - pass +from pypy.rpython.lltypesystem import lltype, llmemory, rstr +from pypy.rpython.ootypesystem import ootype +from pypy.rlib.objectmodel import specialize +from pypy.rlib.rarithmetic import intmask +from pypy.jit.metainterp import history # ____________________________________________________________ -class WarmRunnerDesc: - - def __init__(self, translator, policy=None, backendopt=True, CPUClass=None, - optimizer=None, **kwds): - pyjitpl._warmrunnerdesc = self # this is a global for debugging only! - if policy is None: - policy = JitPolicy() - self.set_translator(translator) - self.find_portal() - self.make_leave_jit_graph() - self.codewriter = codewriter.CodeWriter(self.rtyper) - graphs = self.codewriter.find_all_graphs(self.portal_graph, - self.leave_graph, - policy, - CPUClass.supports_floats) - policy.dump_unsafe_loops() - self.check_access_directly_sanity(graphs) - if backendopt: - self.prejit_optimizations(policy, graphs) - - self.build_meta_interp(CPUClass, **kwds) - self.make_args_specification() - self.rewrite_jit_merge_point(policy) - self.make_driverhook_graph() - if self.jitdriver.virtualizables: - from pypy.jit.metainterp.virtualizable import VirtualizableInfo - self.metainterp_sd.virtualizable_info = VirtualizableInfo(self) - - self.codewriter.generate_bytecode(self.metainterp_sd, - self.portal_graph, - self.leave_graph, - self.portal_runner_ptr - ) - self.make_enter_function() - self.rewrite_can_enter_jit() - self.rewrite_set_param() - self.add_profiler_finish() - self.metainterp_sd.finish_setup(optimizer=optimizer) - - def finish(self): - vinfo = self.metainterp_sd.virtualizable_info - if vinfo is not None: - vinfo.finish() - if self.cpu.translate_support_code: - self.annhelper.finish() - - def _freeze_(self): - return True - - def set_translator(self, translator): - self.translator = translator - self.rtyper = translator.rtyper - self.gcdescr = gc.get_description(translator.config) - - def find_portal(self): - graphs = self.translator.graphs - self.jit_merge_point_pos = find_jit_merge_point(graphs) - graph, block, pos = self.jit_merge_point_pos - op = block.operations[pos] - args = op.args[2:] - s_binding = self.translator.annotator.binding - self.portal_args_s = [s_binding(v) for v in args] - graph = copygraph(graph) - graph.startblock.isstartblock = False - graph.startblock = support.split_before_jit_merge_point( - *find_jit_merge_point([graph])) - graph.startblock.isstartblock = True - # a crash in the following checkgraph() means that you forgot - # to list some variable in greens=[] or reds=[] in JitDriver. - checkgraph(graph) - for v in graph.getargs(): - assert isinstance(v, Variable) - assert len(dict.fromkeys(graph.getargs())) == len(graph.getargs()) - self.translator.graphs.append(graph) - self.portal_graph = graph - # it's a bit unbelievable to have a portal without func - assert hasattr(graph, "func") - graph.func._dont_inline_ = True - graph.func._jit_unroll_safe_ = True - self.jitdriver = block.operations[pos].args[1].value - - def check_access_directly_sanity(self, graphs): - from pypy.translator.backendopt.inline import collect_called_graphs - jit_graphs = set(graphs) - for graph in collect_called_graphs(self.translator.graphs[0], - self.translator): - if graph in jit_graphs: - continue - assert not getattr(graph, 'access_directly', False) - - def prejit_optimizations(self, policy, graphs): - from pypy.translator.backendopt.all import backend_optimizations - backend_optimizations(self.translator, - graphs=graphs, - merge_if_blocks=True, - constfold=True, - raisingop2direct_call=False, - remove_asserts=True, - really_remove_asserts=True) - - def build_meta_interp(self, CPUClass, translate_support_code=False, - view="auto", no_stats=False, - ProfilerClass=EmptyProfiler, **kwds): - assert CPUClass is not None - opt = history.Options(**kwds) - if no_stats: - stats = history.NoStats() - else: - stats = history.Stats() - self.stats = stats - if translate_support_code: - self.annhelper = MixLevelHelperAnnotator(self.translator.rtyper) - annhelper = self.annhelper - else: - annhelper = None - cpu = CPUClass(self.translator.rtyper, self.stats, - translate_support_code, gcdescr=self.gcdescr) - self.cpu = cpu - self.metainterp_sd = MetaInterpStaticData(self.portal_graph, # xxx - cpu, - self.stats, opt, - ProfilerClass=ProfilerClass, - warmrunnerdesc=self) - - def make_enter_function(self): - WarmEnterState = make_state_class(self) - state = WarmEnterState() - self.state = state - - def crash_in_jit(e): - if not we_are_translated(): - print "~~~ Crash in JIT!" - print '~~~ %s: %s' % (e.__class__, e) - if sys.stdout == sys.__stdout__: - import pdb; pdb.post_mortem(sys.exc_info()[2]) - raise - debug_print('~~~ Crash in JIT!') - debug_print('~~~ %s' % (e,)) - raise history.CrashInJIT("crash in JIT") - crash_in_jit._dont_inline_ = True - - if self.translator.rtyper.type_system.name == 'lltypesystem': - def maybe_enter_jit(*args): - try: - state.maybe_compile_and_run(*args) - except JitException: - raise # go through - except Exception, e: - crash_in_jit(e) - maybe_enter_jit._always_inline_ = True - else: - def maybe_enter_jit(*args): - state.maybe_compile_and_run(*args) - maybe_enter_jit._always_inline_ = True - - self.maybe_enter_jit_fn = maybe_enter_jit - - def make_leave_jit_graph(self): - self.leave_graph = None - if self.jitdriver.leave: - args_s = self.portal_args_s - from pypy.annotation import model as annmodel - annhelper = MixLevelHelperAnnotator(self.translator.rtyper) - s_result = annmodel.s_None - self.leave_graph = annhelper.getgraph(self.jitdriver.leave, - args_s, s_result) - annhelper.finish() - - def make_driverhook_graph(self): - self.can_inline_ptr = self._make_hook_graph( - self.jitdriver.can_inline, bool) - self.get_printable_location_ptr = self._make_hook_graph( - self.jitdriver.get_printable_location, str) - - def _make_hook_graph(self, func, rettype): - from pypy.annotation.signature import annotationoftype - if func is None: - return None - annhelper = MixLevelHelperAnnotator(self.translator.rtyper) - s_result = annotationoftype(rettype) - RETTYPE = annhelper.rtyper.getrepr(s_result).lowleveltype - FUNC, PTR = self.cpu.ts.get_FuncType(self.green_args_spec, RETTYPE) - args_s = self.portal_args_s[:len(self.green_args_spec)] - graph = annhelper.getgraph(func, args_s, s_result) - funcptr = annhelper.graph2delayed(graph, FUNC) - annhelper.finish() - return funcptr - - def make_args_specification(self): - graph, block, index = self.jit_merge_point_pos - op = block.operations[index] - args = op.args[2:] - ALLARGS = [] - self.green_args_spec = [] - self.red_args_types = [] - for i, v in enumerate(args): - TYPE = v.concretetype - ALLARGS.append(TYPE) - if i < len(self.jitdriver.greens): - self.green_args_spec.append(TYPE) - else: - self.red_args_types.append(history.getkind(TYPE)) - RESTYPE = graph.getreturnvar().concretetype - (self.JIT_ENTER_FUNCTYPE, - self.PTR_JIT_ENTER_FUNCTYPE) = self.cpu.ts.get_FuncType(ALLARGS, lltype.Void) - (self.PORTAL_FUNCTYPE, - self.PTR_PORTAL_FUNCTYPE) = self.cpu.ts.get_FuncType(ALLARGS, RESTYPE) - - - def rewrite_can_enter_jit(self): - FUNC = self.JIT_ENTER_FUNCTYPE - FUNCPTR = self.PTR_JIT_ENTER_FUNCTYPE - jit_enter_fnptr = self.helper_func(FUNCPTR, self.maybe_enter_jit_fn) - - graphs = self.translator.graphs - can_enter_jits = find_can_enter_jit(graphs) - for graph, block, index in can_enter_jits: - if graph is self.jit_merge_point_pos[0]: - continue - - op = block.operations[index] - greens_v, reds_v = decode_hp_hint_args(op) - args_v = greens_v + reds_v - - vlist = [Constant(jit_enter_fnptr, FUNCPTR)] + args_v - - v_result = Variable() - v_result.concretetype = lltype.Void - newop = SpaceOperation('direct_call', vlist, v_result) - block.operations[index] = newop - - def helper_func(self, FUNCPTR, func): - if not self.cpu.translate_support_code: - return llhelper(FUNCPTR, func) - FUNC = get_functype(FUNCPTR) - args_s = [annmodel.lltype_to_annotation(ARG) for ARG in FUNC.ARGS] - s_result = annmodel.lltype_to_annotation(FUNC.RESULT) - graph = self.annhelper.getgraph(func, args_s, s_result) - return self.annhelper.graph2delayed(graph, FUNC) - - def rewrite_jit_merge_point(self, policy): - # - # Mutate the original portal graph from this: - # - # def original_portal(..): - # stuff - # while 1: - # jit_merge_point(*args) - # more stuff - # - # to that: - # - # def original_portal(..): - # stuff - # return portal_runner(*args) - # - # def portal_runner(*args): - # while 1: - # try: - # return portal(*args) - # except ContinueRunningNormally, e: - # *args = *e.new_args - # except DoneWithThisFrame, e: - # return e.return - # except ExitFrameWithException, e: - # raise Exception, e.value - # - # def portal(*args): - # while 1: - # more stuff - # - origportalgraph = self.jit_merge_point_pos[0] - portalgraph = self.portal_graph - PORTALFUNC = self.PORTAL_FUNCTYPE - - # ____________________________________________________________ - # Prepare the portal_runner() helper - # - portal_ptr = self.cpu.ts.functionptr(PORTALFUNC, 'portal', - graph = portalgraph) - self.portal_ptr = portal_ptr - portalfunc_ARGS = unrolling_iterable( - [(i, 'arg%d' % i, ARG) for i, ARG in enumerate(PORTALFUNC.ARGS)]) - - class DoneWithThisFrameVoid(JitException): - def __str__(self): - return 'DoneWithThisFrameVoid()' - - class DoneWithThisFrameInt(JitException): - def __init__(self, result): - assert lltype.typeOf(result) is lltype.Signed - self.result = result - def __str__(self): - return 'DoneWithThisFrameInt(%s)' % (self.result,) - - class DoneWithThisFrameRef(JitException): - def __init__(self, cpu, result): - assert lltype.typeOf(result) == cpu.ts.BASETYPE - self.result = result - def __str__(self): - return 'DoneWithThisFrameRef(%s)' % (self.result,) - - class DoneWithThisFrameFloat(JitException): - def __init__(self, result): - assert lltype.typeOf(result) is lltype.Float - self.result = result - def __str__(self): - return 'DoneWithThisFrameFloat(%s)' % (self.result,) - - class ExitFrameWithExceptionRef(JitException): - def __init__(self, cpu, value): - assert lltype.typeOf(value) == cpu.ts.BASETYPE - self.value = value - def __str__(self): - return 'ExitFrameWithExceptionRef(%s)' % (self.value,) - - class ContinueRunningNormally(ContinueRunningNormallyBase): - def __init__(self, argboxes): - # accepts boxes as argument, but unpacks them immediately - # before we raise the exception -- the boxes' values will - # be modified in a 'finally' by restore_patched_boxes(). - for i, name, ARG in portalfunc_ARGS: - v = unwrap(ARG, argboxes[i]) - setattr(self, name, v) - - def __str__(self): - return 'ContinueRunningNormally(%s)' % ( - ', '.join(map(str, self.args)),) - - self.DoneWithThisFrameVoid = DoneWithThisFrameVoid - self.DoneWithThisFrameInt = DoneWithThisFrameInt - self.DoneWithThisFrameRef = DoneWithThisFrameRef - self.DoneWithThisFrameFloat = DoneWithThisFrameFloat - self.ExitFrameWithExceptionRef = ExitFrameWithExceptionRef - self.ContinueRunningNormally = ContinueRunningNormally - self.metainterp_sd.DoneWithThisFrameVoid = DoneWithThisFrameVoid - self.metainterp_sd.DoneWithThisFrameInt = DoneWithThisFrameInt - self.metainterp_sd.DoneWithThisFrameRef = DoneWithThisFrameRef - self.metainterp_sd.DoneWithThisFrameFloat = DoneWithThisFrameFloat - self.metainterp_sd.ExitFrameWithExceptionRef = ExitFrameWithExceptionRef - self.metainterp_sd.ContinueRunningNormally = ContinueRunningNormally - rtyper = self.translator.rtyper - RESULT = PORTALFUNC.RESULT - result_kind = history.getkind(RESULT) - ts = self.cpu.ts - - def ll_portal_runner(*args): - while 1: - try: - return support.maybe_on_top_of_llinterp(rtyper, - portal_ptr)(*args) - except ContinueRunningNormally, e: - args = () - for _, name, _ in portalfunc_ARGS: - v = getattr(e, name) - args = args + (v,) - except DoneWithThisFrameVoid: - assert result_kind == 'void' - return - except DoneWithThisFrameInt, e: - assert result_kind == 'int' - return lltype.cast_primitive(RESULT, e.result) - except DoneWithThisFrameRef, e: - assert result_kind == 'ref' - return ts.cast_from_ref(RESULT, e.result) - except DoneWithThisFrameFloat, e: - assert result_kind == 'float' - return e.result - except ExitFrameWithExceptionRef, e: - value = ts.cast_to_baseclass(e.value) - if not we_are_translated(): - raise LLException(ts.get_typeptr(value), value) - else: - value = cast_base_ptr_to_instance(Exception, value) - raise Exception, value - - self.portal_runner_ptr = self.helper_func(self.PTR_PORTAL_FUNCTYPE, - ll_portal_runner) - - # ____________________________________________________________ - # Now mutate origportalgraph to end with a call to portal_runner_ptr - # - _, origblock, origindex = self.jit_merge_point_pos - op = origblock.operations[origindex] - assert op.opname == 'jit_marker' - assert op.args[0].value == 'jit_merge_point' - greens_v, reds_v = decode_hp_hint_args(op) - vlist = [Constant(self.portal_runner_ptr, self.PTR_PORTAL_FUNCTYPE)] - vlist += greens_v - vlist += reds_v - v_result = Variable() - v_result.concretetype = PORTALFUNC.RESULT - newop = SpaceOperation('direct_call', vlist, v_result) - del origblock.operations[origindex:] - origblock.operations.append(newop) - origblock.exitswitch = None - origblock.recloseblock(Link([v_result], origportalgraph.returnblock)) - checkgraph(origportalgraph) - - def add_profiler_finish(self): - def finish_profiler(): - if self.metainterp_sd.profiler.initialized: - self.metainterp_sd.profiler.finish() - - if self.cpu.translate_support_code: - call_final_function(self.translator, finish_profiler, - annhelper = self.annhelper) - - def rewrite_set_param(self): - closures = {} - graphs = self.translator.graphs - _, PTR_SET_PARAM_FUNCTYPE = self.cpu.ts.get_FuncType([lltype.Signed], - lltype.Void) - def make_closure(fullfuncname): - state = self.state - def closure(i): - getattr(state, fullfuncname)(i) - funcptr = self.helper_func(PTR_SET_PARAM_FUNCTYPE, closure) - return Constant(funcptr, PTR_SET_PARAM_FUNCTYPE) - # - for graph, block, i in find_set_param(graphs): - op = block.operations[i] - assert op.args[1].value == self.jitdriver - funcname = op.args[2].value - if funcname not in closures: - closures[funcname] = make_closure('set_param_' + funcname) - op.opname = 'direct_call' - op.args[:3] = [closures[funcname]] - - -def decode_hp_hint_args(op): - # Returns (list-of-green-vars, list-of-red-vars) without Voids. - assert op.opname == 'jit_marker' - jitdriver = op.args[1].value - numgreens = len(jitdriver.greens) - numreds = len(jitdriver.reds) - greens_v = op.args[2:2+numgreens] - reds_v = op.args[2+numgreens:] - assert len(reds_v) == numreds - return ([v for v in greens_v if v.concretetype is not lltype.Void], - [v for v in reds_v if v.concretetype is not lltype.Void]) - + at specialize.arg(0) def unwrap(TYPE, box): if TYPE is lltype.Void: return None @@ -597,8 +19,8 @@ return box.getfloat() else: return lltype.cast_primitive(TYPE, box.getint()) -unwrap._annspecialcase_ = 'specialize:arg(0)' + at specialize.ll() def wrap(cpu, value, in_const_box=False): if isinstance(lltype.typeOf(value), lltype.Ptr): if lltype.typeOf(value).TO._gckind == 'gc': @@ -628,8 +50,8 @@ return history.ConstInt(value) else: return history.BoxInt(value) -wrap._annspecialcase_ = 'specialize:ll' + at specialize.arg(0) def equal_whatever(TYPE, x, y): if isinstance(TYPE, lltype.Ptr): if TYPE.TO is rstr.STR or TYPE.TO is rstr.UNICODE: @@ -637,8 +59,8 @@ if TYPE is ootype.String or TYPE is ootype.Unicode: return x.ll_streq(y) return x == y -equal_whatever._annspecialcase_ = 'specialize:arg(0)' + at specialize.arg(0) def hash_whatever(TYPE, x): # Hash of lltype or ootype object. # Only supports strings, unicodes and regular instances, @@ -660,317 +82,3 @@ return 0 else: return lltype.cast_primitive(lltype.Signed, x) -hash_whatever._annspecialcase_ = 'specialize:arg(0)' - -# ____________________________________________________________ - -def make_state_class(warmrunnerdesc): - jitdriver = warmrunnerdesc.jitdriver - num_green_args = len(jitdriver.greens) - warmrunnerdesc.num_green_args = num_green_args - green_args_spec = unrolling_iterable(warmrunnerdesc.green_args_spec) - green_args_names = unrolling_iterable(jitdriver.greens) - green_args_spec_names = unrolling_iterable(zip( - warmrunnerdesc.green_args_spec, jitdriver.greens)) - red_args_types = unrolling_iterable(warmrunnerdesc.red_args_types) - # - metainterp_sd = warmrunnerdesc.metainterp_sd - vinfo = metainterp_sd.virtualizable_info - if vinfo is None: - vable_static_fields = [] - vable_array_fields = [] - else: - vable_static_fields = unrolling_iterable( - zip(vinfo.static_extra_types, vinfo.static_fields)) - vable_array_fields = unrolling_iterable( - zip(vinfo.arrayitem_extra_types, vinfo.array_fields)) - # - if num_green_args: - MAX_HASH_TABLE_BITS = 28 - else: - MAX_HASH_TABLE_BITS = 1 - THRESHOLD_LIMIT = sys.maxint // 2 - # - getlength = warmrunnerdesc.cpu.ts.getlength - getarrayitem = warmrunnerdesc.cpu.ts.getarrayitem - setarrayitem = warmrunnerdesc.cpu.ts.setarrayitem - # - rtyper = warmrunnerdesc.translator.rtyper - can_inline_ptr = warmrunnerdesc.can_inline_ptr - get_printable_location_ptr = warmrunnerdesc.get_printable_location_ptr - # - class MachineCodeEntryPoint(object): - next = None # linked list - def __init__(self, entry_loop_token, *greenargs): - self.entry_loop_token = entry_loop_token - i = 0 - for name in green_args_names: - setattr(self, 'green_' + name, greenargs[i]) - i = i + 1 - def equalkey(self, *greenargs): - i = 0 - for TYPE, name in green_args_spec_names: - myvalue = getattr(self, 'green_' + name) - if not equal_whatever(TYPE, myvalue, greenargs[i]): - return False - i = i + 1 - return True - def set_future_values(self, *redargs): - i = 0 - for typecode in red_args_types: - set_future_value(i, redargs[i], typecode) - i = i + 1 - if vinfo is not None: - virtualizable = redargs[vinfo.index_of_virtualizable - - num_green_args] - virtualizable = vinfo.cast_to_vtype(virtualizable) - for typecode, fieldname in vable_static_fields: - x = getattr(virtualizable, fieldname) - set_future_value(i, x, typecode) - i = i + 1 - for typecode, fieldname in vable_array_fields: - lst = getattr(virtualizable, fieldname) - for j in range(getlength(lst)): - x = getarrayitem(lst, j) - set_future_value(i, x, typecode) - i = i + 1 - - def set_future_value(j, value, typecode): - cpu = metainterp_sd.cpu - if typecode == 'ref': - refvalue = cpu.ts.cast_to_ref(value) - cpu.set_future_value_ref(j, refvalue) - elif typecode == 'int': - intvalue = lltype.cast_primitive(lltype.Signed, value) - cpu.set_future_value_int(j, intvalue) - elif typecode == 'float': - assert isinstance(value, float) - cpu.set_future_value_float(j, value) - else: - assert False - set_future_value._annspecialcase_ = 'specialize:ll_and_arg(2)' - - class WarmEnterState: - def __init__(self): - # initialize the state with the default values of the - # parameters specified in rlib/jit.py - for name, default_value in PARAMETERS.items(): - meth = getattr(self, 'set_param_' + name) - meth(default_value) - - def set_param_threshold(self, threshold): - if threshold < 2: - threshold = 2 - self.increment_threshold = (THRESHOLD_LIMIT // threshold) + 1 - # the number is at least 1, and at most about half THRESHOLD_LIMIT - - def set_param_trace_eagerness(self, value): - self.trace_eagerness = value - - def set_param_trace_limit(self, value): - self.trace_limit = value - - def set_param_inlining(self, value): - self.inlining = value - - def set_param_hash_bits(self, value): - if value < 1: - value = 1 - elif value > MAX_HASH_TABLE_BITS: - value = MAX_HASH_TABLE_BITS - # the tables are initialized with the correct size only in - # create_tables_now() - self.hashbits = value - self.hashtablemask = 0 - self.mccounters = [0] - self.mcentrypoints = [None] - # invariant: (self.mccounters[j] < 0) if and only if - # (self.mcentrypoints[j] is not None) - - def set_param_optimizer(self, optimizer): - if optimizer == OPTIMIZER_SIMPLE: - from pypy.jit.metainterp import simple_optimize - self.optimize_loop = simple_optimize.optimize_loop - self.optimize_bridge = simple_optimize.optimize_bridge - elif optimizer == OPTIMIZER_FULL: - from pypy.jit.metainterp import optimize - self.optimize_loop = optimize.optimize_loop - self.optimize_bridge = optimize.optimize_bridge - else: - raise ValueError("unknown optimizer") - - def set_param_debug(self, value): - self.debug_level = value - metainterp_sd.profiler.set_printing(value >= DEBUG_PROFILE) - - def create_tables_now(self): - count = 1 << self.hashbits - self.hashtablemask = count - 1 - self.mccounters = [0] * count - self.mcentrypoints = [None] * count - - # Only use the hash of the arguments as the profiling key. - # Indeed, this is all a heuristic, so if things are designed - # correctly, the occasional mistake due to hash collision is - # not too bad. - - def maybe_compile_and_run(self, *args): - globaldata = metainterp_sd.globaldata - if NonConstant(False): - # make sure we always see the saner optimizer from an annotation - # point of view, otherwise we get lots of blocked ops - self.set_param_optimizer(OPTIMIZER_FULL) - - # get the greenargs and look for the cell corresponding to the hash - greenargs = args[:num_green_args] - argshash = self.getkeyhash(*greenargs) & self.hashtablemask - counter = self.mccounters[argshash] - if vinfo: - virtualizable = args[vinfo.index_of_virtualizable] - virtualizable = vinfo.cast_to_vtype(virtualizable) - assert virtualizable != globaldata.blackhole_virtualizable, "reentering same frame via blackhole" - if counter >= 0: - # update the profiling counter - n = counter + self.increment_threshold - if n <= THRESHOLD_LIMIT: # bound not reached - self.mccounters[argshash] = n - return - if self.hashtablemask == 0: # must really create the tables now - self.create_tables_now() - return - metainterp = MetaInterp(metainterp_sd) - try: - loop_token = metainterp.compile_and_run_once(*args) - except warmrunnerdesc.ContinueRunningNormally: - # the trace got too long, reset the counter - self.mccounters[argshash] = 0 - raise - - else: - # machine code was already compiled for these greenargs - # (or we have a hash collision) - cell = self.mcentrypoints[argshash] - if not cell.equalkey(*greenargs): - # hash collision - loop_token = self.handle_hash_collision(cell, argshash, - *args) - if loop_token is None: - return - else: - # get the assembler and fill in the boxes - cell.set_future_values(*args[num_green_args:]) - loop_token = cell.entry_loop_token - # ---------- execute assembler ---------- - while True: # until interrupted by an exception - metainterp_sd.profiler.start_running() - fail_index = metainterp_sd.cpu.execute_token(loop_token) - metainterp_sd.profiler.end_running() - fail_descr = globaldata.get_fail_descr_from_number(fail_index) - loop_token = fail_descr.handle_fail(metainterp_sd) - - maybe_compile_and_run._dont_inline_ = True - - def handle_hash_collision(self, firstcell, argshash, *args): - greenargs = args[:num_green_args] - # search the linked list for the correct cell - cell = firstcell - while cell.next is not None: - nextcell = cell.next - if nextcell.equalkey(*greenargs): - # found, move to the front of the linked list - cell.next = nextcell.next - nextcell.next = firstcell - self.mcentrypoints[argshash] = nextcell - nextcell.set_future_values(*args[num_green_args:]) - return nextcell.entry_loop_token - cell = nextcell - # not found at all, do profiling - counter = self.mccounters[argshash] - assert counter < 0 # by invariant - n = counter + self.increment_threshold - if n < 0: # bound not reached - self.mccounters[argshash] = n - return None - metainterp = MetaInterp(metainterp_sd) - # XXX ContinueRunningNormally => "reset" counters - return metainterp.compile_and_run_once(*args) - handle_hash_collision._dont_inline_ = True - - def unwrap_greenkey(self, greenkey): - greenargs = () - i = 0 - for TYPE in green_args_spec: - value = unwrap(TYPE, greenkey[i]) - greenargs += (value,) - i = i + 1 - return greenargs - unwrap_greenkey._always_inline_ = True - - def comparekey(greenargs1, greenargs2): - i = 0 - for TYPE in green_args_spec: - if not equal_whatever(TYPE, greenargs1[i], greenargs2[i]): - return False - i = i + 1 - return True - comparekey = staticmethod(comparekey) - - def hashkey(greenargs): - return intmask(WarmEnterState.getkeyhash(*greenargs)) - hashkey = staticmethod(hashkey) - - def getkeyhash(*greenargs): - result = r_uint(0x345678) - i = 0 - mult = r_uint(1000003) - for TYPE in green_args_spec: - if i > 0: - result = result * mult - mult = mult + 82520 + 2*len(greenargs) - item = greenargs[i] - result = result ^ hash_whatever(TYPE, item) - i = i + 1 - return result # returns a r_uint - getkeyhash._always_inline_ = True - getkeyhash = staticmethod(getkeyhash) - - def must_compile_from_failure(self, key): - key.counter += 1 - return key.counter >= self.trace_eagerness - - def reset_counter_from_failure(self, key): - key.counter = 0 - - def attach_unoptimized_bridge_from_interp(self, greenkey, - entry_loop_token): - greenargs = self.unwrap_greenkey(greenkey) - newcell = MachineCodeEntryPoint(entry_loop_token, *greenargs) - argshash = self.getkeyhash(*greenargs) & self.hashtablemask - oldcell = self.mcentrypoints[argshash] - newcell.next = oldcell # link - self.mcentrypoints[argshash] = newcell - self.mccounters[argshash] = -THRESHOLD_LIMIT-1 - - if can_inline_ptr is None: - def can_inline_callable(self, greenkey): - return True - else: - def can_inline_callable(self, greenkey): - args = self.unwrap_greenkey(greenkey) - fn = support.maybe_on_top_of_llinterp(rtyper, can_inline_ptr) - return fn(*args) - - if get_printable_location_ptr is None: - def get_location_str(self, greenkey): - return '(no jitdriver.get_printable_location!)' - else: - def get_location_str(self, greenkey): - args = self.unwrap_greenkey(greenkey) - fn = support.maybe_on_top_of_llinterp(rtyper, - get_printable_location_ptr) - res = fn(*args) - if not we_are_translated() and not isinstance(res, str): - res = hlstr(res) - return res - - return WarmEnterState From pedronis at codespeak.net Tue Oct 20 14:58:13 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 20 Oct 2009 14:58:13 +0200 (CEST) Subject: [pypy-svn] r68660 - in pypy/trunk/pypy/translator/c: gcc/test test Message-ID: <20091020125813.0B388168007@codespeak.net> Author: pedronis Date: Tue Oct 20 14:58:12 2009 New Revision: 68660 Modified: pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py pypy/trunk/pypy/translator/c/test/test_newgc.py Log: make test_newgc compile standalone, also the executable takes the (test) name string not a number anymore as first argument. adjust test_asmgcroot, there is some more duplication between the two tests setup code, too bad for now Modified: pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py Tue Oct 20 14:58:12 2009 @@ -18,19 +18,16 @@ should_be_moving = False @classmethod - def _makefunc2(cls, func): + def _makefunc_str_int(cls, func): def main(argv): - arg0 = int(argv[1]) + arg0 = argv[1] arg1 = int(argv[2]) try: res = func(arg0, arg1) except MemoryError: print 'Result: MemoryError' else: - if isinstance(res, int): - print 'Result:', res - else: - print 'Result: "%s"' % (res,) + print 'Result: "%s"' % (res,) return 0 from pypy.config.pypyoption import get_pypy_config config = get_pypy_config(translating=True) @@ -59,7 +56,7 @@ redirect = ' 2> NUL' else: redirect = '' - g = os.popen('"%s" %d %d%s' % (exe_name, arg0, arg1, redirect), 'r') + g = os.popen('"%s" %s %d%s' % (exe_name, arg0, arg1, redirect), 'r') for line in g: print >> sys.stderr, 'RUN:', line.rstrip() lines.append(line) Modified: pypy/trunk/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_newgc.py (original) +++ pypy/trunk/pypy/translator/c/test/test_newgc.py Tue Oct 20 14:58:12 2009 @@ -24,19 +24,39 @@ _isolated_func = None @classmethod - def _makefunc2(cls, f): - t = Translation(f, [int, int], gc=cls.gcpolicy, + def _makefunc_str_int(cls, f): + def main(argv): + arg0 = argv[1] + arg1 = int(argv[2]) + try: + res = f(arg0, arg1) + except MemoryError: + print "MEMORY-ERROR" + else: + print res + return 0 + + t = Translation(main, standalone=True, gc=cls.gcpolicy, policy=annpolicy.StrictAnnotatorPolicy(), taggedpointers=cls.taggedpointers, removetypeptr=cls.removetypeptr, debugprint=True) t.disable(['backendopt']) - t.set_backend_extra_options(c_isolated=True, c_debug_defines=True) + t.set_backend_extra_options(c_debug_defines=True) t.rtype() if conftest.option.view: t.viewcg() - isolated_func = t.compile() - return isolated_func + exename = t.compile() + + def run(s, i): + data = py.process.cmdexec("%s %s %d" % (exename, s, i)) + data = data.strip() + if data == 'MEMORY-ERROR': + raise MemoryError + return data + + return run + def setup_class(cls): funcs0 = [] @@ -66,8 +86,8 @@ funcs1.append(func) assert name not in name_to_func name_to_func[name] = len(name_to_func) - def allfuncs(num, arg): - rgc.collect() + def allfuncs(name, arg): + num = name_to_func[name] func0 = funcs0[num] if func0: return str(func0()) @@ -79,7 +99,7 @@ return funcstr(arg) assert 0, 'unreachable' cls.funcsstr = funcsstr - cls.c_allfuncs = staticmethod(cls._makefunc2(allfuncs)) + cls.c_allfuncs = staticmethod(cls._makefunc_str_int(allfuncs)) cls.allfuncs = staticmethod(allfuncs) cls.name_to_func = name_to_func @@ -90,10 +110,9 @@ def run(self, name, *args): if not args: args = (-1, ) + print 'Running %r)' % name + res = self.c_allfuncs(name, *args) num = self.name_to_func[name] - print - print 'Running %r (test number %d)' % (name, num) - res = self.c_allfuncs(num, *args) if self.funcsstr[num]: return res return int(res) @@ -101,8 +120,8 @@ def run_orig(self, name, *args): if not args: args = (-1, ) - num = self.name_to_func[name] - res = self.allfuncs(num, *args) + res = self.allfuncs(name, *args) + num = self.name_to_func[name] if self.funcsstr[num]: return res return int(res) From pedronis at codespeak.net Tue Oct 20 15:46:50 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 20 Oct 2009 15:46:50 +0200 (CEST) Subject: [pypy-svn] r68661 - pypy/trunk/pypy/jit/backend/x86/test Message-ID: <20091020134650.88330168007@codespeak.net> Author: pedronis Date: Tue Oct 20 15:46:49 2009 New Revision: 68661 Modified: pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py Log: - make the test executable take a name, not a number, - make the initial n an argument defaulting to 2000 Modified: pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py Tue Oct 20 15:46:49 2009 @@ -33,7 +33,7 @@ def get_g(main): main._dont_inline_ = True - def g(num, n): + def g(name, n): x = X() x.foo = 2 main(n, x) @@ -46,12 +46,16 @@ def get_entry(g): def entrypoint(args): - num = 0 - if len(args) == 2: - num = int(args[1]) + name = '' + n = 2000 + argc = len(args) + if argc > 1: + name = args[1] + if argc > 2: + n = int(args[2]) r_list = [] for i in range(20): - r = g(num, 2000) + r = g(name, n) r_list.append(r) rgc.collect() rgc.collect(); rgc.collect() @@ -134,13 +138,14 @@ assert name not in name_to_func name_to_func[name] = len(name_to_func) print name_to_func - def allfuncs(num, n): + def allfuncs(name, n): x = X() x.foo = 2 - main_allfuncs(num, n, x) + main_allfuncs(name, n, x) x.foo = 5 return weakref.ref(x) - def main_allfuncs(num, n, x): + def main_allfuncs(name, n, x): + num = name_to_func[name] n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s = funcs[num][0](n, x) while n > 0: myjitdriver.can_enter_jit(num=num, n=n, x=x, x0=x0, x1=x1, @@ -158,14 +163,12 @@ cls.name_to_func = name_to_func cls.cbuilder = compile(get_entry(allfuncs), "hybrid", gcrootfinder="asmgcc", jit=True) - def run(self, name): - num = self.name_to_func[name] - res = self.cbuilder.cmdexec(str(num)) + def run(self, name, n=2000): + res = self.cbuilder.cmdexec("%s %d" %(name, n)) assert int(res) == 20 def run_orig(self, name, n, x): - num = self.name_to_func[name] - self.main_allfuncs(num, n, x) + self.main_allfuncs(name, n, x) def define_compile_hybrid_1(cls): # a moving GC. Supports malloc_varsize_nonmovable. Simple test, works From afa at codespeak.net Tue Oct 20 17:22:40 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 20 Oct 2009 17:22:40 +0200 (CEST) Subject: [pypy-svn] r68662 - in pypy/branch/msvc-asmgcroot/pypy/translator: c c/gcc c/gcc/test c/gcc/test/msvc platform Message-ID: <20091020152240.852CF168012@codespeak.net> Author: afa Date: Tue Oct 20 17:22:39 2009 New Revision: 68662 Added: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track8.s Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/instruction.py pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_asmgcroot.py pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py pypy/branch/msvc-asmgcroot/pypy/translator/platform/windows.py Log: More progress, now gcroots are tracked, still does not work Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/instruction.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/instruction.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/instruction.py Tue Oct 20 17:22:39 2009 @@ -10,6 +10,7 @@ REG2LOC = {} for _i, _reg in enumerate(CALLEE_SAVE_REGISTERS): REG2LOC[_reg] = LOC_REG | (_i<<2) + REG2LOC[_reg[1:]] = LOC_REG | (_i<<2) def frameloc(base, offset): assert base in (LOC_EBP_BASED, LOC_ESP_BASED) Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s Tue Oct 20 17:22:39 2009 @@ -1,3 +1,10 @@ +; Function compile flags: /Ogtpy +; COMDAT _pypy_g_ll_join_strs__Signed_arrayPtr +_TEXT SEGMENT +_l_result_2$ = -8 ; size = 4 +_l_v405$ = -4 ; size = 4 +_l_num_items_0$ = 8 ; size = 4 +_l_items_2$ = 12 ; size = 4 _pypy_g_ll_join_strs__Signed_arrayPtr PROC ; COMDAT ; 1457 : struct pypy_rpy_string0 *pypy_g_ll_join_strs__Signed_arrayPtr(long l_num_items_0, struct pypy_array0 *l_items_2) { @@ -57,7 +64,7 @@ jl SHORT $LN15 at pypy_g_ll_@139 $LN14 at pypy_g_ll_@139: call _RPyAbort - ;; expected {24(%esp) | %ebx, %esi, %edi, %ebp | } + ;; expected {24(%esp) | %ebx, %esi, %edi, %ebp | 32(%esp)} $LN15 at pypy_g_ll_@139: ; 1529 : l_v420 = l_v419; @@ -69,7 +76,7 @@ test ebx, ebx jne SHORT $LN16 at pypy_g_ll_@139 call _RPyAbort - ;; expected {24(%esp) | %ebx, %esi, %edi, %ebp | } + ;; expected {24(%esp) | %ebx, %esi, %edi, %ebp | 32(%esp)} $LN16 at pypy_g_ll_@139: ; 1531 : OP_INT_ADD(l_v402, l_v421, l_v422); @@ -99,7 +106,7 @@ push edi call _pypy_g_mallocstr__Signed - ;; expected {28(%esp) | %ebx, %esi, %edi, %ebp | } + ;; expected {28(%esp) | %ebx, %esi, %edi, %ebp | 36(%esp)} ; 1486 : l_v405 = (void*)l_items_2; Added: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track8.s ============================================================================== --- (empty file) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track8.s Tue Oct 20 17:22:39 2009 @@ -0,0 +1,528 @@ +; Function compile flags: /Ogtpy +; COMDAT _pypy_g_foo +_TEXT SEGMENT +tv419 = -8 ; size = 4 +_l_v416$ = -4 ; size = 4 +_l_v413$ = -4 ; size = 4 +_l_v410$ = -4 ; size = 4 +_l_v407$ = -4 ; size = 4 +_l_v404$ = -4 ; size = 4 +_l_v401$ = -4 ; size = 4 +_l_v394$ = -4 ; size = 4 +_l_v391$ = -4 ; size = 4 +_l_v388$ = -4 ; size = 4 +_l_v385$ = -4 ; size = 4 +_l_v382$ = -4 ; size = 4 +_l_v379$ = -4 ; size = 4 +_l_v376$ = -4 ; size = 4 +_l_v368$ = -4 ; size = 4 +_l_v365$ = -4 ; size = 4 +_l_v362$ = -4 ; size = 4 +_l_v359$ = -4 ; size = 4 +_l_v356$ = -4 ; size = 4 +_l_v353$ = -4 ; size = 4 +_local$40423 = 8 ; size = 1 +_l_rec_1$ = 8 ; size = 4 +_l_a1_1$ = 12 ; size = 4 +_l_a2_1$ = 16 ; size = 4 +_l_a3_1$ = 20 ; size = 4 +_l_a4_1$ = 24 ; size = 4 +_l_a5_1$ = 28 ; size = 4 +_l_a6_1$ = 32 ; size = 4 +_pypy_g_foo PROC ; COMDAT + +; 1026 : bool_t l_v337; bool_t l_v340; bool_t l_v345; bool_t l_v346; +; 1027 : bool_t l_v371; bool_t l_v398; bool_t l_v420; bool_t l_v426; +; 1028 : long l_v342; long l_v344; long l_v374; long l_v399; long l_v421; +; 1029 : struct pypy_header0 *l_v347; struct pypy_object0 *l_v372; +; 1030 : struct pypy_object_vtable0 *l_v339; +; 1031 : struct pypy_object_vtable0 *l_v397; +; 1032 : struct pypy_object_vtable0 *l_v419; +; 1033 : struct pypy_object_vtable0 *l_v425; struct pypy_src8_A0 *l_v335; +; 1034 : void* l_v336; void* l_v341; void* l_v343; void* l_v349; void* l_v351; +; 1035 : void* l_v352; void* l_v353; void* l_v354; void* l_v356; void* l_v357; +; 1036 : void* l_v359; void* l_v360; void* l_v362; void* l_v363; void* l_v365; +; 1037 : void* l_v366; void* l_v368; void* l_v369; void* l_v376; void* l_v377; +; 1038 : void* l_v379; void* l_v380; void* l_v382; void* l_v383; void* l_v385; +; 1039 : void* l_v386; void* l_v388; void* l_v389; void* l_v391; void* l_v392; +; 1040 : void* l_v394; void* l_v395; void* l_v401; void* l_v402; void* l_v404; +; 1041 : void* l_v405; void* l_v407; void* l_v408; void* l_v410; void* l_v411; +; 1042 : void* l_v413; void* l_v414; void* l_v416; void* l_v417; void* l_v424; +; 1043 : void* l_v428; +; 1044 : +; 1045 : block0: +; 1046 : OP_INT_GT(l_rec_1, 0L, l_v337); + + mov eax, DWORD PTR _l_rec_1$[esp-4] + sub esp, 8 + test eax, eax +$block0$34376: + +; 1047 : if (l_v337) { + + jle $block1$34379 + push ebx + mov ebx, DWORD PTR _l_a2_1$[esp+8] + push ebp + mov ebp, DWORD PTR _l_a1_1$[esp+12] + push edi + mov edi, DWORD PTR _l_a3_1$[esp+16] + add eax, -1 + mov DWORD PTR tv419[esp+20], eax + push esi + npad 10 +$LL63 at pypy_g_foo: + +; 1048 : goto block2; +; 1049 : } +; 1050 : goto block1; +; 1051 : +; 1052 : block1: +; 1053 : RPY_DEBUG_RETURN(); +; 1054 : return /* nothing */; +; 1055 : +; 1056 : block2: +; 1057 : pypy_g_stack_check___(); + + lea eax, DWORD PTR _local$40423[esp+20] + sub eax, DWORD PTR __LLstacktoobig_stack_base_pointer +$block0$40413: + cmp eax, DWORD PTR __LLstacktoobig_stack_min +$block2$34378: + jl SHORT $LN16 at pypy_g_foo + cmp eax, DWORD PTR __LLstacktoobig_stack_max + jle SHORT $LN17 at pypy_g_foo +$LN16 at pypy_g_foo: + call _LL_stack_too_big_slowpath + ;; expected {24(%esp) | %ebx, %esi, %edi, %ebp | %ebx, %edi, %ebp, 44(%esp), 48(%esp), 52(%esp)} + test eax, eax + jne $LN71 at pypy_g_foo +$LN17 at pypy_g_foo: +$block1$40416: + +; 1058 : l_v339 = RPyField((&pypy_g_ExcData), ed_exc_type); +; 1059 : l_v340 = (l_v339 == NULL); + + cmp DWORD PTR _pypy_g_ExcData, 0 + +; 1060 : if (!l_v340) { + + jne $LN75 at pypy_g_foo + +; 1061 : goto block1; +; 1062 : } +; 1063 : goto block3; +; 1064 : +; 1065 : block3: +; 1066 : l_v341 = RPyField((&pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC), ssgc_inst_free); + + mov esi, DWORD PTR _pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC+12 + +; 1067 : OP_RAW_MALLOC_USAGE((0 + ROUND_UP_FOR_ALLOCATION(sizeof(struct pypy_src8_A0), sizeof(struct pypy_forwarding_stub0))), l_v342); +; 1068 : l_v343 = RPyField((&pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC), ssgc_inst_top_of_space); +; 1069 : OP_ADR_DELTA(l_v343, l_v341, l_v344); + + mov eax, DWORD PTR _pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC+80 + sub eax, esi + +; 1070 : OP_INT_GT(l_v342, l_v344, l_v345); + + cmp eax, 8 +$block3$34382: + +; 1071 : if (l_v345) { + + jge SHORT $block4$34395 + +; 1184 : goto block1; +; 1185 : +; 1186 : block10: +; 1187 : abort(); /* debug_llinterpcall should be unreachable */ +; 1188 : goto block5; +; 1189 : +; 1190 : block11: +; 1191 : l_v424 = pypy_g_SemiSpaceGC_obtain_free_space((&pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC), (0 + ROUND_UP_FOR_ALLOCATION(sizeof(struct pypy_src8_A0), sizeof(struct pypy_forwarding_stub0)))); + + push 8 + push OFFSET _pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC +$block11$34394: + call _pypy_g_SemiSpaceGC_obtain_free_space + ;; expected {32(%esp) | %ebx, %esi, %edi, %ebp | %ebx, %edi, %ebp, 52(%esp), 56(%esp), 60(%esp)} + add esp, 8 + +; 1192 : l_v425 = RPyField((&pypy_g_ExcData), ed_exc_type); +; 1193 : l_v426 = (l_v425 == NULL); + + cmp DWORD PTR _pypy_g_ExcData, 0 + +; 1194 : if (!l_v426) { + + je SHORT $LN1 at pypy_g_foo + +; 1195 : l_v428 = NULL; + + xor esi, esi + +; 1196 : goto block6; + + jmp SHORT $block6$34416 +$LN1 at pypy_g_foo: + +; 1197 : } +; 1198 : l_v336 = l_v424; + + mov esi, eax +$block4$34395: + +; 1072 : goto block11; +; 1073 : } +; 1074 : l_v336 = l_v341; +; 1075 : goto block4; +; 1076 : +; 1077 : block4: +; 1078 : OP_INT_IS_TRUE(RUNNING_ON_LLINTERP, l_v346); +; 1079 : if (l_v346) { +; 1080 : goto block10; +; 1081 : } +; 1082 : goto block5; +; 1083 : +; 1084 : block5: +; 1085 : l_v347 = (struct pypy_header0 *)l_v336; +; 1086 : RPyField(l_v347, h_tid) = (GROUP_MEMBER_OFFSET(struct group_pypy_g_typeinfo_s, member1)|0L); +; 1087 : OP_ADR_ADD(l_v336, (0 + ROUND_UP_FOR_ALLOCATION(sizeof(struct pypy_src8_A0), sizeof(struct pypy_forwarding_stub0))), l_v349); + + lea ecx, DWORD PTR [esi+8] + mov DWORD PTR [esi], 1 +$block5$34398: + +; 1088 : RPyField((&pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC), ssgc_inst_free) = l_v349; + + mov DWORD PTR _pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC+12, ecx +$block6$34416: + +; 1089 : OP_ADR_ADD(l_v336, 0, l_v351); +; 1090 : l_v352 = (void*)l_v351; +; 1091 : l_v428 = l_v352; +; 1092 : goto block6; +; 1093 : +; 1094 : block6: +; 1095 : l_v353 = (void*)l_a2_1; + + mov DWORD PTR _l_v353$[esp+24], ebx + +; 1096 : l_v354 = pypy_asm_gcroot(l_v353); + + test DWORD PTR _l_v353$[esp+24], 0 + +; 1097 : l_a2_1 = l_v354; /* for moving GCs */ +; 1098 : l_v356 = (void*)l_a5_1; + + mov edx, DWORD PTR _l_a5_1$[esp+20] + mov DWORD PTR _l_v356$[esp+24], edx + +; 1099 : l_v357 = pypy_asm_gcroot(l_v356); + + test DWORD PTR _l_v356$[esp+24], 0 + +; 1100 : l_a5_1 = l_v357; /* for moving GCs */ +; 1101 : l_v359 = (void*)l_a6_1; + + mov eax, DWORD PTR _l_a6_1$[esp+20] + mov DWORD PTR _l_v359$[esp+24], eax + +; 1102 : l_v360 = pypy_asm_gcroot(l_v359); + + test DWORD PTR _l_v359$[esp+24], 0 + +; 1103 : l_a6_1 = l_v360; /* for moving GCs */ +; 1104 : l_v362 = (void*)l_a1_1; + + mov DWORD PTR _l_v362$[esp+24], ebp + +; 1105 : l_v363 = pypy_asm_gcroot(l_v362); + + test DWORD PTR _l_v362$[esp+24], 0 + +; 1106 : l_a1_1 = l_v363; /* for moving GCs */ +; 1107 : l_v365 = (void*)l_a3_1; + + mov DWORD PTR _l_v365$[esp+24], edi + +; 1108 : l_v366 = pypy_asm_gcroot(l_v365); + + test DWORD PTR _l_v365$[esp+24], 0 + +; 1109 : l_a3_1 = l_v366; /* for moving GCs */ +; 1110 : l_v368 = (void*)l_a4_1; + + mov ecx, DWORD PTR _l_a4_1$[esp+20] + mov DWORD PTR _l_v368$[esp+24], ecx + +; 1111 : l_v369 = pypy_asm_gcroot(l_v368); + + test DWORD PTR _l_v368$[esp+24], 0 + +; 1112 : l_a4_1 = l_v369; /* for moving GCs */ +; 1113 : l_v335 = (struct pypy_src8_A0 *)l_v428; +; 1114 : l_v371 = (l_v335 != NULL); + + test esi, esi + +; 1115 : if (!l_v371) { + + je $LN75 at pypy_g_foo + +; 1116 : goto block1; +; 1117 : } +; 1118 : goto block7; +; 1119 : +; 1120 : block7: +; 1121 : l_v372 = (struct pypy_object0 *)l_v335; +; 1122 : RPyField(l_v372, o_typeptr) = (&pypy_g_src8_A_vtable.a_super); +; 1123 : OP_INT_SUB(l_rec_1, 1L, l_v374); +; 1124 : pypy_g_foo(l_v374, l_v335, l_v335, l_v335, l_v335, l_v335, l_v335); + + mov edx, DWORD PTR tv419[esp+24] + push esi + push esi + push esi + push esi + push esi + push esi + push edx +$block7$34426: + mov DWORD PTR [esi+4], OFFSET _pypy_g_src8_A_vtable + call _pypy_g_foo + ;; expected {52(%esp) | %ebx, %esi, %edi, %ebp | %ebx, %esi, %edi, %ebp, 72(%esp), 76(%esp), 80(%esp)} + add esp, 28 ; 0000001cH + +; 1125 : l_v376 = (void*)l_a2_1; + + mov DWORD PTR _l_v376$[esp+24], ebx + +; 1126 : l_v377 = pypy_asm_gcroot(l_v376); + + test DWORD PTR _l_v376$[esp+24], 0 + +; 1127 : l_a2_1 = l_v377; /* for moving GCs */ +; 1128 : l_v379 = (void*)l_a6_1; + + mov eax, DWORD PTR _l_a6_1$[esp+20] + mov DWORD PTR _l_v379$[esp+24], eax + +; 1129 : l_v380 = pypy_asm_gcroot(l_v379); + + test DWORD PTR _l_v379$[esp+24], 0 + +; 1130 : l_a6_1 = l_v380; /* for moving GCs */ +; 1131 : l_v382 = (void*)l_a1_1; + + mov DWORD PTR _l_v382$[esp+24], ebp + +; 1132 : l_v383 = pypy_asm_gcroot(l_v382); + + test DWORD PTR _l_v382$[esp+24], 0 + +; 1133 : l_a1_1 = l_v383; /* for moving GCs */ +; 1134 : l_v385 = (void*)l_v335; + + mov DWORD PTR _l_v385$[esp+24], esi + +; 1135 : l_v386 = pypy_asm_gcroot(l_v385); + + test DWORD PTR _l_v385$[esp+24], 0 + +; 1136 : l_v335 = l_v386; /* for moving GCs */ +; 1137 : l_v388 = (void*)l_a3_1; + + mov DWORD PTR _l_v388$[esp+24], edi + +; 1138 : l_v389 = pypy_asm_gcroot(l_v388); + + test DWORD PTR _l_v388$[esp+24], 0 + +; 1139 : l_a3_1 = l_v389; /* for moving GCs */ +; 1140 : l_v391 = (void*)l_a5_1; + + mov ecx, DWORD PTR _l_a5_1$[esp+20] + mov DWORD PTR _l_v391$[esp+24], ecx + +; 1141 : l_v392 = pypy_asm_gcroot(l_v391); + + test DWORD PTR _l_v391$[esp+24], 0 + +; 1142 : l_a5_1 = l_v392; /* for moving GCs */ +; 1143 : l_v394 = (void*)l_a4_1; + + mov edx, DWORD PTR _l_a4_1$[esp+20] + mov DWORD PTR _l_v394$[esp+24], edx + +; 1144 : l_v395 = pypy_asm_gcroot(l_v394); + + test DWORD PTR _l_v394$[esp+24], 0 + +; 1145 : l_a4_1 = l_v395; /* for moving GCs */ +; 1146 : l_v397 = RPyField((&pypy_g_ExcData), ed_exc_type); +; 1147 : l_v398 = (l_v397 == NULL); + + cmp DWORD PTR _pypy_g_ExcData, 0 + +; 1148 : if (!l_v398) { + + jne $LN75 at pypy_g_foo + +; 1149 : goto block1; +; 1150 : } +; 1151 : goto block8; +; 1152 : +; 1153 : block8: +; 1154 : OP_INT_SUB(l_rec_1, 1L, l_v399); +; 1155 : pypy_g_foo(l_v399, l_v335, l_v335, l_v335, l_v335, l_v335, l_v335); + + mov eax, DWORD PTR tv419[esp+24] + push esi + push esi + push esi + push esi + push esi + push esi + push eax +$block8$34437: + call _pypy_g_foo + ;; expected {52(%esp) | %ebx, %esi, %edi, %ebp | %ebx, %edi, %ebp, 72(%esp), 76(%esp), 80(%esp)} + add esp, 28 ; 0000001cH + +; 1156 : l_v401 = (void*)l_a2_1; + + mov DWORD PTR _l_v401$[esp+24], ebx + +; 1157 : l_v402 = pypy_asm_gcroot(l_v401); + + test DWORD PTR _l_v401$[esp+24], 0 + +; 1158 : l_a2_1 = l_v402; /* for moving GCs */ +; 1159 : l_v404 = (void*)l_a1_1; + + mov DWORD PTR _l_v404$[esp+24], ebp + +; 1160 : l_v405 = pypy_asm_gcroot(l_v404); + + test DWORD PTR _l_v404$[esp+24], 0 + +; 1161 : l_a1_1 = l_v405; /* for moving GCs */ +; 1162 : l_v407 = (void*)l_a3_1; + + mov DWORD PTR _l_v407$[esp+24], edi + +; 1163 : l_v408 = pypy_asm_gcroot(l_v407); + + test DWORD PTR _l_v407$[esp+24], 0 + +; 1164 : l_a3_1 = l_v408; /* for moving GCs */ +; 1165 : l_v410 = (void*)l_a6_1; + + mov ecx, DWORD PTR _l_a6_1$[esp+20] + mov DWORD PTR _l_v410$[esp+24], ecx + +; 1166 : l_v411 = pypy_asm_gcroot(l_v410); + + test DWORD PTR _l_v410$[esp+24], 0 + +; 1167 : l_a6_1 = l_v411; /* for moving GCs */ +; 1168 : l_v413 = (void*)l_a5_1; + + mov edx, DWORD PTR _l_a5_1$[esp+20] + mov DWORD PTR _l_v413$[esp+24], edx + +; 1169 : l_v414 = pypy_asm_gcroot(l_v413); + + test DWORD PTR _l_v413$[esp+24], 0 + +; 1170 : l_a5_1 = l_v414; /* for moving GCs */ +; 1171 : l_v416 = (void*)l_a4_1; + + mov esi, DWORD PTR _l_a4_1$[esp+20] + mov DWORD PTR _l_v416$[esp+24], esi + +; 1172 : l_v417 = pypy_asm_gcroot(l_v416); + + test DWORD PTR _l_v416$[esp+24], 0 + +; 1173 : l_a4_1 = l_v417; /* for moving GCs */ +; 1174 : l_v419 = RPyField((&pypy_g_ExcData), ed_exc_type); +; 1175 : l_v420 = (l_v419 == NULL); + + cmp DWORD PTR _pypy_g_ExcData, 0 + +; 1176 : if (!l_v420) { + + jne SHORT $LN75 at pypy_g_foo + +; 1177 : goto block1; +; 1178 : } +; 1179 : goto block9; +; 1180 : +; 1181 : block9: +; 1182 : OP_INT_SUB(l_rec_1, 1L, l_v421); +; 1183 : pypy_g_foo(l_v421, l_a6_1, l_a5_1, l_a4_1, l_a3_1, l_a2_1, l_a1_1); + + sub DWORD PTR _l_rec_1$[esp+20], 1 + sub DWORD PTR tv419[esp+24], 1 + cmp DWORD PTR _l_rec_1$[esp+20], 0 + mov eax, ebp + mov ebp, DWORD PTR _l_a6_1$[esp+20] + mov ecx, ebx + mov ebx, DWORD PTR _l_a5_1$[esp+20] + mov edx, edi +$block9$34446: + mov edi, esi + mov DWORD PTR _l_a4_1$[esp+20], edx + mov DWORD PTR _l_a5_1$[esp+20], ecx + mov DWORD PTR _l_a6_1$[esp+20], eax +$block0_1$34376: + jg $LL63 at pypy_g_foo + pop esi + pop edi + pop ebp + pop ebx + +; 1199 : goto block4; +; 1200 : } + + add esp, 8 + ret 0 +$LN71 at pypy_g_foo: +$block0$40426: +$block0$40437: +$block0$40434: +$block2$40415: + +; 1048 : goto block2; +; 1049 : } +; 1050 : goto block1; +; 1051 : +; 1052 : block1: +; 1053 : RPY_DEBUG_RETURN(); +; 1054 : return /* nothing */; +; 1055 : +; 1056 : block2: +; 1057 : pypy_g_stack_check___(); + + mov DWORD PTR _pypy_g_ExcData, OFFSET _pypy_g_exceptions_RuntimeError_vtable + mov DWORD PTR _pypy_g_ExcData+4, OFFSET _pypy_g_exceptions_RuntimeError +$block1$40435: +$block1$40438: +$block1$40427: +$LN75 at pypy_g_foo: + pop esi + pop edi + pop ebp + pop ebx +$block1$34379: + +; 1199 : goto block4; +; 1200 : } + + add esp, 8 + ret 0 +_pypy_g_foo ENDP Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_asmgcroot.py Tue Oct 20 17:22:39 2009 @@ -51,7 +51,7 @@ def run(arg0, arg1): lines = [] - print >> sys.stderr, 'RUN: starting', exe_name + print >> sys.stderr, 'RUN: starting', exe_name, arg0, arg1 g = os.popen('"%s" %d %d' % (exe_name, arg0, arg1), 'r') for line in g: print >> sys.stderr, 'RUN:', line.rstrip() Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Tue Oct 20 17:22:39 2009 @@ -34,16 +34,10 @@ r_jmp_source = re.compile(r"\d*[(](%[\w]+)[,)]") r_jmptable_item = re.compile(r"\t.long\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") r_jmptable_end = re.compile(r"\t.text|\t.section\s+.text|\t\.align|"+LABEL) -LOCALVAR = r"%eax|%edx|%ecx|%ebx|%esi|%edi|%ebp|\d*[(]%esp[)]" -LOCALVARFP = LOCALVAR + r"|-?\d*[(]%ebp[)]" -r_gcroot_marker = re.compile(r"\t/[*] GCROOT ("+LOCALVARFP+") [*]/") r_bottom_marker = re.compile(r"\t/[*] GC_STACK_BOTTOM [*]/") -r_localvarnofp = re.compile(LOCALVAR) -r_localvarfp = re.compile(LOCALVARFP) -r_localvar_esp = re.compile(r"(\d*)[(]%esp[)]") -r_localvar_ebp = re.compile(r"(-?\d*)[(]%ebp[)]") class FunctionGcRootTracker(object): + skip = 0 @classmethod def init_regexp(cls): @@ -57,7 +51,7 @@ self.funcname = funcname self.lines = lines self.uses_frame_pointer = False - self.r_localvar = r_localvarnofp + self.r_localvar = self.r_localvarnofp self.filetag = filetag # a "stack bottom" function is either main() or a callback from C code self.is_stack_bottom = False @@ -77,6 +71,9 @@ self.dump() return self.gettable() + def replace_symbols(self, operand): + return operand + def gettable(self): """Returns a list [(label_after_call, callshape_tuple)] See format_callshape() for more details about callshape_tuple. @@ -145,6 +142,8 @@ self.insns = [InsnFunctionStart()] ignore_insns = False for lineno, line in enumerate(self.lines): + if lineno < self.skip: + continue self.currentlineno = lineno insn = [] match = r_insn.match(line) @@ -158,7 +157,7 @@ meth = getattr(self, 'visit_' + opname) line = line.rsplit(';', 1)[0] insn = meth(line) - elif r_gcroot_marker.match(line): + elif self.r_gcroot_marker.match(line): insn = self._visit_gcroot_marker(line) elif r_bottom_marker.match(line): self.is_stack_bottom = True @@ -235,18 +234,20 @@ elif isinstance(localvar, (list, tuple)): return [fixvar(var) for var in localvar] - match = r_localvar_esp.match(localvar) + match = self.r_localvar_esp.match(localvar) if match: - if localvar == '0(%esp)': # for pushl and popl, by - hint = None # default ebp addressing is - else: # a bit nicer + if localvar == self.TOP_OF_STACK: # for pushl and popl, by + hint = None # default ebp addressing is + else: # a bit nicer hint = 'esp' ofs_from_esp = int(match.group(1) or '0') + if self.format == 'msvc': + ofs_from_esp += int(match.group(2) or '0') localvar = ofs_from_esp - insn.framesize assert localvar != 0 # that's the return address return LocalVar(localvar, hint=hint) elif self.uses_frame_pointer: - match = r_localvar_ebp.match(localvar) + match = self.r_localvar_ebp.match(localvar) if match: ofs_from_ebp = int(match.group(1) or '0') localvar = ofs_from_ebp - 4 @@ -341,9 +342,9 @@ # ____________________________________________________________ def _visit_gcroot_marker(self, line): - match = r_gcroot_marker.match(line) + match = self.r_gcroot_marker.match(line) loc = match.group(1) - return InsnGCROOT(loc) + return InsnGCROOT(self.replace_symbols(loc)) def visit_nop(self, line): return [] @@ -449,7 +450,7 @@ if target == self.ESP: # only for leal -12(%ebp), %esp in function epilogues source = match.group("source") - match = r_localvar_ebp.match(source) + match = self.r_localvar_ebp.match(source) if match: if not self.uses_frame_pointer: raise UnrecognizedOperation('epilogue without prologue') @@ -457,7 +458,7 @@ assert ofs_from_ebp <= 0 framesize = 4 - ofs_from_ebp else: - match = r_localvar_esp.match(source) + match = self.r_localvar_esp.match(source) # leal 12(%esp), %esp if match: return InsnStackAdjust(int(match.group(1))) @@ -468,6 +469,8 @@ return self.binary_insn(line) def insns_for_copy(self, source, target): + source = self.replace_symbols(source) + target = self.replace_symbols(target) if source == self.ESP or target == self.ESP: raise UnrecognizedOperation('%s -> %s' % (source, target)) elif self.r_localvar.match(target): @@ -482,22 +485,22 @@ match = self.r_binaryinsn.match(line) source = match.group("source") target = match.group("target") - if source == self.ESP and target == '%ebp': + if source == self.ESP and target == self.EBP: return self._visit_prologue() - elif source == '%ebp' and target == self.ESP: + elif source == self.EBP and target == self.ESP: return self._visit_epilogue() return self.insns_for_copy(source, target) def visit_pushl(self, line): match = self.r_unaryinsn.match(line) source = match.group(1) - return [InsnStackAdjust(-4)] + self.insns_for_copy(source, '0(%esp)') + return [InsnStackAdjust(-4)] + self.insns_for_copy(source, self.TOP_OF_STACK) def visit_pushw(self, line): return [InsnStackAdjust(-2)] # rare but not impossible def _visit_pop(self, target): - return self.insns_for_copy('0(%esp)', target) + [InsnStackAdjust(+4)] + return self.insns_for_copy(self.TOP_OF_STACK, target) + [InsnStackAdjust(+4)] def visit_popl(self, line): match = self.r_unaryinsn.match(line) @@ -507,7 +510,7 @@ def _visit_prologue(self): # for the prologue of functions that use %ebp as frame pointer self.uses_frame_pointer = True - self.r_localvar = r_localvarfp + self.r_localvar = self.r_localvarfp return [InsnPrologue()] def _visit_epilogue(self): @@ -516,7 +519,7 @@ return [InsnEpilogue(4)] def visit_leave(self, line): - return self._visit_epilogue() + self._visit_pop('%ebp') + return self._visit_epilogue() + self._visit_pop(self.EBP) def visit_ret(self, line): return InsnRet() @@ -542,7 +545,7 @@ # if the source looks like 8(%eax,%edx,4) # %eax is the real source, %edx is an offset. match = r_jmp_source.match(s) - if match and not r_localvar_esp.match(s): + if match and not self.r_localvar_esp.match(s): sources.append(match.group(1)) else: sources.append(s) @@ -648,13 +651,13 @@ assert lineoffset in (1,2) return [InsnStackAdjust(-4)] insns = [InsnCall(self.currentlineno), - InsnSetLocal('%eax')] # the result is there + InsnSetLocal(self.EAX)] # the result is there if sys.platform == 'win32': # handle __stdcall calling convention: # Stack cleanup is performed by the called function, # Function name is decorated with "@N" where N is the stack size if match and '@' in target: - insns.append(InsnStackAdjust(int(target.split('@')[1]))) + insns.append(InsnStackAdjust(int(target.rsplit('@', 1)[1]))) return insns @@ -662,11 +665,21 @@ format = 'elf' ESP = '%esp' + EBP = '%ebp' + EAX = '%eax' OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' + TOP_OF_STACK = '0(%esp)' r_functionstart = re.compile(r"\t.type\s+"+LABEL+",\s*[@]function\s*$") r_functionend = re.compile(r"\t.size\s+"+LABEL+",\s*[.]-"+LABEL+"\s*$") + LOCALVAR = r"%eax|%edx|%ecx|%ebx|%esi|%edi|%ebp|\d*[(]%esp[)]" + LOCALVARFP = LOCALVAR + r"|-?\d*[(]%ebp[)]" + r_localvarnofp = re.compile(LOCALVAR) + r_localvarfp = re.compile(LOCALVARFP) + r_gcroot_marker = re.compile(r"\t/[*] GCROOT ("+LOCALVARFP+") [*]/") + r_localvar_esp = re.compile(r"(\d*)[(]%esp[)]") + r_localvar_ebp = re.compile(r"(-?\d*)[(]%ebp[)]") def __init__(self, lines, filetag=0): match = self.r_functionstart.match(lines[0]) @@ -700,14 +713,27 @@ class MsvcFunctionGcRootTracker(FunctionGcRootTracker): format = 'msvc' - r_functionstart = re.compile(LABEL+r"\s+PROC\s*(:?;.+)\n$") - r_functionend = re.compile(LABEL+r"\s+ENDP\s*$") - ESP = 'esp' + EBP = 'ebp' + EAX = 'eax' + TOP_OF_STACK = 'DWORD PTR [esp]' - OPERAND = r'(?:(:?WORD|DWORD|BYTE) PTR |OFFSET )?[_\w?@$]*(?:[-+0-9]+)?(:?\[[-+*\w0-9]+\])?' + OPERAND = r'(?:(:?WORD|DWORD|BYTE) PTR |OFFSET )?[_\w?:@$]*(?:[-+0-9]+)?(:?\[[-+*\w0-9]+\])?' LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' + r_functionstart = re.compile(r"; Function compile flags: ") + r_codestart = re.compile(LABEL+r"\s+PROC\s*(:?;.+)?\n$") + r_functionend = re.compile(LABEL+r"\s+ENDP\s*$") + r_symboldefine = re.compile(r"([_a-z0-9]+\$) = ([-0-9]+)\s*;.+\n") + + LOCALVAR = r"eax|edx|ecx|ebx|esi|edi|ebp|DWORD PTR [-+]?\d*\[esp[-+]\d+\]" + LOCALVARFP = LOCALVAR + r"|-?\d*\[ebp\]" + r_localvarnofp = re.compile(LOCALVAR) + r_localvarfp = re.compile(LOCALVARFP) + r_gcroot_marker = re.compile(r";.+ = pypy_asm_gcroot\(") + r_localvar_esp = re.compile(r"DWORD PTR ([-+]?\d+)?\[esp([-+]?\d+)?\]") + r_localvar_ebp = re.compile(r"DWORD PTR ([-+]?\d+)?\[ebp([-+]?\d+)?\]") + @classmethod def init_regexp(cls): super(MsvcFunctionGcRootTracker, cls).init_regexp() @@ -715,11 +741,29 @@ cls.r_jump = re.compile(r"\tj\w+\s+(?:SHORT |DWORD PTR )?"+LABEL+"\s*$") def __init__(self, lines, filetag=0): - match = self.r_functionstart.match(lines[0]) + self.defines = {} + for i, line in enumerate(lines): + if self.r_symboldefine.match(line): + match = self.r_symboldefine.match(line) + name = match.group(1) + value = int(match.group(2)) + self.defines[name] = value + continue + + match = self.r_codestart.match(line) + if match: + self.skip = i + break + funcname = match.group(1) super(MsvcFunctionGcRootTracker, self).__init__( funcname, lines, filetag) + def replace_symbols(self, operand): + for name, value in self.defines.items(): + operand = operand.replace(name, str(value)) + return operand + for name in ''' push pop mov lea xor sub add @@ -737,6 +781,13 @@ except ValueError: return None + def _visit_gcroot_marker(self, line): + assert self.lines[self.currentlineno + 1] == "\n" + assert self.lines[self.currentlineno + 2].startswith("\ttest\tDWORD PTR") + match = self.r_binaryinsn.match(self.lines[self.currentlineno + 2]) + loc = match.group("target") + return InsnGCROOT(self.replace_symbols(loc)) + MsvcFunctionGcRootTracker.init_regexp() class AssemblerParser(object): @@ -909,6 +960,9 @@ newlines.append(line) + if line == "\t.model\tflat\n": + newlines.append("\tassume fs:nothing\n") + newfile.writelines(newlines) PARSERS = { @@ -982,13 +1036,13 @@ if self.format == 'msvc': print >> output, "\tpush\t%s\t\t; %s" % (source, comment) else: - print >> output, "\tpushl\t%s\t\t/* %s */ " % (target, comment) + print >> output, "\tpushl\t%s\t\t/* %s */ " % (source, comment) def _popl(source, comment): if self.format == 'msvc': print >> output, "\tpop\t%s\t\t; %s" % (source, comment) else: - print >> output, "\tpopl\t%s\t\t/* %s */ " % (target, comment) + print >> output, "\tpopl\t%s\t\t/* %s */ " % (source, comment) def _register(name, disp=""): @@ -1332,6 +1386,12 @@ verbose = 1 shuffle = False output_raw_table = False + if sys.platform == 'darwin': + format = 'darwin' + elif sys.platform == 'win32': + format = 'mingw32' + else: + format = 'elf' while len(sys.argv) > 1: if sys.argv[1] == '-v': del sys.argv[1] @@ -1342,14 +1402,11 @@ elif sys.argv[1] == '-t': del sys.argv[1] output_raw_table = True + elif sys.argv[1].startswith('-f'): + format = sys.argv[1][2:] + del sys.argv[1] else: break - if sys.platform == 'darwin': - format = 'darwin' - elif sys.platform == 'win32': - format = 'msvc' - else: - format = 'elf' tracker = GcRootTracker(verbose=verbose, shuffle=shuffle, format=format) for fn in sys.argv[1:]: f = open(fn, 'r') Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py Tue Oct 20 17:22:39 2009 @@ -505,13 +505,13 @@ mk.definition('OBJECTS', 'gcmaptable.obj $(ASMLBLOBJFILES)') mk.rule('.SUFFIXES', '.s', []) mk.rule('.s.obj', '', - 'cmd /c $(MASM) /nologo /Cx /Zm /coff /Fo$@ /c $< $(INCLUDEDIRS)') + 'cmd /c $(MASM) /nologo /Cx /Cp /Zm /coff /Fo$@ /c $< $(INCLUDEDIRS)') mk.rule('.c.gcmap', '', ['$(CC) /nologo $(CFLAGS) /c /FAs /Fa$*.s $< $(INCLUDEDIRS)', - 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -t $*.s > $@'] + 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc -t $*.s > $@'] ) mk.rule('gcmaptable.s', '$(GCMAPFILES)', - 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py $(GCMAPFILES) > $@') + 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc $(GCMAPFILES) > $@') else: mk.definition('OBJECTS', '$(ASMLBLFILES) gcmaptable.s') mk.rule('%.s', '%.c', '$(CC) $(CFLAGS) -frandom-seed=$< -o $@ -S $< $(INCLUDEDIRS)') Modified: pypy/branch/msvc-asmgcroot/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/platform/windows.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/platform/windows.py Tue Oct 20 17:22:39 2009 @@ -241,13 +241,21 @@ rules = [ ('all', '$(DEFAULT_TARGET)', []), - ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) /nologo $(LDFLAGS) $(OBJECTS) /out:$@ $(LIBDIRS) $(LIBS)'), ('.c.o', '', '$(CC) $(CFLAGS) /Fo$@ /c $< $(INCLUDEDIRS)'), ] for rule in rules: m.rule(*rule) + if self.version < 80: + m.rule('$(TARGET)', '$(OBJECTS)', + '$(CC_LINK) /nologo $(LDFLAGS) $(OBJECTS) /out:$@ $(LIBDIRS) $(LIBS)') + else: + m.rule('$(TARGET)', '$(OBJECTS)', + ['$(CC_LINK) /nologo $(LDFLAGS) $(OBJECTS) /out:$@ $(LIBDIRS) $(LIBS) /MANIFESTFILE:$*.manifest', + 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', + ]) + return m def execute_makefile(self, path_to_makefile): From afa at codespeak.net Tue Oct 20 17:49:03 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 20 Oct 2009 17:49:03 +0200 (CEST) Subject: [pypy-svn] r68663 - pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc Message-ID: <20091020154903.D77F116800D@codespeak.net> Author: afa Date: Tue Oct 20 17:49:03 2009 New Revision: 68663 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Log: oops - actually generates this instruction. Now the program crashes. This is progress, I suppose. Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Tue Oct 20 17:49:03 2009 @@ -1064,7 +1064,10 @@ return "$%s" % _globalname(name) def _call(arg, comment): - return "call\t%s\t\t;%s" % (arg, comment) + if self.format == 'msvc': + print >> output, "\tcall\t%s\t\t;%s" % (arg, comment) + else: + print >> output, "\tcall\t%s\t\t/* %s */" % (arg, comment) def _indirectjmp(arg): if self.format == 'msvc': @@ -1128,7 +1131,7 @@ _popl(_register("esi"), "restore from ASM_FRAMEDATA[3]") _popl(_register("edi"), "restore from ASM_FRAMEDATA[4]") _popl(_register("ebp"), "restore from ASM_FRAMEDATA[5]") - _popl(_register("ecx"), "restore from ASM_FRAMEDATA[6]") + _popl(_register("ecx"), "ignored ASM_FRAMEDATA[6]") _comment("the return value is the one of the 'call' above,") _comment("because %eax (and possibly %edx) are unmodified") From arigo at codespeak.net Tue Oct 20 17:52:32 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Oct 2009 17:52:32 +0200 (CEST) Subject: [pypy-svn] r68664 - in pypy/branch/warmspot-jitinfo/pypy/jit/metainterp: . test Message-ID: <20091020155232.0ED6B16800D@codespeak.net> Author: arigo Date: Tue Oct 20 17:52:32 2009 New Revision: 68664 Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/support.py pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmstate.py pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmstate.py Log: Found out a reasonable way to write this stuff. Write tests. Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/support.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/support.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/support.py Tue Oct 20 17:52:32 2009 @@ -74,16 +74,15 @@ def maybe_on_top_of_llinterp(rtyper, fnptr): # Run a generated graph on top of the llinterp for testing. # When translated, this just returns the fnptr. - llinterp = LLInterpreter(rtyper) #, exc_data_ptr=exc_data_ptr) funcobj = get_funcobj(fnptr) if hasattr(funcobj, 'graph'): + llinterp = LLInterpreter(rtyper) #, exc_data_ptr=exc_data_ptr) def on_top_of_llinterp(*args): return llinterp.eval_graph(funcobj.graph, list(args)) else: - assert isinstance(fnptr, ootype._meth) - assert hasattr(fnptr, '_callable') + assert hasattr(funcobj, '_callable') def on_top_of_llinterp(*args): - return fnptr._callable(*args) + return funcobj._callable(*args) return on_top_of_llinterp class Entry(ExtRegistryEntry): Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmstate.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmstate.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmstate.py Tue Oct 20 17:52:32 2009 @@ -1,8 +1,10 @@ from pypy.rpython.test.test_llinterp import interpret from pypy.rpython.lltypesystem import lltype, llmemory, rstr from pypy.rpython.ootypesystem import ootype +from pypy.rpython.annlowlevel import llhelper from pypy.jit.metainterp.warmstate import wrap, unwrap from pypy.jit.metainterp.warmstate import equal_whatever, hash_whatever +from pypy.jit.metainterp.warmstate import WarmEnterState from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr from pypy.jit.metainterp.history import ConstInt, ConstFloat, ConstPtr @@ -52,3 +54,82 @@ assert equal_whatever(ootype.typeOf(s1), s1, s2) fn(ord('y')) interpret(fn, [ord('y')], type_system='ootype') + + +def test_make_jitcell_getter_default(): + class FakeWarmRunnerDesc: + green_args_spec = [lltype.Signed, lltype.Float] + class FakeJitCell: + pass + state = WarmEnterState(FakeWarmRunnerDesc()) + get_jitcell = state.make_jitcell_getter_default(FakeJitCell) + cell1 = get_jitcell((42, 42.5)) + assert isinstance(cell1, FakeJitCell) + cell2 = get_jitcell((42, 42.5)) + assert cell1 is cell2 + cell3 = get_jitcell((41, 42.5)) + cell4 = get_jitcell((42, 0.25)) + assert cell1 is not cell3 is not cell4 is not cell1 + +def test_make_jitcell_getter(): + class FakeWarmRunnerDesc: + green_args_spec = [lltype.Float] + get_jitcell_at_ptr = None + state = WarmEnterState(FakeWarmRunnerDesc()) + get_jitcell = state.make_jitcell_getter() + cell1 = get_jitcell((1.75,)) + cell2 = get_jitcell((1.75,)) + assert cell1 is cell2 + +def test_make_jitcell_getter_custom(): + class FakeJitCell: + _TYPE = llmemory.GCREF + celldict = {} + def getter(x, y): + return celldict[x, y] + def setter(newcell, x, y): + newcell.x = x + newcell.y = y + celldict[x, y] = newcell + GETTER = lltype.Ptr(lltype.FuncType([lltype.Signed, lltype.Float], + llmemory.GCREF)) + SETTER = lltype.Ptr(lltype.FuncType([llmemory.GCREF, lltype.Signed, + lltype.Float], lltype.Void)) + class FakeWarmRunnerDesc: + rtyper = None + cpu = None + get_jitcell_at_ptr = llhelper(GETTER, getter) + set_jitcell_at_ptr = llhelper(SETTER, setter) + # + state = WarmEnterState(FakeWarmRunnerDesc()) + get_jitcell = state.make_jitcell_getter_custom(FakeJitCell) + cell1 = get_jitcell((5, 42.5)) + assert isinstance(cell1, FakeJitCell) + assert cell1.x == 5 + assert cell1.y == 42.5 + cell2 = get_jitcell((5, 42.5)) + assert cell2 is cell1 + cell3 = get_jitcell((41, 42.5)) + cell4 = get_jitcell((42, 0.25)) + assert cell1 is not cell3 is not cell4 is not cell1 + +def test_make_set_future_values(): + future_values = {} + class FakeCPU: + def set_future_value_int(self, j, value): + future_values[j] = "int", value + def set_future_value_float(self, j, value): + future_values[j] = "float", value + class FakeWarmRunnerDesc: + cpu = FakeCPU() + red_args_types = ["int", "float"] + class metainterp_sd: + virtualizable_info = None + # + state = WarmEnterState(FakeWarmRunnerDesc()) + set_future_values = state.make_set_future_values() + set_future_values(5, 42.5) + assert future_values == { + 0: ("int", 5), + 1: ("float", 42.5), + } Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmstate.py Tue Oct 20 17:52:32 2009 @@ -1,9 +1,12 @@ - +import sys from pypy.rpython.lltypesystem import lltype, llmemory, rstr from pypy.rpython.ootypesystem import ootype -from pypy.rlib.objectmodel import specialize +from pypy.rlib.objectmodel import specialize, we_are_translated, r_dict from pypy.rlib.rarithmetic import intmask -from pypy.jit.metainterp import history +from pypy.rlib.nonconst import NonConstant +from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.jit import PARAMETERS, OPTIMIZER_SIMPLE, OPTIMIZER_FULL +from pypy.jit.metainterp import support, history # ____________________________________________________________ @@ -82,3 +85,250 @@ return 0 else: return lltype.cast_primitive(lltype.Signed, x) + + at specialize.ll_and_arg(3) +def set_future_value(cpu, j, value, typecode): + if typecode == 'ref': + refvalue = cpu.ts.cast_to_ref(value) + cpu.set_future_value_ref(j, refvalue) + elif typecode == 'int': + intvalue = lltype.cast_primitive(lltype.Signed, value) + cpu.set_future_value_int(j, intvalue) + elif typecode == 'float': + assert isinstance(value, float) + cpu.set_future_value_float(j, value) + else: + assert False + +# ____________________________________________________________ + + +class WarmEnterState(object): + THRESHOLD_LIMIT = sys.maxint // 2 + default_jitcell_dict = None + + def __init__(self, warmrunnerdesc): + "NOT_RPYTHON" + self.warmrunnerdesc = warmrunnerdesc + try: + self.profiler = warmrunnerdesc.metainterp_sd.profiler + except AttributeError: # for tests + self.profiler = None + # initialize the state with the default values of the + # parameters specified in rlib/jit.py + for name, default_value in PARAMETERS.items(): + meth = getattr(self, 'set_param_' + name) + meth(default_value) + + def set_param_threshold(self, threshold): + if threshold < 2: + threshold = 2 + self.increment_threshold = (self.THRESHOLD_LIMIT // threshold) + 1 + # the number is at least 1, and at most about half THRESHOLD_LIMIT + + def set_param_trace_eagerness(self, value): + self.trace_eagerness = value + + def set_param_trace_limit(self, value): + self.trace_limit = value + + def set_param_inlining(self, value): + self.inlining = value + + def set_param_optimizer(self, optimizer): + if optimizer == OPTIMIZER_SIMPLE: + from pypy.jit.metainterp import simple_optimize + self.optimize_loop = simple_optimize.optimize_loop + self.optimize_bridge = simple_optimize.optimize_bridge + elif optimizer == OPTIMIZER_FULL: + from pypy.jit.metainterp import optimize + self.optimize_loop = optimize.optimize_loop + self.optimize_bridge = optimize.optimize_bridge + else: + raise ValueError("unknown optimizer") + + def set_param_debug(self, value): + self.debug_level = value + if self.profiler is not None: + self.profiler.set_printing(value >= DEBUG_PROFILE) + + # ---------- + + def make_entry_point(self): + "NOT_RPYTHON" + metainterp_sd = warmrunnerdesc.metainterp_sd + globaldata = metainterp_sd.globaldata + vinfo = metainterp_sd.virtualizable_info + ContinueRunningNormally = self.warmrunnerdesc.ContinueRunningNormally + get_jitcell = self.make_jitcell_getter() + set_future_values = self.make_set_future_values() + + def maybe_compile_and_run(*args): + """Entry point to the JIT. Called at the point with the + can_enter_jit() hint. + """ + if NonConstant(False): + # make sure we always see the saner optimizer from an + # annotation point of view, otherwise we get lots of + # blocked ops + self.set_param_optimizer(OPTIMIZER_FULL) + + if vinfo is not None: + virtualizable = args[vinfo.index_of_virtualizable] + virtualizable = vinfo.cast_to_vtype(virtualizable) + assert virtualizable != globaldata.blackhole_virtualizable, ( + "reentering same frame via blackhole") + + # look for the cell corresponding to the current greenargs + greenargs = args[:self.num_green_args] + cell = get_jitcell(greenargs) + + if cell.counter >= 0: + # update the profiling counter + n = cell.counter + self.increment_threshold + if n <= self.THRESHOLD_LIMIT: # bound not reached + cell.counter = n + return + # bound reached; start tracing + from pypy.jit.metainterp.pyjitpl import MetaInterp + metainterp = MetaInterp(metainterp_sd) + try: + loop_token = metainterp.compile_and_run_once(*args) + except ContinueRunningNormally: + # the trace got too long, reset the counter + cell.counter = 0 + raise + else: + # machine code was already compiled for these greenargs + # get the assembler and fill in the boxes + set_future_values(*args[self.num_green_args:]) + loop_token = cell.entry_loop_token + + # ---------- execute assembler ---------- + while True: # until interrupted by an exception + metainterp_sd.profiler.start_running() + fail_index = metainterp_sd.cpu.execute_token(loop_token) + metainterp_sd.profiler.end_running() + fail_descr = globaldata.get_fail_descr_from_number(fail_index) + loop_token = fail_descr.handle_fail(metainterp_sd) + + maybe_compile_and_run._dont_inline_ = True + return maybe_compile_and_run + + # ---------- + + def make_jitcell_getter(self): + "NOT_RPYTHON" + # + class JitCell(object): + counter = 0 + # + if self.warmrunnerdesc.get_jitcell_at_ptr is None: + return self.make_jitcell_getter_default(JitCell) + else: + return self.make_jitcell_getter_custom(JitCell) + + def make_jitcell_getter_default(self, JitCell): + "NOT_RPYTHON" + warmrunnerdesc = self.warmrunnerdesc + green_args_spec = unrolling_iterable(warmrunnerdesc.green_args_spec) + # + def comparekey(greenargs1, greenargs2): + i = 0 + for TYPE in green_args_spec: + if not equal_whatever(TYPE, greenargs1[i], greenargs2[i]): + return False + i = i + 1 + return True + # + def hashkey(greenargs): + x = 0x345678 + i = 0 + for TYPE in green_args_spec: + item = greenargs[i] + y = hash_whatever(TYPE, item) + x = intmask((1000003 * x) ^ y) + i = i + 1 + return x + # + jitcell_dict = r_dict(comparekey, hashkey) + # + def get_jitcell(greenargs): + try: + cell = jitcell_dict[greenargs] + except KeyError: + cell = JitCell() + jitcell_dict[greenargs] = cell + return cell + return get_jitcell + + def make_jitcell_getter_custom(self, JitCell): + "NOT_RPYTHON" + rtyper = self.warmrunnerdesc.rtyper + get_jitcell_at_ptr = self.warmrunnerdesc.get_jitcell_at_ptr + set_jitcell_at_ptr = self.warmrunnerdesc.set_jitcell_at_ptr + cpu = self.warmrunnerdesc.cpu + # + def get_jitcell(greenargs): + fn = support.maybe_on_top_of_llinterp(rtyper, get_jitcell_at_ptr) + try: + cellref = fn(*greenargs) + if we_are_translated(): + cell = cast_base_ptr_to_instance(JitCell, cellref) + else: + cell = cellref + except KeyError: + cell = JitCell() + if we_are_translated(): + cellref = cpu.ts.cast_instance_to_base_ref(cell) + else: + cellref = cell + fn = support.maybe_on_top_of_llinterp(rtyper, + set_jitcell_at_ptr) + fn(cellref, *greenargs) + return cell + return get_jitcell + + # ---------- + + def make_set_future_values(self): + "NOT_RPYTHON" + warmrunnerdesc = self.warmrunnerdesc + cpu = warmrunnerdesc.cpu + vinfo = warmrunnerdesc.metainterp_sd.virtualizable_info + red_args_types = unrolling_iterable(warmrunnerdesc.red_args_types) + # + def set_future_values(*redargs): + i = 0 + for typecode in red_args_types: + set_future_value(cpu, i, redargs[i], typecode) + i = i + 1 + if vinfo is not None: + virtualizable = redargs[vinfo.index_of_virtualizable - + num_green_args] + set_future_values_from_vinfo(virtualizable) + # + if vinfo is not None: + vable_static_fields = unrolling_iterable( + zip(vinfo.static_extra_types, vinfo.static_fields)) + vable_array_fields = unrolling_iterable( + zip(vinfo.arrayitem_extra_types, vinfo.array_fields)) + getlength = cpu.ts.getlength + getarrayitem = cpu.ts.getarrayitem + # + def set_future_values_from_vinfo(virtualizable): + virtualizable = vinfo.cast_to_vtype(virtualizable) + for typecode, fieldname in vable_static_fields: + x = getattr(virtualizable, fieldname) + set_future_value(cpu, i, x, typecode) + i = i + 1 + for typecode, fieldname in vable_array_fields: + lst = getattr(virtualizable, fieldname) + for j in range(getlength(lst)): + x = getarrayitem(lst, j) + set_future_value(cpu, i, x, typecode) + i = i + 1 + else: + set_future_values_from_vinfo = None + # + return set_future_values From afa at codespeak.net Tue Oct 20 18:33:13 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 20 Oct 2009 18:33:13 +0200 (CEST) Subject: [pypy-svn] r68665 - pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc Message-ID: <20091020163313.B4F90168005@codespeak.net> Author: afa Date: Tue Oct 20 18:33:13 2009 New Revision: 68665 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Log: Fix trackgcroot when compiled in debug mode try to fix generated assembler. Someone should verify it. Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Tue Oct 20 18:33:13 2009 @@ -250,6 +250,8 @@ match = self.r_localvar_ebp.match(localvar) if match: ofs_from_ebp = int(match.group(1) or '0') + if self.format == 'msvc': + ofs_from_ebp += int(match.group(2) or '0') localvar = ofs_from_ebp - 4 assert localvar != 0 # that's the return address return LocalVar(localvar, hint='ebp') @@ -727,7 +729,7 @@ r_symboldefine = re.compile(r"([_a-z0-9]+\$) = ([-0-9]+)\s*;.+\n") LOCALVAR = r"eax|edx|ecx|ebx|esi|edi|ebp|DWORD PTR [-+]?\d*\[esp[-+]\d+\]" - LOCALVARFP = LOCALVAR + r"|-?\d*\[ebp\]" + LOCALVARFP = LOCALVAR + r"|DWORD PTR -?\d*\[ebp\]" r_localvarnofp = re.compile(LOCALVAR) r_localvarfp = re.compile(LOCALVARFP) r_gcroot_marker = re.compile(r";.+ = pypy_asm_gcroot\(") @@ -1004,10 +1006,16 @@ if self.format in ('darwin', 'mingw32', 'msvc'): name = '_' + name - if disp: - return "%s + %s" % (name, disp) + if self.format == 'msvc': + if disp: + return "DWORD PTR [%s+%s]" % (name, disp) + else: + return name else: - return name + if disp: + return "%s + %s" % (name, disp) + else: + return name def _globl(name): if self.format == 'msvc': @@ -1059,7 +1067,7 @@ def _offset(name): if self.format == 'msvc': - return "OFFSET %s" % _globalname(name) + return "DWORD PTR [%s]" % _globalname(name) else: return "$%s" % _globalname(name) @@ -1071,7 +1079,7 @@ def _indirectjmp(arg): if self.format == 'msvc': - return arg + return "DWORD PTR " + arg else: return "*%" + arg From arigo at codespeak.net Tue Oct 20 18:44:10 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Oct 2009 18:44:10 +0200 (CEST) Subject: [pypy-svn] r68666 - in pypy/branch/warmspot-jitinfo/pypy/jit/metainterp: . test Message-ID: <20091020164410.AD9A5168005@codespeak.net> Author: arigo Date: Tue Oct 20 18:44:09 2009 New Revision: 68666 Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmstate.py pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmstate.py Log: Port the rest of the code in WarmEnterState and write tests. Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmstate.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmstate.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmstate.py Tue Oct 20 18:44:09 2009 @@ -62,13 +62,13 @@ class FakeJitCell: pass state = WarmEnterState(FakeWarmRunnerDesc()) - get_jitcell = state.make_jitcell_getter_default(FakeJitCell) - cell1 = get_jitcell((42, 42.5)) + get_jitcell = state._make_jitcell_getter_default(FakeJitCell) + cell1 = get_jitcell(42, 42.5) assert isinstance(cell1, FakeJitCell) - cell2 = get_jitcell((42, 42.5)) + cell2 = get_jitcell(42, 42.5) assert cell1 is cell2 - cell3 = get_jitcell((41, 42.5)) - cell4 = get_jitcell((42, 0.25)) + cell3 = get_jitcell(41, 42.5) + cell4 = get_jitcell(42, 0.25) assert cell1 is not cell3 is not cell4 is not cell1 def test_make_jitcell_getter(): @@ -77,9 +77,10 @@ get_jitcell_at_ptr = None state = WarmEnterState(FakeWarmRunnerDesc()) get_jitcell = state.make_jitcell_getter() - cell1 = get_jitcell((1.75,)) - cell2 = get_jitcell((1.75,)) + cell1 = get_jitcell(1.75) + cell2 = get_jitcell(1.75) assert cell1 is cell2 + assert get_jitcell is state.make_jitcell_getter() def test_make_jitcell_getter_custom(): class FakeJitCell: @@ -102,15 +103,15 @@ set_jitcell_at_ptr = llhelper(SETTER, setter) # state = WarmEnterState(FakeWarmRunnerDesc()) - get_jitcell = state.make_jitcell_getter_custom(FakeJitCell) - cell1 = get_jitcell((5, 42.5)) + get_jitcell = state._make_jitcell_getter_custom(FakeJitCell) + cell1 = get_jitcell(5, 42.5) assert isinstance(cell1, FakeJitCell) assert cell1.x == 5 assert cell1.y == 42.5 - cell2 = get_jitcell((5, 42.5)) + cell2 = get_jitcell(5, 42.5) assert cell2 is cell1 - cell3 = get_jitcell((41, 42.5)) - cell4 = get_jitcell((42, 0.25)) + cell3 = get_jitcell(41, 42.5) + cell4 = get_jitcell(42, 0.25) assert cell1 is not cell3 is not cell4 is not cell1 def test_make_set_future_values(): @@ -133,3 +134,70 @@ 0: ("int", 5), 1: ("float", 42.5), } + assert set_future_values is state.make_set_future_values() + +def test_make_unwrap_greenkey(): + class FakeWarmRunnerDesc: + green_args_spec = [lltype.Signed, lltype.Float] + state = WarmEnterState(FakeWarmRunnerDesc()) + unwrap_greenkey = state.make_unwrap_greenkey() + greenargs = unwrap_greenkey([BoxInt(42), BoxFloat(42.5)]) + assert greenargs == (42, 42.5) + assert type(greenargs[0]) is int + +def test_attach_unoptimized_bridge_from_interp(): + class FakeWarmRunnerDesc: + green_args_spec = [lltype.Signed, lltype.Float] + get_jitcell_at_ptr = None + state = WarmEnterState(FakeWarmRunnerDesc()) + get_jitcell = state.make_jitcell_getter() + state.attach_unoptimized_bridge_from_interp([BoxInt(5), BoxFloat(2.25)], + "entry loop token") + cell1 = get_jitcell(5, 2.25) + assert cell1.counter < 0 + assert cell1.entry_loop_token == "entry loop token" + +def test_make_jitdriver_callbacks_1(): + class FakeWarmRunnerDesc: + can_inline_ptr = None + get_printable_location_ptr = None + state = WarmEnterState(FakeWarmRunnerDesc()) + state.make_jitdriver_callbacks() + res = state.can_inline_callable([BoxInt(5), BoxFloat(42.5)]) + assert res is True + res = state.get_location_str([BoxInt(5), BoxFloat(42.5)]) + assert res == '(no jitdriver.get_printable_location!)' + +def test_make_jitdriver_callbacks_2(): + def can_inline(x, y): + assert x == 5 + assert y == 42.5 + return False + CAN_INLINE = lltype.Ptr(lltype.FuncType([lltype.Signed, lltype.Float], + lltype.Bool)) + class FakeWarmRunnerDesc: + rtyper = None + green_args_spec = [lltype.Signed, lltype.Float] + can_inline_ptr = llhelper(CAN_INLINE, can_inline) + get_printable_location_ptr = None + state = WarmEnterState(FakeWarmRunnerDesc()) + state.make_jitdriver_callbacks() + res = state.can_inline_callable([BoxInt(5), BoxFloat(42.5)]) + assert res is False + +def test_make_jitdriver_callbacks_3(): + def get_location(x, y): + assert x == 5 + assert y == 42.5 + return "hi there" # abuse the return type, but nobody checks it + GET_LOCATION = lltype.Ptr(lltype.FuncType([lltype.Signed, lltype.Float], + lltype.Ptr(rstr.STR))) + class FakeWarmRunnerDesc: + rtyper = None + green_args_spec = [lltype.Signed, lltype.Float] + can_inline_ptr = None + get_printable_location_ptr = llhelper(GET_LOCATION, get_location) + state = WarmEnterState(FakeWarmRunnerDesc()) + state.make_jitdriver_callbacks() + res = state.get_location_str([BoxInt(5), BoxFloat(42.5)]) + assert res == "hi there" Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmstate.py Tue Oct 20 18:44:09 2009 @@ -1,6 +1,7 @@ import sys from pypy.rpython.lltypesystem import lltype, llmemory, rstr from pypy.rpython.ootypesystem import ootype +from pypy.rpython.annlowlevel import hlstr from pypy.rlib.objectmodel import specialize, we_are_translated, r_dict from pypy.rlib.rarithmetic import intmask from pypy.rlib.nonconst import NonConstant @@ -152,16 +153,33 @@ if self.profiler is not None: self.profiler.set_printing(value >= DEBUG_PROFILE) + def must_compile_from_failure(self, key): + key.counter += 1 + return key.counter >= self.trace_eagerness + + def reset_counter_from_failure(self, key): + key.counter = 0 + + def attach_unoptimized_bridge_from_interp(self, greenkey, + entry_loop_token): + cell = self.jit_cell_at_key(greenkey) + cell.counter = -1 + cell.entry_loop_token = entry_loop_token + # ---------- def make_entry_point(self): "NOT_RPYTHON" - metainterp_sd = warmrunnerdesc.metainterp_sd + if hasattr(self, 'maybe_compile_and_run'): + return self.maybe_compile_and_run + + metainterp_sd = self.warmrunnerdesc.metainterp_sd globaldata = metainterp_sd.globaldata vinfo = metainterp_sd.virtualizable_info ContinueRunningNormally = self.warmrunnerdesc.ContinueRunningNormally get_jitcell = self.make_jitcell_getter() set_future_values = self.make_set_future_values() + self.make_jitdriver_callbacks() def maybe_compile_and_run(*args): """Entry point to the JIT. Called at the point with the @@ -181,7 +199,7 @@ # look for the cell corresponding to the current greenargs greenargs = args[:self.num_green_args] - cell = get_jitcell(greenargs) + cell = get_jitcell(*greenargs) if cell.counter >= 0: # update the profiling counter @@ -213,22 +231,58 @@ loop_token = fail_descr.handle_fail(metainterp_sd) maybe_compile_and_run._dont_inline_ = True + self.maybe_compile_and_run = maybe_compile_and_run return maybe_compile_and_run # ---------- + def make_unwrap_greenkey(self): + "NOT_RPYTHON" + if hasattr(self, 'unwrap_greenkey'): + return self.unwrap_greenkey + # + warmrunnerdesc = self.warmrunnerdesc + green_args_spec = unrolling_iterable(warmrunnerdesc.green_args_spec) + # + def unwrap_greenkey(greenkey): + greenargs = () + i = 0 + for TYPE in green_args_spec: + value = unwrap(TYPE, greenkey[i]) + greenargs += (value,) + i = i + 1 + return greenargs + # + unwrap_greenkey._always_inline_ = True + self.unwrap_greenkey = unwrap_greenkey + return unwrap_greenkey + + # ---------- + def make_jitcell_getter(self): "NOT_RPYTHON" + if hasattr(self, 'jit_getter'): + return self.jit_getter # class JitCell(object): counter = 0 # if self.warmrunnerdesc.get_jitcell_at_ptr is None: - return self.make_jitcell_getter_default(JitCell) + jit_getter = self._make_jitcell_getter_default(JitCell) else: - return self.make_jitcell_getter_custom(JitCell) + jit_getter = self._make_jitcell_getter_custom(JitCell) + # + unwrap_greenkey = self.make_unwrap_greenkey() + # + def jit_cell_at_key(greenkey): + greenargs = unwrap_greenkey(greenkey) + return jit_getter(*greenargs) + self.jit_cell_at_key = jit_cell_at_key + self.jit_getter = jit_getter + # + return jit_getter - def make_jitcell_getter_default(self, JitCell): + def _make_jitcell_getter_default(self, JitCell): "NOT_RPYTHON" warmrunnerdesc = self.warmrunnerdesc green_args_spec = unrolling_iterable(warmrunnerdesc.green_args_spec) @@ -253,7 +307,7 @@ # jitcell_dict = r_dict(comparekey, hashkey) # - def get_jitcell(greenargs): + def get_jitcell(*greenargs): try: cell = jitcell_dict[greenargs] except KeyError: @@ -262,14 +316,14 @@ return cell return get_jitcell - def make_jitcell_getter_custom(self, JitCell): + def _make_jitcell_getter_custom(self, JitCell): "NOT_RPYTHON" rtyper = self.warmrunnerdesc.rtyper get_jitcell_at_ptr = self.warmrunnerdesc.get_jitcell_at_ptr set_jitcell_at_ptr = self.warmrunnerdesc.set_jitcell_at_ptr cpu = self.warmrunnerdesc.cpu # - def get_jitcell(greenargs): + def get_jitcell(*greenargs): fn = support.maybe_on_top_of_llinterp(rtyper, get_jitcell_at_ptr) try: cellref = fn(*greenargs) @@ -293,6 +347,9 @@ def make_set_future_values(self): "NOT_RPYTHON" + if hasattr(self, 'set_future_values'): + return self.set_future_values + warmrunnerdesc = self.warmrunnerdesc cpu = warmrunnerdesc.cpu vinfo = warmrunnerdesc.metainterp_sd.virtualizable_info @@ -331,4 +388,42 @@ else: set_future_values_from_vinfo = None # + self.set_future_values = set_future_values return set_future_values + + # ---------- + + def make_jitdriver_callbacks(self): + if hasattr(self, 'get_location_str'): + return + # + can_inline_ptr = self.warmrunnerdesc.can_inline_ptr + if can_inline_ptr is None: + def can_inline_callable(greenkey): + return True + else: + rtyper = self.warmrunnerdesc.rtyper + unwrap_greenkey = self.make_unwrap_greenkey() + # + def can_inline_callable(greenkey): + greenargs = unwrap_greenkey(greenkey) + fn = support.maybe_on_top_of_llinterp(rtyper, can_inline_ptr) + return fn(*greenargs) + self.can_inline_callable = can_inline_callable + # + get_location_ptr = self.warmrunnerdesc.get_printable_location_ptr + if get_location_ptr is None: + def get_location_str(greenkey): + return '(no jitdriver.get_printable_location!)' + else: + rtyper = self.warmrunnerdesc.rtyper + unwrap_greenkey = self.make_unwrap_greenkey() + # + def get_location_str(greenkey): + greenargs = unwrap_greenkey(greenkey) + fn = support.maybe_on_top_of_llinterp(rtyper, get_location_ptr) + res = fn(*greenargs) + if not we_are_translated() and not isinstance(res, str): + res = hlstr(res) + return res + self.get_location_str = get_location_str From arigo at codespeak.net Tue Oct 20 19:14:43 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Oct 2009 19:14:43 +0200 (CEST) Subject: [pypy-svn] r68667 - in pypy/branch/warmspot-jitinfo/pypy/jit: backend/llgraph metainterp metainterp/test Message-ID: <20091020171443.DE1F8168007@codespeak.net> Author: arigo Date: Tue Oct 20 19:14:43 2009 New Revision: 68667 Modified: pypy/branch/warmspot-jitinfo/pypy/jit/backend/llgraph/runner.py pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/compile.py pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/pyjitpl.py pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmspot.py pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmspot.py pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmstate.py Log: Start fixing the rest of the code. Modified: pypy/branch/warmspot-jitinfo/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/backend/llgraph/runner.py Tue Oct 20 19:14:43 2009 @@ -9,7 +9,7 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.jit.metainterp import history from pypy.jit.metainterp.history import REF, INT, FLOAT -from pypy.jit.metainterp.warmspot import unwrap +from pypy.jit.metainterp.warmstate import unwrap from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.backend import model from pypy.jit.backend.llgraph import llimpl, symbolic Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/compile.py Tue Oct 20 19:14:43 2009 @@ -259,8 +259,8 @@ new_loop_token) # store the new loop in compiled_merge_points too glob = metainterp_sd.globaldata - greenargs = glob.unpack_greenkey(self.original_greenkey) - old_loop_tokens = glob.compiled_merge_points.setdefault(greenargs, []) + old_loop_tokens = glob.get_compiled_merge_points( + self.original_greenkey) # it always goes at the end of the list, as it is the most # general loop token old_loop_tokens.append(new_loop_token) Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/pyjitpl.py Tue Oct 20 19:14:43 2009 @@ -1,7 +1,7 @@ import py from pypy.rpython.lltypesystem import llmemory from pypy.rpython.ootypesystem import ootype -from pypy.rlib.objectmodel import we_are_translated, r_dict +from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.debug import debug_print @@ -1097,15 +1097,30 @@ # state = staticdata.state if state is not None: - self.unpack_greenkey = state.unwrap_greenkey - self.compiled_merge_points = r_dict(state.comparekey,state.hashkey) - # { (greenargs): [MergePoints] } + self.jit_cell_at_key = state.jit_cell_at_key else: - self.compiled_merge_points = {} # for tests only; not RPython - self.unpack_greenkey = tuple + # for tests only; not RPython + class JitCell: + compiled_merge_points = None + _jitcell_dict = {} + def jit_cell_at_key(greenkey): + greenkey = tuple(greenkey) + try: + cell = _jitcell_dict[greenkey] + except KeyError: + cell = JitCell() + _jitcell_dict[greenkey] = cell + return cell + self.jit_cell_at_key = jit_cell_at_key if staticdata.virtualizable_info: self.blackhole_virtualizable = staticdata.virtualizable_info.null_vable + def get_compiled_merge_points(self, greenkey): + cell = self.jit_cell_at_key(greenkey) + if cell.compiled_merge_points is None: + cell.compiled_merge_points = [] + return cell.compiled_merge_points + def get_fail_descr_number(self, descr): assert isinstance(descr, history.AbstractFailDescr) lst = self.fail_descr_list @@ -1486,8 +1501,7 @@ self.history.inputargs = original_boxes[num_green_args:] greenkey = original_boxes[:num_green_args] glob = self.staticdata.globaldata - greenargs = glob.unpack_greenkey(greenkey) - old_loop_tokens = glob.compiled_merge_points.setdefault(greenargs, []) + old_loop_tokens = glob.get_compiled_merge_points(greenkey) self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None) loop_token = compile.compile_new_loop(self, old_loop_tokens, greenkey, start) @@ -1498,10 +1512,8 @@ num_green_args = self.staticdata.num_green_args greenkey = live_arg_boxes[:num_green_args] glob = self.staticdata.globaldata - greenargs = glob.unpack_greenkey(greenkey) - try: - old_loop_tokens = glob.compiled_merge_points[greenargs] - except KeyError: + old_loop_tokens = glob.get_compiled_merge_points(greenkey) + if len(old_loop_tokens) == 0: return self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None) target_loop_token = compile.compile_new_bridge(self, old_loop_tokens, @@ -1555,7 +1567,7 @@ def _initialize_from_start(self, original_boxes, num_green_args, *args): if args: - from pypy.jit.metainterp.warmspot import wrap + from pypy.jit.metainterp.warmstate import wrap box = wrap(self.cpu, args[0], num_green_args > 0) original_boxes.append(box) self._initialize_from_start(original_boxes, num_green_args-1, Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmspot.py Tue Oct 20 19:14:43 2009 @@ -1,39 +1,13 @@ import py -from pypy.jit.metainterp.warmspot import ll_meta_interp, hash_whatever -from pypy.jit.metainterp.warmspot import get_stats, equal_whatever +from pypy.jit.metainterp.warmspot import ll_meta_interp +from pypy.jit.metainterp.warmspot import get_stats from pypy.rlib.jit import JitDriver, OPTIMIZER_FULL, OPTIMIZER_SIMPLE from pypy.rlib.jit import unroll_safe from pypy.jit.backend.llgraph import runner -from pypy.rpython.test.test_llinterp import interpret from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin -def test_hash_equal_whatever_lltype(): - from pypy.rpython.lltypesystem import lltype, rstr - s1 = rstr.mallocstr(2) - s2 = rstr.mallocstr(2) - s1.chars[0] = 'x'; s1.chars[1] = 'y' - s2.chars[0] = 'x'; s2.chars[1] = 'y' - def fn(x): - assert hash_whatever(lltype.typeOf(x), x) == 42 - assert (hash_whatever(lltype.typeOf(s1), s1) == - hash_whatever(lltype.typeOf(s2), s2)) - assert equal_whatever(lltype.typeOf(s1), s1, s2) - fn(42) - interpret(fn, [42], type_system='lltype') - -def test_hash_equal_whatever_ootype(): - from pypy.rpython.ootypesystem import ootype - def fn(c): - s1 = ootype.oostring("xy", -1) - s2 = ootype.oostring("x" + chr(c), -1) - assert (hash_whatever(ootype.typeOf(s1), s1) == - hash_whatever(ootype.typeOf(s2), s2)) - assert equal_whatever(ootype.typeOf(s1), s1, s2) - fn(ord('y')) - interpret(fn, [ord('y')], type_system='ootype') - class Exit(Exception): def __init__(self, result): self.result = result @@ -90,20 +64,6 @@ res = self.meta_interp(f, [60]) assert res == f(30) - def test_hash_collision(self): - mydriver = JitDriver(reds = ['n'], greens = ['m']) - def f(n): - m = 0 - while n > 0: - mydriver.can_enter_jit(n=n, m=m) - mydriver.jit_merge_point(n=n, m=m) - n -= 1 - if not (n % 11): - m = (m+n) & 3 - return m - res = self.meta_interp(f, [110], hash_bits=1) - assert res == f(110) - def test_location(self): def get_printable_location(n): return 'GREEN IS %d.' % n Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmspot.py Tue Oct 20 19:14:43 2009 @@ -10,7 +10,6 @@ from pypy.objspace.flow.model import checkgraph, Link, copygraph from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib.jit import PARAMETERS, OPTIMIZER_SIMPLE, OPTIMIZER_FULL from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.debug import debug_print from pypy.rpython.lltypesystem.lloperation import llop @@ -24,7 +23,6 @@ from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper from pypy.jit.metainterp.jitprof import Profiler, EmptyProfiler from pypy.rlib.jit import DEBUG_STEPS, DEBUG_DETAILED, DEBUG_OFF, DEBUG_PROFILE -from pypy.rlib.nonconst import NonConstant # ____________________________________________________________ # Bootstrapping @@ -63,7 +61,7 @@ clear_tcache() return jittify_and_run(interp, graph, args, backendopt=backendopt, **kwds) -def jittify_and_run(interp, graph, args, repeat=1, hash_bits=None, +def jittify_and_run(interp, graph, args, repeat=1, backendopt=False, trace_limit=sys.maxint, debug_level=DEBUG_STEPS, inline=False, **kwds): translator = interp.typer.annotator.translator @@ -74,9 +72,6 @@ warmrunnerdesc.state.set_param_trace_limit(trace_limit) warmrunnerdesc.state.set_param_inlining(inline) warmrunnerdesc.state.set_param_debug(debug_level) - warmrunnerdesc.state.create_tables_now() # for tests - if hash_bits: - warmrunnerdesc.state.set_param_hash_bits(hash_bits) warmrunnerdesc.finish() res = interp.eval_graph(graph, args) if not kwds.get('translate_support_code', False): @@ -167,7 +162,7 @@ self.build_meta_interp(CPUClass, **kwds) self.make_args_specification() self.rewrite_jit_merge_point(policy) - self.make_driverhook_graph() + self.make_driverhook_graphs() if self.jitdriver.virtualizables: from pypy.jit.metainterp.virtualizable import VirtualizableInfo self.metainterp_sd.virtualizable_info = VirtualizableInfo(self) @@ -269,8 +264,9 @@ warmrunnerdesc=self) def make_enter_function(self): - WarmEnterState = make_state_class(self) - state = WarmEnterState() + from pypy.jit.metainterp.warmstate import WarmEnterState + state = WarmEnterState(self) + maybe_compile_and_run = state.make_entry_point() self.state = state def crash_in_jit(e): @@ -288,7 +284,7 @@ if self.translator.rtyper.type_system.name == 'lltypesystem': def maybe_enter_jit(*args): try: - state.maybe_compile_and_run(*args) + maybe_compile_and_run(*args) except JitException: raise # go through except Exception, e: @@ -296,7 +292,7 @@ maybe_enter_jit._always_inline_ = True else: def maybe_enter_jit(*args): - state.maybe_compile_and_run(*args) + maybe_compile_and_run(*args) maybe_enter_jit._always_inline_ = True self.maybe_enter_jit_fn = maybe_enter_jit @@ -312,24 +308,34 @@ args_s, s_result) annhelper.finish() - def make_driverhook_graph(self): + def make_driverhook_graphs(self): + annhelper = MixLevelHelperAnnotator(self.translator.rtyper) + blackboxtype = self.cpu.ts.BASETYPE + self.set_jitcell_at_ptr = self._make_hook_graph( + annhelper, self.jitdriver.set_jitcell_at, None, blackboxtype) + self.get_jitcell_at_ptr = self._make_hook_graph( + annhelper, self.jitdriver.get_jitcell_at, blackboxtype) self.can_inline_ptr = self._make_hook_graph( - self.jitdriver.can_inline, bool) + annhelper, self.jitdriver.can_inline, bool) self.get_printable_location_ptr = self._make_hook_graph( - self.jitdriver.get_printable_location, str) + annhelper, self.jitdriver.get_printable_location, str) + annhelper.finish() - def _make_hook_graph(self, func, rettype): + def _make_hook_graph(self, annhelper, func, rettype, first_arg_type=None): from pypy.annotation.signature import annotationoftype if func is None: return None - annhelper = MixLevelHelperAnnotator(self.translator.rtyper) - s_result = annotationoftype(rettype) - RETTYPE = annhelper.rtyper.getrepr(s_result).lowleveltype + if rettype is not None: + s_result = annotationoftype(rettype) + RETTYPE = annhelper.rtyper.getrepr(s_result).lowleveltype + else: + RETTYPE = lltype.Void FUNC, PTR = self.cpu.ts.get_FuncType(self.green_args_spec, RETTYPE) args_s = self.portal_args_s[:len(self.green_args_spec)] + if first_arg_type is not None: + args_s.insert(0, annotationoftype(first_arg_type)) graph = annhelper.getgraph(func, args_s, s_result) funcptr = annhelper.graph2delayed(graph, FUNC) - annhelper.finish() return funcptr def make_args_specification(self): @@ -346,6 +352,7 @@ self.green_args_spec.append(TYPE) else: self.red_args_types.append(history.getkind(TYPE)) + self.num_green_args = len(self.green_args_spec) RESTYPE = graph.getreturnvar().concretetype (self.JIT_ENTER_FUNCTYPE, self.PTR_JIT_ENTER_FUNCTYPE) = self.cpu.ts.get_FuncType(ALLARGS, lltype.Void) @@ -465,6 +472,7 @@ # accepts boxes as argument, but unpacks them immediately # before we raise the exception -- the boxes' values will # be modified in a 'finally' by restore_patched_boxes(). + from pypy.jit.metainterp.warmstate import unwrap for i, name, ARG in portalfunc_ARGS: v = unwrap(ARG, argboxes[i]) setattr(self, name, v) @@ -585,392 +593,3 @@ assert len(reds_v) == numreds return ([v for v in greens_v if v.concretetype is not lltype.Void], [v for v in reds_v if v.concretetype is not lltype.Void]) - -def unwrap(TYPE, box): - if TYPE is lltype.Void: - return None - if isinstance(TYPE, lltype.Ptr): - return box.getref(TYPE) - if isinstance(TYPE, ootype.OOType): - return box.getref(TYPE) - if TYPE == lltype.Float: - return box.getfloat() - else: - return lltype.cast_primitive(TYPE, box.getint()) -unwrap._annspecialcase_ = 'specialize:arg(0)' - -def wrap(cpu, value, in_const_box=False): - if isinstance(lltype.typeOf(value), lltype.Ptr): - if lltype.typeOf(value).TO._gckind == 'gc': - value = lltype.cast_opaque_ptr(llmemory.GCREF, value) - if in_const_box: - return history.ConstPtr(value) - else: - return history.BoxPtr(value) - else: - adr = llmemory.cast_ptr_to_adr(value) - value = cpu.cast_adr_to_int(adr) - # fall through to the end of the function - elif isinstance(lltype.typeOf(value), ootype.OOType): - value = ootype.cast_to_object(value) - if in_const_box: - return history.ConstObj(value) - else: - return history.BoxObj(value) - elif isinstance(value, float): - if in_const_box: - return history.ConstFloat(value) - else: - return history.BoxFloat(value) - else: - value = intmask(value) - if in_const_box: - return history.ConstInt(value) - else: - return history.BoxInt(value) -wrap._annspecialcase_ = 'specialize:ll' - -def equal_whatever(TYPE, x, y): - if isinstance(TYPE, lltype.Ptr): - if TYPE.TO is rstr.STR or TYPE.TO is rstr.UNICODE: - return rstr.LLHelpers.ll_streq(x, y) - if TYPE is ootype.String or TYPE is ootype.Unicode: - return x.ll_streq(y) - return x == y -equal_whatever._annspecialcase_ = 'specialize:arg(0)' - -def hash_whatever(TYPE, x): - # Hash of lltype or ootype object. - # Only supports strings, unicodes and regular instances, - # as well as primitives that can meaningfully be cast to Signed. - if isinstance(TYPE, lltype.Ptr): - if TYPE.TO is rstr.STR or TYPE.TO is rstr.UNICODE: - return rstr.LLHelpers.ll_strhash(x) # assumed not null - else: - if x: - return lltype.identityhash(x) - else: - return 0 - elif TYPE is ootype.String or TYPE is ootype.Unicode: - return x.ll_hash() - elif isinstance(TYPE, ootype.OOType): - if x: - return ootype.identityhash(x) - else: - return 0 - else: - return lltype.cast_primitive(lltype.Signed, x) -hash_whatever._annspecialcase_ = 'specialize:arg(0)' - -# ____________________________________________________________ - -def make_state_class(warmrunnerdesc): - jitdriver = warmrunnerdesc.jitdriver - num_green_args = len(jitdriver.greens) - warmrunnerdesc.num_green_args = num_green_args - green_args_spec = unrolling_iterable(warmrunnerdesc.green_args_spec) - green_args_names = unrolling_iterable(jitdriver.greens) - green_args_spec_names = unrolling_iterable(zip( - warmrunnerdesc.green_args_spec, jitdriver.greens)) - red_args_types = unrolling_iterable(warmrunnerdesc.red_args_types) - # - metainterp_sd = warmrunnerdesc.metainterp_sd - vinfo = metainterp_sd.virtualizable_info - if vinfo is None: - vable_static_fields = [] - vable_array_fields = [] - else: - vable_static_fields = unrolling_iterable( - zip(vinfo.static_extra_types, vinfo.static_fields)) - vable_array_fields = unrolling_iterable( - zip(vinfo.arrayitem_extra_types, vinfo.array_fields)) - # - if num_green_args: - MAX_HASH_TABLE_BITS = 28 - else: - MAX_HASH_TABLE_BITS = 1 - THRESHOLD_LIMIT = sys.maxint // 2 - # - getlength = warmrunnerdesc.cpu.ts.getlength - getarrayitem = warmrunnerdesc.cpu.ts.getarrayitem - setarrayitem = warmrunnerdesc.cpu.ts.setarrayitem - # - rtyper = warmrunnerdesc.translator.rtyper - can_inline_ptr = warmrunnerdesc.can_inline_ptr - get_printable_location_ptr = warmrunnerdesc.get_printable_location_ptr - # - class MachineCodeEntryPoint(object): - next = None # linked list - def __init__(self, entry_loop_token, *greenargs): - self.entry_loop_token = entry_loop_token - i = 0 - for name in green_args_names: - setattr(self, 'green_' + name, greenargs[i]) - i = i + 1 - def equalkey(self, *greenargs): - i = 0 - for TYPE, name in green_args_spec_names: - myvalue = getattr(self, 'green_' + name) - if not equal_whatever(TYPE, myvalue, greenargs[i]): - return False - i = i + 1 - return True - def set_future_values(self, *redargs): - i = 0 - for typecode in red_args_types: - set_future_value(i, redargs[i], typecode) - i = i + 1 - if vinfo is not None: - virtualizable = redargs[vinfo.index_of_virtualizable - - num_green_args] - virtualizable = vinfo.cast_to_vtype(virtualizable) - for typecode, fieldname in vable_static_fields: - x = getattr(virtualizable, fieldname) - set_future_value(i, x, typecode) - i = i + 1 - for typecode, fieldname in vable_array_fields: - lst = getattr(virtualizable, fieldname) - for j in range(getlength(lst)): - x = getarrayitem(lst, j) - set_future_value(i, x, typecode) - i = i + 1 - - def set_future_value(j, value, typecode): - cpu = metainterp_sd.cpu - if typecode == 'ref': - refvalue = cpu.ts.cast_to_ref(value) - cpu.set_future_value_ref(j, refvalue) - elif typecode == 'int': - intvalue = lltype.cast_primitive(lltype.Signed, value) - cpu.set_future_value_int(j, intvalue) - elif typecode == 'float': - assert isinstance(value, float) - cpu.set_future_value_float(j, value) - else: - assert False - set_future_value._annspecialcase_ = 'specialize:ll_and_arg(2)' - - class WarmEnterState: - def __init__(self): - # initialize the state with the default values of the - # parameters specified in rlib/jit.py - for name, default_value in PARAMETERS.items(): - meth = getattr(self, 'set_param_' + name) - meth(default_value) - - def set_param_threshold(self, threshold): - if threshold < 2: - threshold = 2 - self.increment_threshold = (THRESHOLD_LIMIT // threshold) + 1 - # the number is at least 1, and at most about half THRESHOLD_LIMIT - - def set_param_trace_eagerness(self, value): - self.trace_eagerness = value - - def set_param_trace_limit(self, value): - self.trace_limit = value - - def set_param_inlining(self, value): - self.inlining = value - - def set_param_hash_bits(self, value): - if value < 1: - value = 1 - elif value > MAX_HASH_TABLE_BITS: - value = MAX_HASH_TABLE_BITS - # the tables are initialized with the correct size only in - # create_tables_now() - self.hashbits = value - self.hashtablemask = 0 - self.mccounters = [0] - self.mcentrypoints = [None] - # invariant: (self.mccounters[j] < 0) if and only if - # (self.mcentrypoints[j] is not None) - - def set_param_optimizer(self, optimizer): - if optimizer == OPTIMIZER_SIMPLE: - from pypy.jit.metainterp import simple_optimize - self.optimize_loop = simple_optimize.optimize_loop - self.optimize_bridge = simple_optimize.optimize_bridge - elif optimizer == OPTIMIZER_FULL: - from pypy.jit.metainterp import optimize - self.optimize_loop = optimize.optimize_loop - self.optimize_bridge = optimize.optimize_bridge - else: - raise ValueError("unknown optimizer") - - def set_param_debug(self, value): - self.debug_level = value - metainterp_sd.profiler.set_printing(value >= DEBUG_PROFILE) - - def create_tables_now(self): - count = 1 << self.hashbits - self.hashtablemask = count - 1 - self.mccounters = [0] * count - self.mcentrypoints = [None] * count - - # Only use the hash of the arguments as the profiling key. - # Indeed, this is all a heuristic, so if things are designed - # correctly, the occasional mistake due to hash collision is - # not too bad. - - def maybe_compile_and_run(self, *args): - globaldata = metainterp_sd.globaldata - if NonConstant(False): - # make sure we always see the saner optimizer from an annotation - # point of view, otherwise we get lots of blocked ops - self.set_param_optimizer(OPTIMIZER_FULL) - - # get the greenargs and look for the cell corresponding to the hash - greenargs = args[:num_green_args] - argshash = self.getkeyhash(*greenargs) & self.hashtablemask - counter = self.mccounters[argshash] - if vinfo: - virtualizable = args[vinfo.index_of_virtualizable] - virtualizable = vinfo.cast_to_vtype(virtualizable) - assert virtualizable != globaldata.blackhole_virtualizable, "reentering same frame via blackhole" - if counter >= 0: - # update the profiling counter - n = counter + self.increment_threshold - if n <= THRESHOLD_LIMIT: # bound not reached - self.mccounters[argshash] = n - return - if self.hashtablemask == 0: # must really create the tables now - self.create_tables_now() - return - metainterp = MetaInterp(metainterp_sd) - try: - loop_token = metainterp.compile_and_run_once(*args) - except warmrunnerdesc.ContinueRunningNormally: - # the trace got too long, reset the counter - self.mccounters[argshash] = 0 - raise - - else: - # machine code was already compiled for these greenargs - # (or we have a hash collision) - cell = self.mcentrypoints[argshash] - if not cell.equalkey(*greenargs): - # hash collision - loop_token = self.handle_hash_collision(cell, argshash, - *args) - if loop_token is None: - return - else: - # get the assembler and fill in the boxes - cell.set_future_values(*args[num_green_args:]) - loop_token = cell.entry_loop_token - # ---------- execute assembler ---------- - while True: # until interrupted by an exception - metainterp_sd.profiler.start_running() - fail_index = metainterp_sd.cpu.execute_token(loop_token) - metainterp_sd.profiler.end_running() - fail_descr = globaldata.get_fail_descr_from_number(fail_index) - loop_token = fail_descr.handle_fail(metainterp_sd) - - maybe_compile_and_run._dont_inline_ = True - - def handle_hash_collision(self, firstcell, argshash, *args): - greenargs = args[:num_green_args] - # search the linked list for the correct cell - cell = firstcell - while cell.next is not None: - nextcell = cell.next - if nextcell.equalkey(*greenargs): - # found, move to the front of the linked list - cell.next = nextcell.next - nextcell.next = firstcell - self.mcentrypoints[argshash] = nextcell - nextcell.set_future_values(*args[num_green_args:]) - return nextcell.entry_loop_token - cell = nextcell - # not found at all, do profiling - counter = self.mccounters[argshash] - assert counter < 0 # by invariant - n = counter + self.increment_threshold - if n < 0: # bound not reached - self.mccounters[argshash] = n - return None - metainterp = MetaInterp(metainterp_sd) - # XXX ContinueRunningNormally => "reset" counters - return metainterp.compile_and_run_once(*args) - handle_hash_collision._dont_inline_ = True - - def unwrap_greenkey(self, greenkey): - greenargs = () - i = 0 - for TYPE in green_args_spec: - value = unwrap(TYPE, greenkey[i]) - greenargs += (value,) - i = i + 1 - return greenargs - unwrap_greenkey._always_inline_ = True - - def comparekey(greenargs1, greenargs2): - i = 0 - for TYPE in green_args_spec: - if not equal_whatever(TYPE, greenargs1[i], greenargs2[i]): - return False - i = i + 1 - return True - comparekey = staticmethod(comparekey) - - def hashkey(greenargs): - return intmask(WarmEnterState.getkeyhash(*greenargs)) - hashkey = staticmethod(hashkey) - - def getkeyhash(*greenargs): - result = r_uint(0x345678) - i = 0 - mult = r_uint(1000003) - for TYPE in green_args_spec: - if i > 0: - result = result * mult - mult = mult + 82520 + 2*len(greenargs) - item = greenargs[i] - result = result ^ hash_whatever(TYPE, item) - i = i + 1 - return result # returns a r_uint - getkeyhash._always_inline_ = True - getkeyhash = staticmethod(getkeyhash) - - def must_compile_from_failure(self, key): - key.counter += 1 - return key.counter >= self.trace_eagerness - - def reset_counter_from_failure(self, key): - key.counter = 0 - - def attach_unoptimized_bridge_from_interp(self, greenkey, - entry_loop_token): - greenargs = self.unwrap_greenkey(greenkey) - newcell = MachineCodeEntryPoint(entry_loop_token, *greenargs) - argshash = self.getkeyhash(*greenargs) & self.hashtablemask - oldcell = self.mcentrypoints[argshash] - newcell.next = oldcell # link - self.mcentrypoints[argshash] = newcell - self.mccounters[argshash] = -THRESHOLD_LIMIT-1 - - if can_inline_ptr is None: - def can_inline_callable(self, greenkey): - return True - else: - def can_inline_callable(self, greenkey): - args = self.unwrap_greenkey(greenkey) - fn = support.maybe_on_top_of_llinterp(rtyper, can_inline_ptr) - return fn(*args) - - if get_printable_location_ptr is None: - def get_location_str(self, greenkey): - return '(no jitdriver.get_printable_location!)' - else: - def get_location_str(self, greenkey): - args = self.unwrap_greenkey(greenkey) - fn = support.maybe_on_top_of_llinterp(rtyper, - get_printable_location_ptr) - res = fn(*args) - if not we_are_translated() and not isinstance(res, str): - res = hlstr(res) - return res - - return WarmEnterState Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmstate.py Tue Oct 20 19:14:43 2009 @@ -7,6 +7,7 @@ from pypy.rlib.nonconst import NonConstant from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.jit import PARAMETERS, OPTIMIZER_SIMPLE, OPTIMIZER_FULL +from pypy.rlib.jit import DEBUG_PROFILE from pypy.jit.metainterp import support, history # ____________________________________________________________ @@ -174,9 +175,9 @@ return self.maybe_compile_and_run metainterp_sd = self.warmrunnerdesc.metainterp_sd - globaldata = metainterp_sd.globaldata vinfo = metainterp_sd.virtualizable_info ContinueRunningNormally = self.warmrunnerdesc.ContinueRunningNormally + num_green_args = self.warmrunnerdesc.num_green_args get_jitcell = self.make_jitcell_getter() set_future_values = self.make_set_future_values() self.make_jitdriver_callbacks() @@ -185,6 +186,7 @@ """Entry point to the JIT. Called at the point with the can_enter_jit() hint. """ + globaldata = metainterp_sd.globaldata if NonConstant(False): # make sure we always see the saner optimizer from an # annotation point of view, otherwise we get lots of @@ -198,7 +200,7 @@ "reentering same frame via blackhole") # look for the cell corresponding to the current greenargs - greenargs = args[:self.num_green_args] + greenargs = args[:num_green_args] cell = get_jitcell(*greenargs) if cell.counter >= 0: @@ -219,7 +221,7 @@ else: # machine code was already compiled for these greenargs # get the assembler and fill in the boxes - set_future_values(*args[self.num_green_args:]) + set_future_values(*args[num_green_args:]) loop_token = cell.entry_loop_token # ---------- execute assembler ---------- @@ -266,6 +268,7 @@ # class JitCell(object): counter = 0 + compiled_merge_points = None # if self.warmrunnerdesc.get_jitcell_at_ptr is None: jit_getter = self._make_jitcell_getter_default(JitCell) @@ -361,11 +364,10 @@ set_future_value(cpu, i, redargs[i], typecode) i = i + 1 if vinfo is not None: - virtualizable = redargs[vinfo.index_of_virtualizable - - num_green_args] - set_future_values_from_vinfo(virtualizable) + set_future_values_from_vinfo(*redargs) # if vinfo is not None: + num_green_args = warmrunnerdesc.num_green_args vable_static_fields = unrolling_iterable( zip(vinfo.static_extra_types, vinfo.static_fields)) vable_array_fields = unrolling_iterable( @@ -373,7 +375,9 @@ getlength = cpu.ts.getlength getarrayitem = cpu.ts.getarrayitem # - def set_future_values_from_vinfo(virtualizable): + def set_future_values_from_vinfo(*redargs): + virtualizable = redargs[vinfo.index_of_virtualizable - + num_green_args] virtualizable = vinfo.cast_to_vtype(virtualizable) for typecode, fieldname in vable_static_fields: x = getattr(virtualizable, fieldname) From arigo at codespeak.net Tue Oct 20 19:16:24 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Oct 2009 19:16:24 +0200 (CEST) Subject: [pypy-svn] r68668 - in pypy/branch/warmspot-jitinfo/pypy/jit: backend/llgraph metainterp Message-ID: <20091020171624.2D814168007@codespeak.net> Author: arigo Date: Tue Oct 20 19:16:23 2009 New Revision: 68668 Modified: pypy/branch/warmspot-jitinfo/pypy/jit/backend/llgraph/llimpl.py pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/pyjitpl.py Log: Minor fixes. Modified: pypy/branch/warmspot-jitinfo/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/backend/llgraph/llimpl.py Tue Oct 20 19:16:23 2009 @@ -807,7 +807,6 @@ op_getarrayitem_gc_pure = op_getarrayitem_gc def op_setarrayitem_gc(self, typedescr, obj, index, objnewvalue): - from pypy.jit.metainterp.warmspot import unwrap array = ootype.cast_from_object(typedescr.ARRAY, obj) if ootype.typeOf(objnewvalue) == ootype.Object: newvalue = ootype.cast_from_object(typedescr.TYPE, objnewvalue) Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/pyjitpl.py Tue Oct 20 19:16:23 2009 @@ -1105,12 +1105,7 @@ _jitcell_dict = {} def jit_cell_at_key(greenkey): greenkey = tuple(greenkey) - try: - cell = _jitcell_dict[greenkey] - except KeyError: - cell = JitCell() - _jitcell_dict[greenkey] = cell - return cell + return _jitcell_dict.setdefaut(greenkey, JitCell()) self.jit_cell_at_key = jit_cell_at_key if staticdata.virtualizable_info: self.blackhole_virtualizable = staticdata.virtualizable_info.null_vable From arigo at codespeak.net Tue Oct 20 19:17:02 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Oct 2009 19:17:02 +0200 (CEST) Subject: [pypy-svn] r68669 - pypy/branch/warmspot-jitinfo/pypy/jit/metainterp Message-ID: <20091020171702.EF0F2168007@codespeak.net> Author: arigo Date: Tue Oct 20 19:17:02 2009 New Revision: 68669 Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/pyjitpl.py Log: Argh. Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/pyjitpl.py Tue Oct 20 19:17:02 2009 @@ -1105,7 +1105,7 @@ _jitcell_dict = {} def jit_cell_at_key(greenkey): greenkey = tuple(greenkey) - return _jitcell_dict.setdefaut(greenkey, JitCell()) + return _jitcell_dict.setdefault(greenkey, JitCell()) self.jit_cell_at_key = jit_cell_at_key if staticdata.virtualizable_info: self.blackhole_virtualizable = staticdata.virtualizable_info.null_vable From arigo at codespeak.net Tue Oct 20 19:17:35 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Oct 2009 19:17:35 +0200 (CEST) Subject: [pypy-svn] r68670 - pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test Message-ID: <20091020171735.94328168007@codespeak.net> Author: arigo Date: Tue Oct 20 19:17:34 2009 New Revision: 68670 Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_compile.py Log: Fix for Python 2.4. Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_compile.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_compile.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_compile.py Tue Oct 20 19:17:34 2009 @@ -41,7 +41,7 @@ optimize_loop = staticmethod(optimize.optimize_loop) debug_level = 0 -class FakeGlobalData(): +class FakeGlobalData: loopnumbering = 0 class FakeMetaInterpStaticData: From arigo at codespeak.net Tue Oct 20 21:36:35 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Oct 2009 21:36:35 +0200 (CEST) Subject: [pypy-svn] r68671 - in pypy/branch/warmspot-jitinfo/pypy: jit/metainterp jit/metainterp/test module/pypyjit rlib Message-ID: <20091020193635.D5213168007@codespeak.net> Author: arigo Date: Tue Oct 20 21:36:33 2009 New Revision: 68671 Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmstate.py pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_ztranslation.py pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/virtualizable.py pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmspot.py pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmstate.py pypy/branch/warmspot-jitinfo/pypy/module/pypyjit/interp_jit.py pypy/branch/warmspot-jitinfo/pypy/rlib/jit.py Log: Fix test_ztranslation. Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmstate.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmstate.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmstate.py Tue Oct 20 21:36:33 2009 @@ -87,7 +87,7 @@ _TYPE = llmemory.GCREF celldict = {} def getter(x, y): - return celldict[x, y] + return celldict.get((x, y)) def setter(newcell, x, y): newcell.x = x newcell.y = y Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_ztranslation.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_ztranslation.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_ztranslation.py Tue Oct 20 21:36:33 2009 @@ -4,6 +4,8 @@ from pypy.rlib.jit import JitDriver, OPTIMIZER_FULL, unroll_parameters from pypy.rlib.jit import PARAMETERS, dont_look_inside from pypy.jit.metainterp.jitprof import Profiler +from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.ootypesystem import ootype class TranslationTest: @@ -18,6 +20,7 @@ # - set_param interface # - profiler # - full optimizer + # - jitdriver hooks class Frame(object): _virtualizable2_ = ['i'] @@ -25,8 +28,24 @@ def __init__(self, i): self.i = i + class JitCellCache: + entry = None + jitcellcache = JitCellCache() + def set_jitcell_at(entry): + jitcellcache.entry = entry + def get_jitcell_at(): + return jitcellcache.entry + def get_printable_location(): + return '(hello world)' + def can_inline(): + return False + jitdriver = JitDriver(greens = [], reds = ['frame', 'total'], - virtualizables = ['frame']) + virtualizables = ['frame'], + get_jitcell_at=get_jitcell_at, + set_jitcell_at=set_jitcell_at, + get_printable_location=get_printable_location, + can_inline=can_inline) def f(i): for param in unroll_parameters: defl = PARAMETERS[param] Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/virtualizable.py Tue Oct 20 21:36:33 2009 @@ -6,7 +6,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.jit.metainterp.typesystem import deref, fieldType, arrayItem from pypy.jit.metainterp import history -from pypy.jit.metainterp.warmspot import wrap, unwrap +from pypy.jit.metainterp.warmstate import wrap, unwrap class VirtualizableInfo: Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmspot.py Tue Oct 20 21:36:33 2009 @@ -309,33 +309,37 @@ annhelper.finish() def make_driverhook_graphs(self): + from pypy.rlib.jit import BaseJitCell + bk = self.rtyper.annotator.bookkeeper + classdef = bk.getuniqueclassdef(BaseJitCell) + s_BaseJitCell_or_None = annmodel.SomeInstance(classdef, + can_be_None=True) + s_BaseJitCell_not_None = annmodel.SomeInstance(classdef) + s_Str = annmodel.SomeString() + # annhelper = MixLevelHelperAnnotator(self.translator.rtyper) - blackboxtype = self.cpu.ts.BASETYPE self.set_jitcell_at_ptr = self._make_hook_graph( - annhelper, self.jitdriver.set_jitcell_at, None, blackboxtype) + annhelper, self.jitdriver.set_jitcell_at, annmodel.s_None, + s_BaseJitCell_not_None) self.get_jitcell_at_ptr = self._make_hook_graph( - annhelper, self.jitdriver.get_jitcell_at, blackboxtype) + annhelper, self.jitdriver.get_jitcell_at, s_BaseJitCell_or_None) self.can_inline_ptr = self._make_hook_graph( - annhelper, self.jitdriver.can_inline, bool) + annhelper, self.jitdriver.can_inline, annmodel.s_Bool) self.get_printable_location_ptr = self._make_hook_graph( - annhelper, self.jitdriver.get_printable_location, str) + annhelper, self.jitdriver.get_printable_location, s_Str) annhelper.finish() - def _make_hook_graph(self, annhelper, func, rettype, first_arg_type=None): - from pypy.annotation.signature import annotationoftype + def _make_hook_graph(self, annhelper, func, s_result, s_first_arg=None): if func is None: return None - if rettype is not None: - s_result = annotationoftype(rettype) - RETTYPE = annhelper.rtyper.getrepr(s_result).lowleveltype - else: - RETTYPE = lltype.Void - FUNC, PTR = self.cpu.ts.get_FuncType(self.green_args_spec, RETTYPE) + # + extra_args_s = [] + if s_first_arg is not None: + extra_args_s.append(s_first_arg) + # args_s = self.portal_args_s[:len(self.green_args_spec)] - if first_arg_type is not None: - args_s.insert(0, annotationoftype(first_arg_type)) - graph = annhelper.getgraph(func, args_s, s_result) - funcptr = annhelper.graph2delayed(graph, FUNC) + graph = annhelper.getgraph(func, extra_args_s + args_s, s_result) + funcptr = annhelper.graph2delayed(graph) return funcptr def make_args_specification(self): Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmstate.py Tue Oct 20 21:36:33 2009 @@ -1,13 +1,15 @@ import sys from pypy.rpython.lltypesystem import lltype, llmemory, rstr from pypy.rpython.ootypesystem import ootype -from pypy.rpython.annlowlevel import hlstr +from pypy.rpython.annlowlevel import hlstr, cast_base_ptr_to_instance +from pypy.rpython.annlowlevel import cast_object_to_ptr from pypy.rlib.objectmodel import specialize, we_are_translated, r_dict from pypy.rlib.rarithmetic import intmask from pypy.rlib.nonconst import NonConstant from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.jit import PARAMETERS, OPTIMIZER_SIMPLE, OPTIMIZER_FULL from pypy.rlib.jit import DEBUG_PROFILE +from pypy.rlib.jit import BaseJitCell from pypy.jit.metainterp import support, history # ____________________________________________________________ @@ -266,7 +268,7 @@ if hasattr(self, 'jit_getter'): return self.jit_getter # - class JitCell(object): + class JitCell(BaseJitCell): counter = 0 compiled_merge_points = None # @@ -324,22 +326,41 @@ rtyper = self.warmrunnerdesc.rtyper get_jitcell_at_ptr = self.warmrunnerdesc.get_jitcell_at_ptr set_jitcell_at_ptr = self.warmrunnerdesc.set_jitcell_at_ptr - cpu = self.warmrunnerdesc.cpu + lltohlhack = {} # def get_jitcell(*greenargs): fn = support.maybe_on_top_of_llinterp(rtyper, get_jitcell_at_ptr) - try: - cellref = fn(*greenargs) - if we_are_translated(): - cell = cast_base_ptr_to_instance(JitCell, cellref) + cellref = fn(*greenargs) + # + if we_are_translated(): + BASEJITCELL = lltype.typeOf(cellref) + cell = cast_base_ptr_to_instance(JitCell, cellref) + elif isinstance(cellref, (BaseJitCell, type(None))): + BASEJITCELL = None + cell = cellref + else: + BASEJITCELL = lltype.typeOf(cellref) + if cellref: + cell = lltohlhack[rtyper.type_system.deref(cellref)] else: - cell = cellref - except KeyError: + cell = None + # + if cell is None: cell = JitCell() + # if we_are_translated(): - cellref = cpu.ts.cast_instance_to_base_ref(cell) - else: + cellref = cast_object_to_ptr(BASEJITCELL, cell) + elif BASEJITCELL is None: cellref = cell + else: + if isinstance(BASEJITCELL, lltype.Ptr): + cellref = lltype.malloc(BASEJITCELL.TO) + elif isinstance(BASEJITCELL, ootype.Instance): + cellref = ootype.new(BASEJITCELL) + else: + assert False, "no clue" + lltohlhack[rtyper.type_system.deref(cellref)] = cell + # fn = support.maybe_on_top_of_llinterp(rtyper, set_jitcell_at_ptr) fn(cellref, *greenargs) @@ -367,6 +388,7 @@ set_future_values_from_vinfo(*redargs) # if vinfo is not None: + i0 = len(warmrunnerdesc.red_args_types) num_green_args = warmrunnerdesc.num_green_args vable_static_fields = unrolling_iterable( zip(vinfo.static_extra_types, vinfo.static_fields)) @@ -376,6 +398,7 @@ getarrayitem = cpu.ts.getarrayitem # def set_future_values_from_vinfo(*redargs): + i = i0 virtualizable = redargs[vinfo.index_of_virtualizable - num_green_args] virtualizable = vinfo.cast_to_vtype(virtualizable) Modified: pypy/branch/warmspot-jitinfo/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/module/pypyjit/interp_jit.py Tue Oct 20 21:36:33 2009 @@ -50,7 +50,7 @@ def leave(next_instr, pycode, frame, ec): from pypy.interpreter.executioncontext import ExecutionContext # can't use a method here, since this function is seen later than the main - # annotation + # annotation XXX no longer true, could be fixed ExecutionContext._jit_rechain_frame(ec, frame) class PyPyJitDriver(JitDriver): Modified: pypy/branch/warmspot-jitinfo/pypy/rlib/jit.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/rlib/jit.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/rlib/jit.py Tue Oct 20 21:36:33 2009 @@ -94,7 +94,6 @@ PARAMETERS = {'threshold': 1000, 'trace_eagerness': 200, - 'hash_bits': 14, 'trace_limit': 10000, 'inlining': False, 'optimizer': OPTIMIZER_FULL, @@ -114,6 +113,7 @@ virtualizables = [] def __init__(self, greens=None, reds=None, virtualizables=None, + get_jitcell_at=None, set_jitcell_at=None, can_inline=None, get_printable_location=None, leave=None): if greens is not None: @@ -128,6 +128,8 @@ assert v in self.reds self._alllivevars = dict.fromkeys(self.greens + self.reds) self._make_extregistryentries() + self.get_jitcell_at = get_jitcell_at + self.set_jitcell_at = set_jitcell_at self.get_printable_location = get_printable_location self.can_inline = can_inline self.leave = leave @@ -194,6 +196,10 @@ # # Annotation and rtyping of some of the JitDriver methods +class BaseJitCell(object): + __slots__ = () + + class ExtEnterLeaveMarker(ExtRegistryEntry): # Replace a call to myjitdriver.jit_merge_point(**livevars) # with an operation jit_marker('jit_merge_point', myjitdriver, livevars...) @@ -218,17 +224,21 @@ def annotate_hooks(self, **kwds_s): driver = self.instance.im_self + s_jitcell = self.bookkeeper.valueoftype(BaseJitCell) + self.annotate_hook(driver.get_jitcell_at, driver.greens, **kwds_s) + self.annotate_hook(driver.set_jitcell_at, driver.greens, [s_jitcell], + **kwds_s) self.annotate_hook(driver.can_inline, driver.greens, **kwds_s) self.annotate_hook(driver.get_printable_location, driver.greens, **kwds_s) self.annotate_hook(driver.leave, driver.greens + driver.reds, **kwds_s) - def annotate_hook(self, func, variables, **kwds_s): + def annotate_hook(self, func, variables, args_s=[], **kwds_s): if func is None: return bk = self.bookkeeper s_func = bk.immutablevalue(func) uniquekey = 'jitdriver.%s' % func.func_name - args_s = [] + args_s = args_s[:] for name in variables: s_arg = kwds_s['s_' + name] args_s.append(s_arg) From arigo at codespeak.net Tue Oct 20 21:47:38 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Oct 2009 21:47:38 +0200 (CEST) Subject: [pypy-svn] r68672 - pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test Message-ID: <20091020194738.A637D168007@codespeak.net> Author: arigo Date: Tue Oct 20 21:47:38 2009 New Revision: 68672 Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmstate.py Log: Fix the test. Modified: pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmstate.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmstate.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmstate.py Tue Oct 20 21:47:38 2009 @@ -7,6 +7,7 @@ from pypy.jit.metainterp.warmstate import WarmEnterState from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr from pypy.jit.metainterp.history import ConstInt, ConstFloat, ConstPtr +from pypy.rlib.jit import BaseJitCell def test_unwrap(): @@ -59,7 +60,7 @@ def test_make_jitcell_getter_default(): class FakeWarmRunnerDesc: green_args_spec = [lltype.Signed, lltype.Float] - class FakeJitCell: + class FakeJitCell(BaseJitCell): pass state = WarmEnterState(FakeWarmRunnerDesc()) get_jitcell = state._make_jitcell_getter_default(FakeJitCell) @@ -83,8 +84,11 @@ assert get_jitcell is state.make_jitcell_getter() def test_make_jitcell_getter_custom(): - class FakeJitCell: - _TYPE = llmemory.GCREF + from pypy.rpython.typesystem import LowLevelTypeSystem + class FakeRTyper: + type_system = LowLevelTypeSystem.instance + class FakeJitCell(BaseJitCell): + pass celldict = {} def getter(x, y): return celldict.get((x, y)) @@ -97,7 +101,7 @@ SETTER = lltype.Ptr(lltype.FuncType([llmemory.GCREF, lltype.Signed, lltype.Float], lltype.Void)) class FakeWarmRunnerDesc: - rtyper = None + rtyper = FakeRTyper() cpu = None get_jitcell_at_ptr = llhelper(GETTER, getter) set_jitcell_at_ptr = llhelper(SETTER, setter) From arigo at codespeak.net Tue Oct 20 22:23:44 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Oct 2009 22:23:44 +0200 (CEST) Subject: [pypy-svn] r68673 - in pypy/branch/warmspot-jitinfo/pypy/rpython: . test Message-ID: <20091020202344.DF9C6168005@codespeak.net> Author: arigo Date: Tue Oct 20 22:23:44 2009 New Revision: 68673 Modified: pypy/branch/warmspot-jitinfo/pypy/rpython/rint.py pypy/branch/warmspot-jitinfo/pypy/rpython/test/test_rdict.py Log: Fix ll_hash_int. Can you believe it. Modified: pypy/branch/warmspot-jitinfo/pypy/rpython/rint.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/rpython/rint.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/rpython/rint.py Tue Oct 20 22:23:44 2009 @@ -408,10 +408,8 @@ fn = hop.rtyper.type_system.ll_str.ll_int2oct return hop.gendirectcall(fn, varg, true) -def ll_identity(n): - return n - -ll_hash_int = ll_identity +def ll_hash_int(n): + return intmask(n) def ll_check_chr(n): if 0 <= n <= 255: Modified: pypy/branch/warmspot-jitinfo/pypy/rpython/test/test_rdict.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/rpython/test/test_rdict.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/rpython/test/test_rdict.py Tue Oct 20 22:23:44 2009 @@ -4,6 +4,7 @@ from pypy.rpython.lltypesystem import rdict, rstr from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin from pypy.rlib.objectmodel import r_dict +from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong import py py.log.setconsumer("rtyper", py.log.STDOUT) @@ -564,6 +565,17 @@ res = self.interpret(fn, [3.0]) assert res == 42 + def test_dict_of_r_uint(self): + for r_t in [r_uint, r_longlong, r_ulonglong]: + d = {r_t(2): 3, r_t(4): 5} + def fn(x, y): + d[r_t(x)] = 123 + return d[r_t(y)] + res = self.interpret(fn, [4, 2]) + assert res == 3 + res = self.interpret(fn, [3, 3]) + assert res == 123 + class TestLLtype(BaseTestRdict, LLRtypeMixin): def test_dict_but_not_with_char_keys(self): From arigo at codespeak.net Tue Oct 20 22:41:54 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 20 Oct 2009 22:41:54 +0200 (CEST) Subject: [pypy-svn] r68674 - pypy/branch/warmspot-jitinfo/pypy/jit/backend/cli Message-ID: <20091020204154.7516F168005@codespeak.net> Author: arigo Date: Tue Oct 20 22:41:54 2009 New Revision: 68674 Modified: pypy/branch/warmspot-jitinfo/pypy/jit/backend/cli/runner.py Log: Fix imports. Modified: pypy/branch/warmspot-jitinfo/pypy/jit/backend/cli/runner.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/jit/backend/cli/runner.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/jit/backend/cli/runner.py Tue Oct 20 22:41:54 2009 @@ -319,7 +319,7 @@ def __init__(self, TYPE): DescrWithKey.__init__(self, TYPE) from pypy.jit.backend.llgraph.runner import boxresult - from pypy.jit.metainterp.warmspot import unwrap + from pypy.jit.metainterp.warmstate import unwrap ARRAY = ootype.Array(TYPE) def create(): if isinstance(TYPE, ootype.OOType): @@ -482,7 +482,7 @@ def __init__(self, TYPE, fieldname): DescrWithKey.__init__(self, (TYPE, fieldname)) from pypy.jit.backend.llgraph.runner import boxresult - from pypy.jit.metainterp.warmspot import unwrap + from pypy.jit.metainterp.warmstate import unwrap _, T = TYPE._lookup_field(fieldname) def getfield(objbox): obj = objbox.getref(TYPE) From afa at codespeak.net Wed Oct 21 01:25:49 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 21 Oct 2009 01:25:49 +0200 (CEST) Subject: [pypy-svn] r68675 - pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc Message-ID: <20091020232549.E57AA168009@codespeak.net> Author: afa Date: Wed Oct 21 01:25:49 2009 New Revision: 68675 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Log: Fix the assembler generated for gcmap tables. now, our call shapes are probably wrong... Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Wed Oct 21 01:25:49 2009 @@ -1160,7 +1160,7 @@ _comment("A circular doubly-linked list of all") _comment("the ASM_FRAMEDATAs currently alive") if self.format == 'msvc': - print >> output, '%s DD ?' % _globalname("__gcrootanchor") + print >> output, '%s:' % _globalname("__gcrootanchor") print >> output, '\tDD FLAT:___gcrootanchor ; prev' print >> output, '\tDD FLAT:___gcrootanchor ; next' else: @@ -1174,7 +1174,7 @@ _globl('__gcmapstart') if self.format == 'msvc': - print >> output, '%s DD ?' % _globalname('__gcmapstart') + print >> output, '%s:' % _globalname('__gcmapstart') else: _label('__gcmapstart') for label, state, is_range in self.gcmaptable: @@ -1184,7 +1184,7 @@ n = shapes[state] = shapeofs bytes = [str(b) for b in compress_callshape(state)] if self.format == 'msvc': - shapelines.append('\tDD\t%s\t;%s\n' % ( + shapelines.append('\tDB\t%s\t;%s\n' % ( ', '.join(bytes), shapeofs)) else: @@ -1195,7 +1195,7 @@ if is_range: n = ~ n if self.format == 'msvc': - print >> output, '\tDD\t%s-%d' % (label, OFFSET_LABELS) + print >> output, '\tDD\t%s' % (label,) print >> output, '\tDD\t%d' % (n,) else: print >> output, '\t.long\t%s-%d' % (label, OFFSET_LABELS) @@ -1210,7 +1210,7 @@ _globl('__gccallshapes') if self.format == 'msvc': - print >> output, '%s DD ?' % _globalname('__gccallshapes') + print >> output, '%s:' % _globalname('__gccallshapes') else: _label('__gccallshapes') output.writelines(shapelines) From arigo at codespeak.net Wed Oct 21 10:55:58 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Oct 2009 10:55:58 +0200 (CEST) Subject: [pypy-svn] r68676 - pypy/branch/warmspot-jitinfo/pypy/module/pypyjit Message-ID: <20091021085558.75B09168007@codespeak.net> Author: arigo Date: Wed Oct 21 10:55:56 2009 New Revision: 68676 Modified: pypy/branch/warmspot-jitinfo/pypy/module/pypyjit/interp_jit.py Log: Add get_jitcell_at and set_jitcell_at to the PyPy jit. Still tracking segfaults... Modified: pypy/branch/warmspot-jitinfo/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/module/pypyjit/interp_jit.py Wed Oct 21 10:55:56 2009 @@ -20,6 +20,8 @@ from opcode import opmap from pypy.rlib.objectmodel import we_are_translated +PyCode.jit_cells = None + PyFrame._virtualizable2_ = ['last_instr', 'pycode', 'valuestackdepth', 'valuestack_w[*]', 'fastlocals_w[*]', 'f_forward', @@ -53,6 +55,19 @@ # annotation XXX no longer true, could be fixed ExecutionContext._jit_rechain_frame(ec, frame) +def get_jitcell_at(next_instr, bytecode): + d = bytecode.jit_cells + if d is None: + return None + return d.get(next_instr, None) + +def set_jitcell_at(newcell, next_instr, bytecode): + d = bytecode.jit_cells + if d is None: + d = bytecode.jit_cells = {} + d[next_instr] = newcell + + class PyPyJitDriver(JitDriver): reds = ['frame', 'ec'] greens = ['next_instr', 'pycode'] @@ -68,7 +83,9 @@ pypyjitdriver = PyPyJitDriver(can_inline = can_inline, get_printable_location = get_printable_location, - leave = leave) + leave = leave, + get_jitcell_at = get_jitcell_at, + set_jitcell_at = set_jitcell_at) class __extend__(PyFrame): From fijal at codespeak.net Wed Oct 21 13:32:55 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 21 Oct 2009 13:32:55 +0200 (CEST) Subject: [pypy-svn] r68677 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091021113255.145ED168007@codespeak.net> Author: fijal Date: Wed Oct 21 13:32:55 2009 New Revision: 68677 Modified: pypy/trunk/pypy/jit/metainterp/compile.py pypy/trunk/pypy/jit/metainterp/history.py pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_recursive.py Log: Check in raising GiveUp when number of fail args for guards exceeds some limit (1000 for now) Modified: pypy/trunk/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/compile.py (original) +++ pypy/trunk/pypy/jit/metainterp/compile.py Wed Oct 21 13:32:55 2009 @@ -13,6 +13,9 @@ from pypy.jit.metainterp.typesystem import llhelper, oohelper from pypy.jit.metainterp.optimizeutil import InvalidLoop +class GiveUp(Exception): + pass + def show_loop(metainterp_sd, loop=None, error=None): # debugging if option.view or option.viewloops: @@ -284,7 +287,6 @@ old_loop_tokens, new_loop) except InvalidLoop: - assert 0, "InvalidLoop in optimize_bridge?" return None # Did it work? if target_loop_token is not None: Modified: pypy/trunk/pypy/jit/metainterp/history.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/history.py (original) +++ pypy/trunk/pypy/jit/metainterp/history.py Wed Oct 21 13:32:55 2009 @@ -21,6 +21,8 @@ REF = 'r' FLOAT = 'f' +FAILARGS_LIMIT = 1000 + def getkind(TYPE, supports_floats=True): if TYPE is lltype.Void: return "void" @@ -117,6 +119,9 @@ def repr_rpython(self): return '%s' % self + def _get_str(self): + raise NotImplementedError + class AbstractDescr(AbstractValue): __slots__ = () @@ -930,8 +935,9 @@ # ---------------------------------------------------------------- class Options: - def __init__(self, listops=False): + def __init__(self, listops=False, failargs_limit=FAILARGS_LIMIT): self.listops = listops + self.failargs_limit = failargs_limit def _freeze_(self): return True Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Wed Oct 21 13:32:55 2009 @@ -520,6 +520,8 @@ assert isinstance(descr, compile.ResumeGuardDescr) modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo) newboxes = modifier.finish(self.values) + if len(newboxes) > self.metainterp_sd.options.failargs_limit: + raise compile.GiveUp descr.store_final_boxes(op, newboxes) def optimize_default(self, op): Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Wed Oct 21 13:32:55 2009 @@ -16,6 +16,7 @@ from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import specialize from pypy.rlib.jit import DEBUG_OFF, DEBUG_PROFILE, DEBUG_STEPS, DEBUG_DETAILED +from pypy.jit.metainterp.compile import GiveUp # ____________________________________________________________ @@ -1154,7 +1155,11 @@ return True else: if not self.is_blackholing(): - self.compile_done_with_this_frame(resultbox) + try: + self.compile_done_with_this_frame(resultbox) + except GiveUp: + self.staticdata.profiler.count(ABORT_BRIDGE) + self.switch_to_blackhole() sd = self.staticdata if sd.result_type == 'void': assert resultbox is None @@ -1188,7 +1193,11 @@ return True self.popframe() if not self.is_blackholing(): - self.compile_exit_frame_with_exception(excvaluebox) + try: + self.compile_exit_frame_with_exception(excvaluebox) + except GiveUp: + self.staticdata.profiler.count(ABORT_BRIDGE) + self.switch_to_blackhole() raise self.staticdata.ExitFrameWithExceptionRef(self.cpu, excvaluebox.getref_base()) def check_recursion_invariant(self): @@ -1801,6 +1810,3 @@ assert target_loop_token is not None self.argboxes = args self.target_loop_token = target_loop_token - -class GiveUp(Exception): - pass Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Wed Oct 21 13:32:55 2009 @@ -23,11 +23,15 @@ self.pc = pc self.exception_target = exc_target +class FakeOptions(object): + failargs_limit = 1000 + class FakeMetaInterpStaticData(object): def __init__(self, cpu): self.cpu = cpu self.profiler = EmptyProfiler() + self.options = FakeOptions() def test_store_final_boxes_in_guard(): from pypy.jit.metainterp.compile import ResumeGuardDescr Modified: pypy/trunk/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_recursive.py Wed Oct 21 13:32:55 2009 @@ -393,6 +393,80 @@ self.check_aborted_count(8) self.check_enter_count_at_most(30) + def test_max_failure_args(self): + FAILARGS_LIMIT = 10 + jitdriver = JitDriver(greens = [], reds = ['o', 'i', 'n']) + + class A(object): + def __init__(self, i0, i1, i2, i3, i4, i5, i6, i7, i8, i9): + self.i0 = i0 + self.i1 = i1 + self.i2 = i2 + self.i3 = i3 + self.i4 = i4 + self.i5 = i5 + self.i6 = i6 + self.i7 = i7 + self.i8 = i8 + self.i9 = i9 + + + def loop(n): + i = 0 + o = A(0, 1, 2, 3, 4, 5, 6, 7, 8, 9) + while i < n: + jitdriver.can_enter_jit(o=o, i=i, n=n) + jitdriver.jit_merge_point(o=o, i=i, n=n) + o = A(i, i + 1, i + 2, i + 3, i + 4, i + 5, + i + 6, i + 7, i + 8, i + 9) + i += 1 + return o + + res = self.meta_interp(loop, [20], failargs_limit=FAILARGS_LIMIT, + listops=True) + self.check_aborted_count(5) + + def test_max_failure_args_exc(self): + FAILARGS_LIMIT = 10 + jitdriver = JitDriver(greens = [], reds = ['o', 'i', 'n']) + + class A(object): + def __init__(self, i0, i1, i2, i3, i4, i5, i6, i7, i8, i9): + self.i0 = i0 + self.i1 = i1 + self.i2 = i2 + self.i3 = i3 + self.i4 = i4 + self.i5 = i5 + self.i6 = i6 + self.i7 = i7 + self.i8 = i8 + self.i9 = i9 + + + def loop(n): + i = 0 + o = A(0, 1, 2, 3, 4, 5, 6, 7, 8, 9) + while i < n: + jitdriver.can_enter_jit(o=o, i=i, n=n) + jitdriver.jit_merge_point(o=o, i=i, n=n) + o = A(i, i + 1, i + 2, i + 3, i + 4, i + 5, + i + 6, i + 7, i + 8, i + 9) + i += 1 + raise ValueError + + def main(n): + try: + loop(n) + return 1 + except ValueError: + return 0 + + res = self.meta_interp(main, [20], failargs_limit=FAILARGS_LIMIT, + listops=True) + assert not res + self.check_aborted_count(5) + def test_set_param_inlining(self): myjitdriver = JitDriver(greens=[], reds=['n', 'recurse']) def loop(n, recurse=False): From arigo at codespeak.net Wed Oct 21 14:39:04 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Oct 2009 14:39:04 +0200 (CEST) Subject: [pypy-svn] r68679 - in pypy/branch/warmspot-jitinfo/pypy: rpython/lltypesystem rpython/lltypesystem/test translator/backendopt translator/backendopt/test Message-ID: <20091021123904.B0389168007@codespeak.net> Author: arigo Date: Wed Oct 21 14:39:04 2009 New Revision: 68679 Modified: pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem/lloperation.py pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem/opimpl.py pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem/test/test_lloperation.py pypy/branch/warmspot-jitinfo/pypy/translator/backendopt/mallocv.py pypy/branch/warmspot-jitinfo/pypy/translator/backendopt/test/test_constfold.py Log: Improve support of 'immutable_fields' in optimizations. Modified: pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem/lloperation.py Wed Oct 21 14:39:04 2009 @@ -84,12 +84,16 @@ return op_impl fold = roproperty(get_fold_impl) - def is_pure(self, *ARGTYPES): + def is_pure(self, args_v): return (self.canfold or # canfold => pure operation self is llop.debug_assert or # debug_assert is pure enough # reading from immutable (self in (llop.getfield, llop.getarrayitem) and - ARGTYPES[0].TO._hints.get('immutable'))) + args_v[0].concretetype.TO._hints.get('immutable')) or + (self is llop.getfield and # reading from immutable_field + 'immutable_fields' in args_v[0].concretetype.TO._hints and + args_v[1].value in args_v[0].concretetype.TO + ._hints['immutable_fields'].fields)) def __repr__(self): return '' % (getattr(self, 'opname', '?'),) Modified: pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem/opimpl.py Wed Oct 21 14:39:04 2009 @@ -390,7 +390,13 @@ def op_getfield(p, name): checkptr(p) - if not lltype.typeOf(p).TO._hints.get('immutable'): + TYPE = lltype.typeOf(p).TO + if TYPE._hints.get('immutable'): + pass + elif ('immutable_fields' in TYPE._hints and + name in TYPE._hints['immutable_fields'].fields): + pass + else: raise TypeError("cannot fold getfield on mutable struct") return getattr(p, name) Modified: pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem/test/test_lloperation.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem/test/test_lloperation.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem/test/test_lloperation.py Wed Oct 21 14:39:04 2009 @@ -4,6 +4,7 @@ from pypy.rpython.ootypesystem import ootype, ooopimpl from pypy.rpython.llinterp import LLFrame from pypy.rpython.test.test_llinterp import interpret +from pypy.rpython import rclass LL_INTERP_OPERATIONS = [name[3:] for name in LLFrame.__dict__.keys() if name.startswith('op_')] @@ -50,7 +51,51 @@ return s.x + s.y res = interpret(llf, [], policy=LowLevelAnnotatorPolicy()) assert res == 5 - + +def test_is_pure(): + from pypy.objspace.flow.model import Variable, Constant + assert llop.bool_not.is_pure([Variable()]) + assert llop.debug_assert.is_pure([Variable()]) + assert not llop.int_add_ovf.is_pure([Variable(), Variable()]) + # + S1 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed)) + v_s1 = Variable() + v_s1.concretetype = lltype.Ptr(S1) + assert not llop.setfield.is_pure([v_s1, Constant('x'), Variable()]) + assert not llop.getfield.is_pure([v_s1, Constant('y')]) + # + A1 = lltype.GcArray(lltype.Signed) + v_a1 = Variable() + v_a1.concretetype = lltype.Ptr(A1) + assert not llop.setarrayitem.is_pure([v_a1, Variable(), Variable()]) + assert not llop.getarrayitem.is_pure([v_a1, Variable()]) + assert llop.getarraysize.is_pure([v_a1]) + # + S2 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed), + hints={'immutable': True}) + v_s2 = Variable() + v_s2.concretetype = lltype.Ptr(S2) + assert not llop.setfield.is_pure([v_s2, Constant('x'), Variable()]) + assert llop.getfield.is_pure([v_s2, Constant('y')]) + # + A2 = lltype.GcArray(lltype.Signed, hints={'immutable': True}) + v_a2 = Variable() + v_a2.concretetype = lltype.Ptr(A2) + assert not llop.setarrayitem.is_pure([v_a2, Variable(), Variable()]) + assert llop.getarrayitem.is_pure([v_a2, Variable()]) + assert llop.getarraysize.is_pure([v_a2]) + # + accessor = rclass.FieldListAccessor() + S3 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed), + hints={'immutable_fields': accessor}) + accessor.initialize(S3, ['x']) + v_s3 = Variable() + v_s3.concretetype = lltype.Ptr(S3) + assert not llop.setfield.is_pure([v_s3, Constant('x'), Variable()]) + assert not llop.setfield.is_pure([v_s3, Constant('y'), Variable()]) + assert llop.getfield.is_pure([v_s3, Constant('x')]) + assert not llop.getfield.is_pure([v_s3, Constant('y')]) + # ___________________________________________________________________________ # This tests that the LLInterpreter and the LL_OPERATIONS tables are in sync. Modified: pypy/branch/warmspot-jitinfo/pypy/translator/backendopt/mallocv.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/translator/backendopt/mallocv.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/translator/backendopt/mallocv.py Wed Oct 21 14:39:04 2009 @@ -1046,7 +1046,7 @@ op = getattr(llop, opname) except AttributeError: return - if not op.is_pure(*[v.concretetype for v in args_v]): + if not op.is_pure(args_v): return try: result = op(RESTYPE, *args) Modified: pypy/branch/warmspot-jitinfo/pypy/translator/backendopt/test/test_constfold.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/translator/backendopt/test/test_constfold.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/translator/backendopt/test/test_constfold.py Wed Oct 21 14:39:04 2009 @@ -4,6 +4,7 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rpython import rclass from pypy.rlib import objectmodel from pypy.translator.backendopt.constfold import constant_fold_graph from pypy import conftest @@ -26,8 +27,10 @@ assert res == expected_result -def test_simple(): - S1 = lltype.GcStruct('S1', ('x', lltype.Signed), hints={'immutable': True}) +def test_simple(S1=None): + if S1 is None: + S1 = lltype.GcStruct('S1', ('x', lltype.Signed), + hints={'immutable': True}) s1 = lltype.malloc(S1) s1.x = 123 def g(y): @@ -42,6 +45,14 @@ check_graph(graph, [], 124, t) +def test_immutable_fields(): + accessor = rclass.FieldListAccessor() + S2 = lltype.GcStruct('S2', ('x', lltype.Signed), + hints={'immutable_fields': accessor}) + accessor.initialize(S2, ['x']) + test_simple(S2) + + def test_along_link(): S1 = lltype.GcStruct('S1', ('x', lltype.Signed), hints={'immutable': True}) s1 = lltype.malloc(S1) From arigo at codespeak.net Wed Oct 21 14:45:33 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Oct 2009 14:45:33 +0200 (CEST) Subject: [pypy-svn] r68680 - in pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem: . test Message-ID: <20091021124533.9FF12168007@codespeak.net> Author: arigo Date: Wed Oct 21 14:45:32 2009 New Revision: 68680 Modified: pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem/opimpl.py pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem/test/test_lloperation.py Log: Add a test, and a missing case. Modified: pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem/opimpl.py Wed Oct 21 14:45:32 2009 @@ -142,7 +142,12 @@ # we can constant-fold this if the innermost structure from which we # read the final field is immutable. T = lltype.typeOf(innermostcontainer).TO - if not T._hints.get('immutable'): + if T._hints.get('immutable'): + pass + elif ('immutable_fields' in T._hints and + offsets[-1] in T._hints['immutable_fields'].fields): + pass + else: raise TypeError("cannot fold getinteriorfield on mutable struct") assert not isinstance(ob, lltype._interior_ptr) return ob Modified: pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem/test/test_lloperation.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem/test/test_lloperation.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/rpython/lltypesystem/test/test_lloperation.py Wed Oct 21 14:45:32 2009 @@ -96,6 +96,27 @@ assert llop.getfield.is_pure([v_s3, Constant('x')]) assert not llop.getfield.is_pure([v_s3, Constant('y')]) +def test_getfield_pure(): + S1 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed)) + S2 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed), + hints={'immutable': True}) + accessor = rclass.FieldListAccessor() + S3 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed), + hints={'immutable_fields': accessor}) + accessor.initialize(S3, ['x']) + # + s1 = lltype.malloc(S1); s1.x = 45 + py.test.raises(TypeError, llop.getfield, lltype.Signed, s1, 'x') + s2 = lltype.malloc(S2); s2.x = 45 + assert llop.getfield(lltype.Signed, s2, 'x') == 45 + s3 = lltype.malloc(S3); s3.x = 46; s3.y = 47 + assert llop.getfield(lltype.Signed, s3, 'x') == 46 + py.test.raises(TypeError, llop.getfield, lltype.Signed, s3, 'y') + # + py.test.raises(TypeError, llop.getinteriorfield, lltype.Signed, s1, 'x') + assert llop.getinteriorfield(lltype.Signed, s2, 'x') == 45 + assert llop.getinteriorfield(lltype.Signed, s3, 'x') == 46 + py.test.raises(TypeError, llop.getinteriorfield, lltype.Signed, s3, 'y') # ___________________________________________________________________________ # This tests that the LLInterpreter and the LL_OPERATIONS tables are in sync. From arigo at codespeak.net Wed Oct 21 15:04:39 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Oct 2009 15:04:39 +0200 (CEST) Subject: [pypy-svn] r68681 - in pypy/branch/warmspot-jitinfo/pypy/rpython/memory: . test Message-ID: <20091021130439.345F4168007@codespeak.net> Author: arigo Date: Wed Oct 21 15:04:38 2009 New Revision: 68681 Modified: pypy/branch/warmspot-jitinfo/pypy/rpython/memory/gctypelayout.py pypy/branch/warmspot-jitinfo/pypy/rpython/memory/test/test_gctypelayout.py Log: Support 'immutable_fields' here too. Modified: pypy/branch/warmspot-jitinfo/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/rpython/memory/gctypelayout.py Wed Oct 21 15:04:38 2009 @@ -350,11 +350,16 @@ def gc_pointers_inside(v, adr, mutable_only=False): t = lltype.typeOf(v) if isinstance(t, lltype.Struct): - if mutable_only and t._hints.get('immutable'): - return + skip = () + if mutable_only: + if t._hints.get('immutable'): + return + if 'immutable_fields' in t._hints: + skip = t._hints['immutable_fields'].fields for n, t2 in t._flds.iteritems(): if isinstance(t2, lltype.Ptr) and t2.TO._gckind == 'gc': - yield adr + llmemory.offsetof(t, n) + if n not in skip: + yield adr + llmemory.offsetof(t, n) elif isinstance(t2, (lltype.Array, lltype.Struct)): for a in gc_pointers_inside(getattr(v, n), adr + llmemory.offsetof(t, n), Modified: pypy/branch/warmspot-jitinfo/pypy/rpython/memory/test/test_gctypelayout.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/rpython/memory/test/test_gctypelayout.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/rpython/memory/test/test_gctypelayout.py Wed Oct 21 15:04:38 2009 @@ -1,7 +1,8 @@ import py from pypy.rpython.memory.gctypelayout import TypeLayoutBuilder, GCData from pypy.rpython.memory.gctypelayout import offsets_to_gc_pointers -from pypy.rpython.lltypesystem import lltype, rclass +from pypy.rpython.memory.gctypelayout import gc_pointers_inside +from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.test.test_llinterp import get_interpreter from pypy.objspace.flow.model import Constant @@ -90,3 +91,31 @@ interp, graph = get_interpreter(f, [], backendopt=True) assert interp.eval_graph(graph, []) == 11001 assert graph.startblock.exits[0].args == [Constant(11001, lltype.Signed)] + +def test_gc_pointers_inside(): + from pypy.rpython import rclass + PT = lltype.Ptr(lltype.GcStruct('T')) + S1 = lltype.GcStruct('S', ('x', PT), ('y', PT)) + S2 = lltype.GcStruct('S', ('x', PT), ('y', PT), + hints={'immutable': True}) + accessor = rclass.FieldListAccessor() + S3 = lltype.GcStruct('S', ('x', PT), ('y', PT), + hints={'immutable_fields': accessor}) + accessor.initialize(S3, ['x']) + # + s1 = lltype.malloc(S1) + adr = llmemory.cast_ptr_to_adr(s1) + lst = list(gc_pointers_inside(s1._obj, adr, mutable_only=True)) + expected = [adr + llmemory.offsetof(S1, 'x'), + adr + llmemory.offsetof(S1, 'y')] + assert lst == expected or lst == expected[::-1] + # + s2 = lltype.malloc(S2) + adr = llmemory.cast_ptr_to_adr(s2) + lst = list(gc_pointers_inside(s2._obj, adr, mutable_only=True)) + assert lst == [] + # + s3 = lltype.malloc(S3) + adr = llmemory.cast_ptr_to_adr(s3) + lst = list(gc_pointers_inside(s3._obj, adr, mutable_only=True)) + assert lst == [adr + llmemory.offsetof(S3, 'y')] From antocuni at codespeak.net Wed Oct 21 15:15:29 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 21 Oct 2009 15:15:29 +0200 (CEST) Subject: [pypy-svn] r68682 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20091021131529.F0872168012@codespeak.net> Author: antocuni Date: Wed Oct 21 15:15:29 2009 New Revision: 68682 Modified: pypy/trunk/pypy/rpython/lltypesystem/lloperation.py Log: add an XXX to remind that an ootype optimization is missing Modified: pypy/trunk/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/lloperation.py Wed Oct 21 15:15:29 2009 @@ -89,7 +89,7 @@ self is llop.debug_assert or # debug_assert is pure enough # reading from immutable (self in (llop.getfield, llop.getarrayitem) and - ARGTYPES[0].TO._hints.get('immutable'))) + ARGTYPES[0].TO._hints.get('immutable'))) # XXX: what about ootype immutable arrays? def __repr__(self): return '' % (getattr(self, 'opname', '?'),) From verte at codespeak.net Wed Oct 21 15:21:24 2009 From: verte at codespeak.net (verte at codespeak.net) Date: Wed, 21 Oct 2009 15:21:24 +0200 (CEST) Subject: [pypy-svn] r68683 - pypy/branch/effect-analysis Message-ID: <20091021132124.B9800168007@codespeak.net> Author: verte Date: Wed Oct 21 15:21:24 2009 New Revision: 68683 Added: pypy/branch/effect-analysis/ - copied from r68682, pypy/trunk/ Log: Experiment with adding extensive effect anaysis to pypy. From antocuni at codespeak.net Wed Oct 21 15:34:23 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 21 Oct 2009 15:34:23 +0200 (CEST) Subject: [pypy-svn] r68684 - pypy/trunk/pypy/rpython/ootypesystem Message-ID: <20091021133423.D0974168007@codespeak.net> Author: antocuni Date: Wed Oct 21 15:34:22 2009 New Revision: 68684 Modified: pypy/trunk/pypy/rpython/ootypesystem/ootype.py Log: revert to the old behaviour of ootype.{_record,_array}._identityhash: actually hash by identity, not by value Modified: pypy/trunk/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/trunk/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/trunk/pypy/rpython/ootypesystem/ootype.py Wed Oct 21 15:34:22 2009 @@ -1629,7 +1629,10 @@ self._array[index] = item def _identityhash(self): - return hash(self) + if self: + return intmask(id(self)) + else: + return 0 # for all null arrays class _null_array(_null_mixin(_array), _array): @@ -1776,7 +1779,10 @@ self.__dict__[name] = value def _identityhash(self): - return hash(self) + if self: + return intmask(id(self)) + else: + return 0 # for all null records def _items_in_order(self): return [self._items[name] for name in self._TYPE._fields_in_order] From arigo at codespeak.net Wed Oct 21 15:43:45 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Oct 2009 15:43:45 +0200 (CEST) Subject: [pypy-svn] r68685 - pypy/branch/warmspot-jitinfo/pypy/module/pypyjit/test Message-ID: <20091021134345.74C78168009@codespeak.net> Author: arigo Date: Wed Oct 21 15:43:44 2009 New Revision: 68685 Modified: pypy/branch/warmspot-jitinfo/pypy/module/pypyjit/test/test_jit_setup.py Log: Fix test. Modified: pypy/branch/warmspot-jitinfo/pypy/module/pypyjit/test/test_jit_setup.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/module/pypyjit/test/test_jit_setup.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/module/pypyjit/test/test_jit_setup.py Wed Oct 21 15:43:44 2009 @@ -9,8 +9,8 @@ # this just checks that the module is setting up things correctly, and # the resulting code makes sense on top of CPython. import pypyjit - pypyjit.set_param(threshold=5, hash_bits=9) - pypyjit.set_param("trace_eagerness=3,hash_bits=7") + pypyjit.set_param(threshold=5, inlining=1) + pypyjit.set_param("trace_eagerness=3,inlining=0") def f(x, y): return x*y+1 From fijal at codespeak.net Wed Oct 21 16:49:56 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 21 Oct 2009 16:49:56 +0200 (CEST) Subject: [pypy-svn] r68686 - pypy/trunk/pypy/rlib Message-ID: <20091021144956.41E68168009@codespeak.net> Author: fijal Date: Wed Oct 21 16:49:55 2009 New Revision: 68686 Modified: pypy/trunk/pypy/rlib/rmmap.py Log: Annotate argument to c_mmap as non-const, should fix some issues on some platforms Modified: pypy/trunk/pypy/rlib/rmmap.py ============================================================================== --- pypy/trunk/pypy/rlib/rmmap.py (original) +++ pypy/trunk/pypy/rlib/rmmap.py Wed Oct 21 16:49:55 2009 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem import rffi, lltype, llmemory from pypy.rlib import rposix from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.rlib.nonconst import NonConstant import sys import os @@ -612,7 +613,10 @@ else: m.fd = os.dup(fd) - res = c_mmap(NULL, map_size, prot, flags, fd, 0) + # XXX if we use hintp below in alloc, the NonConstant + # is necessary since we want a general version of c_mmap + # to be annotated with a non-constant pointer. + res = c_mmap(NonConstant(NULL), map_size, prot, flags, fd, 0) if res == rffi.cast(PTR, -1): errno = _get_error_no() raise OSError(errno, os.strerror(errno)) From antocuni at codespeak.net Wed Oct 21 17:10:26 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 21 Oct 2009 17:10:26 +0200 (CEST) Subject: [pypy-svn] r68687 - pypy/trunk/pypy/translator/cli Message-ID: <20091021151026.BA388168009@codespeak.net> Author: antocuni Date: Wed Oct 21 17:10:24 2009 New Revision: 68687 Modified: pypy/trunk/pypy/translator/cli/ilgenerator.py Log: load the correct value when you use -INFINITY in your code. So far, it didn't distinguish between positive and negative inf Modified: pypy/trunk/pypy/translator/cli/ilgenerator.py ============================================================================== --- pypy/trunk/pypy/translator/cli/ilgenerator.py (original) +++ pypy/trunk/pypy/translator/cli/ilgenerator.py Wed Oct 21 17:10:24 2009 @@ -400,7 +400,10 @@ ilasm.opcode('ldc.i4', ord(value)) elif TYPE is ootype.Float: if isinf(value): - ilasm.opcode('ldc.r8', '(00 00 00 00 00 00 f0 7f)') + if value < 0.0: + ilasm.opcode('ldc.r8', '(00 00 00 00 00 00 f0 ff)') + else: + ilasm.opcode('ldc.r8', '(00 00 00 00 00 00 f0 7f)') elif isnan(value): ilasm.opcode('ldc.r8', '(00 00 00 00 00 00 f8 ff)') else: From fijal at codespeak.net Wed Oct 21 17:42:51 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 21 Oct 2009 17:42:51 +0200 (CEST) Subject: [pypy-svn] r68688 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091021154251.08CB2168009@codespeak.net> Author: fijal Date: Wed Oct 21 17:42:46 2009 New Revision: 68688 Modified: pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/jit/metainterp/test/test_loop.py Log: use env variable instead of hardcoded debug flag for dump_storage and test it Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Wed Oct 21 17:42:46 2009 @@ -1,4 +1,4 @@ -import sys +import sys, os from pypy.jit.metainterp.history import Box, Const, ConstInt, INT, REF from pypy.jit.metainterp.resoperation import rop from pypy.rpython.lltypesystem import rffi @@ -10,8 +10,6 @@ # because it needs to support optimize.py which encodes virtuals with # arbitrary cycles and also to compress the information -debug = False - class Snapshot(object): __slots__ = ('prev', 'boxes') @@ -259,8 +257,9 @@ self._number_virtuals(liveboxes) storage.rd_consts = self.memo.consts - if debug: - dump_storage(storage, liveboxes) + logname = os.environ.get('PYPYJITRESUMELOG') + if logname: + dump_storage(logname, storage, liveboxes) return liveboxes[:] def _number_virtuals(self, liveboxes): @@ -426,11 +425,11 @@ # ____________________________________________________________ -def dump_storage(storage, liveboxes): +def dump_storage(logname, storage, liveboxes): "For profiling only." import os from pypy.rlib import objectmodel - fd = os.open('log.storage', os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0666) + fd = os.open(logname, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0666) os.write(fd, 'Log(%d, [\n' % objectmodel.compute_unique_id(storage)) frameinfo = storage.rd_frame_info_list while True: Modified: pypy/trunk/pypy/jit/metainterp/test/test_loop.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_loop.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_loop.py Wed Oct 21 17:42:46 2009 @@ -594,6 +594,26 @@ res = self.meta_interp(f, [200]) + def test_dump_storage(self): + import os + from pypy.tool.udir import udir + logfile = udir.join('resume.log') + os.environ['PYPYJITRESUMELOG'] = str(logfile) + try: + jitdriver = JitDriver(greens = [], reds = ['i', 'n']) + + def f(n): + i = 0 + while i < n: + jitdriver.can_enter_jit(n=n, i=i) + jitdriver.jit_merge_point(n=n, i=i) + i += 1 + + res = self.meta_interp(f, [10]) + data = logfile.read() # assert did not crash + finally: + del os.environ['PYPYJITRESUMELOG'] + class TestOOtype(LoopTest, OOJitMixin): pass From antocuni at codespeak.net Wed Oct 21 17:49:21 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 21 Oct 2009 17:49:21 +0200 (CEST) Subject: [pypy-svn] r68689 - pypy/trunk/pypy/translator/cli Message-ID: <20091021154921.6FCAC168009@codespeak.net> Author: antocuni Date: Wed Oct 21 17:49:21 2009 New Revision: 68689 Modified: pypy/trunk/pypy/translator/cli/sdk.py Log: workaround for a mono bug present in some versions Modified: pypy/trunk/pypy/translator/cli/sdk.py ============================================================================== --- pypy/trunk/pypy/translator/cli/sdk.py (original) +++ pypy/trunk/pypy/translator/cli/sdk.py Wed Oct 21 17:49:21 2009 @@ -34,12 +34,35 @@ CSC = 'csc' PEVERIFY = 'peverify' +def get_mono_version(): + from commands import getoutput + lines = getoutput('mono -V').splitlines() + parts = lines[0].split() + # something like ['Mono', 'JIT', 'compiler', 'version', '2.4.2.3', ...] + iversion = parts.index('version') + ver = parts[iversion+1] # '2.4.2.3' + ver = ver.split('.') # ['2', '4', '2', '3'] + return tuple(map(int, ver)) # (2, 4, 2, 3) + + class MonoSDK(AbstractSDK): RUNTIME = ['mono'] ILASM = 'ilasm2' CSC = 'gmcs' PEVERIFY = 'peverify' # it's not part of mono, but we get a meaningful skip message + # this is a workaround for this bug: + # https://bugzilla.novell.com/show_bug.cgi?id=474718 they promised that it + # should be fixed in versions after 2.4.3.x, in the meanwhile pass + # -O=-branch + @classmethod + def runtime(cls): + cls._check_helper('mono') + ver = get_mono_version() + if (2, 1) < ver < (2, 4, 3): + return ['mono', '-O=-branch'] + return ['mono'] + def key_as_dict(handle): import _winreg i = 0 From antocuni at codespeak.net Wed Oct 21 17:58:23 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 21 Oct 2009 17:58:23 +0200 (CEST) Subject: [pypy-svn] r68690 - pypy/trunk/pypy/jit/backend/cli/test Message-ID: <20091021155823.02E91168009@codespeak.net> Author: antocuni Date: Wed Oct 21 17:58:23 2009 New Revision: 68690 Modified: pypy/trunk/pypy/jit/backend/cli/test/test_zrpy_vlist.py Log: this test does not fail, since check_loops does nothing when compiled to cli Modified: pypy/trunk/pypy/jit/backend/cli/test/test_zrpy_vlist.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/test/test_zrpy_vlist.py (original) +++ pypy/trunk/pypy/jit/backend/cli/test/test_zrpy_vlist.py Wed Oct 21 17:58:23 2009 @@ -7,4 +7,6 @@ # for the individual tests see # ====> ../../../metainterp/test/test_vlist.py - pass + # disable the xfail() + def test_vlist_alloc_and_set(self): + test_vlist.TestOOtype.test_vlist_alloc_and_set(self) From fijal at codespeak.net Wed Oct 21 18:18:06 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 21 Oct 2009 18:18:06 +0200 (CEST) Subject: [pypy-svn] r68691 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091021161806.4EAFF168009@codespeak.net> Author: fijal Date: Wed Oct 21 18:18:05 2009 New Revision: 68691 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/jit/metainterp/simple_optimize.py Log: Move checking environment up to initializing of global data Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Wed Oct 21 18:18:05 2009 @@ -518,7 +518,8 @@ def store_final_boxes_in_guard(self, op): descr = op.descr assert isinstance(descr, compile.ResumeGuardDescr) - modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo) + modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo, + self.metainterp_sd.globaldata.storedebug) newboxes = modifier.finish(self.values) if len(newboxes) > self.metainterp_sd.options.failargs_limit: raise compile.GiveUp Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Wed Oct 21 18:18:05 2009 @@ -1,4 +1,4 @@ -import py +import py, os from pypy.rpython.lltypesystem import llmemory from pypy.rpython.ootypesystem import ootype from pypy.rlib.objectmodel import we_are_translated, r_dict @@ -1029,6 +1029,7 @@ self.globaldata.initialized = True self.logger_noopt.create_log('.noopt') self.logger_ops.create_log('.ops') + self.globaldata.storedebug = os.environ.get('PYPYJITRESUMELOG') def _setup_class_sizes(self): class_sizes = {} Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Wed Oct 21 18:18:05 2009 @@ -179,9 +179,10 @@ class ResumeDataVirtualAdder(object): - def __init__(self, storage, memo): + def __init__(self, storage, memo, debug_storage=None): self.storage = storage self.memo = memo + self.debug_storage = debug_storage #self.virtuals = [] #self.vfieldboxes = [] @@ -257,9 +258,8 @@ self._number_virtuals(liveboxes) storage.rd_consts = self.memo.consts - logname = os.environ.get('PYPYJITRESUMELOG') - if logname: - dump_storage(logname, storage, liveboxes) + if self.debug_storage: + dump_storage(self.debug_storage, storage, liveboxes) return liveboxes[:] def _number_virtuals(self, liveboxes): @@ -433,8 +433,9 @@ os.write(fd, 'Log(%d, [\n' % objectmodel.compute_unique_id(storage)) frameinfo = storage.rd_frame_info_list while True: - os.write(fd, '\t("%s", %d, %d),\n' % ( - frameinfo.jitcode, frameinfo.pc, frameinfo.exception_target)) + os.write(fd, '\t("%s", %d, %d, %xd),\n' % ( + frameinfo.jitcode, frameinfo.pc, frameinfo.exception_target, + objectmodel.compute_unique_id(frameinfo))) frameinfo = frameinfo.prev if frameinfo is None: break Modified: pypy/trunk/pypy/jit/metainterp/simple_optimize.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/simple_optimize.py (original) +++ pypy/trunk/pypy/jit/metainterp/simple_optimize.py Wed Oct 21 18:18:05 2009 @@ -22,7 +22,8 @@ if op.is_guard(): descr = op.descr assert isinstance(descr, compile.ResumeGuardDescr) - modifier = resume.ResumeDataVirtualAdder(descr, memo) + modifier = resume.ResumeDataVirtualAdder(descr, memo, + metainterp_sd.globaldata.storedebug) newboxes = modifier.finish(EMPTY_VALUES) descr.store_final_boxes(op, newboxes) newoperations.append(op) From arigo at codespeak.net Wed Oct 21 18:25:55 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Oct 2009 18:25:55 +0200 (CEST) Subject: [pypy-svn] r68692 - in pypy/branch/warmspot-jitinfo/pypy: interpreter module/pypyjit Message-ID: <20091021162555.F13CB168009@codespeak.net> Author: arigo Date: Wed Oct 21 18:25:55 2009 New Revision: 68692 Modified: pypy/branch/warmspot-jitinfo/pypy/interpreter/pycode.py pypy/branch/warmspot-jitinfo/pypy/module/pypyjit/interp_jit.py Log: In this version, pypy-c-jit seems to work. Modified: pypy/branch/warmspot-jitinfo/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/interpreter/pycode.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/interpreter/pycode.py Wed Oct 21 18:25:55 2009 @@ -84,6 +84,9 @@ self.hidden_applevel = hidden_applevel self.magic = magic self._signature = cpython_code_signature(self) + self._initialize() + + def _initialize(self): # Precompute what arguments need to be copied into cellvars self._args_as_cellvars = [] @@ -116,7 +119,7 @@ self._compute_flatcall() - if space.config.objspace.std.withcelldict: + if self.space.config.objspace.std.withcelldict: from pypy.objspace.std.celldict import init_code init_code(self) Modified: pypy/branch/warmspot-jitinfo/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/warmspot-jitinfo/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/warmspot-jitinfo/pypy/module/pypyjit/interp_jit.py Wed Oct 21 18:25:55 2009 @@ -20,8 +20,6 @@ from opcode import opmap from pypy.rlib.objectmodel import we_are_translated -PyCode.jit_cells = None - PyFrame._virtualizable2_ = ['last_instr', 'pycode', 'valuestackdepth', 'valuestack_w[*]', 'fastlocals_w[*]', 'f_forward', @@ -56,16 +54,10 @@ ExecutionContext._jit_rechain_frame(ec, frame) def get_jitcell_at(next_instr, bytecode): - d = bytecode.jit_cells - if d is None: - return None - return d.get(next_instr, None) + return bytecode.jit_cells.get(next_instr, None) def set_jitcell_at(newcell, next_instr, bytecode): - d = bytecode.jit_cells - if d is None: - d = bytecode.jit_cells = {} - d[next_instr] = newcell + bytecode.jit_cells[next_instr] = newcell class PyPyJitDriver(JitDriver): @@ -111,6 +103,20 @@ pycode=f.getcode()) return jumpto + +PyCode__initialize = PyCode._initialize + +class __extend__(PyCode): + __metaclass__ = extendabletype + + def _initialize(self): + PyCode__initialize(self) + self.jit_cells = {} + + def _freeze_(self): + self.jit_cells = {} + return False + # ____________________________________________________________ # # Public interface From arigo at codespeak.net Wed Oct 21 18:32:20 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Oct 2009 18:32:20 +0200 (CEST) Subject: [pypy-svn] r68693 - in pypy/trunk/pypy/rpython: . test Message-ID: <20091021163220.C19D5168009@codespeak.net> Author: arigo Date: Wed Oct 21 18:32:17 2009 New Revision: 68693 Modified: pypy/trunk/pypy/rpython/rint.py pypy/trunk/pypy/rpython/test/test_rdict.py Log: Fix ll_hash_int. Can you believe it. (merged from r68673) Modified: pypy/trunk/pypy/rpython/rint.py ============================================================================== --- pypy/trunk/pypy/rpython/rint.py (original) +++ pypy/trunk/pypy/rpython/rint.py Wed Oct 21 18:32:17 2009 @@ -408,10 +408,8 @@ fn = hop.rtyper.type_system.ll_str.ll_int2oct return hop.gendirectcall(fn, varg, true) -def ll_identity(n): - return n - -ll_hash_int = ll_identity +def ll_hash_int(n): + return intmask(n) def ll_check_chr(n): if 0 <= n <= 255: Modified: pypy/trunk/pypy/rpython/test/test_rdict.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_rdict.py (original) +++ pypy/trunk/pypy/rpython/test/test_rdict.py Wed Oct 21 18:32:17 2009 @@ -4,6 +4,7 @@ from pypy.rpython.lltypesystem import rdict, rstr from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin from pypy.rlib.objectmodel import r_dict +from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong import py py.log.setconsumer("rtyper", py.log.STDOUT) @@ -564,6 +565,17 @@ res = self.interpret(fn, [3.0]) assert res == 42 + def test_dict_of_r_uint(self): + for r_t in [r_uint, r_longlong, r_ulonglong]: + d = {r_t(2): 3, r_t(4): 5} + def fn(x, y): + d[r_t(x)] = 123 + return d[r_t(y)] + res = self.interpret(fn, [4, 2]) + assert res == 3 + res = self.interpret(fn, [3, 3]) + assert res == 123 + class TestLLtype(BaseTestRdict, LLRtypeMixin): def test_dict_but_not_with_char_keys(self): From arigo at codespeak.net Wed Oct 21 18:34:54 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Oct 2009 18:34:54 +0200 (CEST) Subject: [pypy-svn] r68694 - in pypy/trunk/pypy: rpython/lltypesystem rpython/lltypesystem/test rpython/memory rpython/memory/test translator/backendopt translator/backendopt/test Message-ID: <20091021163454.5AB01168009@codespeak.net> Author: arigo Date: Wed Oct 21 18:34:53 2009 New Revision: 68694 Modified: pypy/trunk/pypy/rpython/lltypesystem/lloperation.py pypy/trunk/pypy/rpython/lltypesystem/opimpl.py pypy/trunk/pypy/rpython/lltypesystem/test/test_lloperation.py pypy/trunk/pypy/rpython/memory/gctypelayout.py pypy/trunk/pypy/rpython/memory/test/test_gctypelayout.py pypy/trunk/pypy/translator/backendopt/mallocv.py pypy/trunk/pypy/translator/backendopt/test/test_constfold.py Log: Improve support of 'immutable_fields' wherever 'immutable' is supported, notably in optimizations. Add tests. (merge r68679 to r68681) Modified: pypy/trunk/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/lloperation.py Wed Oct 21 18:34:53 2009 @@ -84,12 +84,17 @@ return op_impl fold = roproperty(get_fold_impl) - def is_pure(self, *ARGTYPES): + def is_pure(self, args_v): return (self.canfold or # canfold => pure operation self is llop.debug_assert or # debug_assert is pure enough # reading from immutable (self in (llop.getfield, llop.getarrayitem) and - ARGTYPES[0].TO._hints.get('immutable'))) # XXX: what about ootype immutable arrays? + args_v[0].concretetype.TO._hints.get('immutable')) or + (self is llop.getfield and # reading from immutable_field + 'immutable_fields' in args_v[0].concretetype.TO._hints and + args_v[1].value in args_v[0].concretetype.TO + ._hints['immutable_fields'].fields)) + # XXX: what about ootype immutable arrays? def __repr__(self): return '' % (getattr(self, 'opname', '?'),) Modified: pypy/trunk/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/opimpl.py Wed Oct 21 18:34:53 2009 @@ -142,7 +142,12 @@ # we can constant-fold this if the innermost structure from which we # read the final field is immutable. T = lltype.typeOf(innermostcontainer).TO - if not T._hints.get('immutable'): + if T._hints.get('immutable'): + pass + elif ('immutable_fields' in T._hints and + offsets[-1] in T._hints['immutable_fields'].fields): + pass + else: raise TypeError("cannot fold getinteriorfield on mutable struct") assert not isinstance(ob, lltype._interior_ptr) return ob @@ -390,7 +395,13 @@ def op_getfield(p, name): checkptr(p) - if not lltype.typeOf(p).TO._hints.get('immutable'): + TYPE = lltype.typeOf(p).TO + if TYPE._hints.get('immutable'): + pass + elif ('immutable_fields' in TYPE._hints and + name in TYPE._hints['immutable_fields'].fields): + pass + else: raise TypeError("cannot fold getfield on mutable struct") return getattr(p, name) Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_lloperation.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/test/test_lloperation.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/test/test_lloperation.py Wed Oct 21 18:34:53 2009 @@ -4,6 +4,7 @@ from pypy.rpython.ootypesystem import ootype, ooopimpl from pypy.rpython.llinterp import LLFrame from pypy.rpython.test.test_llinterp import interpret +from pypy.rpython import rclass LL_INTERP_OPERATIONS = [name[3:] for name in LLFrame.__dict__.keys() if name.startswith('op_')] @@ -50,7 +51,72 @@ return s.x + s.y res = interpret(llf, [], policy=LowLevelAnnotatorPolicy()) assert res == 5 - + +def test_is_pure(): + from pypy.objspace.flow.model import Variable, Constant + assert llop.bool_not.is_pure([Variable()]) + assert llop.debug_assert.is_pure([Variable()]) + assert not llop.int_add_ovf.is_pure([Variable(), Variable()]) + # + S1 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed)) + v_s1 = Variable() + v_s1.concretetype = lltype.Ptr(S1) + assert not llop.setfield.is_pure([v_s1, Constant('x'), Variable()]) + assert not llop.getfield.is_pure([v_s1, Constant('y')]) + # + A1 = lltype.GcArray(lltype.Signed) + v_a1 = Variable() + v_a1.concretetype = lltype.Ptr(A1) + assert not llop.setarrayitem.is_pure([v_a1, Variable(), Variable()]) + assert not llop.getarrayitem.is_pure([v_a1, Variable()]) + assert llop.getarraysize.is_pure([v_a1]) + # + S2 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed), + hints={'immutable': True}) + v_s2 = Variable() + v_s2.concretetype = lltype.Ptr(S2) + assert not llop.setfield.is_pure([v_s2, Constant('x'), Variable()]) + assert llop.getfield.is_pure([v_s2, Constant('y')]) + # + A2 = lltype.GcArray(lltype.Signed, hints={'immutable': True}) + v_a2 = Variable() + v_a2.concretetype = lltype.Ptr(A2) + assert not llop.setarrayitem.is_pure([v_a2, Variable(), Variable()]) + assert llop.getarrayitem.is_pure([v_a2, Variable()]) + assert llop.getarraysize.is_pure([v_a2]) + # + accessor = rclass.FieldListAccessor() + S3 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed), + hints={'immutable_fields': accessor}) + accessor.initialize(S3, ['x']) + v_s3 = Variable() + v_s3.concretetype = lltype.Ptr(S3) + assert not llop.setfield.is_pure([v_s3, Constant('x'), Variable()]) + assert not llop.setfield.is_pure([v_s3, Constant('y'), Variable()]) + assert llop.getfield.is_pure([v_s3, Constant('x')]) + assert not llop.getfield.is_pure([v_s3, Constant('y')]) + +def test_getfield_pure(): + S1 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed)) + S2 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed), + hints={'immutable': True}) + accessor = rclass.FieldListAccessor() + S3 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed), + hints={'immutable_fields': accessor}) + accessor.initialize(S3, ['x']) + # + s1 = lltype.malloc(S1); s1.x = 45 + py.test.raises(TypeError, llop.getfield, lltype.Signed, s1, 'x') + s2 = lltype.malloc(S2); s2.x = 45 + assert llop.getfield(lltype.Signed, s2, 'x') == 45 + s3 = lltype.malloc(S3); s3.x = 46; s3.y = 47 + assert llop.getfield(lltype.Signed, s3, 'x') == 46 + py.test.raises(TypeError, llop.getfield, lltype.Signed, s3, 'y') + # + py.test.raises(TypeError, llop.getinteriorfield, lltype.Signed, s1, 'x') + assert llop.getinteriorfield(lltype.Signed, s2, 'x') == 45 + assert llop.getinteriorfield(lltype.Signed, s3, 'x') == 46 + py.test.raises(TypeError, llop.getinteriorfield, lltype.Signed, s3, 'y') # ___________________________________________________________________________ # This tests that the LLInterpreter and the LL_OPERATIONS tables are in sync. Modified: pypy/trunk/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/trunk/pypy/rpython/memory/gctypelayout.py Wed Oct 21 18:34:53 2009 @@ -350,11 +350,16 @@ def gc_pointers_inside(v, adr, mutable_only=False): t = lltype.typeOf(v) if isinstance(t, lltype.Struct): - if mutable_only and t._hints.get('immutable'): - return + skip = () + if mutable_only: + if t._hints.get('immutable'): + return + if 'immutable_fields' in t._hints: + skip = t._hints['immutable_fields'].fields for n, t2 in t._flds.iteritems(): if isinstance(t2, lltype.Ptr) and t2.TO._gckind == 'gc': - yield adr + llmemory.offsetof(t, n) + if n not in skip: + yield adr + llmemory.offsetof(t, n) elif isinstance(t2, (lltype.Array, lltype.Struct)): for a in gc_pointers_inside(getattr(v, n), adr + llmemory.offsetof(t, n), Modified: pypy/trunk/pypy/rpython/memory/test/test_gctypelayout.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/test/test_gctypelayout.py (original) +++ pypy/trunk/pypy/rpython/memory/test/test_gctypelayout.py Wed Oct 21 18:34:53 2009 @@ -1,7 +1,8 @@ import py from pypy.rpython.memory.gctypelayout import TypeLayoutBuilder, GCData from pypy.rpython.memory.gctypelayout import offsets_to_gc_pointers -from pypy.rpython.lltypesystem import lltype, rclass +from pypy.rpython.memory.gctypelayout import gc_pointers_inside +from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.test.test_llinterp import get_interpreter from pypy.objspace.flow.model import Constant @@ -90,3 +91,31 @@ interp, graph = get_interpreter(f, [], backendopt=True) assert interp.eval_graph(graph, []) == 11001 assert graph.startblock.exits[0].args == [Constant(11001, lltype.Signed)] + +def test_gc_pointers_inside(): + from pypy.rpython import rclass + PT = lltype.Ptr(lltype.GcStruct('T')) + S1 = lltype.GcStruct('S', ('x', PT), ('y', PT)) + S2 = lltype.GcStruct('S', ('x', PT), ('y', PT), + hints={'immutable': True}) + accessor = rclass.FieldListAccessor() + S3 = lltype.GcStruct('S', ('x', PT), ('y', PT), + hints={'immutable_fields': accessor}) + accessor.initialize(S3, ['x']) + # + s1 = lltype.malloc(S1) + adr = llmemory.cast_ptr_to_adr(s1) + lst = list(gc_pointers_inside(s1._obj, adr, mutable_only=True)) + expected = [adr + llmemory.offsetof(S1, 'x'), + adr + llmemory.offsetof(S1, 'y')] + assert lst == expected or lst == expected[::-1] + # + s2 = lltype.malloc(S2) + adr = llmemory.cast_ptr_to_adr(s2) + lst = list(gc_pointers_inside(s2._obj, adr, mutable_only=True)) + assert lst == [] + # + s3 = lltype.malloc(S3) + adr = llmemory.cast_ptr_to_adr(s3) + lst = list(gc_pointers_inside(s3._obj, adr, mutable_only=True)) + assert lst == [adr + llmemory.offsetof(S3, 'y')] Modified: pypy/trunk/pypy/translator/backendopt/mallocv.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/mallocv.py (original) +++ pypy/trunk/pypy/translator/backendopt/mallocv.py Wed Oct 21 18:34:53 2009 @@ -1046,7 +1046,7 @@ op = getattr(llop, opname) except AttributeError: return - if not op.is_pure(*[v.concretetype for v in args_v]): + if not op.is_pure(args_v): return try: result = op(RESTYPE, *args) Modified: pypy/trunk/pypy/translator/backendopt/test/test_constfold.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/test/test_constfold.py (original) +++ pypy/trunk/pypy/translator/backendopt/test/test_constfold.py Wed Oct 21 18:34:53 2009 @@ -4,6 +4,7 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rpython import rclass from pypy.rlib import objectmodel from pypy.translator.backendopt.constfold import constant_fold_graph from pypy import conftest @@ -26,8 +27,10 @@ assert res == expected_result -def test_simple(): - S1 = lltype.GcStruct('S1', ('x', lltype.Signed), hints={'immutable': True}) +def test_simple(S1=None): + if S1 is None: + S1 = lltype.GcStruct('S1', ('x', lltype.Signed), + hints={'immutable': True}) s1 = lltype.malloc(S1) s1.x = 123 def g(y): @@ -42,6 +45,14 @@ check_graph(graph, [], 124, t) +def test_immutable_fields(): + accessor = rclass.FieldListAccessor() + S2 = lltype.GcStruct('S2', ('x', lltype.Signed), + hints={'immutable_fields': accessor}) + accessor.initialize(S2, ['x']) + test_simple(S2) + + def test_along_link(): S1 = lltype.GcStruct('S1', ('x', lltype.Signed), hints={'immutable': True}) s1 = lltype.malloc(S1) From arigo at codespeak.net Wed Oct 21 18:35:40 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Oct 2009 18:35:40 +0200 (CEST) Subject: [pypy-svn] r68695 - in pypy/trunk/pypy: interpreter jit/backend/cli jit/backend/llgraph jit/metainterp jit/metainterp/test module/pypyjit module/pypyjit/test rlib Message-ID: <20091021163540.2EAD8168012@codespeak.net> Author: arigo Date: Wed Oct 21 18:35:39 2009 New Revision: 68695 Added: pypy/trunk/pypy/jit/metainterp/test/test_warmstate.py - copied unchanged from r68692, pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/test/test_warmstate.py pypy/trunk/pypy/jit/metainterp/warmstate.py - copied unchanged from r68692, pypy/branch/warmspot-jitinfo/pypy/jit/metainterp/warmstate.py Modified: pypy/trunk/pypy/interpreter/pycode.py pypy/trunk/pypy/jit/backend/cli/runner.py pypy/trunk/pypy/jit/backend/llgraph/llimpl.py pypy/trunk/pypy/jit/backend/llgraph/runner.py pypy/trunk/pypy/jit/metainterp/compile.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/support.py pypy/trunk/pypy/jit/metainterp/test/test_compile.py pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py pypy/trunk/pypy/jit/metainterp/test/test_ztranslation.py pypy/trunk/pypy/jit/metainterp/virtualizable.py pypy/trunk/pypy/jit/metainterp/warmspot.py pypy/trunk/pypy/module/pypyjit/interp_jit.py pypy/trunk/pypy/module/pypyjit/test/test_jit_setup.py pypy/trunk/pypy/rlib/jit.py Log: Finish the merge of the 'warmspot-jitinfo' branch. Clean up the WarmEnterState class, move it to its own file, test it, and add support for a JitDriver hook to replace the approximate hashtable logic. Implement the corresponding hook for pypy-c-jit. Modified: pypy/trunk/pypy/interpreter/pycode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pycode.py (original) +++ pypy/trunk/pypy/interpreter/pycode.py Wed Oct 21 18:35:39 2009 @@ -84,6 +84,9 @@ self.hidden_applevel = hidden_applevel self.magic = magic self._signature = cpython_code_signature(self) + self._initialize() + + def _initialize(self): # Precompute what arguments need to be copied into cellvars self._args_as_cellvars = [] @@ -116,7 +119,7 @@ self._compute_flatcall() - if space.config.objspace.std.withcelldict: + if self.space.config.objspace.std.withcelldict: from pypy.objspace.std.celldict import init_code init_code(self) Modified: pypy/trunk/pypy/jit/backend/cli/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/runner.py (original) +++ pypy/trunk/pypy/jit/backend/cli/runner.py Wed Oct 21 18:35:39 2009 @@ -319,7 +319,7 @@ def __init__(self, TYPE): DescrWithKey.__init__(self, TYPE) from pypy.jit.backend.llgraph.runner import boxresult - from pypy.jit.metainterp.warmspot import unwrap + from pypy.jit.metainterp.warmstate import unwrap ARRAY = ootype.Array(TYPE) def create(): if isinstance(TYPE, ootype.OOType): @@ -482,7 +482,7 @@ def __init__(self, TYPE, fieldname): DescrWithKey.__init__(self, (TYPE, fieldname)) from pypy.jit.backend.llgraph.runner import boxresult - from pypy.jit.metainterp.warmspot import unwrap + from pypy.jit.metainterp.warmstate import unwrap _, T = TYPE._lookup_field(fieldname) def getfield(objbox): obj = objbox.getref(TYPE) Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/llimpl.py Wed Oct 21 18:35:39 2009 @@ -807,7 +807,6 @@ op_getarrayitem_gc_pure = op_getarrayitem_gc def op_setarrayitem_gc(self, typedescr, obj, index, objnewvalue): - from pypy.jit.metainterp.warmspot import unwrap array = ootype.cast_from_object(typedescr.ARRAY, obj) if ootype.typeOf(objnewvalue) == ootype.Object: newvalue = ootype.cast_from_object(typedescr.TYPE, objnewvalue) Modified: pypy/trunk/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/runner.py Wed Oct 21 18:35:39 2009 @@ -9,7 +9,7 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.jit.metainterp import history from pypy.jit.metainterp.history import REF, INT, FLOAT -from pypy.jit.metainterp.warmspot import unwrap +from pypy.jit.metainterp.warmstate import unwrap from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.backend import model from pypy.jit.backend.llgraph import llimpl, symbolic Modified: pypy/trunk/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/compile.py (original) +++ pypy/trunk/pypy/jit/metainterp/compile.py Wed Oct 21 18:35:39 2009 @@ -262,8 +262,8 @@ new_loop_token) # store the new loop in compiled_merge_points too glob = metainterp_sd.globaldata - greenargs = glob.unpack_greenkey(self.original_greenkey) - old_loop_tokens = glob.compiled_merge_points.setdefault(greenargs, []) + old_loop_tokens = glob.get_compiled_merge_points( + self.original_greenkey) # it always goes at the end of the list, as it is the most # general loop token old_loop_tokens.append(new_loop_token) Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Wed Oct 21 18:35:39 2009 @@ -1,7 +1,7 @@ import py, os from pypy.rpython.lltypesystem import llmemory from pypy.rpython.ootypesystem import ootype -from pypy.rlib.objectmodel import we_are_translated, r_dict +from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.debug import debug_print @@ -1099,15 +1099,25 @@ # state = staticdata.state if state is not None: - self.unpack_greenkey = state.unwrap_greenkey - self.compiled_merge_points = r_dict(state.comparekey,state.hashkey) - # { (greenargs): [MergePoints] } + self.jit_cell_at_key = state.jit_cell_at_key else: - self.compiled_merge_points = {} # for tests only; not RPython - self.unpack_greenkey = tuple + # for tests only; not RPython + class JitCell: + compiled_merge_points = None + _jitcell_dict = {} + def jit_cell_at_key(greenkey): + greenkey = tuple(greenkey) + return _jitcell_dict.setdefault(greenkey, JitCell()) + self.jit_cell_at_key = jit_cell_at_key if staticdata.virtualizable_info: self.blackhole_virtualizable = staticdata.virtualizable_info.null_vable + def get_compiled_merge_points(self, greenkey): + cell = self.jit_cell_at_key(greenkey) + if cell.compiled_merge_points is None: + cell.compiled_merge_points = [] + return cell.compiled_merge_points + def get_fail_descr_number(self, descr): assert isinstance(descr, history.AbstractFailDescr) lst = self.fail_descr_list @@ -1496,8 +1506,7 @@ self.history.inputargs = original_boxes[num_green_args:] greenkey = original_boxes[:num_green_args] glob = self.staticdata.globaldata - greenargs = glob.unpack_greenkey(greenkey) - old_loop_tokens = glob.compiled_merge_points.setdefault(greenargs, []) + old_loop_tokens = glob.get_compiled_merge_points(greenkey) self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None) loop_token = compile.compile_new_loop(self, old_loop_tokens, greenkey, start) @@ -1508,10 +1517,8 @@ num_green_args = self.staticdata.num_green_args greenkey = live_arg_boxes[:num_green_args] glob = self.staticdata.globaldata - greenargs = glob.unpack_greenkey(greenkey) - try: - old_loop_tokens = glob.compiled_merge_points[greenargs] - except KeyError: + old_loop_tokens = glob.get_compiled_merge_points(greenkey) + if len(old_loop_tokens) == 0: return self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None) target_loop_token = compile.compile_new_bridge(self, old_loop_tokens, @@ -1565,7 +1572,7 @@ def _initialize_from_start(self, original_boxes, num_green_args, *args): if args: - from pypy.jit.metainterp.warmspot import wrap + from pypy.jit.metainterp.warmstate import wrap box = wrap(self.cpu, args[0], num_green_args > 0) original_boxes.append(box) self._initialize_from_start(original_boxes, num_green_args-1, Modified: pypy/trunk/pypy/jit/metainterp/support.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/support.py (original) +++ pypy/trunk/pypy/jit/metainterp/support.py Wed Oct 21 18:35:39 2009 @@ -74,16 +74,15 @@ def maybe_on_top_of_llinterp(rtyper, fnptr): # Run a generated graph on top of the llinterp for testing. # When translated, this just returns the fnptr. - llinterp = LLInterpreter(rtyper) #, exc_data_ptr=exc_data_ptr) funcobj = get_funcobj(fnptr) if hasattr(funcobj, 'graph'): + llinterp = LLInterpreter(rtyper) #, exc_data_ptr=exc_data_ptr) def on_top_of_llinterp(*args): return llinterp.eval_graph(funcobj.graph, list(args)) else: - assert isinstance(fnptr, ootype._meth) - assert hasattr(fnptr, '_callable') + assert hasattr(funcobj, '_callable') def on_top_of_llinterp(*args): - return fnptr._callable(*args) + return funcobj._callable(*args) return on_top_of_llinterp class Entry(ExtRegistryEntry): Modified: pypy/trunk/pypy/jit/metainterp/test/test_compile.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_compile.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_compile.py Wed Oct 21 18:35:39 2009 @@ -41,7 +41,7 @@ optimize_loop = staticmethod(optimize.optimize_loop) debug_level = 0 -class FakeGlobalData(): +class FakeGlobalData: loopnumbering = 0 class FakeMetaInterpStaticData: Modified: pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py Wed Oct 21 18:35:39 2009 @@ -1,39 +1,13 @@ import py -from pypy.jit.metainterp.warmspot import ll_meta_interp, hash_whatever -from pypy.jit.metainterp.warmspot import get_stats, equal_whatever +from pypy.jit.metainterp.warmspot import ll_meta_interp +from pypy.jit.metainterp.warmspot import get_stats from pypy.rlib.jit import JitDriver, OPTIMIZER_FULL, OPTIMIZER_SIMPLE from pypy.rlib.jit import unroll_safe from pypy.jit.backend.llgraph import runner -from pypy.rpython.test.test_llinterp import interpret from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin -def test_hash_equal_whatever_lltype(): - from pypy.rpython.lltypesystem import lltype, rstr - s1 = rstr.mallocstr(2) - s2 = rstr.mallocstr(2) - s1.chars[0] = 'x'; s1.chars[1] = 'y' - s2.chars[0] = 'x'; s2.chars[1] = 'y' - def fn(x): - assert hash_whatever(lltype.typeOf(x), x) == 42 - assert (hash_whatever(lltype.typeOf(s1), s1) == - hash_whatever(lltype.typeOf(s2), s2)) - assert equal_whatever(lltype.typeOf(s1), s1, s2) - fn(42) - interpret(fn, [42], type_system='lltype') - -def test_hash_equal_whatever_ootype(): - from pypy.rpython.ootypesystem import ootype - def fn(c): - s1 = ootype.oostring("xy", -1) - s2 = ootype.oostring("x" + chr(c), -1) - assert (hash_whatever(ootype.typeOf(s1), s1) == - hash_whatever(ootype.typeOf(s2), s2)) - assert equal_whatever(ootype.typeOf(s1), s1, s2) - fn(ord('y')) - interpret(fn, [ord('y')], type_system='ootype') - class Exit(Exception): def __init__(self, result): self.result = result @@ -90,20 +64,6 @@ res = self.meta_interp(f, [60]) assert res == f(30) - def test_hash_collision(self): - mydriver = JitDriver(reds = ['n'], greens = ['m']) - def f(n): - m = 0 - while n > 0: - mydriver.can_enter_jit(n=n, m=m) - mydriver.jit_merge_point(n=n, m=m) - n -= 1 - if not (n % 11): - m = (m+n) & 3 - return m - res = self.meta_interp(f, [110], hash_bits=1) - assert res == f(110) - def test_location(self): def get_printable_location(n): return 'GREEN IS %d.' % n Modified: pypy/trunk/pypy/jit/metainterp/test/test_ztranslation.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_ztranslation.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_ztranslation.py Wed Oct 21 18:35:39 2009 @@ -4,6 +4,8 @@ from pypy.rlib.jit import JitDriver, OPTIMIZER_FULL, unroll_parameters from pypy.rlib.jit import PARAMETERS, dont_look_inside from pypy.jit.metainterp.jitprof import Profiler +from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.ootypesystem import ootype class TranslationTest: @@ -18,6 +20,7 @@ # - set_param interface # - profiler # - full optimizer + # - jitdriver hooks class Frame(object): _virtualizable2_ = ['i'] @@ -25,8 +28,24 @@ def __init__(self, i): self.i = i + class JitCellCache: + entry = None + jitcellcache = JitCellCache() + def set_jitcell_at(entry): + jitcellcache.entry = entry + def get_jitcell_at(): + return jitcellcache.entry + def get_printable_location(): + return '(hello world)' + def can_inline(): + return False + jitdriver = JitDriver(greens = [], reds = ['frame', 'total'], - virtualizables = ['frame']) + virtualizables = ['frame'], + get_jitcell_at=get_jitcell_at, + set_jitcell_at=set_jitcell_at, + get_printable_location=get_printable_location, + can_inline=can_inline) def f(i): for param in unroll_parameters: defl = PARAMETERS[param] Modified: pypy/trunk/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/trunk/pypy/jit/metainterp/virtualizable.py Wed Oct 21 18:35:39 2009 @@ -6,7 +6,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.jit.metainterp.typesystem import deref, fieldType, arrayItem from pypy.jit.metainterp import history -from pypy.jit.metainterp.warmspot import wrap, unwrap +from pypy.jit.metainterp.warmstate import wrap, unwrap class VirtualizableInfo: Modified: pypy/trunk/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmspot.py Wed Oct 21 18:35:39 2009 @@ -10,7 +10,6 @@ from pypy.objspace.flow.model import checkgraph, Link, copygraph from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib.jit import PARAMETERS, OPTIMIZER_SIMPLE, OPTIMIZER_FULL from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.debug import debug_print from pypy.rpython.lltypesystem.lloperation import llop @@ -24,7 +23,6 @@ from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper from pypy.jit.metainterp.jitprof import Profiler, EmptyProfiler from pypy.rlib.jit import DEBUG_STEPS, DEBUG_DETAILED, DEBUG_OFF, DEBUG_PROFILE -from pypy.rlib.nonconst import NonConstant # ____________________________________________________________ # Bootstrapping @@ -63,7 +61,7 @@ clear_tcache() return jittify_and_run(interp, graph, args, backendopt=backendopt, **kwds) -def jittify_and_run(interp, graph, args, repeat=1, hash_bits=None, +def jittify_and_run(interp, graph, args, repeat=1, backendopt=False, trace_limit=sys.maxint, debug_level=DEBUG_STEPS, inline=False, **kwds): translator = interp.typer.annotator.translator @@ -74,9 +72,6 @@ warmrunnerdesc.state.set_param_trace_limit(trace_limit) warmrunnerdesc.state.set_param_inlining(inline) warmrunnerdesc.state.set_param_debug(debug_level) - warmrunnerdesc.state.create_tables_now() # for tests - if hash_bits: - warmrunnerdesc.state.set_param_hash_bits(hash_bits) warmrunnerdesc.finish() res = interp.eval_graph(graph, args) if not kwds.get('translate_support_code', False): @@ -167,7 +162,7 @@ self.build_meta_interp(CPUClass, **kwds) self.make_args_specification() self.rewrite_jit_merge_point(policy) - self.make_driverhook_graph() + self.make_driverhook_graphs() if self.jitdriver.virtualizables: from pypy.jit.metainterp.virtualizable import VirtualizableInfo self.metainterp_sd.virtualizable_info = VirtualizableInfo(self) @@ -269,8 +264,9 @@ warmrunnerdesc=self) def make_enter_function(self): - WarmEnterState = make_state_class(self) - state = WarmEnterState() + from pypy.jit.metainterp.warmstate import WarmEnterState + state = WarmEnterState(self) + maybe_compile_and_run = state.make_entry_point() self.state = state def crash_in_jit(e): @@ -288,7 +284,7 @@ if self.translator.rtyper.type_system.name == 'lltypesystem': def maybe_enter_jit(*args): try: - state.maybe_compile_and_run(*args) + maybe_compile_and_run(*args) except JitException: raise # go through except Exception, e: @@ -296,7 +292,7 @@ maybe_enter_jit._always_inline_ = True else: def maybe_enter_jit(*args): - state.maybe_compile_and_run(*args) + maybe_compile_and_run(*args) maybe_enter_jit._always_inline_ = True self.maybe_enter_jit_fn = maybe_enter_jit @@ -312,24 +308,38 @@ args_s, s_result) annhelper.finish() - def make_driverhook_graph(self): + def make_driverhook_graphs(self): + from pypy.rlib.jit import BaseJitCell + bk = self.rtyper.annotator.bookkeeper + classdef = bk.getuniqueclassdef(BaseJitCell) + s_BaseJitCell_or_None = annmodel.SomeInstance(classdef, + can_be_None=True) + s_BaseJitCell_not_None = annmodel.SomeInstance(classdef) + s_Str = annmodel.SomeString() + # + annhelper = MixLevelHelperAnnotator(self.translator.rtyper) + self.set_jitcell_at_ptr = self._make_hook_graph( + annhelper, self.jitdriver.set_jitcell_at, annmodel.s_None, + s_BaseJitCell_not_None) + self.get_jitcell_at_ptr = self._make_hook_graph( + annhelper, self.jitdriver.get_jitcell_at, s_BaseJitCell_or_None) self.can_inline_ptr = self._make_hook_graph( - self.jitdriver.can_inline, bool) + annhelper, self.jitdriver.can_inline, annmodel.s_Bool) self.get_printable_location_ptr = self._make_hook_graph( - self.jitdriver.get_printable_location, str) + annhelper, self.jitdriver.get_printable_location, s_Str) + annhelper.finish() - def _make_hook_graph(self, func, rettype): - from pypy.annotation.signature import annotationoftype + def _make_hook_graph(self, annhelper, func, s_result, s_first_arg=None): if func is None: return None - annhelper = MixLevelHelperAnnotator(self.translator.rtyper) - s_result = annotationoftype(rettype) - RETTYPE = annhelper.rtyper.getrepr(s_result).lowleveltype - FUNC, PTR = self.cpu.ts.get_FuncType(self.green_args_spec, RETTYPE) + # + extra_args_s = [] + if s_first_arg is not None: + extra_args_s.append(s_first_arg) + # args_s = self.portal_args_s[:len(self.green_args_spec)] - graph = annhelper.getgraph(func, args_s, s_result) - funcptr = annhelper.graph2delayed(graph, FUNC) - annhelper.finish() + graph = annhelper.getgraph(func, extra_args_s + args_s, s_result) + funcptr = annhelper.graph2delayed(graph) return funcptr def make_args_specification(self): @@ -346,6 +356,7 @@ self.green_args_spec.append(TYPE) else: self.red_args_types.append(history.getkind(TYPE)) + self.num_green_args = len(self.green_args_spec) RESTYPE = graph.getreturnvar().concretetype (self.JIT_ENTER_FUNCTYPE, self.PTR_JIT_ENTER_FUNCTYPE) = self.cpu.ts.get_FuncType(ALLARGS, lltype.Void) @@ -465,6 +476,7 @@ # accepts boxes as argument, but unpacks them immediately # before we raise the exception -- the boxes' values will # be modified in a 'finally' by restore_patched_boxes(). + from pypy.jit.metainterp.warmstate import unwrap for i, name, ARG in portalfunc_ARGS: v = unwrap(ARG, argboxes[i]) setattr(self, name, v) @@ -585,392 +597,3 @@ assert len(reds_v) == numreds return ([v for v in greens_v if v.concretetype is not lltype.Void], [v for v in reds_v if v.concretetype is not lltype.Void]) - -def unwrap(TYPE, box): - if TYPE is lltype.Void: - return None - if isinstance(TYPE, lltype.Ptr): - return box.getref(TYPE) - if isinstance(TYPE, ootype.OOType): - return box.getref(TYPE) - if TYPE == lltype.Float: - return box.getfloat() - else: - return lltype.cast_primitive(TYPE, box.getint()) -unwrap._annspecialcase_ = 'specialize:arg(0)' - -def wrap(cpu, value, in_const_box=False): - if isinstance(lltype.typeOf(value), lltype.Ptr): - if lltype.typeOf(value).TO._gckind == 'gc': - value = lltype.cast_opaque_ptr(llmemory.GCREF, value) - if in_const_box: - return history.ConstPtr(value) - else: - return history.BoxPtr(value) - else: - adr = llmemory.cast_ptr_to_adr(value) - value = cpu.cast_adr_to_int(adr) - # fall through to the end of the function - elif isinstance(lltype.typeOf(value), ootype.OOType): - value = ootype.cast_to_object(value) - if in_const_box: - return history.ConstObj(value) - else: - return history.BoxObj(value) - elif isinstance(value, float): - if in_const_box: - return history.ConstFloat(value) - else: - return history.BoxFloat(value) - else: - value = intmask(value) - if in_const_box: - return history.ConstInt(value) - else: - return history.BoxInt(value) -wrap._annspecialcase_ = 'specialize:ll' - -def equal_whatever(TYPE, x, y): - if isinstance(TYPE, lltype.Ptr): - if TYPE.TO is rstr.STR or TYPE.TO is rstr.UNICODE: - return rstr.LLHelpers.ll_streq(x, y) - if TYPE is ootype.String or TYPE is ootype.Unicode: - return x.ll_streq(y) - return x == y -equal_whatever._annspecialcase_ = 'specialize:arg(0)' - -def hash_whatever(TYPE, x): - # Hash of lltype or ootype object. - # Only supports strings, unicodes and regular instances, - # as well as primitives that can meaningfully be cast to Signed. - if isinstance(TYPE, lltype.Ptr): - if TYPE.TO is rstr.STR or TYPE.TO is rstr.UNICODE: - return rstr.LLHelpers.ll_strhash(x) # assumed not null - else: - if x: - return lltype.identityhash(x) - else: - return 0 - elif TYPE is ootype.String or TYPE is ootype.Unicode: - return x.ll_hash() - elif isinstance(TYPE, ootype.OOType): - if x: - return ootype.identityhash(x) - else: - return 0 - else: - return lltype.cast_primitive(lltype.Signed, x) -hash_whatever._annspecialcase_ = 'specialize:arg(0)' - -# ____________________________________________________________ - -def make_state_class(warmrunnerdesc): - jitdriver = warmrunnerdesc.jitdriver - num_green_args = len(jitdriver.greens) - warmrunnerdesc.num_green_args = num_green_args - green_args_spec = unrolling_iterable(warmrunnerdesc.green_args_spec) - green_args_names = unrolling_iterable(jitdriver.greens) - green_args_spec_names = unrolling_iterable(zip( - warmrunnerdesc.green_args_spec, jitdriver.greens)) - red_args_types = unrolling_iterable(warmrunnerdesc.red_args_types) - # - metainterp_sd = warmrunnerdesc.metainterp_sd - vinfo = metainterp_sd.virtualizable_info - if vinfo is None: - vable_static_fields = [] - vable_array_fields = [] - else: - vable_static_fields = unrolling_iterable( - zip(vinfo.static_extra_types, vinfo.static_fields)) - vable_array_fields = unrolling_iterable( - zip(vinfo.arrayitem_extra_types, vinfo.array_fields)) - # - if num_green_args: - MAX_HASH_TABLE_BITS = 28 - else: - MAX_HASH_TABLE_BITS = 1 - THRESHOLD_LIMIT = sys.maxint // 2 - # - getlength = warmrunnerdesc.cpu.ts.getlength - getarrayitem = warmrunnerdesc.cpu.ts.getarrayitem - setarrayitem = warmrunnerdesc.cpu.ts.setarrayitem - # - rtyper = warmrunnerdesc.translator.rtyper - can_inline_ptr = warmrunnerdesc.can_inline_ptr - get_printable_location_ptr = warmrunnerdesc.get_printable_location_ptr - # - class MachineCodeEntryPoint(object): - next = None # linked list - def __init__(self, entry_loop_token, *greenargs): - self.entry_loop_token = entry_loop_token - i = 0 - for name in green_args_names: - setattr(self, 'green_' + name, greenargs[i]) - i = i + 1 - def equalkey(self, *greenargs): - i = 0 - for TYPE, name in green_args_spec_names: - myvalue = getattr(self, 'green_' + name) - if not equal_whatever(TYPE, myvalue, greenargs[i]): - return False - i = i + 1 - return True - def set_future_values(self, *redargs): - i = 0 - for typecode in red_args_types: - set_future_value(i, redargs[i], typecode) - i = i + 1 - if vinfo is not None: - virtualizable = redargs[vinfo.index_of_virtualizable - - num_green_args] - virtualizable = vinfo.cast_to_vtype(virtualizable) - for typecode, fieldname in vable_static_fields: - x = getattr(virtualizable, fieldname) - set_future_value(i, x, typecode) - i = i + 1 - for typecode, fieldname in vable_array_fields: - lst = getattr(virtualizable, fieldname) - for j in range(getlength(lst)): - x = getarrayitem(lst, j) - set_future_value(i, x, typecode) - i = i + 1 - - def set_future_value(j, value, typecode): - cpu = metainterp_sd.cpu - if typecode == 'ref': - refvalue = cpu.ts.cast_to_ref(value) - cpu.set_future_value_ref(j, refvalue) - elif typecode == 'int': - intvalue = lltype.cast_primitive(lltype.Signed, value) - cpu.set_future_value_int(j, intvalue) - elif typecode == 'float': - assert isinstance(value, float) - cpu.set_future_value_float(j, value) - else: - assert False - set_future_value._annspecialcase_ = 'specialize:ll_and_arg(2)' - - class WarmEnterState: - def __init__(self): - # initialize the state with the default values of the - # parameters specified in rlib/jit.py - for name, default_value in PARAMETERS.items(): - meth = getattr(self, 'set_param_' + name) - meth(default_value) - - def set_param_threshold(self, threshold): - if threshold < 2: - threshold = 2 - self.increment_threshold = (THRESHOLD_LIMIT // threshold) + 1 - # the number is at least 1, and at most about half THRESHOLD_LIMIT - - def set_param_trace_eagerness(self, value): - self.trace_eagerness = value - - def set_param_trace_limit(self, value): - self.trace_limit = value - - def set_param_inlining(self, value): - self.inlining = value - - def set_param_hash_bits(self, value): - if value < 1: - value = 1 - elif value > MAX_HASH_TABLE_BITS: - value = MAX_HASH_TABLE_BITS - # the tables are initialized with the correct size only in - # create_tables_now() - self.hashbits = value - self.hashtablemask = 0 - self.mccounters = [0] - self.mcentrypoints = [None] - # invariant: (self.mccounters[j] < 0) if and only if - # (self.mcentrypoints[j] is not None) - - def set_param_optimizer(self, optimizer): - if optimizer == OPTIMIZER_SIMPLE: - from pypy.jit.metainterp import simple_optimize - self.optimize_loop = simple_optimize.optimize_loop - self.optimize_bridge = simple_optimize.optimize_bridge - elif optimizer == OPTIMIZER_FULL: - from pypy.jit.metainterp import optimize - self.optimize_loop = optimize.optimize_loop - self.optimize_bridge = optimize.optimize_bridge - else: - raise ValueError("unknown optimizer") - - def set_param_debug(self, value): - self.debug_level = value - metainterp_sd.profiler.set_printing(value >= DEBUG_PROFILE) - - def create_tables_now(self): - count = 1 << self.hashbits - self.hashtablemask = count - 1 - self.mccounters = [0] * count - self.mcentrypoints = [None] * count - - # Only use the hash of the arguments as the profiling key. - # Indeed, this is all a heuristic, so if things are designed - # correctly, the occasional mistake due to hash collision is - # not too bad. - - def maybe_compile_and_run(self, *args): - globaldata = metainterp_sd.globaldata - if NonConstant(False): - # make sure we always see the saner optimizer from an annotation - # point of view, otherwise we get lots of blocked ops - self.set_param_optimizer(OPTIMIZER_FULL) - - # get the greenargs and look for the cell corresponding to the hash - greenargs = args[:num_green_args] - argshash = self.getkeyhash(*greenargs) & self.hashtablemask - counter = self.mccounters[argshash] - if vinfo: - virtualizable = args[vinfo.index_of_virtualizable] - virtualizable = vinfo.cast_to_vtype(virtualizable) - assert virtualizable != globaldata.blackhole_virtualizable, "reentering same frame via blackhole" - if counter >= 0: - # update the profiling counter - n = counter + self.increment_threshold - if n <= THRESHOLD_LIMIT: # bound not reached - self.mccounters[argshash] = n - return - if self.hashtablemask == 0: # must really create the tables now - self.create_tables_now() - return - metainterp = MetaInterp(metainterp_sd) - try: - loop_token = metainterp.compile_and_run_once(*args) - except warmrunnerdesc.ContinueRunningNormally: - # the trace got too long, reset the counter - self.mccounters[argshash] = 0 - raise - - else: - # machine code was already compiled for these greenargs - # (or we have a hash collision) - cell = self.mcentrypoints[argshash] - if not cell.equalkey(*greenargs): - # hash collision - loop_token = self.handle_hash_collision(cell, argshash, - *args) - if loop_token is None: - return - else: - # get the assembler and fill in the boxes - cell.set_future_values(*args[num_green_args:]) - loop_token = cell.entry_loop_token - # ---------- execute assembler ---------- - while True: # until interrupted by an exception - metainterp_sd.profiler.start_running() - fail_index = metainterp_sd.cpu.execute_token(loop_token) - metainterp_sd.profiler.end_running() - fail_descr = globaldata.get_fail_descr_from_number(fail_index) - loop_token = fail_descr.handle_fail(metainterp_sd) - - maybe_compile_and_run._dont_inline_ = True - - def handle_hash_collision(self, firstcell, argshash, *args): - greenargs = args[:num_green_args] - # search the linked list for the correct cell - cell = firstcell - while cell.next is not None: - nextcell = cell.next - if nextcell.equalkey(*greenargs): - # found, move to the front of the linked list - cell.next = nextcell.next - nextcell.next = firstcell - self.mcentrypoints[argshash] = nextcell - nextcell.set_future_values(*args[num_green_args:]) - return nextcell.entry_loop_token - cell = nextcell - # not found at all, do profiling - counter = self.mccounters[argshash] - assert counter < 0 # by invariant - n = counter + self.increment_threshold - if n < 0: # bound not reached - self.mccounters[argshash] = n - return None - metainterp = MetaInterp(metainterp_sd) - # XXX ContinueRunningNormally => "reset" counters - return metainterp.compile_and_run_once(*args) - handle_hash_collision._dont_inline_ = True - - def unwrap_greenkey(self, greenkey): - greenargs = () - i = 0 - for TYPE in green_args_spec: - value = unwrap(TYPE, greenkey[i]) - greenargs += (value,) - i = i + 1 - return greenargs - unwrap_greenkey._always_inline_ = True - - def comparekey(greenargs1, greenargs2): - i = 0 - for TYPE in green_args_spec: - if not equal_whatever(TYPE, greenargs1[i], greenargs2[i]): - return False - i = i + 1 - return True - comparekey = staticmethod(comparekey) - - def hashkey(greenargs): - return intmask(WarmEnterState.getkeyhash(*greenargs)) - hashkey = staticmethod(hashkey) - - def getkeyhash(*greenargs): - result = r_uint(0x345678) - i = 0 - mult = r_uint(1000003) - for TYPE in green_args_spec: - if i > 0: - result = result * mult - mult = mult + 82520 + 2*len(greenargs) - item = greenargs[i] - result = result ^ hash_whatever(TYPE, item) - i = i + 1 - return result # returns a r_uint - getkeyhash._always_inline_ = True - getkeyhash = staticmethod(getkeyhash) - - def must_compile_from_failure(self, key): - key.counter += 1 - return key.counter >= self.trace_eagerness - - def reset_counter_from_failure(self, key): - key.counter = 0 - - def attach_unoptimized_bridge_from_interp(self, greenkey, - entry_loop_token): - greenargs = self.unwrap_greenkey(greenkey) - newcell = MachineCodeEntryPoint(entry_loop_token, *greenargs) - argshash = self.getkeyhash(*greenargs) & self.hashtablemask - oldcell = self.mcentrypoints[argshash] - newcell.next = oldcell # link - self.mcentrypoints[argshash] = newcell - self.mccounters[argshash] = -THRESHOLD_LIMIT-1 - - if can_inline_ptr is None: - def can_inline_callable(self, greenkey): - return True - else: - def can_inline_callable(self, greenkey): - args = self.unwrap_greenkey(greenkey) - fn = support.maybe_on_top_of_llinterp(rtyper, can_inline_ptr) - return fn(*args) - - if get_printable_location_ptr is None: - def get_location_str(self, greenkey): - return '(no jitdriver.get_printable_location!)' - else: - def get_location_str(self, greenkey): - args = self.unwrap_greenkey(greenkey) - fn = support.maybe_on_top_of_llinterp(rtyper, - get_printable_location_ptr) - res = fn(*args) - if not we_are_translated() and not isinstance(res, str): - res = hlstr(res) - return res - - return WarmEnterState Modified: pypy/trunk/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/trunk/pypy/module/pypyjit/interp_jit.py Wed Oct 21 18:35:39 2009 @@ -50,9 +50,16 @@ def leave(next_instr, pycode, frame, ec): from pypy.interpreter.executioncontext import ExecutionContext # can't use a method here, since this function is seen later than the main - # annotation + # annotation XXX no longer true, could be fixed ExecutionContext._jit_rechain_frame(ec, frame) +def get_jitcell_at(next_instr, bytecode): + return bytecode.jit_cells.get(next_instr, None) + +def set_jitcell_at(newcell, next_instr, bytecode): + bytecode.jit_cells[next_instr] = newcell + + class PyPyJitDriver(JitDriver): reds = ['frame', 'ec'] greens = ['next_instr', 'pycode'] @@ -68,7 +75,9 @@ pypyjitdriver = PyPyJitDriver(can_inline = can_inline, get_printable_location = get_printable_location, - leave = leave) + leave = leave, + get_jitcell_at = get_jitcell_at, + set_jitcell_at = set_jitcell_at) class __extend__(PyFrame): @@ -94,6 +103,20 @@ pycode=f.getcode()) return jumpto + +PyCode__initialize = PyCode._initialize + +class __extend__(PyCode): + __metaclass__ = extendabletype + + def _initialize(self): + PyCode__initialize(self) + self.jit_cells = {} + + def _freeze_(self): + self.jit_cells = {} + return False + # ____________________________________________________________ # # Public interface Modified: pypy/trunk/pypy/module/pypyjit/test/test_jit_setup.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_jit_setup.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_jit_setup.py Wed Oct 21 18:35:39 2009 @@ -9,8 +9,8 @@ # this just checks that the module is setting up things correctly, and # the resulting code makes sense on top of CPython. import pypyjit - pypyjit.set_param(threshold=5, hash_bits=9) - pypyjit.set_param("trace_eagerness=3,hash_bits=7") + pypyjit.set_param(threshold=5, inlining=1) + pypyjit.set_param("trace_eagerness=3,inlining=0") def f(x, y): return x*y+1 Modified: pypy/trunk/pypy/rlib/jit.py ============================================================================== --- pypy/trunk/pypy/rlib/jit.py (original) +++ pypy/trunk/pypy/rlib/jit.py Wed Oct 21 18:35:39 2009 @@ -94,7 +94,6 @@ PARAMETERS = {'threshold': 1000, 'trace_eagerness': 200, - 'hash_bits': 14, 'trace_limit': 10000, 'inlining': False, 'optimizer': OPTIMIZER_FULL, @@ -114,6 +113,7 @@ virtualizables = [] def __init__(self, greens=None, reds=None, virtualizables=None, + get_jitcell_at=None, set_jitcell_at=None, can_inline=None, get_printable_location=None, leave=None): if greens is not None: @@ -128,6 +128,8 @@ assert v in self.reds self._alllivevars = dict.fromkeys(self.greens + self.reds) self._make_extregistryentries() + self.get_jitcell_at = get_jitcell_at + self.set_jitcell_at = set_jitcell_at self.get_printable_location = get_printable_location self.can_inline = can_inline self.leave = leave @@ -194,6 +196,10 @@ # # Annotation and rtyping of some of the JitDriver methods +class BaseJitCell(object): + __slots__ = () + + class ExtEnterLeaveMarker(ExtRegistryEntry): # Replace a call to myjitdriver.jit_merge_point(**livevars) # with an operation jit_marker('jit_merge_point', myjitdriver, livevars...) @@ -218,17 +224,21 @@ def annotate_hooks(self, **kwds_s): driver = self.instance.im_self + s_jitcell = self.bookkeeper.valueoftype(BaseJitCell) + self.annotate_hook(driver.get_jitcell_at, driver.greens, **kwds_s) + self.annotate_hook(driver.set_jitcell_at, driver.greens, [s_jitcell], + **kwds_s) self.annotate_hook(driver.can_inline, driver.greens, **kwds_s) self.annotate_hook(driver.get_printable_location, driver.greens, **kwds_s) self.annotate_hook(driver.leave, driver.greens + driver.reds, **kwds_s) - def annotate_hook(self, func, variables, **kwds_s): + def annotate_hook(self, func, variables, args_s=[], **kwds_s): if func is None: return bk = self.bookkeeper s_func = bk.immutablevalue(func) uniquekey = 'jitdriver.%s' % func.func_name - args_s = [] + args_s = args_s[:] for name in variables: s_arg = kwds_s['s_' + name] args_s.append(s_arg) From arigo at codespeak.net Wed Oct 21 18:35:58 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Oct 2009 18:35:58 +0200 (CEST) Subject: [pypy-svn] r68696 - pypy/branch/warmspot-jitinfo Message-ID: <20091021163558.9C3B3168010@codespeak.net> Author: arigo Date: Wed Oct 21 18:35:58 2009 New Revision: 68696 Removed: pypy/branch/warmspot-jitinfo/ Log: Branch merged. From arigo at codespeak.net Wed Oct 21 18:50:52 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Oct 2009 18:50:52 +0200 (CEST) Subject: [pypy-svn] r68697 - pypy/branch/gc-arena/pypy/rpython/memory/gc Message-ID: <20091021165052.12B40168009@codespeak.net> Author: arigo Date: Wed Oct 21 18:50:51 2009 New Revision: 68697 Modified: pypy/branch/gc-arena/pypy/rpython/memory/gc/markcompact.py Log: Update the comments. Modified: pypy/branch/gc-arena/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/gc-arena/pypy/rpython/memory/gc/markcompact.py Wed Oct 21 18:50:51 2009 @@ -34,24 +34,24 @@ # differencies. The main difference is that we have separate phases of # marking and assigning pointers, hence order of objects is preserved. # This means we can reuse the same space if it did not grow enough. -# More importantly, in case we need to resize space we can copy it bit by -# bit, hence avoiding double memory consumption at peak times -# so the algorithm itself is performed in 3 stages (module weakrefs and +# so the algorithm itself is performed in 3 stages (modulo weakrefs and # finalizers) # 1. We mark alive objects # 2. We walk all objects and assign forward pointers in the same order, # also updating all references -# 3. We compact the space by moving. In case we move to the same space, -# we use arena_new_view trick, which looks like new space to tests, -# but compiles to the same pointer. Also we use raw_memmove in case -# objects overlap. - -# Exact algorithm for space resizing: we keep allocated more space than needed -# (2x, can be even more), but it's full of zeroes. After each collection, -# we bump next_collect_after which is a marker where to start each collection. -# It should be exponential (but less than 2) from the size occupied by objects +# 3. We compact the space by moving. In order to move to the same space, +# we use the 'arena_new_view' trick, which looks like a new space to +# tests, but compiles to the same pointer. Also we use raw_memmove in +# case the object overlaps. + +# Exact algorithm for space resizing: at process start, we "reserve" a +# large range of pages, mostly unused (on Unix it is full of zeroes; on +# Windows it is obtained with VirtualAlloc MEM_RESERVE, not MEM_COMMIT). +# A marker, 'next_collect_after', tells us when to start each +# collection. It should be initialized at a multiple (e.g. 2x) of the +# size occupied by live objects after a collection. # field optimization - we don't need forward pointer and flags at the same # time. Instead we copy list of tids when we know how many objects are alive From arigo at codespeak.net Wed Oct 21 18:52:58 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Oct 2009 18:52:58 +0200 (CEST) Subject: [pypy-svn] r68698 - in pypy/branch/gc-arena/pypy: interpreter interpreter/astcompiler jit/backend/cli jit/backend/cli/test jit/backend/llgraph jit/backend/x86/test jit/metainterp jit/metainterp/test module/pypyjit module/pypyjit/test module/pypyjit/test/loops rlib rpython rpython/lltypesystem rpython/lltypesystem/test rpython/memory rpython/memory/test rpython/ootypesystem rpython/test translator/backendopt translator/backendopt/test translator/c/gcc/test translator/c/test translator/cli Message-ID: <20091021165258.EB1EE168009@codespeak.net> Author: arigo Date: Wed Oct 21 18:52:56 2009 New Revision: 68698 Added: pypy/branch/gc-arena/pypy/jit/metainterp/test/test_warmstate.py - copied unchanged from r68697, pypy/trunk/pypy/jit/metainterp/test/test_warmstate.py pypy/branch/gc-arena/pypy/jit/metainterp/warmstate.py - copied unchanged from r68697, pypy/trunk/pypy/jit/metainterp/warmstate.py Removed: pypy/branch/gc-arena/pypy/module/pypyjit/test/loops/ Modified: pypy/branch/gc-arena/pypy/interpreter/astcompiler/consts.py pypy/branch/gc-arena/pypy/interpreter/pycode.py pypy/branch/gc-arena/pypy/jit/backend/cli/runner.py pypy/branch/gc-arena/pypy/jit/backend/cli/test/test_zrpy_vlist.py pypy/branch/gc-arena/pypy/jit/backend/llgraph/llimpl.py pypy/branch/gc-arena/pypy/jit/backend/llgraph/runner.py pypy/branch/gc-arena/pypy/jit/backend/x86/test/test_zrpy_gc.py pypy/branch/gc-arena/pypy/jit/metainterp/compile.py pypy/branch/gc-arena/pypy/jit/metainterp/history.py pypy/branch/gc-arena/pypy/jit/metainterp/optimizeopt.py pypy/branch/gc-arena/pypy/jit/metainterp/pyjitpl.py pypy/branch/gc-arena/pypy/jit/metainterp/resume.py pypy/branch/gc-arena/pypy/jit/metainterp/simple_optimize.py pypy/branch/gc-arena/pypy/jit/metainterp/support.py pypy/branch/gc-arena/pypy/jit/metainterp/test/test_compile.py pypy/branch/gc-arena/pypy/jit/metainterp/test/test_loop.py pypy/branch/gc-arena/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/gc-arena/pypy/jit/metainterp/test/test_recursive.py pypy/branch/gc-arena/pypy/jit/metainterp/test/test_warmspot.py pypy/branch/gc-arena/pypy/jit/metainterp/test/test_ztranslation.py pypy/branch/gc-arena/pypy/jit/metainterp/virtualizable.py pypy/branch/gc-arena/pypy/jit/metainterp/warmspot.py pypy/branch/gc-arena/pypy/module/pypyjit/interp_jit.py pypy/branch/gc-arena/pypy/module/pypyjit/test/conftest.py pypy/branch/gc-arena/pypy/module/pypyjit/test/test_jit_setup.py pypy/branch/gc-arena/pypy/module/pypyjit/test/test_pypy_c.py pypy/branch/gc-arena/pypy/rlib/jit.py pypy/branch/gc-arena/pypy/rlib/rmmap.py pypy/branch/gc-arena/pypy/rpython/lltypesystem/lloperation.py pypy/branch/gc-arena/pypy/rpython/lltypesystem/opimpl.py pypy/branch/gc-arena/pypy/rpython/lltypesystem/test/test_lloperation.py pypy/branch/gc-arena/pypy/rpython/memory/gctypelayout.py pypy/branch/gc-arena/pypy/rpython/memory/test/test_gctypelayout.py pypy/branch/gc-arena/pypy/rpython/ootypesystem/ootype.py pypy/branch/gc-arena/pypy/rpython/rint.py pypy/branch/gc-arena/pypy/rpython/test/test_rdict.py pypy/branch/gc-arena/pypy/translator/backendopt/mallocv.py pypy/branch/gc-arena/pypy/translator/backendopt/test/test_constfold.py pypy/branch/gc-arena/pypy/translator/c/gcc/test/test_asmgcroot.py pypy/branch/gc-arena/pypy/translator/c/test/test_newgc.py pypy/branch/gc-arena/pypy/translator/cli/ilgenerator.py pypy/branch/gc-arena/pypy/translator/cli/sdk.py Log: svn merge -r68634:68697 svn+ssh://codespeak.net/svn/pypy/trunk . Modified: pypy/branch/gc-arena/pypy/interpreter/astcompiler/consts.py ============================================================================== --- pypy/branch/gc-arena/pypy/interpreter/astcompiler/consts.py (original) +++ pypy/branch/gc-arena/pypy/interpreter/astcompiler/consts.py Wed Oct 21 18:52:56 2009 @@ -1,15 +1,6 @@ -# operation flags -OP_ASSIGN = 0 # 'OP_ASSIGN' -OP_DELETE = 1 # 'OP_DELETE' -OP_APPLY = 2 # 'OP_APPLY' -OP_NONE = 3 - -SC_LOCAL = 1 -SC_GLOBAL = 2 -SC_FREE = 3 -SC_CELL = 4 -SC_UNKNOWN = 5 -SC_DEFAULT = 6 +""" +Various flags used during the compilation process. +""" CO_OPTIMIZED = 0x0001 CO_NEWLOCALS = 0x0002 Modified: pypy/branch/gc-arena/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/gc-arena/pypy/interpreter/pycode.py (original) +++ pypy/branch/gc-arena/pypy/interpreter/pycode.py Wed Oct 21 18:52:56 2009 @@ -84,6 +84,9 @@ self.hidden_applevel = hidden_applevel self.magic = magic self._signature = cpython_code_signature(self) + self._initialize() + + def _initialize(self): # Precompute what arguments need to be copied into cellvars self._args_as_cellvars = [] @@ -116,7 +119,7 @@ self._compute_flatcall() - if space.config.objspace.std.withcelldict: + if self.space.config.objspace.std.withcelldict: from pypy.objspace.std.celldict import init_code init_code(self) Modified: pypy/branch/gc-arena/pypy/jit/backend/cli/runner.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/backend/cli/runner.py (original) +++ pypy/branch/gc-arena/pypy/jit/backend/cli/runner.py Wed Oct 21 18:52:56 2009 @@ -319,7 +319,7 @@ def __init__(self, TYPE): DescrWithKey.__init__(self, TYPE) from pypy.jit.backend.llgraph.runner import boxresult - from pypy.jit.metainterp.warmspot import unwrap + from pypy.jit.metainterp.warmstate import unwrap ARRAY = ootype.Array(TYPE) def create(): if isinstance(TYPE, ootype.OOType): @@ -482,7 +482,7 @@ def __init__(self, TYPE, fieldname): DescrWithKey.__init__(self, (TYPE, fieldname)) from pypy.jit.backend.llgraph.runner import boxresult - from pypy.jit.metainterp.warmspot import unwrap + from pypy.jit.metainterp.warmstate import unwrap _, T = TYPE._lookup_field(fieldname) def getfield(objbox): obj = objbox.getref(TYPE) Modified: pypy/branch/gc-arena/pypy/jit/backend/cli/test/test_zrpy_vlist.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/backend/cli/test/test_zrpy_vlist.py (original) +++ pypy/branch/gc-arena/pypy/jit/backend/cli/test/test_zrpy_vlist.py Wed Oct 21 18:52:56 2009 @@ -7,4 +7,6 @@ # for the individual tests see # ====> ../../../metainterp/test/test_vlist.py - pass + # disable the xfail() + def test_vlist_alloc_and_set(self): + test_vlist.TestOOtype.test_vlist_alloc_and_set(self) Modified: pypy/branch/gc-arena/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/gc-arena/pypy/jit/backend/llgraph/llimpl.py Wed Oct 21 18:52:56 2009 @@ -807,7 +807,6 @@ op_getarrayitem_gc_pure = op_getarrayitem_gc def op_setarrayitem_gc(self, typedescr, obj, index, objnewvalue): - from pypy.jit.metainterp.warmspot import unwrap array = ootype.cast_from_object(typedescr.ARRAY, obj) if ootype.typeOf(objnewvalue) == ootype.Object: newvalue = ootype.cast_from_object(typedescr.TYPE, objnewvalue) Modified: pypy/branch/gc-arena/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/gc-arena/pypy/jit/backend/llgraph/runner.py Wed Oct 21 18:52:56 2009 @@ -9,7 +9,7 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.jit.metainterp import history from pypy.jit.metainterp.history import REF, INT, FLOAT -from pypy.jit.metainterp.warmspot import unwrap +from pypy.jit.metainterp.warmstate import unwrap from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.backend import model from pypy.jit.backend.llgraph import llimpl, symbolic Modified: pypy/branch/gc-arena/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/gc-arena/pypy/jit/backend/x86/test/test_zrpy_gc.py Wed Oct 21 18:52:56 2009 @@ -33,7 +33,7 @@ def get_g(main): main._dont_inline_ = True - def g(num, n): + def g(name, n): x = X() x.foo = 2 main(n, x) @@ -46,12 +46,16 @@ def get_entry(g): def entrypoint(args): - num = 0 - if len(args) == 2: - num = int(args[1]) + name = '' + n = 2000 + argc = len(args) + if argc > 1: + name = args[1] + if argc > 2: + n = int(args[2]) r_list = [] for i in range(20): - r = g(num, 2000) + r = g(name, n) r_list.append(r) rgc.collect() rgc.collect(); rgc.collect() @@ -134,13 +138,14 @@ assert name not in name_to_func name_to_func[name] = len(name_to_func) print name_to_func - def allfuncs(num, n): + def allfuncs(name, n): x = X() x.foo = 2 - main_allfuncs(num, n, x) + main_allfuncs(name, n, x) x.foo = 5 return weakref.ref(x) - def main_allfuncs(num, n, x): + def main_allfuncs(name, n, x): + num = name_to_func[name] n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s = funcs[num][0](n, x) while n > 0: myjitdriver.can_enter_jit(num=num, n=n, x=x, x0=x0, x1=x1, @@ -158,14 +163,12 @@ cls.name_to_func = name_to_func cls.cbuilder = compile(get_entry(allfuncs), "hybrid", gcrootfinder="asmgcc", jit=True) - def run(self, name): - num = self.name_to_func[name] - res = self.cbuilder.cmdexec(str(num)) + def run(self, name, n=2000): + res = self.cbuilder.cmdexec("%s %d" %(name, n)) assert int(res) == 20 def run_orig(self, name, n, x): - num = self.name_to_func[name] - self.main_allfuncs(num, n, x) + self.main_allfuncs(name, n, x) def define_compile_hybrid_1(cls): # a moving GC. Supports malloc_varsize_nonmovable. Simple test, works Modified: pypy/branch/gc-arena/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/gc-arena/pypy/jit/metainterp/compile.py Wed Oct 21 18:52:56 2009 @@ -13,6 +13,9 @@ from pypy.jit.metainterp.typesystem import llhelper, oohelper from pypy.jit.metainterp.optimizeutil import InvalidLoop +class GiveUp(Exception): + pass + def show_loop(metainterp_sd, loop=None, error=None): # debugging if option.view or option.viewloops: @@ -259,8 +262,8 @@ new_loop_token) # store the new loop in compiled_merge_points too glob = metainterp_sd.globaldata - greenargs = glob.unpack_greenkey(self.original_greenkey) - old_loop_tokens = glob.compiled_merge_points.setdefault(greenargs, []) + old_loop_tokens = glob.get_compiled_merge_points( + self.original_greenkey) # it always goes at the end of the list, as it is the most # general loop token old_loop_tokens.append(new_loop_token) @@ -284,7 +287,6 @@ old_loop_tokens, new_loop) except InvalidLoop: - assert 0, "InvalidLoop in optimize_bridge?" return None # Did it work? if target_loop_token is not None: Modified: pypy/branch/gc-arena/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/metainterp/history.py (original) +++ pypy/branch/gc-arena/pypy/jit/metainterp/history.py Wed Oct 21 18:52:56 2009 @@ -21,6 +21,8 @@ REF = 'r' FLOAT = 'f' +FAILARGS_LIMIT = 1000 + def getkind(TYPE, supports_floats=True): if TYPE is lltype.Void: return "void" @@ -117,6 +119,9 @@ def repr_rpython(self): return '%s' % self + def _get_str(self): + raise NotImplementedError + class AbstractDescr(AbstractValue): __slots__ = () @@ -930,8 +935,9 @@ # ---------------------------------------------------------------- class Options: - def __init__(self, listops=False): + def __init__(self, listops=False, failargs_limit=FAILARGS_LIMIT): self.listops = listops + self.failargs_limit = failargs_limit def _freeze_(self): return True Modified: pypy/branch/gc-arena/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/gc-arena/pypy/jit/metainterp/optimizeopt.py Wed Oct 21 18:52:56 2009 @@ -518,8 +518,11 @@ def store_final_boxes_in_guard(self, op): descr = op.descr assert isinstance(descr, compile.ResumeGuardDescr) - modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo) + modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo, + self.metainterp_sd.globaldata.storedebug) newboxes = modifier.finish(self.values) + if len(newboxes) > self.metainterp_sd.options.failargs_limit: + raise compile.GiveUp descr.store_final_boxes(op, newboxes) def optimize_default(self, op): Modified: pypy/branch/gc-arena/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/gc-arena/pypy/jit/metainterp/pyjitpl.py Wed Oct 21 18:52:56 2009 @@ -1,7 +1,7 @@ -import py +import py, os from pypy.rpython.lltypesystem import llmemory from pypy.rpython.ootypesystem import ootype -from pypy.rlib.objectmodel import we_are_translated, r_dict +from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.debug import debug_print @@ -16,6 +16,7 @@ from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import specialize from pypy.rlib.jit import DEBUG_OFF, DEBUG_PROFILE, DEBUG_STEPS, DEBUG_DETAILED +from pypy.jit.metainterp.compile import GiveUp # ____________________________________________________________ @@ -1028,6 +1029,7 @@ self.globaldata.initialized = True self.logger_noopt.create_log('.noopt') self.logger_ops.create_log('.ops') + self.globaldata.storedebug = os.environ.get('PYPYJITRESUMELOG') def _setup_class_sizes(self): class_sizes = {} @@ -1097,15 +1099,25 @@ # state = staticdata.state if state is not None: - self.unpack_greenkey = state.unwrap_greenkey - self.compiled_merge_points = r_dict(state.comparekey,state.hashkey) - # { (greenargs): [MergePoints] } + self.jit_cell_at_key = state.jit_cell_at_key else: - self.compiled_merge_points = {} # for tests only; not RPython - self.unpack_greenkey = tuple + # for tests only; not RPython + class JitCell: + compiled_merge_points = None + _jitcell_dict = {} + def jit_cell_at_key(greenkey): + greenkey = tuple(greenkey) + return _jitcell_dict.setdefault(greenkey, JitCell()) + self.jit_cell_at_key = jit_cell_at_key if staticdata.virtualizable_info: self.blackhole_virtualizable = staticdata.virtualizable_info.null_vable + def get_compiled_merge_points(self, greenkey): + cell = self.jit_cell_at_key(greenkey) + if cell.compiled_merge_points is None: + cell.compiled_merge_points = [] + return cell.compiled_merge_points + def get_fail_descr_number(self, descr): assert isinstance(descr, history.AbstractFailDescr) lst = self.fail_descr_list @@ -1154,7 +1166,11 @@ return True else: if not self.is_blackholing(): - self.compile_done_with_this_frame(resultbox) + try: + self.compile_done_with_this_frame(resultbox) + except GiveUp: + self.staticdata.profiler.count(ABORT_BRIDGE) + self.switch_to_blackhole() sd = self.staticdata if sd.result_type == 'void': assert resultbox is None @@ -1188,7 +1204,11 @@ return True self.popframe() if not self.is_blackholing(): - self.compile_exit_frame_with_exception(excvaluebox) + try: + self.compile_exit_frame_with_exception(excvaluebox) + except GiveUp: + self.staticdata.profiler.count(ABORT_BRIDGE) + self.switch_to_blackhole() raise self.staticdata.ExitFrameWithExceptionRef(self.cpu, excvaluebox.getref_base()) def check_recursion_invariant(self): @@ -1486,8 +1506,7 @@ self.history.inputargs = original_boxes[num_green_args:] greenkey = original_boxes[:num_green_args] glob = self.staticdata.globaldata - greenargs = glob.unpack_greenkey(greenkey) - old_loop_tokens = glob.compiled_merge_points.setdefault(greenargs, []) + old_loop_tokens = glob.get_compiled_merge_points(greenkey) self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None) loop_token = compile.compile_new_loop(self, old_loop_tokens, greenkey, start) @@ -1498,10 +1517,8 @@ num_green_args = self.staticdata.num_green_args greenkey = live_arg_boxes[:num_green_args] glob = self.staticdata.globaldata - greenargs = glob.unpack_greenkey(greenkey) - try: - old_loop_tokens = glob.compiled_merge_points[greenargs] - except KeyError: + old_loop_tokens = glob.get_compiled_merge_points(greenkey) + if len(old_loop_tokens) == 0: return self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None) target_loop_token = compile.compile_new_bridge(self, old_loop_tokens, @@ -1555,7 +1572,7 @@ def _initialize_from_start(self, original_boxes, num_green_args, *args): if args: - from pypy.jit.metainterp.warmspot import wrap + from pypy.jit.metainterp.warmstate import wrap box = wrap(self.cpu, args[0], num_green_args > 0) original_boxes.append(box) self._initialize_from_start(original_boxes, num_green_args-1, @@ -1801,6 +1818,3 @@ assert target_loop_token is not None self.argboxes = args self.target_loop_token = target_loop_token - -class GiveUp(Exception): - pass Modified: pypy/branch/gc-arena/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/gc-arena/pypy/jit/metainterp/resume.py Wed Oct 21 18:52:56 2009 @@ -1,4 +1,4 @@ -import sys +import sys, os from pypy.jit.metainterp.history import Box, Const, ConstInt, INT, REF from pypy.jit.metainterp.resoperation import rop from pypy.rpython.lltypesystem import rffi @@ -10,8 +10,6 @@ # because it needs to support optimize.py which encodes virtuals with # arbitrary cycles and also to compress the information -debug = False - class Snapshot(object): __slots__ = ('prev', 'boxes') @@ -181,9 +179,10 @@ class ResumeDataVirtualAdder(object): - def __init__(self, storage, memo): + def __init__(self, storage, memo, debug_storage=None): self.storage = storage self.memo = memo + self.debug_storage = debug_storage #self.virtuals = [] #self.vfieldboxes = [] @@ -259,8 +258,8 @@ self._number_virtuals(liveboxes) storage.rd_consts = self.memo.consts - if debug: - dump_storage(storage, liveboxes) + if self.debug_storage: + dump_storage(self.debug_storage, storage, liveboxes) return liveboxes[:] def _number_virtuals(self, liveboxes): @@ -426,16 +425,17 @@ # ____________________________________________________________ -def dump_storage(storage, liveboxes): +def dump_storage(logname, storage, liveboxes): "For profiling only." import os from pypy.rlib import objectmodel - fd = os.open('log.storage', os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0666) + fd = os.open(logname, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0666) os.write(fd, 'Log(%d, [\n' % objectmodel.compute_unique_id(storage)) frameinfo = storage.rd_frame_info_list while True: - os.write(fd, '\t("%s", %d, %d),\n' % ( - frameinfo.jitcode, frameinfo.pc, frameinfo.exception_target)) + os.write(fd, '\t("%s", %d, %d, %xd),\n' % ( + frameinfo.jitcode, frameinfo.pc, frameinfo.exception_target, + objectmodel.compute_unique_id(frameinfo))) frameinfo = frameinfo.prev if frameinfo is None: break Modified: pypy/branch/gc-arena/pypy/jit/metainterp/simple_optimize.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/metainterp/simple_optimize.py (original) +++ pypy/branch/gc-arena/pypy/jit/metainterp/simple_optimize.py Wed Oct 21 18:52:56 2009 @@ -22,7 +22,8 @@ if op.is_guard(): descr = op.descr assert isinstance(descr, compile.ResumeGuardDescr) - modifier = resume.ResumeDataVirtualAdder(descr, memo) + modifier = resume.ResumeDataVirtualAdder(descr, memo, + metainterp_sd.globaldata.storedebug) newboxes = modifier.finish(EMPTY_VALUES) descr.store_final_boxes(op, newboxes) newoperations.append(op) Modified: pypy/branch/gc-arena/pypy/jit/metainterp/support.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/metainterp/support.py (original) +++ pypy/branch/gc-arena/pypy/jit/metainterp/support.py Wed Oct 21 18:52:56 2009 @@ -74,16 +74,15 @@ def maybe_on_top_of_llinterp(rtyper, fnptr): # Run a generated graph on top of the llinterp for testing. # When translated, this just returns the fnptr. - llinterp = LLInterpreter(rtyper) #, exc_data_ptr=exc_data_ptr) funcobj = get_funcobj(fnptr) if hasattr(funcobj, 'graph'): + llinterp = LLInterpreter(rtyper) #, exc_data_ptr=exc_data_ptr) def on_top_of_llinterp(*args): return llinterp.eval_graph(funcobj.graph, list(args)) else: - assert isinstance(fnptr, ootype._meth) - assert hasattr(fnptr, '_callable') + assert hasattr(funcobj, '_callable') def on_top_of_llinterp(*args): - return fnptr._callable(*args) + return funcobj._callable(*args) return on_top_of_llinterp class Entry(ExtRegistryEntry): Modified: pypy/branch/gc-arena/pypy/jit/metainterp/test/test_compile.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/metainterp/test/test_compile.py (original) +++ pypy/branch/gc-arena/pypy/jit/metainterp/test/test_compile.py Wed Oct 21 18:52:56 2009 @@ -41,7 +41,7 @@ optimize_loop = staticmethod(optimize.optimize_loop) debug_level = 0 -class FakeGlobalData(): +class FakeGlobalData: loopnumbering = 0 class FakeMetaInterpStaticData: Modified: pypy/branch/gc-arena/pypy/jit/metainterp/test/test_loop.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/metainterp/test/test_loop.py (original) +++ pypy/branch/gc-arena/pypy/jit/metainterp/test/test_loop.py Wed Oct 21 18:52:56 2009 @@ -594,6 +594,26 @@ res = self.meta_interp(f, [200]) + def test_dump_storage(self): + import os + from pypy.tool.udir import udir + logfile = udir.join('resume.log') + os.environ['PYPYJITRESUMELOG'] = str(logfile) + try: + jitdriver = JitDriver(greens = [], reds = ['i', 'n']) + + def f(n): + i = 0 + while i < n: + jitdriver.can_enter_jit(n=n, i=i) + jitdriver.jit_merge_point(n=n, i=i) + i += 1 + + res = self.meta_interp(f, [10]) + data = logfile.read() # assert did not crash + finally: + del os.environ['PYPYJITRESUMELOG'] + class TestOOtype(LoopTest, OOJitMixin): pass Modified: pypy/branch/gc-arena/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/gc-arena/pypy/jit/metainterp/test/test_optimizeopt.py Wed Oct 21 18:52:56 2009 @@ -23,11 +23,15 @@ self.pc = pc self.exception_target = exc_target +class FakeOptions(object): + failargs_limit = 1000 + class FakeMetaInterpStaticData(object): def __init__(self, cpu): self.cpu = cpu self.profiler = EmptyProfiler() + self.options = FakeOptions() def test_store_final_boxes_in_guard(): from pypy.jit.metainterp.compile import ResumeGuardDescr Modified: pypy/branch/gc-arena/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/gc-arena/pypy/jit/metainterp/test/test_recursive.py Wed Oct 21 18:52:56 2009 @@ -393,6 +393,80 @@ self.check_aborted_count(8) self.check_enter_count_at_most(30) + def test_max_failure_args(self): + FAILARGS_LIMIT = 10 + jitdriver = JitDriver(greens = [], reds = ['o', 'i', 'n']) + + class A(object): + def __init__(self, i0, i1, i2, i3, i4, i5, i6, i7, i8, i9): + self.i0 = i0 + self.i1 = i1 + self.i2 = i2 + self.i3 = i3 + self.i4 = i4 + self.i5 = i5 + self.i6 = i6 + self.i7 = i7 + self.i8 = i8 + self.i9 = i9 + + + def loop(n): + i = 0 + o = A(0, 1, 2, 3, 4, 5, 6, 7, 8, 9) + while i < n: + jitdriver.can_enter_jit(o=o, i=i, n=n) + jitdriver.jit_merge_point(o=o, i=i, n=n) + o = A(i, i + 1, i + 2, i + 3, i + 4, i + 5, + i + 6, i + 7, i + 8, i + 9) + i += 1 + return o + + res = self.meta_interp(loop, [20], failargs_limit=FAILARGS_LIMIT, + listops=True) + self.check_aborted_count(5) + + def test_max_failure_args_exc(self): + FAILARGS_LIMIT = 10 + jitdriver = JitDriver(greens = [], reds = ['o', 'i', 'n']) + + class A(object): + def __init__(self, i0, i1, i2, i3, i4, i5, i6, i7, i8, i9): + self.i0 = i0 + self.i1 = i1 + self.i2 = i2 + self.i3 = i3 + self.i4 = i4 + self.i5 = i5 + self.i6 = i6 + self.i7 = i7 + self.i8 = i8 + self.i9 = i9 + + + def loop(n): + i = 0 + o = A(0, 1, 2, 3, 4, 5, 6, 7, 8, 9) + while i < n: + jitdriver.can_enter_jit(o=o, i=i, n=n) + jitdriver.jit_merge_point(o=o, i=i, n=n) + o = A(i, i + 1, i + 2, i + 3, i + 4, i + 5, + i + 6, i + 7, i + 8, i + 9) + i += 1 + raise ValueError + + def main(n): + try: + loop(n) + return 1 + except ValueError: + return 0 + + res = self.meta_interp(main, [20], failargs_limit=FAILARGS_LIMIT, + listops=True) + assert not res + self.check_aborted_count(5) + def test_set_param_inlining(self): myjitdriver = JitDriver(greens=[], reds=['n', 'recurse']) def loop(n, recurse=False): Modified: pypy/branch/gc-arena/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/branch/gc-arena/pypy/jit/metainterp/test/test_warmspot.py Wed Oct 21 18:52:56 2009 @@ -1,5 +1,5 @@ import py -from pypy.jit.metainterp.warmspot import ll_meta_interp, hash_whatever +from pypy.jit.metainterp.warmspot import ll_meta_interp from pypy.jit.metainterp.warmspot import get_stats from pypy.rlib.jit import JitDriver, OPTIMIZER_FULL, OPTIMIZER_SIMPLE from pypy.rlib.jit import unroll_safe @@ -8,15 +8,6 @@ from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin -def test_translate_hash_whatever(): - from pypy.rpython.test.test_llinterp import interpret - from pypy.rpython.lltypesystem import lltype - def fn(x): - return hash_whatever(lltype.typeOf(x), x) - for type_system in ('lltype', 'ootype'): - res = interpret(fn, [42], type_system=type_system) - assert res == 42 - class Exit(Exception): def __init__(self, result): self.result = result @@ -73,20 +64,6 @@ res = self.meta_interp(f, [60]) assert res == f(30) - def test_hash_collision(self): - mydriver = JitDriver(reds = ['n'], greens = ['m']) - def f(n): - m = 0 - while n > 0: - mydriver.can_enter_jit(n=n, m=m) - mydriver.jit_merge_point(n=n, m=m) - n -= 1 - if not (n % 11): - m = (m+n) & 3 - return m - res = self.meta_interp(f, [110], hash_bits=1) - assert res == f(110) - def test_location(self): def get_printable_location(n): return 'GREEN IS %d.' % n Modified: pypy/branch/gc-arena/pypy/jit/metainterp/test/test_ztranslation.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/metainterp/test/test_ztranslation.py (original) +++ pypy/branch/gc-arena/pypy/jit/metainterp/test/test_ztranslation.py Wed Oct 21 18:52:56 2009 @@ -4,6 +4,8 @@ from pypy.rlib.jit import JitDriver, OPTIMIZER_FULL, unroll_parameters from pypy.rlib.jit import PARAMETERS, dont_look_inside from pypy.jit.metainterp.jitprof import Profiler +from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.ootypesystem import ootype class TranslationTest: @@ -18,6 +20,7 @@ # - set_param interface # - profiler # - full optimizer + # - jitdriver hooks class Frame(object): _virtualizable2_ = ['i'] @@ -25,8 +28,24 @@ def __init__(self, i): self.i = i + class JitCellCache: + entry = None + jitcellcache = JitCellCache() + def set_jitcell_at(entry): + jitcellcache.entry = entry + def get_jitcell_at(): + return jitcellcache.entry + def get_printable_location(): + return '(hello world)' + def can_inline(): + return False + jitdriver = JitDriver(greens = [], reds = ['frame', 'total'], - virtualizables = ['frame']) + virtualizables = ['frame'], + get_jitcell_at=get_jitcell_at, + set_jitcell_at=set_jitcell_at, + get_printable_location=get_printable_location, + can_inline=can_inline) def f(i): for param in unroll_parameters: defl = PARAMETERS[param] Modified: pypy/branch/gc-arena/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/branch/gc-arena/pypy/jit/metainterp/virtualizable.py Wed Oct 21 18:52:56 2009 @@ -6,7 +6,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.jit.metainterp.typesystem import deref, fieldType, arrayItem from pypy.jit.metainterp import history -from pypy.jit.metainterp.warmspot import wrap, unwrap +from pypy.jit.metainterp.warmstate import wrap, unwrap class VirtualizableInfo: Modified: pypy/branch/gc-arena/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/gc-arena/pypy/jit/metainterp/warmspot.py Wed Oct 21 18:52:56 2009 @@ -10,7 +10,6 @@ from pypy.objspace.flow.model import checkgraph, Link, copygraph from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib.jit import PARAMETERS, OPTIMIZER_SIMPLE, OPTIMIZER_FULL from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.debug import debug_print from pypy.rpython.lltypesystem.lloperation import llop @@ -24,7 +23,6 @@ from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper from pypy.jit.metainterp.jitprof import Profiler, EmptyProfiler from pypy.rlib.jit import DEBUG_STEPS, DEBUG_DETAILED, DEBUG_OFF, DEBUG_PROFILE -from pypy.rlib.nonconst import NonConstant # ____________________________________________________________ # Bootstrapping @@ -63,7 +61,7 @@ clear_tcache() return jittify_and_run(interp, graph, args, backendopt=backendopt, **kwds) -def jittify_and_run(interp, graph, args, repeat=1, hash_bits=None, +def jittify_and_run(interp, graph, args, repeat=1, backendopt=False, trace_limit=sys.maxint, debug_level=DEBUG_STEPS, inline=False, **kwds): translator = interp.typer.annotator.translator @@ -74,9 +72,6 @@ warmrunnerdesc.state.set_param_trace_limit(trace_limit) warmrunnerdesc.state.set_param_inlining(inline) warmrunnerdesc.state.set_param_debug(debug_level) - warmrunnerdesc.state.create_tables_now() # for tests - if hash_bits: - warmrunnerdesc.state.set_param_hash_bits(hash_bits) warmrunnerdesc.finish() res = interp.eval_graph(graph, args) if not kwds.get('translate_support_code', False): @@ -167,7 +162,7 @@ self.build_meta_interp(CPUClass, **kwds) self.make_args_specification() self.rewrite_jit_merge_point(policy) - self.make_driverhook_graph() + self.make_driverhook_graphs() if self.jitdriver.virtualizables: from pypy.jit.metainterp.virtualizable import VirtualizableInfo self.metainterp_sd.virtualizable_info = VirtualizableInfo(self) @@ -269,8 +264,9 @@ warmrunnerdesc=self) def make_enter_function(self): - WarmEnterState = make_state_class(self) - state = WarmEnterState() + from pypy.jit.metainterp.warmstate import WarmEnterState + state = WarmEnterState(self) + maybe_compile_and_run = state.make_entry_point() self.state = state def crash_in_jit(e): @@ -288,7 +284,7 @@ if self.translator.rtyper.type_system.name == 'lltypesystem': def maybe_enter_jit(*args): try: - state.maybe_compile_and_run(*args) + maybe_compile_and_run(*args) except JitException: raise # go through except Exception, e: @@ -296,7 +292,7 @@ maybe_enter_jit._always_inline_ = True else: def maybe_enter_jit(*args): - state.maybe_compile_and_run(*args) + maybe_compile_and_run(*args) maybe_enter_jit._always_inline_ = True self.maybe_enter_jit_fn = maybe_enter_jit @@ -312,24 +308,38 @@ args_s, s_result) annhelper.finish() - def make_driverhook_graph(self): + def make_driverhook_graphs(self): + from pypy.rlib.jit import BaseJitCell + bk = self.rtyper.annotator.bookkeeper + classdef = bk.getuniqueclassdef(BaseJitCell) + s_BaseJitCell_or_None = annmodel.SomeInstance(classdef, + can_be_None=True) + s_BaseJitCell_not_None = annmodel.SomeInstance(classdef) + s_Str = annmodel.SomeString() + # + annhelper = MixLevelHelperAnnotator(self.translator.rtyper) + self.set_jitcell_at_ptr = self._make_hook_graph( + annhelper, self.jitdriver.set_jitcell_at, annmodel.s_None, + s_BaseJitCell_not_None) + self.get_jitcell_at_ptr = self._make_hook_graph( + annhelper, self.jitdriver.get_jitcell_at, s_BaseJitCell_or_None) self.can_inline_ptr = self._make_hook_graph( - self.jitdriver.can_inline, bool) + annhelper, self.jitdriver.can_inline, annmodel.s_Bool) self.get_printable_location_ptr = self._make_hook_graph( - self.jitdriver.get_printable_location, str) + annhelper, self.jitdriver.get_printable_location, s_Str) + annhelper.finish() - def _make_hook_graph(self, func, rettype): - from pypy.annotation.signature import annotationoftype + def _make_hook_graph(self, annhelper, func, s_result, s_first_arg=None): if func is None: return None - annhelper = MixLevelHelperAnnotator(self.translator.rtyper) - s_result = annotationoftype(rettype) - RETTYPE = annhelper.rtyper.getrepr(s_result).lowleveltype - FUNC, PTR = self.cpu.ts.get_FuncType(self.green_args_spec, RETTYPE) + # + extra_args_s = [] + if s_first_arg is not None: + extra_args_s.append(s_first_arg) + # args_s = self.portal_args_s[:len(self.green_args_spec)] - graph = annhelper.getgraph(func, args_s, s_result) - funcptr = annhelper.graph2delayed(graph, FUNC) - annhelper.finish() + graph = annhelper.getgraph(func, extra_args_s + args_s, s_result) + funcptr = annhelper.graph2delayed(graph) return funcptr def make_args_specification(self): @@ -346,6 +356,7 @@ self.green_args_spec.append(TYPE) else: self.red_args_types.append(history.getkind(TYPE)) + self.num_green_args = len(self.green_args_spec) RESTYPE = graph.getreturnvar().concretetype (self.JIT_ENTER_FUNCTYPE, self.PTR_JIT_ENTER_FUNCTYPE) = self.cpu.ts.get_FuncType(ALLARGS, lltype.Void) @@ -465,6 +476,7 @@ # accepts boxes as argument, but unpacks them immediately # before we raise the exception -- the boxes' values will # be modified in a 'finally' by restore_patched_boxes(). + from pypy.jit.metainterp.warmstate import unwrap for i, name, ARG in portalfunc_ARGS: v = unwrap(ARG, argboxes[i]) setattr(self, name, v) @@ -585,390 +597,3 @@ assert len(reds_v) == numreds return ([v for v in greens_v if v.concretetype is not lltype.Void], [v for v in reds_v if v.concretetype is not lltype.Void]) - -def unwrap(TYPE, box): - if TYPE is lltype.Void: - return None - if isinstance(TYPE, lltype.Ptr): - return box.getref(TYPE) - if isinstance(TYPE, ootype.OOType): - return box.getref(TYPE) - if TYPE == lltype.Float: - return box.getfloat() - else: - return lltype.cast_primitive(TYPE, box.getint()) -unwrap._annspecialcase_ = 'specialize:arg(0)' - -def wrap(cpu, value, in_const_box=False): - if isinstance(lltype.typeOf(value), lltype.Ptr): - if lltype.typeOf(value).TO._gckind == 'gc': - value = lltype.cast_opaque_ptr(llmemory.GCREF, value) - if in_const_box: - return history.ConstPtr(value) - else: - return history.BoxPtr(value) - else: - adr = llmemory.cast_ptr_to_adr(value) - value = cpu.cast_adr_to_int(adr) - # fall through to the end of the function - elif isinstance(lltype.typeOf(value), ootype.OOType): - value = ootype.cast_to_object(value) - if in_const_box: - return history.ConstObj(value) - else: - return history.BoxObj(value) - elif isinstance(value, float): - if in_const_box: - return history.ConstFloat(value) - else: - return history.BoxFloat(value) - else: - value = intmask(value) - if in_const_box: - return history.ConstInt(value) - else: - return history.BoxInt(value) -wrap._annspecialcase_ = 'specialize:ll' - -def equal_whatever(TYPE, x, y): - if isinstance(TYPE, lltype.Ptr): - if TYPE.TO is rstr.STR or TYPE.TO is rstr.UNICODE: - return rstr.LLHelpers.ll_streq(x, y) - return x == y -equal_whatever._annspecialcase_ = 'specialize:arg(0)' - -def hash_whatever(TYPE, x): - # Hash of lltype or ootype object. - # Only supports strings, unicodes and regular instances, - # as well as primitives that can meaningfully be cast to Signed. - if isinstance(TYPE, lltype.Ptr): - if TYPE.TO is rstr.STR or TYPE.TO is rstr.UNICODE: - return rstr.LLHelpers.ll_strhash(x) # assumed not null - else: - if x: - return lltype.identityhash(x) - else: - return 0 - elif TYPE is ootype.String or TYPE is ootype.Unicode: - return x.ll_hash() - elif isinstance(TYPE, ootype.OOType): - if x: - return ootype.identityhash(x) - else: - return 0 - else: - return lltype.cast_primitive(lltype.Signed, x) -hash_whatever._annspecialcase_ = 'specialize:arg(0)' - -# ____________________________________________________________ - -def make_state_class(warmrunnerdesc): - jitdriver = warmrunnerdesc.jitdriver - num_green_args = len(jitdriver.greens) - warmrunnerdesc.num_green_args = num_green_args - green_args_spec = unrolling_iterable(warmrunnerdesc.green_args_spec) - green_args_names = unrolling_iterable(jitdriver.greens) - green_args_spec_names = unrolling_iterable(zip( - warmrunnerdesc.green_args_spec, jitdriver.greens)) - red_args_types = unrolling_iterable(warmrunnerdesc.red_args_types) - # - metainterp_sd = warmrunnerdesc.metainterp_sd - vinfo = metainterp_sd.virtualizable_info - if vinfo is None: - vable_static_fields = [] - vable_array_fields = [] - else: - vable_static_fields = unrolling_iterable( - zip(vinfo.static_extra_types, vinfo.static_fields)) - vable_array_fields = unrolling_iterable( - zip(vinfo.arrayitem_extra_types, vinfo.array_fields)) - # - if num_green_args: - MAX_HASH_TABLE_BITS = 28 - else: - MAX_HASH_TABLE_BITS = 1 - THRESHOLD_LIMIT = sys.maxint // 2 - # - getlength = warmrunnerdesc.cpu.ts.getlength - getarrayitem = warmrunnerdesc.cpu.ts.getarrayitem - setarrayitem = warmrunnerdesc.cpu.ts.setarrayitem - # - rtyper = warmrunnerdesc.translator.rtyper - can_inline_ptr = warmrunnerdesc.can_inline_ptr - get_printable_location_ptr = warmrunnerdesc.get_printable_location_ptr - # - class MachineCodeEntryPoint(object): - next = None # linked list - def __init__(self, entry_loop_token, *greenargs): - self.entry_loop_token = entry_loop_token - i = 0 - for name in green_args_names: - setattr(self, 'green_' + name, greenargs[i]) - i = i + 1 - def equalkey(self, *greenargs): - i = 0 - for TYPE, name in green_args_spec_names: - myvalue = getattr(self, 'green_' + name) - if not equal_whatever(TYPE, myvalue, greenargs[i]): - return False - i = i + 1 - return True - def set_future_values(self, *redargs): - i = 0 - for typecode in red_args_types: - set_future_value(i, redargs[i], typecode) - i = i + 1 - if vinfo is not None: - virtualizable = redargs[vinfo.index_of_virtualizable - - num_green_args] - virtualizable = vinfo.cast_to_vtype(virtualizable) - for typecode, fieldname in vable_static_fields: - x = getattr(virtualizable, fieldname) - set_future_value(i, x, typecode) - i = i + 1 - for typecode, fieldname in vable_array_fields: - lst = getattr(virtualizable, fieldname) - for j in range(getlength(lst)): - x = getarrayitem(lst, j) - set_future_value(i, x, typecode) - i = i + 1 - - def set_future_value(j, value, typecode): - cpu = metainterp_sd.cpu - if typecode == 'ref': - refvalue = cpu.ts.cast_to_ref(value) - cpu.set_future_value_ref(j, refvalue) - elif typecode == 'int': - intvalue = lltype.cast_primitive(lltype.Signed, value) - cpu.set_future_value_int(j, intvalue) - elif typecode == 'float': - assert isinstance(value, float) - cpu.set_future_value_float(j, value) - else: - assert False - set_future_value._annspecialcase_ = 'specialize:ll_and_arg(2)' - - class WarmEnterState: - def __init__(self): - # initialize the state with the default values of the - # parameters specified in rlib/jit.py - for name, default_value in PARAMETERS.items(): - meth = getattr(self, 'set_param_' + name) - meth(default_value) - - def set_param_threshold(self, threshold): - if threshold < 2: - threshold = 2 - self.increment_threshold = (THRESHOLD_LIMIT // threshold) + 1 - # the number is at least 1, and at most about half THRESHOLD_LIMIT - - def set_param_trace_eagerness(self, value): - self.trace_eagerness = value - - def set_param_trace_limit(self, value): - self.trace_limit = value - - def set_param_inlining(self, value): - self.inlining = value - - def set_param_hash_bits(self, value): - if value < 1: - value = 1 - elif value > MAX_HASH_TABLE_BITS: - value = MAX_HASH_TABLE_BITS - # the tables are initialized with the correct size only in - # create_tables_now() - self.hashbits = value - self.hashtablemask = 0 - self.mccounters = [0] - self.mcentrypoints = [None] - # invariant: (self.mccounters[j] < 0) if and only if - # (self.mcentrypoints[j] is not None) - - def set_param_optimizer(self, optimizer): - if optimizer == OPTIMIZER_SIMPLE: - from pypy.jit.metainterp import simple_optimize - self.optimize_loop = simple_optimize.optimize_loop - self.optimize_bridge = simple_optimize.optimize_bridge - elif optimizer == OPTIMIZER_FULL: - from pypy.jit.metainterp import optimize - self.optimize_loop = optimize.optimize_loop - self.optimize_bridge = optimize.optimize_bridge - else: - raise ValueError("unknown optimizer") - - def set_param_debug(self, value): - self.debug_level = value - metainterp_sd.profiler.set_printing(value >= DEBUG_PROFILE) - - def create_tables_now(self): - count = 1 << self.hashbits - self.hashtablemask = count - 1 - self.mccounters = [0] * count - self.mcentrypoints = [None] * count - - # Only use the hash of the arguments as the profiling key. - # Indeed, this is all a heuristic, so if things are designed - # correctly, the occasional mistake due to hash collision is - # not too bad. - - def maybe_compile_and_run(self, *args): - globaldata = metainterp_sd.globaldata - if NonConstant(False): - # make sure we always see the saner optimizer from an annotation - # point of view, otherwise we get lots of blocked ops - self.set_param_optimizer(OPTIMIZER_FULL) - - # get the greenargs and look for the cell corresponding to the hash - greenargs = args[:num_green_args] - argshash = self.getkeyhash(*greenargs) & self.hashtablemask - counter = self.mccounters[argshash] - if vinfo: - virtualizable = args[vinfo.index_of_virtualizable] - virtualizable = vinfo.cast_to_vtype(virtualizable) - assert virtualizable != globaldata.blackhole_virtualizable, "reentering same frame via blackhole" - if counter >= 0: - # update the profiling counter - n = counter + self.increment_threshold - if n <= THRESHOLD_LIMIT: # bound not reached - self.mccounters[argshash] = n - return - if self.hashtablemask == 0: # must really create the tables now - self.create_tables_now() - return - metainterp = MetaInterp(metainterp_sd) - try: - loop_token = metainterp.compile_and_run_once(*args) - except warmrunnerdesc.ContinueRunningNormally: - # the trace got too long, reset the counter - self.mccounters[argshash] = 0 - raise - - else: - # machine code was already compiled for these greenargs - # (or we have a hash collision) - cell = self.mcentrypoints[argshash] - if not cell.equalkey(*greenargs): - # hash collision - loop_token = self.handle_hash_collision(cell, argshash, - *args) - if loop_token is None: - return - else: - # get the assembler and fill in the boxes - cell.set_future_values(*args[num_green_args:]) - loop_token = cell.entry_loop_token - # ---------- execute assembler ---------- - while True: # until interrupted by an exception - metainterp_sd.profiler.start_running() - fail_index = metainterp_sd.cpu.execute_token(loop_token) - metainterp_sd.profiler.end_running() - fail_descr = globaldata.get_fail_descr_from_number(fail_index) - loop_token = fail_descr.handle_fail(metainterp_sd) - - maybe_compile_and_run._dont_inline_ = True - - def handle_hash_collision(self, firstcell, argshash, *args): - greenargs = args[:num_green_args] - # search the linked list for the correct cell - cell = firstcell - while cell.next is not None: - nextcell = cell.next - if nextcell.equalkey(*greenargs): - # found, move to the front of the linked list - cell.next = nextcell.next - nextcell.next = firstcell - self.mcentrypoints[argshash] = nextcell - nextcell.set_future_values(*args[num_green_args:]) - return nextcell.entry_loop_token - cell = nextcell - # not found at all, do profiling - counter = self.mccounters[argshash] - assert counter < 0 # by invariant - n = counter + self.increment_threshold - if n < 0: # bound not reached - self.mccounters[argshash] = n - return None - metainterp = MetaInterp(metainterp_sd) - # XXX ContinueRunningNormally => "reset" counters - return metainterp.compile_and_run_once(*args) - handle_hash_collision._dont_inline_ = True - - def unwrap_greenkey(self, greenkey): - greenargs = () - i = 0 - for TYPE in green_args_spec: - value = unwrap(TYPE, greenkey[i]) - greenargs += (value,) - i = i + 1 - return greenargs - unwrap_greenkey._always_inline_ = True - - def comparekey(greenargs1, greenargs2): - i = 0 - for TYPE in green_args_spec: - if not equal_whatever(TYPE, greenargs1[i], greenargs2[i]): - return False - i = i + 1 - return True - comparekey = staticmethod(comparekey) - - def hashkey(greenargs): - return intmask(WarmEnterState.getkeyhash(*greenargs)) - hashkey = staticmethod(hashkey) - - def getkeyhash(*greenargs): - result = r_uint(0x345678) - i = 0 - mult = r_uint(1000003) - for TYPE in green_args_spec: - if i > 0: - result = result * mult - mult = mult + 82520 + 2*len(greenargs) - item = greenargs[i] - result = result ^ hash_whatever(TYPE, item) - i = i + 1 - return result # returns a r_uint - getkeyhash._always_inline_ = True - getkeyhash = staticmethod(getkeyhash) - - def must_compile_from_failure(self, key): - key.counter += 1 - return key.counter >= self.trace_eagerness - - def reset_counter_from_failure(self, key): - key.counter = 0 - - def attach_unoptimized_bridge_from_interp(self, greenkey, - entry_loop_token): - greenargs = self.unwrap_greenkey(greenkey) - newcell = MachineCodeEntryPoint(entry_loop_token, *greenargs) - argshash = self.getkeyhash(*greenargs) & self.hashtablemask - oldcell = self.mcentrypoints[argshash] - newcell.next = oldcell # link - self.mcentrypoints[argshash] = newcell - self.mccounters[argshash] = -THRESHOLD_LIMIT-1 - - if can_inline_ptr is None: - def can_inline_callable(self, greenkey): - return True - else: - def can_inline_callable(self, greenkey): - args = self.unwrap_greenkey(greenkey) - fn = support.maybe_on_top_of_llinterp(rtyper, can_inline_ptr) - return fn(*args) - - if get_printable_location_ptr is None: - def get_location_str(self, greenkey): - return '(no jitdriver.get_printable_location!)' - else: - def get_location_str(self, greenkey): - args = self.unwrap_greenkey(greenkey) - fn = support.maybe_on_top_of_llinterp(rtyper, - get_printable_location_ptr) - res = fn(*args) - if not we_are_translated() and not isinstance(res, str): - res = hlstr(res) - return res - - return WarmEnterState Modified: pypy/branch/gc-arena/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/gc-arena/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/gc-arena/pypy/module/pypyjit/interp_jit.py Wed Oct 21 18:52:56 2009 @@ -50,9 +50,16 @@ def leave(next_instr, pycode, frame, ec): from pypy.interpreter.executioncontext import ExecutionContext # can't use a method here, since this function is seen later than the main - # annotation + # annotation XXX no longer true, could be fixed ExecutionContext._jit_rechain_frame(ec, frame) +def get_jitcell_at(next_instr, bytecode): + return bytecode.jit_cells.get(next_instr, None) + +def set_jitcell_at(newcell, next_instr, bytecode): + bytecode.jit_cells[next_instr] = newcell + + class PyPyJitDriver(JitDriver): reds = ['frame', 'ec'] greens = ['next_instr', 'pycode'] @@ -68,7 +75,9 @@ pypyjitdriver = PyPyJitDriver(can_inline = can_inline, get_printable_location = get_printable_location, - leave = leave) + leave = leave, + get_jitcell_at = get_jitcell_at, + set_jitcell_at = set_jitcell_at) class __extend__(PyFrame): @@ -94,6 +103,20 @@ pycode=f.getcode()) return jumpto + +PyCode__initialize = PyCode._initialize + +class __extend__(PyCode): + __metaclass__ = extendabletype + + def _initialize(self): + PyCode__initialize(self) + self.jit_cells = {} + + def _freeze_(self): + self.jit_cells = {} + return False + # ____________________________________________________________ # # Public interface Modified: pypy/branch/gc-arena/pypy/module/pypyjit/test/conftest.py ============================================================================== --- pypy/branch/gc-arena/pypy/module/pypyjit/test/conftest.py (original) +++ pypy/branch/gc-arena/pypy/module/pypyjit/test/conftest.py Wed Oct 21 18:52:56 2009 @@ -1,5 +1,5 @@ def pytest_addoption(parser): group = parser.addgroup("pypyjit options") - group.addoption("--pypy-c", action="store", default=None, dest="pypy_c", + group.addoption("--pypy", action="store", default=None, dest="pypy_c", help="the location of the JIT enabled pypy-c") Modified: pypy/branch/gc-arena/pypy/module/pypyjit/test/test_jit_setup.py ============================================================================== --- pypy/branch/gc-arena/pypy/module/pypyjit/test/test_jit_setup.py (original) +++ pypy/branch/gc-arena/pypy/module/pypyjit/test/test_jit_setup.py Wed Oct 21 18:52:56 2009 @@ -9,8 +9,8 @@ # this just checks that the module is setting up things correctly, and # the resulting code makes sense on top of CPython. import pypyjit - pypyjit.set_param(threshold=5, hash_bits=9) - pypyjit.set_param("trace_eagerness=3,hash_bits=7") + pypyjit.set_param(threshold=5, inlining=1) + pypyjit.set_param("trace_eagerness=3,inlining=0") def f(x, y): return x*y+1 Modified: pypy/branch/gc-arena/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/gc-arena/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/gc-arena/pypy/module/pypyjit/test/test_pypy_c.py Wed Oct 21 18:52:56 2009 @@ -235,6 +235,7 @@ i = 0 while i < n: a = A() + assert isinstance(a, A) a.x = 2 i = i + a.x return i @@ -242,10 +243,13 @@ ([20], 20), ([31], 32)) - bytecode, = self.get_by_bytecode("CALL_FUNCTION") - assert not bytecode.get_opnames("call") - assert not bytecode.get_opnames("new") - assert len(bytecode.get_opnames("guard")) <= 9 + callA, callisinstance = self.get_by_bytecode("CALL_FUNCTION") + assert not callA.get_opnames("call") + assert not callA.get_opnames("new") + assert len(callA.get_opnames("guard")) <= 9 + assert not callisinstance.get_opnames("call") + assert not callisinstance.get_opnames("new") + assert len(callisinstance.get_opnames("guard")) <= 2 bytecode, = self.get_by_bytecode("STORE_ATTR") # XXX where does that come from? Modified: pypy/branch/gc-arena/pypy/rlib/jit.py ============================================================================== --- pypy/branch/gc-arena/pypy/rlib/jit.py (original) +++ pypy/branch/gc-arena/pypy/rlib/jit.py Wed Oct 21 18:52:56 2009 @@ -94,7 +94,6 @@ PARAMETERS = {'threshold': 1000, 'trace_eagerness': 200, - 'hash_bits': 14, 'trace_limit': 10000, 'inlining': False, 'optimizer': OPTIMIZER_FULL, @@ -114,6 +113,7 @@ virtualizables = [] def __init__(self, greens=None, reds=None, virtualizables=None, + get_jitcell_at=None, set_jitcell_at=None, can_inline=None, get_printable_location=None, leave=None): if greens is not None: @@ -128,6 +128,8 @@ assert v in self.reds self._alllivevars = dict.fromkeys(self.greens + self.reds) self._make_extregistryentries() + self.get_jitcell_at = get_jitcell_at + self.set_jitcell_at = set_jitcell_at self.get_printable_location = get_printable_location self.can_inline = can_inline self.leave = leave @@ -194,6 +196,10 @@ # # Annotation and rtyping of some of the JitDriver methods +class BaseJitCell(object): + __slots__ = () + + class ExtEnterLeaveMarker(ExtRegistryEntry): # Replace a call to myjitdriver.jit_merge_point(**livevars) # with an operation jit_marker('jit_merge_point', myjitdriver, livevars...) @@ -218,17 +224,21 @@ def annotate_hooks(self, **kwds_s): driver = self.instance.im_self + s_jitcell = self.bookkeeper.valueoftype(BaseJitCell) + self.annotate_hook(driver.get_jitcell_at, driver.greens, **kwds_s) + self.annotate_hook(driver.set_jitcell_at, driver.greens, [s_jitcell], + **kwds_s) self.annotate_hook(driver.can_inline, driver.greens, **kwds_s) self.annotate_hook(driver.get_printable_location, driver.greens, **kwds_s) self.annotate_hook(driver.leave, driver.greens + driver.reds, **kwds_s) - def annotate_hook(self, func, variables, **kwds_s): + def annotate_hook(self, func, variables, args_s=[], **kwds_s): if func is None: return bk = self.bookkeeper s_func = bk.immutablevalue(func) uniquekey = 'jitdriver.%s' % func.func_name - args_s = [] + args_s = args_s[:] for name in variables: s_arg = kwds_s['s_' + name] args_s.append(s_arg) Modified: pypy/branch/gc-arena/pypy/rlib/rmmap.py ============================================================================== --- pypy/branch/gc-arena/pypy/rlib/rmmap.py (original) +++ pypy/branch/gc-arena/pypy/rlib/rmmap.py Wed Oct 21 18:52:56 2009 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem import rffi, lltype, llmemory from pypy.rlib import rposix from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.rlib.nonconst import NonConstant import sys import os @@ -612,7 +613,10 @@ else: m.fd = os.dup(fd) - res = c_mmap(NULL, map_size, prot, flags, fd, 0) + # XXX if we use hintp below in alloc, the NonConstant + # is necessary since we want a general version of c_mmap + # to be annotated with a non-constant pointer. + res = c_mmap(NonConstant(NULL), map_size, prot, flags, fd, 0) if res == rffi.cast(PTR, -1): errno = _get_error_no() raise OSError(errno, os.strerror(errno)) Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/lloperation.py Wed Oct 21 18:52:56 2009 @@ -84,12 +84,17 @@ return op_impl fold = roproperty(get_fold_impl) - def is_pure(self, *ARGTYPES): + def is_pure(self, args_v): return (self.canfold or # canfold => pure operation self is llop.debug_assert or # debug_assert is pure enough # reading from immutable (self in (llop.getfield, llop.getarrayitem) and - ARGTYPES[0].TO._hints.get('immutable'))) + args_v[0].concretetype.TO._hints.get('immutable')) or + (self is llop.getfield and # reading from immutable_field + 'immutable_fields' in args_v[0].concretetype.TO._hints and + args_v[1].value in args_v[0].concretetype.TO + ._hints['immutable_fields'].fields)) + # XXX: what about ootype immutable arrays? def __repr__(self): return '' % (getattr(self, 'opname', '?'),) Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/opimpl.py Wed Oct 21 18:52:56 2009 @@ -142,7 +142,12 @@ # we can constant-fold this if the innermost structure from which we # read the final field is immutable. T = lltype.typeOf(innermostcontainer).TO - if not T._hints.get('immutable'): + if T._hints.get('immutable'): + pass + elif ('immutable_fields' in T._hints and + offsets[-1] in T._hints['immutable_fields'].fields): + pass + else: raise TypeError("cannot fold getinteriorfield on mutable struct") assert not isinstance(ob, lltype._interior_ptr) return ob @@ -390,7 +395,13 @@ def op_getfield(p, name): checkptr(p) - if not lltype.typeOf(p).TO._hints.get('immutable'): + TYPE = lltype.typeOf(p).TO + if TYPE._hints.get('immutable'): + pass + elif ('immutable_fields' in TYPE._hints and + name in TYPE._hints['immutable_fields'].fields): + pass + else: raise TypeError("cannot fold getfield on mutable struct") return getattr(p, name) Modified: pypy/branch/gc-arena/pypy/rpython/lltypesystem/test/test_lloperation.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/lltypesystem/test/test_lloperation.py (original) +++ pypy/branch/gc-arena/pypy/rpython/lltypesystem/test/test_lloperation.py Wed Oct 21 18:52:56 2009 @@ -4,6 +4,7 @@ from pypy.rpython.ootypesystem import ootype, ooopimpl from pypy.rpython.llinterp import LLFrame from pypy.rpython.test.test_llinterp import interpret +from pypy.rpython import rclass LL_INTERP_OPERATIONS = [name[3:] for name in LLFrame.__dict__.keys() if name.startswith('op_')] @@ -50,7 +51,72 @@ return s.x + s.y res = interpret(llf, [], policy=LowLevelAnnotatorPolicy()) assert res == 5 - + +def test_is_pure(): + from pypy.objspace.flow.model import Variable, Constant + assert llop.bool_not.is_pure([Variable()]) + assert llop.debug_assert.is_pure([Variable()]) + assert not llop.int_add_ovf.is_pure([Variable(), Variable()]) + # + S1 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed)) + v_s1 = Variable() + v_s1.concretetype = lltype.Ptr(S1) + assert not llop.setfield.is_pure([v_s1, Constant('x'), Variable()]) + assert not llop.getfield.is_pure([v_s1, Constant('y')]) + # + A1 = lltype.GcArray(lltype.Signed) + v_a1 = Variable() + v_a1.concretetype = lltype.Ptr(A1) + assert not llop.setarrayitem.is_pure([v_a1, Variable(), Variable()]) + assert not llop.getarrayitem.is_pure([v_a1, Variable()]) + assert llop.getarraysize.is_pure([v_a1]) + # + S2 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed), + hints={'immutable': True}) + v_s2 = Variable() + v_s2.concretetype = lltype.Ptr(S2) + assert not llop.setfield.is_pure([v_s2, Constant('x'), Variable()]) + assert llop.getfield.is_pure([v_s2, Constant('y')]) + # + A2 = lltype.GcArray(lltype.Signed, hints={'immutable': True}) + v_a2 = Variable() + v_a2.concretetype = lltype.Ptr(A2) + assert not llop.setarrayitem.is_pure([v_a2, Variable(), Variable()]) + assert llop.getarrayitem.is_pure([v_a2, Variable()]) + assert llop.getarraysize.is_pure([v_a2]) + # + accessor = rclass.FieldListAccessor() + S3 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed), + hints={'immutable_fields': accessor}) + accessor.initialize(S3, ['x']) + v_s3 = Variable() + v_s3.concretetype = lltype.Ptr(S3) + assert not llop.setfield.is_pure([v_s3, Constant('x'), Variable()]) + assert not llop.setfield.is_pure([v_s3, Constant('y'), Variable()]) + assert llop.getfield.is_pure([v_s3, Constant('x')]) + assert not llop.getfield.is_pure([v_s3, Constant('y')]) + +def test_getfield_pure(): + S1 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed)) + S2 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed), + hints={'immutable': True}) + accessor = rclass.FieldListAccessor() + S3 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed), + hints={'immutable_fields': accessor}) + accessor.initialize(S3, ['x']) + # + s1 = lltype.malloc(S1); s1.x = 45 + py.test.raises(TypeError, llop.getfield, lltype.Signed, s1, 'x') + s2 = lltype.malloc(S2); s2.x = 45 + assert llop.getfield(lltype.Signed, s2, 'x') == 45 + s3 = lltype.malloc(S3); s3.x = 46; s3.y = 47 + assert llop.getfield(lltype.Signed, s3, 'x') == 46 + py.test.raises(TypeError, llop.getfield, lltype.Signed, s3, 'y') + # + py.test.raises(TypeError, llop.getinteriorfield, lltype.Signed, s1, 'x') + assert llop.getinteriorfield(lltype.Signed, s2, 'x') == 45 + assert llop.getinteriorfield(lltype.Signed, s3, 'x') == 46 + py.test.raises(TypeError, llop.getinteriorfield, lltype.Signed, s3, 'y') # ___________________________________________________________________________ # This tests that the LLInterpreter and the LL_OPERATIONS tables are in sync. Modified: pypy/branch/gc-arena/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/gc-arena/pypy/rpython/memory/gctypelayout.py Wed Oct 21 18:52:56 2009 @@ -350,11 +350,16 @@ def gc_pointers_inside(v, adr, mutable_only=False): t = lltype.typeOf(v) if isinstance(t, lltype.Struct): - if mutable_only and t._hints.get('immutable'): - return + skip = () + if mutable_only: + if t._hints.get('immutable'): + return + if 'immutable_fields' in t._hints: + skip = t._hints['immutable_fields'].fields for n, t2 in t._flds.iteritems(): if isinstance(t2, lltype.Ptr) and t2.TO._gckind == 'gc': - yield adr + llmemory.offsetof(t, n) + if n not in skip: + yield adr + llmemory.offsetof(t, n) elif isinstance(t2, (lltype.Array, lltype.Struct)): for a in gc_pointers_inside(getattr(v, n), adr + llmemory.offsetof(t, n), Modified: pypy/branch/gc-arena/pypy/rpython/memory/test/test_gctypelayout.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/memory/test/test_gctypelayout.py (original) +++ pypy/branch/gc-arena/pypy/rpython/memory/test/test_gctypelayout.py Wed Oct 21 18:52:56 2009 @@ -1,7 +1,8 @@ import py from pypy.rpython.memory.gctypelayout import TypeLayoutBuilder, GCData from pypy.rpython.memory.gctypelayout import offsets_to_gc_pointers -from pypy.rpython.lltypesystem import lltype, rclass +from pypy.rpython.memory.gctypelayout import gc_pointers_inside +from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.test.test_llinterp import get_interpreter from pypy.objspace.flow.model import Constant @@ -90,3 +91,31 @@ interp, graph = get_interpreter(f, [], backendopt=True) assert interp.eval_graph(graph, []) == 11001 assert graph.startblock.exits[0].args == [Constant(11001, lltype.Signed)] + +def test_gc_pointers_inside(): + from pypy.rpython import rclass + PT = lltype.Ptr(lltype.GcStruct('T')) + S1 = lltype.GcStruct('S', ('x', PT), ('y', PT)) + S2 = lltype.GcStruct('S', ('x', PT), ('y', PT), + hints={'immutable': True}) + accessor = rclass.FieldListAccessor() + S3 = lltype.GcStruct('S', ('x', PT), ('y', PT), + hints={'immutable_fields': accessor}) + accessor.initialize(S3, ['x']) + # + s1 = lltype.malloc(S1) + adr = llmemory.cast_ptr_to_adr(s1) + lst = list(gc_pointers_inside(s1._obj, adr, mutable_only=True)) + expected = [adr + llmemory.offsetof(S1, 'x'), + adr + llmemory.offsetof(S1, 'y')] + assert lst == expected or lst == expected[::-1] + # + s2 = lltype.malloc(S2) + adr = llmemory.cast_ptr_to_adr(s2) + lst = list(gc_pointers_inside(s2._obj, adr, mutable_only=True)) + assert lst == [] + # + s3 = lltype.malloc(S3) + adr = llmemory.cast_ptr_to_adr(s3) + lst = list(gc_pointers_inside(s3._obj, adr, mutable_only=True)) + assert lst == [adr + llmemory.offsetof(S3, 'y')] Modified: pypy/branch/gc-arena/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/branch/gc-arena/pypy/rpython/ootypesystem/ootype.py Wed Oct 21 18:52:56 2009 @@ -1629,7 +1629,10 @@ self._array[index] = item def _identityhash(self): - return hash(self) + if self: + return intmask(id(self)) + else: + return 0 # for all null arrays class _null_array(_null_mixin(_array), _array): @@ -1776,7 +1779,10 @@ self.__dict__[name] = value def _identityhash(self): - return hash(self) + if self: + return intmask(id(self)) + else: + return 0 # for all null records def _items_in_order(self): return [self._items[name] for name in self._TYPE._fields_in_order] Modified: pypy/branch/gc-arena/pypy/rpython/rint.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/rint.py (original) +++ pypy/branch/gc-arena/pypy/rpython/rint.py Wed Oct 21 18:52:56 2009 @@ -408,10 +408,8 @@ fn = hop.rtyper.type_system.ll_str.ll_int2oct return hop.gendirectcall(fn, varg, true) -def ll_identity(n): - return n - -ll_hash_int = ll_identity +def ll_hash_int(n): + return intmask(n) def ll_check_chr(n): if 0 <= n <= 255: Modified: pypy/branch/gc-arena/pypy/rpython/test/test_rdict.py ============================================================================== --- pypy/branch/gc-arena/pypy/rpython/test/test_rdict.py (original) +++ pypy/branch/gc-arena/pypy/rpython/test/test_rdict.py Wed Oct 21 18:52:56 2009 @@ -4,6 +4,7 @@ from pypy.rpython.lltypesystem import rdict, rstr from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin from pypy.rlib.objectmodel import r_dict +from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong import py py.log.setconsumer("rtyper", py.log.STDOUT) @@ -564,6 +565,17 @@ res = self.interpret(fn, [3.0]) assert res == 42 + def test_dict_of_r_uint(self): + for r_t in [r_uint, r_longlong, r_ulonglong]: + d = {r_t(2): 3, r_t(4): 5} + def fn(x, y): + d[r_t(x)] = 123 + return d[r_t(y)] + res = self.interpret(fn, [4, 2]) + assert res == 3 + res = self.interpret(fn, [3, 3]) + assert res == 123 + class TestLLtype(BaseTestRdict, LLRtypeMixin): def test_dict_but_not_with_char_keys(self): Modified: pypy/branch/gc-arena/pypy/translator/backendopt/mallocv.py ============================================================================== --- pypy/branch/gc-arena/pypy/translator/backendopt/mallocv.py (original) +++ pypy/branch/gc-arena/pypy/translator/backendopt/mallocv.py Wed Oct 21 18:52:56 2009 @@ -1046,7 +1046,7 @@ op = getattr(llop, opname) except AttributeError: return - if not op.is_pure(*[v.concretetype for v in args_v]): + if not op.is_pure(args_v): return try: result = op(RESTYPE, *args) Modified: pypy/branch/gc-arena/pypy/translator/backendopt/test/test_constfold.py ============================================================================== --- pypy/branch/gc-arena/pypy/translator/backendopt/test/test_constfold.py (original) +++ pypy/branch/gc-arena/pypy/translator/backendopt/test/test_constfold.py Wed Oct 21 18:52:56 2009 @@ -4,6 +4,7 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rpython import rclass from pypy.rlib import objectmodel from pypy.translator.backendopt.constfold import constant_fold_graph from pypy import conftest @@ -26,8 +27,10 @@ assert res == expected_result -def test_simple(): - S1 = lltype.GcStruct('S1', ('x', lltype.Signed), hints={'immutable': True}) +def test_simple(S1=None): + if S1 is None: + S1 = lltype.GcStruct('S1', ('x', lltype.Signed), + hints={'immutable': True}) s1 = lltype.malloc(S1) s1.x = 123 def g(y): @@ -42,6 +45,14 @@ check_graph(graph, [], 124, t) +def test_immutable_fields(): + accessor = rclass.FieldListAccessor() + S2 = lltype.GcStruct('S2', ('x', lltype.Signed), + hints={'immutable_fields': accessor}) + accessor.initialize(S2, ['x']) + test_simple(S2) + + def test_along_link(): S1 = lltype.GcStruct('S1', ('x', lltype.Signed), hints={'immutable': True}) s1 = lltype.malloc(S1) Modified: pypy/branch/gc-arena/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/branch/gc-arena/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/branch/gc-arena/pypy/translator/c/gcc/test/test_asmgcroot.py Wed Oct 21 18:52:56 2009 @@ -18,19 +18,16 @@ should_be_moving = False @classmethod - def _makefunc2(cls, func): + def _makefunc_str_int(cls, func): def main(argv): - arg0 = int(argv[1]) + arg0 = argv[1] arg1 = int(argv[2]) try: res = func(arg0, arg1) except MemoryError: print 'Result: MemoryError' else: - if isinstance(res, int): - print 'Result:', res - else: - print 'Result: "%s"' % (res,) + print 'Result: "%s"' % (res,) return 0 from pypy.config.pypyoption import get_pypy_config config = get_pypy_config(translating=True) @@ -59,7 +56,7 @@ redirect = ' 2> NUL' else: redirect = '' - g = os.popen('"%s" %d %d%s' % (exe_name, arg0, arg1, redirect), 'r') + g = os.popen('"%s" %s %d%s' % (exe_name, arg0, arg1, redirect), 'r') for line in g: print >> sys.stderr, 'RUN:', line.rstrip() lines.append(line) Modified: pypy/branch/gc-arena/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/gc-arena/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/gc-arena/pypy/translator/c/test/test_newgc.py Wed Oct 21 18:52:56 2009 @@ -24,19 +24,39 @@ _isolated_func = None @classmethod - def _makefunc2(cls, f): - t = Translation(f, [int, int], gc=cls.gcpolicy, + def _makefunc_str_int(cls, f): + def main(argv): + arg0 = argv[1] + arg1 = int(argv[2]) + try: + res = f(arg0, arg1) + except MemoryError: + print "MEMORY-ERROR" + else: + print res + return 0 + + t = Translation(main, standalone=True, gc=cls.gcpolicy, policy=annpolicy.StrictAnnotatorPolicy(), taggedpointers=cls.taggedpointers, removetypeptr=cls.removetypeptr, debugprint=True) t.disable(['backendopt']) - t.set_backend_extra_options(c_isolated=True, c_debug_defines=True) + t.set_backend_extra_options(c_debug_defines=True) t.rtype() if conftest.option.view: t.viewcg() - isolated_func = t.compile() - return isolated_func + exename = t.compile() + + def run(s, i): + data = py.process.cmdexec("%s %s %d" % (exename, s, i)) + data = data.strip() + if data == 'MEMORY-ERROR': + raise MemoryError + return data + + return run + def setup_class(cls): funcs0 = [] @@ -66,8 +86,8 @@ funcs1.append(func) assert name not in name_to_func name_to_func[name] = len(name_to_func) - def allfuncs(num, arg): - rgc.collect() + def allfuncs(name, arg): + num = name_to_func[name] func0 = funcs0[num] if func0: return str(func0()) @@ -79,7 +99,7 @@ return funcstr(arg) assert 0, 'unreachable' cls.funcsstr = funcsstr - cls.c_allfuncs = staticmethod(cls._makefunc2(allfuncs)) + cls.c_allfuncs = staticmethod(cls._makefunc_str_int(allfuncs)) cls.allfuncs = staticmethod(allfuncs) cls.name_to_func = name_to_func @@ -90,10 +110,9 @@ def run(self, name, *args): if not args: args = (-1, ) + print 'Running %r)' % name + res = self.c_allfuncs(name, *args) num = self.name_to_func[name] - print - print 'Running %r (test number %d)' % (name, num) - res = self.c_allfuncs(num, *args) if self.funcsstr[num]: return res return int(res) @@ -101,8 +120,8 @@ def run_orig(self, name, *args): if not args: args = (-1, ) - num = self.name_to_func[name] - res = self.allfuncs(num, *args) + res = self.allfuncs(name, *args) + num = self.name_to_func[name] if self.funcsstr[num]: return res return int(res) Modified: pypy/branch/gc-arena/pypy/translator/cli/ilgenerator.py ============================================================================== --- pypy/branch/gc-arena/pypy/translator/cli/ilgenerator.py (original) +++ pypy/branch/gc-arena/pypy/translator/cli/ilgenerator.py Wed Oct 21 18:52:56 2009 @@ -400,7 +400,10 @@ ilasm.opcode('ldc.i4', ord(value)) elif TYPE is ootype.Float: if isinf(value): - ilasm.opcode('ldc.r8', '(00 00 00 00 00 00 f0 7f)') + if value < 0.0: + ilasm.opcode('ldc.r8', '(00 00 00 00 00 00 f0 ff)') + else: + ilasm.opcode('ldc.r8', '(00 00 00 00 00 00 f0 7f)') elif isnan(value): ilasm.opcode('ldc.r8', '(00 00 00 00 00 00 f8 ff)') else: Modified: pypy/branch/gc-arena/pypy/translator/cli/sdk.py ============================================================================== --- pypy/branch/gc-arena/pypy/translator/cli/sdk.py (original) +++ pypy/branch/gc-arena/pypy/translator/cli/sdk.py Wed Oct 21 18:52:56 2009 @@ -34,12 +34,35 @@ CSC = 'csc' PEVERIFY = 'peverify' +def get_mono_version(): + from commands import getoutput + lines = getoutput('mono -V').splitlines() + parts = lines[0].split() + # something like ['Mono', 'JIT', 'compiler', 'version', '2.4.2.3', ...] + iversion = parts.index('version') + ver = parts[iversion+1] # '2.4.2.3' + ver = ver.split('.') # ['2', '4', '2', '3'] + return tuple(map(int, ver)) # (2, 4, 2, 3) + + class MonoSDK(AbstractSDK): RUNTIME = ['mono'] ILASM = 'ilasm2' CSC = 'gmcs' PEVERIFY = 'peverify' # it's not part of mono, but we get a meaningful skip message + # this is a workaround for this bug: + # https://bugzilla.novell.com/show_bug.cgi?id=474718 they promised that it + # should be fixed in versions after 2.4.3.x, in the meanwhile pass + # -O=-branch + @classmethod + def runtime(cls): + cls._check_helper('mono') + ver = get_mono_version() + if (2, 1) < ver < (2, 4, 3): + return ['mono', '-O=-branch'] + return ['mono'] + def key_as_dict(handle): import _winreg i = 0 From arigo at codespeak.net Wed Oct 21 19:04:43 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Oct 2009 19:04:43 +0200 (CEST) Subject: [pypy-svn] r68699 - pypy/branch/gc-arena/pypy/translator/c/test Message-ID: <20091021170443.42E79168009@codespeak.net> Author: arigo Date: Wed Oct 21 19:04:42 2009 New Revision: 68699 Modified: pypy/branch/gc-arena/pypy/translator/c/test/test_newgc.py Log: Fix the test. Modified: pypy/branch/gc-arena/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/gc-arena/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/gc-arena/pypy/translator/c/test/test_newgc.py Wed Oct 21 19:04:42 2009 @@ -931,7 +931,7 @@ def test_string_builder_over_allocation(self): res = self.run('string_builder_over_allocation') - assert res == 'abcddefgrty' * 1000*'z' + 1000*'u' + assert res == 'abcddefgrty' + 1000*'z' + 1000*'u' class TestGenerationalGC(TestSemiSpaceGC): gcpolicy = "generation" From arigo at codespeak.net Wed Oct 21 19:11:02 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Oct 2009 19:11:02 +0200 (CEST) Subject: [pypy-svn] r68700 - pypy/trunk/pypy/doc/config Message-ID: <20091021171102.6F8A5168009@codespeak.net> Author: arigo Date: Wed Oct 21 19:11:01 2009 New Revision: 68700 Modified: pypy/trunk/pypy/doc/config/objspace.std.withshadowtracking.txt Log: Kill the link to 'withmultidict'. Modified: pypy/trunk/pypy/doc/config/objspace.std.withshadowtracking.txt ============================================================================== --- pypy/trunk/pypy/doc/config/objspace.std.withshadowtracking.txt (original) +++ pypy/trunk/pypy/doc/config/objspace.std.withshadowtracking.txt Wed Oct 21 19:11:01 2009 @@ -1,5 +1,5 @@ Enable "shadow tracking". This means a special dict representation is used -together with `multidicts`_. This dict representation is used only for instance +-- but only for instance dictionaries. The instance dictionary tracks whether an instance attribute shadows an attribute of its class. This makes method calls slightly faster in the following way: When calling a method the first thing that is checked is the @@ -7,5 +7,3 @@ instance dictionary is then checked for instance attributes shadowing the class attribute. If we know that there is no shadowing (since our instance dict tells us that) we can save this lookup on the instance dictionary. - -.. _`multidicts`: objspace.std.withmultidict.html From pedronis at codespeak.net Wed Oct 21 19:17:55 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 21 Oct 2009 19:17:55 +0200 (CEST) Subject: [pypy-svn] r68701 - in pypy/trunk/pypy/rpython/memory: gc gctransform test Message-ID: <20091021171755.870AA168009@codespeak.net> Author: pedronis Date: Wed Oct 21 19:17:54 2009 New Revision: 68701 Modified: pypy/trunk/pypy/rpython/memory/gc/base.py pypy/trunk/pypy/rpython/memory/gc/generation.py pypy/trunk/pypy/rpython/memory/gc/hybrid.py pypy/trunk/pypy/rpython/memory/gc/markcompact.py pypy/trunk/pypy/rpython/memory/gc/marksweep.py pypy/trunk/pypy/rpython/memory/gc/semispace.py pypy/trunk/pypy/rpython/memory/gctransform/framework.py pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py Log: - rewrite test_transformed_gc to compile the gcs only once per test class - make sure all runtime values are setup in the gc setup methods - add a _teardown() logic to some gc classes both the latter for the benefit of test_transformed_gc probably at some point need to do better about the cleanup calls in case of failures, also the global_list test didn't make it Modified: pypy/trunk/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/base.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/base.py Wed Oct 21 19:17:54 2009 @@ -20,9 +20,17 @@ self.AddressStack = get_address_stack(chunk_size) self.AddressDeque = get_address_deque(chunk_size) self.AddressDict = AddressDict - self.finalizer_lock_count = 0 self.config = config + def setup(self): + # all runtime mutable values' setup should happen here + # and in its overriden versions! for the benefit of test_transformed_gc + self.finalizer_lock_count = 0 + self.run_finalizers = self.AddressDeque() + + def _teardown(self): + pass + def can_malloc_nonmovable(self): return not self.moving_gc @@ -59,9 +67,6 @@ def write_barrier(self, newvalue, addr_struct): pass - def setup(self): - self.run_finalizers = self.AddressDeque() - def statistics(self, index): return -1 Modified: pypy/trunk/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/generation.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/generation.py Wed Oct 21 19:17:54 2009 @@ -55,12 +55,8 @@ self.initial_nursery_size = nursery_size self.auto_nursery_size = auto_nursery_size self.min_nursery_size = min_nursery_size - self.old_objects_pointing_to_young = self.AddressStack() - # ^^^ a list of addresses inside the old objects space; it - # may contain static prebuilt objects as well. More precisely, - # it lists exactly the old and static objects whose - # GCFLAG_NO_YOUNG_PTRS bit is not set. - self.young_objects_with_weakrefs = self.AddressStack() + + # define nursery fields self.reset_nursery() self._setup_wb() @@ -74,6 +70,13 @@ self.lb_young_var_basesize = sz def setup(self): + self.old_objects_pointing_to_young = self.AddressStack() + # ^^^ a list of addresses inside the old objects space; it + # may contain static prebuilt objects as well. More precisely, + # it lists exactly the old and static objects whose + # GCFLAG_NO_YOUNG_PTRS bit is not set. + self.young_objects_with_weakrefs = self.AddressStack() + self.last_generation_root_objects = self.AddressStack() self.young_objects_with_id = self.AddressDict() SemiSpaceGC.setup(self) @@ -87,6 +90,12 @@ if newsize > 0: self.set_nursery_size(newsize) + self.reset_nursery() + + def _teardown(self): + self.collect() # should restore last gen objects flags + SemiSpaceGC._teardown(self) + def reset_nursery(self): self.nursery = NULL self.nursery_top = NULL Modified: pypy/trunk/pypy/rpython/memory/gc/hybrid.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/hybrid.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/hybrid.py Wed Oct 21 19:17:54 2009 @@ -114,13 +114,14 @@ self.nonlarge_gcptrs_max = large_object_gcptrs - 1 assert self.nonlarge_gcptrs_max <= self.lb_young_var_basesize assert self.nonlarge_max <= self.nonlarge_gcptrs_max - self.large_objects_collect_trigger = self.space_size + + def setup(self): + self.large_objects_collect_trigger = self.param_space_size if self.config.gcconfig.debugprint: self._initial_trigger = self.large_objects_collect_trigger self.rawmalloced_objects_to_trace = self.AddressStack() self.count_semispaceonly_collects = 0 - def setup(self): self.gen2_rawmalloced_objects = self.AddressStack() self.gen3_rawmalloced_objects = self.AddressStack() self.gen2_resizable_objects = self.AddressStack() Modified: pypy/trunk/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/markcompact.py Wed Oct 21 19:17:54 2009 @@ -91,11 +91,13 @@ def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096): import py; py.test.skip("Disabled for now, sorry") + self.param_space_size = space_size MovingGCBase.__init__(self, config, chunk_size) - self.space_size = space_size - self.next_collect_after = space_size/2 # whatever... def setup(self): + self.space_size = self.param_space_size + self.next_collect_after = self.param_space_size/2 # whatever... + if self.config.gcconfig.debugprint: self.program_start_time = time.time() self.space = llarena.arena_malloc(self.space_size, True) Modified: pypy/trunk/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/marksweep.py Wed Oct 21 19:17:54 2009 @@ -48,10 +48,14 @@ TRANSLATION_PARAMS = {'start_heap_size': 8*1024*1024} # XXX adjust def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, start_heap_size=4096): + self.param_start_heap_size = start_heap_size GCBase.__init__(self, config, chunk_size) + + def setup(self): + GCBase.setup(self) self.heap_usage = 0 # at the end of the latest collection self.bytes_malloced = 0 # since the latest collection - self.bytes_malloced_threshold = start_heap_size + self.bytes_malloced_threshold = self.param_start_heap_size self.total_collection_time = 0.0 self.malloced_objects = lltype.nullptr(self.HDR) self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR) Modified: pypy/trunk/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/semispace.py Wed Oct 21 19:17:54 2009 @@ -31,8 +31,6 @@ inline_simple_malloc_varsize = True malloc_zero_filled = True first_unused_gcflag = first_gcflag << 5 - total_collection_time = 0.0 - total_collection_count = 0 HDR = lltype.Struct('header', ('tid', lltype.Signed)) # XXX or rffi.INT? typeid_is_in_field = 'tid' @@ -51,12 +49,18 @@ def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096, max_space_size=sys.maxint//2+1): + self.param_space_size = space_size + self.param_max_space_size = max_space_size MovingGCBase.__init__(self, config, chunk_size) - self.space_size = space_size - self.max_space_size = max_space_size - self.red_zone = 0 def setup(self): + self.total_collection_time = 0.0 + self.total_collection_count = 0 + + self.space_size = self.param_space_size + self.max_space_size = self.param_max_space_size + self.red_zone = 0 + if self.config.gcconfig.debugprint: self.program_start_time = time.time() self.tospace = llarena.arena_malloc(self.space_size, True) @@ -69,6 +73,11 @@ self.objects_with_finalizers = self.AddressDeque() self.objects_with_weakrefs = self.AddressStack() + def _teardown(self): + llop.debug_print(lltype.Void, "Teardown") + llarena.arena_free(self.fromspace) + llarena.arena_free(self.tospace) + # This class only defines the malloc_{fixed,var}size_clear() methods # because the spaces are filled with zeroes in advance. Modified: pypy/trunk/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/framework.py Wed Oct 21 19:17:54 2009 @@ -168,6 +168,10 @@ root_walker.setup_root_walker() gcdata.gc.setup() + def frameworkgc__teardown(): + # run-time teardown code for tests! + gcdata.gc._teardown() + bk = self.translator.annotator.bookkeeper r_typeid16 = rffi.platform.numbertype_to_rclass[TYPE_ID] s_typeid16 = annmodel.SomeInteger(knowntype=r_typeid16) @@ -198,6 +202,10 @@ self.frameworkgc_setup_ptr = getfn(frameworkgc_setup, [], annmodel.s_None) + # for tests + self.frameworkgc__teardown_ptr = getfn(frameworkgc__teardown, [], + annmodel.s_None) + if root_walker.need_root_stack: self.incr_stack_ptr = getfn(root_walker.incr_stack, [annmodel.SomeInteger()], Modified: pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py Wed Oct 21 19:17:54 2009 @@ -1,9 +1,10 @@ import py import sys -import struct +import struct, inspect from pypy.translator.c import gc from pypy.annotation import model as annmodel -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.annotation import policy as annpolicy +from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rpython.memory.gctransform import framework from pypy.rpython.lltypesystem.lloperation import llop, void from pypy.rpython.memory.gc.marksweep import X_CLONE, X_POOL, X_POOL_PTR @@ -26,7 +27,9 @@ if stacklessgc: t.config.translation.gcrootfinder = "stackless" t.config.set(**extraconfigopts) - t.buildannotator().build_types(func, inputtypes) + ann = t.buildannotator(policy=annpolicy.StrictAnnotatorPolicy()) + ann.build_types(func, inputtypes) + if specialize: t.buildrtyper().specialize() if backendopt: @@ -36,64 +39,124 @@ t.viewcg() return t +ARGS = lltype.FixedSizeArray(lltype.Signed, 3) + class GCTest(object): gcpolicy = None stacklessgc = False GC_CAN_MOVE = False GC_CANNOT_MALLOC_NONMOVABLE = False + taggedpointers = False + + def setup_class(cls): + funcs0 = [] + funcs2 = [] + cleanups = [] + name_to_func = {} + mixlevelstuff = [] + for fullname in dir(cls): + if not fullname.startswith('define'): + continue + definefunc = getattr(cls, fullname) + _, name = fullname.split('_', 1) + func_fixup = definefunc.im_func(cls) + cleanup = None + if isinstance(func_fixup, tuple): + func, cleanup, fixup = func_fixup + mixlevelstuff.append(fixup) + else: + func = func_fixup + func.func_name = "f_%s" % name + if cleanup: + cleanup.func_name = "clean_%s" % name + + nargs = len(inspect.getargspec(func)[0]) + name_to_func[name] = len(funcs0) + if nargs == 2: + funcs2.append(func) + funcs0.append(None) + elif nargs == 0: + funcs0.append(func) + funcs2.append(None) + else: + raise NotImplementedError( + "defined test functions should have 0/2 arguments") + # used to let test cleanup static root pointing to runtime + # allocated stuff + cleanups.append(cleanup) + + def entrypoint(args): + num = args[0] + func = funcs0[num] + if func: + res = func() + else: + func = funcs2[num] + res = func(args[1], args[2]) + cleanup = cleanups[num] + if cleanup: + cleanup() + return res - def runner(self, f, nbargs=0, statistics=False, transformer=False, - mixlevelstuff=None, **extraconfigopts): - if nbargs == 2: - def entrypoint(args): - x = args[0] - y = args[1] - r = f(x, y) - return r - elif nbargs == 0: - def entrypoint(args): - return f() - else: - raise NotImplementedError("pure laziness") - - from pypy.rpython.llinterp import LLInterpreter from pypy.translator.c.genc import CStandaloneBuilder - ARGS = lltype.FixedSizeArray(lltype.Signed, nbargs) s_args = annmodel.SomePtr(lltype.Ptr(ARGS)) - t = rtype(entrypoint, [s_args], gcname=self.gcname, - stacklessgc=self.stacklessgc, - **extraconfigopts) - if mixlevelstuff: - mixlevelstuff(t) + t = rtype(entrypoint, [s_args], gcname=cls.gcname, + stacklessgc=cls.stacklessgc, + taggedpointers=cls.taggedpointers) + + for fixup in mixlevelstuff: + if fixup: + fixup(t) + cbuild = CStandaloneBuilder(t, entrypoint, config=t.config, - gcpolicy=self.gcpolicy) + gcpolicy=cls.gcpolicy) db = cbuild.generate_graphs_for_llinterp() entrypointptr = cbuild.getentrypointptr() entrygraph = entrypointptr._obj.graph if conftest.option.view: t.viewcg() - llinterp = LLInterpreter(t.rtyper) + cls.name_to_func = name_to_func + cls.entrygraph = entrygraph + cls.rtyper = t.rtyper + cls.db = db + + def runner(self, name, statistics=False, transformer=False): + db = self.db + name_to_func = self.name_to_func + entrygraph = self.entrygraph + from pypy.rpython.llinterp import LLInterpreter + + llinterp = LLInterpreter(self.rtyper) + + gct = db.gctransformer + + if self.__class__.__dict__.get('_used', False): + teardowngraph = gct.frameworkgc__teardown_ptr.value._obj.graph + llinterp.eval_graph(teardowngraph, []) + self.__class__._used = True # FIIIIISH - setupgraph = db.gctransformer.frameworkgc_setup_ptr.value._obj.graph + setupgraph = gct.frameworkgc_setup_ptr.value._obj.graph + # setup => resets the gc llinterp.eval_graph(setupgraph, []) def run(args): ll_args = lltype.malloc(ARGS, immortal=True) - for i in range(nbargs): - ll_args[i] = args[i] + ll_args[0] = name_to_func[name] + for i in range(len(args)): + ll_args[1+i] = args[i] res = llinterp.eval_graph(entrygraph, [ll_args]) return res if statistics: - statisticsgraph = db.gctransformer.statistics_ptr.value._obj.graph - ll_gc = db.gctransformer.c_const_gc.value + statisticsgraph = gct.statistics_ptr.value._obj.graph + ll_gc = gct.c_const_gc.value def statistics(index): return llinterp.eval_graph(statisticsgraph, [ll_gc, index]) return run, statistics elif transformer: - return run, db.gctransformer + return run, gct else: return run @@ -109,7 +172,7 @@ else: return -1 # xxx - def test_instances(self): + def define_instances(cls): class A(object): pass class B(A): @@ -129,12 +192,15 @@ b.last = first j += 1 return 0 - run, statistics = self.runner(malloc_a_lot, statistics=True) + return malloc_a_lot + + def test_instances(self): + run, statistics = self.runner("instances", statistics=True) run([]) heap_size = self.heap_usage(statistics) - def test_llinterp_lists(self): + def define_llinterp_lists(cls): def malloc_a_lot(): i = 0 while i < 10: @@ -145,12 +211,15 @@ j += 1 a.append(j) return 0 - run, statistics = self.runner(malloc_a_lot, statistics=True) + return malloc_a_lot + + def test_llinterp_lists(self): + run, statistics = self.runner("llinterp_lists", statistics=True) run([]) heap_size = self.heap_usage(statistics) assert heap_size < 16000 * INT_SIZE / 4 # xxx - def test_llinterp_tuples(self): + def define_llinterp_tuples(cls): def malloc_a_lot(): i = 0 while i < 10: @@ -162,41 +231,51 @@ j += 1 b.append((1, j, i)) return 0 - run, statistics = self.runner(malloc_a_lot, statistics=True) + return malloc_a_lot + + def test_llinterp_tuples(self): + run, statistics = self.runner("llinterp_tuples", statistics=True) run([]) heap_size = self.heap_usage(statistics) assert heap_size < 16000 * INT_SIZE / 4 # xxx - def test_global_list(self): + def skipdefine_global_list(cls): + gl = [] class Box: def __init__(self): - self.lst = [] + self.lst = gl box = Box() def append_to_list(i, j): box.lst.append([i] * 50) llop.gc__collect(lltype.Void) return box.lst[j][0] - run = self.runner(append_to_list, nbargs=2) + return append_to_list, None, None + + def test_global_list(self): + py.test.skip("doesn't fit in the model, tested elsewhere too") + run = self.runner("global_list") res = run([0, 0]) assert res == 0 for i in range(1, 5): res = run([i, i - 1]) assert res == i - 1 # crashes if constants are not considered roots - def test_string_concatenation(self): - + def define_string_concatenation(cls): def concat(j, dummy): lst = [] for i in range(j): lst.append(str(i)) return len("".join(lst)) - run, statistics = self.runner(concat, nbargs=2, statistics=True) + return concat + + def test_string_concatenation(self): + run, statistics = self.runner("string_concatenation", statistics=True) res = run([100, 0]) - assert res == concat(100, 0) + assert res == len(''.join([str(x) for x in range(100)])) heap_size = self.heap_usage(statistics) assert heap_size < 16000 * INT_SIZE / 4 # xxx - def test_nongc_static_root(self): + def define_nongc_static_root(cls): T1 = lltype.GcStruct("C", ('x', lltype.Signed)) T2 = lltype.Struct("C", ('p', lltype.Ptr(T1))) static = lltype.malloc(T2, immortal=True) @@ -206,11 +285,16 @@ static.p = t1 llop.gc__collect(lltype.Void) return static.p.x - run = self.runner(f, nbargs=0) + def cleanup(): + static.p = lltype.nullptr(T1) + return f, cleanup, None + + def test_nongc_static_root(self): + run = self.runner("nongc_static_root") res = run([]) assert res == 42 - def test_finalizer(self): + def define_finalizer(cls): class B(object): pass b = B() @@ -231,11 +315,14 @@ llop.gc__collect(lltype.Void) llop.gc__collect(lltype.Void) return b.num_deleted - run = self.runner(f, nbargs=2) + return f + + def test_finalizer(self): + run = self.runner("finalizer") res = run([5, 42]) #XXX pure lazyness here too assert res == 6 - def test_finalizer_calls_malloc(self): + def define_finalizer_calls_malloc(cls): class B(object): pass b = B() @@ -260,11 +347,14 @@ llop.gc__collect(lltype.Void) llop.gc__collect(lltype.Void) return b.num_deleted - run = self.runner(f, nbargs=2) + return f + + def test_finalizer_calls_malloc(self): + run = self.runner("finalizer_calls_malloc") res = run([5, 42]) #XXX pure lazyness here too assert res == 12 - def test_finalizer_resurrects(self): + def define_finalizer_resurrects(cls): class B(object): pass b = B() @@ -291,11 +381,14 @@ llop.gc__collect(lltype.Void) llop.gc__collect(lltype.Void) return b.num_deleted * 10 + aid + 100 * (b.a is None) - run = self.runner(f, nbargs=2) + return f + + def test_finalizer_resurrects(self): + run = self.runner("finalizer_resurrects") res = run([5, 42]) #XXX pure lazyness here too assert 160 <= res <= 165 - def test_weakref(self): + def define_weakref(cls): import weakref, gc class A(object): pass @@ -313,11 +406,14 @@ llop.gc__collect(lltype.Void) result = result and (ref() is None) return result - run = self.runner(f) + return f + + def test_weakref(self): + run = self.runner("weakref") res = run([]) assert res - def test_weakref_to_object_with_finalizer(self): + def define_weakref_to_object_with_finalizer(cls): import weakref, gc class A(object): count = 0 @@ -334,11 +430,14 @@ llop.gc__collect(lltype.Void) result = a.count == 1 and (ref() is None) return result - run = self.runner(f) + return f + + def test_weakref_to_object_with_finalizer(self): + run = self.runner("weakref_to_object_with_finalizer") res = run([]) assert res - def test_collect_during_collect(self): + def define_collect_during_collect(cls): class B(object): pass b = B() @@ -370,14 +469,18 @@ llop.gc__collect(lltype.Void) llop.gc__collect(lltype.Void) b.bla = persistent_a1.id + persistent_a2.id + persistent_a3.id + persistent_a4.id - print b.num_deleted_c + # NB print would create a static root! + llop.debug_print(lltype.Void, b.num_deleted_c) return b.num_deleted - run = self.runner(f, nbargs=2) + return f + + def test_collect_during_collect(self): + run = self.runner("collect_during_collect") # runs collect recursively 4 times res = run([4, 42]) #XXX pure lazyness here too assert res == 12 - def test_collect_0(self): + def define_collect_0(cls): def concat(j, dummy): lst = [] for i in range(j): @@ -386,11 +489,14 @@ if we_are_translated(): llop.gc__collect(lltype.Void, 0) return result - run = self.runner(concat, nbargs=2) + return concat + + def test_collect_0(self): + run = self.runner("collect_0") res = run([100, 0]) - assert res == concat(100, 0) + assert res == len(''.join([str(x) for x in range(100)])) - def test_interior_ptrs(self): + def define_interior_ptrs(cls): from pypy.rpython.lltypesystem.lltype import Struct, GcStruct, GcArray from pypy.rpython.lltypesystem.lltype import Array, Signed, malloc @@ -444,11 +550,14 @@ f6()) assert func() == 111111 - run = self.runner(func) + return func + + def test_interior_ptrs(self): + run = self.runner("interior_ptrs") res = run([]) assert res == 111111 - def test_id(self): + def define_id(cls): class A(object): pass a1 = A() @@ -464,19 +573,25 @@ if id2 != compute_unique_id(a2): error += 2 if id3 != compute_unique_id(a3): error += 4 return error - run = self.runner(func) + return func + + def test_id(self): + run = self.runner("id") res = run([]) assert res == 0 - def test_can_move(self): + def define_can_move(cls): TP = lltype.GcArray(lltype.Float) def func(): return rgc.can_move(lltype.malloc(TP, 1)) - run = self.runner(func) + return func + + def test_can_move(self): + run = self.runner("can_move") res = run([]) assert res == self.GC_CAN_MOVE - def test_malloc_nonmovable(self): + def define_malloc_nonmovable(cls): TP = lltype.GcArray(lltype.Char) def func(): #try: @@ -489,10 +604,13 @@ #except Exception, e: # return 2 - run = self.runner(func) + return func + + def test_malloc_nonmovable(self): + run = self.runner("malloc_nonmovable") assert int(self.GC_CANNOT_MALLOC_NONMOVABLE) == run([]) - def test_malloc_nonmovable_fixsize(self): + def define_malloc_nonmovable_fixsize(cls): S = lltype.GcStruct('S', ('x', lltype.Float)) TP = lltype.GcStruct('T', ('s', lltype.Ptr(S))) def func(): @@ -506,10 +624,13 @@ except Exception, e: return 2 - run = self.runner(func) + return func + + def test_malloc_nonmovable_fixsize(self): + run = self.runner("malloc_nonmovable_fixsize") assert run([]) == int(self.GC_CANNOT_MALLOC_NONMOVABLE) - def test_resizable_buffer(self): + def define_resizable_buffer(cls): from pypy.rpython.lltypesystem.rstr import STR from pypy.rpython.annlowlevel import hlstr @@ -520,10 +641,13 @@ ptr.chars[1] = 'b' return hlstr(rgc.finish_building_buffer(ptr, 2)) == "ab" - run = self.runner(f) + return f + + def test_resizable_buffer(self): + run = self.runner("resizable_buffer") assert run([]) == 1 - def test_string_builder_over_allocation(self): + def define_string_builder_over_allocation(cls): import gc def fn(): s = StringBuilder(4) @@ -535,79 +659,19 @@ s.append_multiple_char('y', 1000) res = s.build()[1000] gc.collect() - return res - fn = self.runner(fn) - res = fn([]) - assert res == 'y' - - def test_tagged_simple(self): - class Unrelated(object): - pass - - u = Unrelated() - u.x = UnboxedObject(47) - def fn(n): - rgc.collect() # check that a prebuilt tagged pointer doesn't explode - if n > 0: - x = BoxedObject(n) - else: - x = UnboxedObject(n) - u.x = x # invoke write barrier - rgc.collect() - return x.meth(100) - def func(): - return fn(1000) + fn(-1000) - func = self.runner(func, taggedpointers=True) - res = func([]) - assert res == fn(1000) + fn(-1000) - - def test_tagged_prebuilt(self): - - class F: - pass - - f = F() - f.l = [UnboxedObject(10)] - def fn(n): - if n > 0: - x = BoxedObject(n) - else: - x = UnboxedObject(n) - f.l.append(x) - rgc.collect() - return f.l[-1].meth(100) - def func(): - return fn(1000) ^ fn(-1000) - func = self.runner(func, taggedpointers=True) - res = func([]) - assert res == fn(1000) ^ fn(-1000) - -from pypy.rlib.objectmodel import UnboxedValue - -class TaggedBase(object): - __slots__ = () - def meth(self, x): - raise NotImplementedError - -class BoxedObject(TaggedBase): - attrvalue = 66 - def __init__(self, normalint): - self.normalint = normalint - def meth(self, x): - return self.normalint + x + 2 - -class UnboxedObject(TaggedBase, UnboxedValue): - __slots__ = 'smallint' - def meth(self, x): - return self.smallint + x + 3 + return ord(res) + return fn + def test_string_builder_over_allocation(self): + fn = self.runner("string_builder_over_allocation") + res = fn([]) + assert res == ord('y') class GenericMovingGCTests(GenericGCTests): GC_CAN_MOVE = True GC_CANNOT_MALLOC_NONMOVABLE = True - def test_many_ids(self): - py.test.skip("fails for bad reasons in lltype.py :-(") + def define_many_ids(cls): class A(object): pass def f(): @@ -631,10 +695,30 @@ i += 1 j += 1 lltype.free(idarray, flavor='raw') - run = self.runner(f) + return 0 + return f + + def test_many_ids(self): + py.test.skip("fails for bad reasons in lltype.py :-(") + run = self.runner("many_ids") run([]) - def test_do_malloc_operations(self): + @classmethod + def ensure_layoutbuilder(cls, translator): + jit2gc = getattr(translator, '_jit2gc', None) + if jit2gc: + return jit2gc['layoutbuilder'] + GCClass = cls.gcpolicy.transformerclass.GCClass + lltype2vtable = translator.rtyper.lltype2vtable + layoutbuilder = framework.TransformerLayoutBuilder(GCClass, + lltype2vtable) + layoutbuilder.delay_encoding() + translator._jit2gc = { + 'layoutbuilder': layoutbuilder, + } + return layoutbuilder + + def define_do_malloc_operations(cls): P = lltype.GcStruct('P', ('x', lltype.Signed)) def g(): r = lltype.malloc(P) @@ -648,18 +732,13 @@ while i < 40: g() i += 1 + return 0 def fix_graph_of_g(translator): from pypy.translator.translator import graphof from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem import rffi - GCClass = self.gcpolicy.transformerclass.GCClass - lltype2vtable = translator.rtyper.lltype2vtable - layoutbuilder = framework.TransformerLayoutBuilder(GCClass, - lltype2vtable) - layoutbuilder.delay_encoding() - translator._jit2gc = { - 'layoutbuilder': layoutbuilder, - } + layoutbuilder = cls.ensure_layoutbuilder(translator) + type_id = layoutbuilder.get_type_id(P) # # now fix the do_malloc_fixedsize_clear in the graph of g @@ -674,10 +753,13 @@ break else: assert 0, "oups, not found" - run = self.runner(f, mixlevelstuff=fix_graph_of_g) + return f, None, fix_graph_of_g + + def test_do_malloc_operations(self): + run = self.runner("do_malloc_operations") run([]) - def test_do_malloc_operations_in_call(self): + def define_do_malloc_operations_in_call(cls): P = lltype.GcStruct('P', ('x', lltype.Signed)) def g(): llop.do_malloc_fixedsize_clear(llmemory.GCREF) # placeholder @@ -688,18 +770,12 @@ while i < 40: g() i += q.x + return 0 def fix_graph_of_g(translator): from pypy.translator.translator import graphof from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem import rffi - GCClass = self.gcpolicy.transformerclass.GCClass - lltype2vtable = translator.rtyper.lltype2vtable - layoutbuilder = framework.TransformerLayoutBuilder(GCClass, - lltype2vtable) - layoutbuilder.delay_encoding() - translator._jit2gc = { - 'layoutbuilder': layoutbuilder, - } + layoutbuilder = cls.ensure_layoutbuilder(translator) type_id = layoutbuilder.get_type_id(P) # # now fix the do_malloc_fixedsize_clear in the graph of g @@ -714,9 +790,14 @@ break else: assert 0, "oups, not found" - run = self.runner(f, mixlevelstuff=fix_graph_of_g) + return f, None, fix_graph_of_g + + def test_do_malloc_operations_in_call(self): + run = self.runner("do_malloc_operations_in_call") run([]) +# ________________________________________________________________ + class TestMarkSweepGC(GenericGCTests): gcname = "marksweep" class gcpolicy(gc.FrameworkGcPolicy): @@ -725,7 +806,7 @@ root_stack_depth = 200 - def test_cloning(self): + def define_cloning(cls): B = lltype.GcStruct('B', ('x', lltype.Signed)) A = lltype.GcStruct('A', ('b', lltype.Ptr(B)), ('unused', lltype.Ptr(B))) @@ -755,11 +836,14 @@ a2copy.b.x = 444 return a1.b.x * 1000000 + a2.b.x * 1000 + a3.b.x - run = self.runner(func) + return func + + def test_cloning(self): + run = self.runner("cloning") res = run([]) assert res == 111222333 - def test_cloning_varsize(self): + def define_cloning_varsize(cls): B = lltype.GcStruct('B', ('x', lltype.Signed)) A = lltype.GcStruct('A', ('b', lltype.Ptr(B)), ('more', lltype.Array(lltype.Ptr(B)))) @@ -790,11 +874,14 @@ a2copy.more[1].x = 441 return a2.b.x * 1000000 + a2.more[0].x * 1000 + a2.more[1].x - run = self.runner(func) + return func + + def test_cloning_varsize(self): + run = self.runner("cloning_varsize") res = run([]) assert res == 22220221 - def test_cloning_highlevel(self): + def define_cloning_highlevel(cls): class A: pass class B(A): @@ -821,13 +908,16 @@ assert n > 5 return 1 - run = self.runner(func, nbargs=2) + return func + + def test_cloning_highlevel(self): + run = self.runner("cloning_highlevel") res = run([3, 0]) assert res == 1 res = run([7, 0]) assert res == 1 - def test_cloning_highlevel_varsize(self): + def define_cloning_highlevel_varsize(cls): class A: pass def func(n, dummy): @@ -849,11 +939,14 @@ n = n*10 + a.value return n - run = self.runner(func, nbargs=2) + return func + + def test_cloning_highlevel_varsize(self): + run = self.runner("cloning_highlevel_varsize") res = run([3, 0]) assert res == 456012789 - def test_tree_cloning(self): + def define_tree_cloning(cls): import os # this makes a tree of calls. Each leaf stores its path (a linked # list) in 'result'. Paths are mutated in-place but the leaves don't @@ -933,7 +1026,10 @@ check(result[i], i, 0, depth) os.write(2, 'ok\n') return 1 - run = self.runner(func, nbargs=2) + return func + + def test_tree_cloning(self): + run = self.runner("tree_cloning") res = run([3, 0]) assert res == 1 @@ -959,6 +1055,9 @@ class TestMarkCompactGC(GenericMovingGCTests): gcname = 'markcompact' + def setup_class(cls): + py.test.skip("Disabled for now, sorry") + class gcpolicy(gc.FrameworkGcPolicy): class transformerclass(framework.FrameworkGCTransformer): from pypy.rpython.memory.gc.markcompact import MarkCompactGC as GCClass @@ -976,7 +1075,7 @@ 'nursery_size': 128} root_stack_depth = 200 - def test_weakref_across_minor_collection(self): + def define_weakref_across_minor_collection(cls): import weakref class A: pass @@ -994,11 +1093,14 @@ llop.gc__collect(lltype.Void) assert ref() is a return a.foo + len(all) - run = self.runner(f) + return f + + def test_weakref_across_minor_collection(self): + run = self.runner("weakref_across_minor_collection") res = run([]) assert res == 20 + 20 - def test_nongc_static_root_minor_collect(self): + def define_nongc_static_root_minor_collect(cls): T1 = lltype.GcStruct("C", ('x', lltype.Signed)) T2 = lltype.Struct("C", ('p', lltype.Ptr(T1))) static = lltype.malloc(T2, immortal=True) @@ -1015,12 +1117,17 @@ i = static.p.x llop.gc__collect(lltype.Void) return static.p.x + i - run = self.runner(f, nbargs=0) + def cleanup(): + static.p = lltype.nullptr(T1) + return f, cleanup, None + + def test_nongc_static_root_minor_collect(self): + run = self.runner("nongc_static_root_minor_collect") res = run([]) assert res == 84 - def test_static_root_minor_collect(self): + def define_static_root_minor_collect(cls): class A: pass class B: @@ -1040,12 +1147,17 @@ i = static.p.x llop.gc__collect(lltype.Void) return static.p.x + i - run = self.runner(f, nbargs=0) + def cleanup(): + static.p = None + return f, cleanup, None + + def test_static_root_minor_collect(self): + run = self.runner("static_root_minor_collect") res = run([]) assert res == 84 - def test_many_weakrefs(self): + def define_many_weakrefs(cls): # test for the case where allocating the weakref itself triggers # a collection import weakref @@ -1058,10 +1170,15 @@ ref = weakref.ref(a) assert ref() is a i += 1 - run = self.runner(f, nbargs=0) + return 0 + + return f + + def test_many_weakrefs(self): + run = self.runner("many_weakrefs") run([]) - def test_immutable_to_old_promotion(self): + def define_immutable_to_old_promotion(cls): T_CHILD = lltype.Ptr(lltype.GcStruct('Child', ('field', lltype.Signed))) T_PARENT = lltype.Ptr(lltype.GcStruct('Parent', ('sub', T_CHILD))) child = lltype.malloc(T_CHILD.TO) @@ -1083,7 +1200,10 @@ #all[x] = lltype.nullptr(T_PARENT.TO) return res.sub.field - run, transformer = self.runner(f, nbargs=2, transformer=True) + return f + + def test_immutable_to_old_promotion(self): + run, transformer = self.runner("immutable_to_old_promotion", transformer=True) run([1, 4]) if not transformer.GCClass.prebuilt_gc_objects_are_static_roots: assert len(transformer.layoutbuilder.addresses_of_static_ptrs) == 0 @@ -1100,7 +1220,7 @@ # * the GcArray pointer from gc.wr_to_objects_with_id # * the GcArray pointer from gc.object_id_dict. - def test_adr_of_nursery(self): + def define_adr_of_nursery(cls): class A(object): pass @@ -1118,7 +1238,12 @@ assert nf1 > nf0 assert nt1 > nf1 assert nt1 == nt0 - run = self.runner(f, nbargs=0) + return 0 + + return f + + def test_adr_of_nursery(self): + run = self.runner("adr_of_nursery") res = run([]) class TestGenerationalNoFullCollectGC(GCTest): @@ -1139,11 +1264,15 @@ def semispace_collect(self, size_changing=False): ll_assert(not self.__ready, "no full collect should occur in this test") + def _teardown(self): + self.__ready = False # collecting here is expected + GenerationGC._teardown(self) + GC_PARAMS = {'space_size': 2048, 'nursery_size': 512} root_stack_depth = 200 - def test_working_nursery(self): + def define_working_nursery(cls): def f(): total = 0 i = 0 @@ -1156,7 +1285,10 @@ total += len(lst) i += 1 return total - run = self.runner(f, nbargs=0) + return f + + def test_working_nursery(self): + run = self.runner("working_nursery") res = run([]) assert res == 40 * 5 @@ -1172,7 +1304,7 @@ 'large_object': 32} root_stack_depth = 200 - def test_ref_from_rawmalloced_to_regular(self): + def define_ref_from_rawmalloced_to_regular(cls): import gc S = lltype.GcStruct('S', ('x', lltype.Signed)) A = lltype.GcStruct('A', ('p', lltype.Ptr(S)), @@ -1190,11 +1322,14 @@ lst = setup(j) gc.collect() return lst.p.x - run = self.runner(f, nbargs=2) + return f + + def test_ref_from_rawmalloced_to_regular(self): + run = self.runner("ref_from_rawmalloced_to_regular") res = run([100, 100]) assert res == 200 - def test_assume_young_pointers(self): + def define_assume_young_pointers(cls): from pypy.rlib import rgc S = lltype.GcForwardReference() S.become(lltype.GcStruct('S', @@ -1211,9 +1346,109 @@ rgc.collect(0) return s0.next.x - run = self.runner(f, nbargs=0) + def cleanup(): + s0.next = lltype.nullptr(S) + + return f, cleanup, None + + def test_assume_young_pointers(self): + run = self.runner("assume_young_pointers") res = run([]) assert res == 42 def test_malloc_nonmovable_fixsize(self): py.test.skip("not supported") + +# ________________________________________________________________ +# tagged pointers + +class TaggedPointerGCTests(GCTest): + taggedpointers = True + + def define_tagged_simple(cls): + class Unrelated(object): + pass + + u = Unrelated() + u.x = UnboxedObject(47) + def fn(n): + rgc.collect() # check that a prebuilt tagged pointer doesn't explode + if n > 0: + x = BoxedObject(n) + else: + x = UnboxedObject(n) + u.x = x # invoke write barrier + rgc.collect() + return x.meth(100) + def func(): + return fn(1000) + fn(-1000) + assert func() == 205 + return func + + def test_tagged_simple(self): + func = self.runner("tagged_simple") + res = func([]) + assert res == 205 + + def define_tagged_prebuilt(cls): + + class F: + pass + + f = F() + f.l = [UnboxedObject(10)] + def fn(n): + if n > 0: + x = BoxedObject(n) + else: + x = UnboxedObject(n) + f.l.append(x) + rgc.collect() + return f.l[-1].meth(100) + def func(): + return fn(1000) ^ fn(-1000) + assert func() == -1999 + return func + + def test_tagged_prebuilt(self): + func = self.runner("tagged_prebuilt") + res = func([]) + assert res == -1999 + +from pypy.rlib.objectmodel import UnboxedValue + +class TaggedBase(object): + __slots__ = () + def meth(self, x): + raise NotImplementedError + +class BoxedObject(TaggedBase): + attrvalue = 66 + def __init__(self, normalint): + self.normalint = normalint + def meth(self, x): + return self.normalint + x + 2 + +class UnboxedObject(TaggedBase, UnboxedValue): + __slots__ = 'smallint' + def meth(self, x): + return self.smallint + x + 3 + + +class TestMarkSweepTaggedPointerGC(TaggedPointerGCTests): + gcname = "marksweep" + class gcpolicy(gc.FrameworkGcPolicy): + class transformerclass(framework.FrameworkGCTransformer): + GC_PARAMS = {'start_heap_size': 4096 } + root_stack_depth = 200 + +class TestHybridTaggedPointerGC(TaggedPointerGCTests): + gcname = "hybrid" + + class gcpolicy(gc.FrameworkGcPolicy): + class transformerclass(framework.FrameworkGCTransformer): + from pypy.rpython.memory.gc.generation import GenerationGC as \ + GCClass + GC_PARAMS = {'space_size': 2048, + 'nursery_size': 128} + root_stack_depth = 200 From arigo at codespeak.net Wed Oct 21 19:30:29 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Oct 2009 19:30:29 +0200 (CEST) Subject: [pypy-svn] r68702 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091021173029.44DB8168009@codespeak.net> Author: arigo Date: Wed Oct 21 19:30:28 2009 New Revision: 68702 Modified: pypy/trunk/pypy/jit/metainterp/resume.py Log: Fix annotation. Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Wed Oct 21 19:30:28 2009 @@ -429,6 +429,7 @@ "For profiling only." import os from pypy.rlib import objectmodel + assert logname is not None # annotator hack fd = os.open(logname, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0666) os.write(fd, 'Log(%d, [\n' % objectmodel.compute_unique_id(storage)) frameinfo = storage.rd_frame_info_list From arigo at codespeak.net Wed Oct 21 19:33:12 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Oct 2009 19:33:12 +0200 (CEST) Subject: [pypy-svn] r68703 - pypy/branch/gc-arena/pypy/jit/metainterp Message-ID: <20091021173312.5F712168009@codespeak.net> Author: arigo Date: Wed Oct 21 19:33:11 2009 New Revision: 68703 Modified: pypy/branch/gc-arena/pypy/jit/metainterp/resume.py Log: Merge r68702 from trunk. Modified: pypy/branch/gc-arena/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/gc-arena/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/gc-arena/pypy/jit/metainterp/resume.py Wed Oct 21 19:33:11 2009 @@ -429,6 +429,7 @@ "For profiling only." import os from pypy.rlib import objectmodel + assert logname is not None # annotator hack fd = os.open(logname, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0666) os.write(fd, 'Log(%d, [\n' % objectmodel.compute_unique_id(storage)) frameinfo = storage.rd_frame_info_list From arigo at codespeak.net Wed Oct 21 21:56:46 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 21 Oct 2009 21:56:46 +0200 (CEST) Subject: [pypy-svn] r68704 - in pypy/trunk/pypy/interpreter: . test Message-ID: <20091021195646.EC280168005@codespeak.net> Author: arigo Date: Wed Oct 21 21:56:43 2009 New Revision: 68704 Modified: pypy/trunk/pypy/interpreter/function.py pypy/trunk/pypy/interpreter/test/test_function.py Log: issue475 resolved Test and minimal fix. Modified: pypy/trunk/pypy/interpreter/function.py ============================================================================== --- pypy/trunk/pypy/interpreter/function.py (original) +++ pypy/trunk/pypy/interpreter/function.py Wed Oct 21 21:56:43 2009 @@ -309,7 +309,7 @@ def fget_func_defaults(space, self): values_w = self.defs_w - if not values_w: + if not values_w or None in values_w: return space.w_None return space.newtuple(values_w) Modified: pypy/trunk/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_function.py (original) +++ pypy/trunk/pypy/interpreter/test/test_function.py Wed Oct 21 21:56:43 2009 @@ -644,3 +644,15 @@ assert space.eq_w(w_res, space.wrap(44)) + +class TestFunction: + + def test_func_defaults(self): + from pypy.interpreter import gateway + def g(w_a=gateway.NoneNotWrapped): + pass + app_g = gateway.interp2app_temp(g) + space = self.space + w_g = space.wrap(app_g) + w_defs = space.getattr(w_g, space.wrap("func_defaults")) + assert space.is_w(w_defs, space.w_None) From benjamin at codespeak.net Thu Oct 22 01:54:25 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 22 Oct 2009 01:54:25 +0200 (CEST) Subject: [pypy-svn] r68705 - pypy/trunk/pypy/doc/jit Message-ID: <20091021235425.90672168007@codespeak.net> Author: benjamin Date: Thu Oct 22 01:54:24 2009 New Revision: 68705 Modified: pypy/trunk/pypy/doc/jit/pyjitpl5.txt Log: a little paragraph I had in my wc Modified: pypy/trunk/pypy/doc/jit/pyjitpl5.txt ============================================================================== --- pypy/trunk/pypy/doc/jit/pyjitpl5.txt (original) +++ pypy/trunk/pypy/doc/jit/pyjitpl5.txt Thu Oct 22 01:54:24 2009 @@ -126,6 +126,17 @@ ``do_something_exciting``. +Optimization +------------ + +Once the meta-interpreter has verified that it has traced a loop, it decides how +to compile what it has. There is an optional optimization phase between these +actions which is covered future down this page. The backend converts the trace +operations into assembly for the particular machine. It then hands it back to +the frontend. The next time the loop is seen in application code, the optimized +version can be run instead of the normal intepreter! + + More resources ============== From benjamin at codespeak.net Thu Oct 22 01:55:36 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 22 Oct 2009 01:55:36 +0200 (CEST) Subject: [pypy-svn] r68706 - in pypy/trunk/pypy/interpreter/astcompiler: . test Message-ID: <20091021235536.88C42168007@codespeak.net> Author: benjamin Date: Thu Oct 22 01:55:35 2009 New Revision: 68706 Modified: pypy/trunk/pypy/interpreter/astcompiler/codegen.py pypy/trunk/pypy/interpreter/astcompiler/test/test_compiler.py Log: prevent a LOAD_CONST from being generated for function docstrings Modified: pypy/trunk/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/codegen.py Thu Oct 22 01:55:35 2009 @@ -1264,9 +1264,11 @@ assert isinstance(func, ast.FunctionDef) # If there's a docstring, store it as the first constant. if self.is_docstring(func.body[0]): - doc_string = func.body[0] - assert isinstance(doc_string, ast.Expr) - doc_string.value.walkabout(self) + doc_expr = func.body[0] + assert isinstance(doc_expr, ast.Expr) + doc_str = doc_expr.value + assert isinstance(doc_str, ast.Str) + self.add_const(doc_str.s) start = 1 else: self.add_const(self.space.w_None) Modified: pypy/trunk/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/test/test_compiler.py Thu Oct 22 01:55:35 2009 @@ -725,7 +725,20 @@ yield self.st, test, "X.__name__", "X" -class AppTestPrint: +class AppTestCompiler: + + def test_docstring_not_loaded(self): + import StringIO, dis, sys + ns = {} + exec "def f():\n 'hi'" in ns + f = ns["f"] + save = sys.stdout + sys.stdout = output = StringIO.StringIO() + try: + dis.dis(f) + finally: + sys.stdout = save + assert "0 ('hi')" not in output.getvalue() def test_print_to(self): exec """from StringIO import StringIO From fijal at codespeak.net Thu Oct 22 12:17:27 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 22 Oct 2009 12:17:27 +0200 (CEST) Subject: [pypy-svn] r68708 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20091022101727.AD922168009@codespeak.net> Author: fijal Date: Thu Oct 22 12:17:26 2009 New Revision: 68708 Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: Oops, sorry, fix the test Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Thu Oct 22 12:17:26 2009 @@ -23,15 +23,17 @@ self.pc = pc self.exception_target = exc_target -class FakeOptions(object): +class Fake(object): failargs_limit = 1000 + storedebug = None class FakeMetaInterpStaticData(object): def __init__(self, cpu): self.cpu = cpu self.profiler = EmptyProfiler() - self.options = FakeOptions() + self.options = Fake() + self.globaldata = Fake() def test_store_final_boxes_in_guard(): from pypy.jit.metainterp.compile import ResumeGuardDescr From fijal at codespeak.net Thu Oct 22 13:11:55 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 22 Oct 2009 13:11:55 +0200 (CEST) Subject: [pypy-svn] r68709 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091022111155.40D36168010@codespeak.net> Author: fijal Date: Thu Oct 22 13:11:54 2009 New Revision: 68709 Modified: pypy/trunk/pypy/jit/metainterp/resume.py Log: Store identities Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Thu Oct 22 13:11:54 2009 @@ -434,7 +434,7 @@ os.write(fd, 'Log(%d, [\n' % objectmodel.compute_unique_id(storage)) frameinfo = storage.rd_frame_info_list while True: - os.write(fd, '\t("%s", %d, %d, %xd),\n' % ( + os.write(fd, '\t("%s", %d, %d) at %xd,\n' % ( frameinfo.jitcode, frameinfo.pc, frameinfo.exception_target, objectmodel.compute_unique_id(frameinfo))) frameinfo = frameinfo.prev @@ -443,7 +443,8 @@ os.write(fd, '\t],\n\t[\n') numb = storage.rd_numb while True: - os.write(fd, '\t\t%s,\n' % ([untag(i) for i in numb.nums],)) + os.write(fd, '\t\t%s at %xd,\n' % ([untag(i) for i in numb.nums], + objectmodel.compute_unique_id(numb))) numb = numb.prev if numb is None: break From afa at codespeak.net Thu Oct 22 13:13:23 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Oct 2009 13:13:23 +0200 (CEST) Subject: [pypy-svn] r68710 - in pypy/branch/msvc-asmgcroot/pypy/translator/c: gcc gcc/test/msvc src Message-ID: <20091022111323.4671A168010@codespeak.net> Author: afa Date: Thu Oct 22 13:13:22 2009 New Revision: 68710 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/instruction.py pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track8.s pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py pypy/branch/msvc-asmgcroot/pypy/translator/c/src/mem.h Log: More progress: generate usable tables in assembler, and really track saved registers... The call shapes seem to be correct now Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/instruction.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/instruction.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/instruction.py Thu Oct 22 13:13:22 2009 @@ -1,24 +1,15 @@ -CALLEE_SAVE_REGISTERS_NOEBP = ['%ebx', '%esi', '%edi'] -CALLEE_SAVE_REGISTERS = CALLEE_SAVE_REGISTERS_NOEBP + ['%ebp'] - LOC_NOWHERE = 0 LOC_REG = 1 LOC_EBP_BASED = 2 LOC_ESP_BASED = 3 LOC_MASK = 0x03 -REG2LOC = {} -for _i, _reg in enumerate(CALLEE_SAVE_REGISTERS): - REG2LOC[_reg] = LOC_REG | (_i<<2) - REG2LOC[_reg[1:]] = LOC_REG | (_i<<2) - def frameloc(base, offset): assert base in (LOC_EBP_BASED, LOC_ESP_BASED) assert offset % 4 == 0 return base | offset - class SomeNewValue(object): pass somenewvalue = SomeNewValue() @@ -84,9 +75,9 @@ class InsnFunctionStart(Insn): framesize = 0 previous_insns = () - def __init__(self): + def __init__(self, registers): self.arguments = {} - for reg in CALLEE_SAVE_REGISTERS: + for reg in registers: self.arguments[reg] = somenewvalue def source_of(self, localvar, tag): @@ -160,6 +151,9 @@ class InsnRet(InsnStop): framesize = 0 + def __init__(self, registers): + self.registers = registers + def requestgcroots(self, tracker): # no need to track the value of these registers in the caller # function if we are the main(), or if we are flagged as a @@ -167,7 +161,7 @@ if tracker.is_stack_bottom: return {} else: - return dict(zip(CALLEE_SAVE_REGISTERS, CALLEE_SAVE_REGISTERS)) + return dict(zip(self.registers, self.registers)) class InsnCall(Insn): _args_ = ['lineno', 'gcroots'] Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s Thu Oct 22 13:13:22 2009 @@ -64,7 +64,7 @@ jl SHORT $LN15 at pypy_g_ll_@139 $LN14 at pypy_g_ll_@139: call _RPyAbort - ;; expected {24(%esp) | %ebx, %esi, %edi, %ebp | 32(%esp)} + ;; expected {24(%esp) | 12(%esp), 4(%esp), (%esp), 8(%esp) | 32(%esp)} $LN15 at pypy_g_ll_@139: ; 1529 : l_v420 = l_v419; @@ -76,7 +76,7 @@ test ebx, ebx jne SHORT $LN16 at pypy_g_ll_@139 call _RPyAbort - ;; expected {24(%esp) | %ebx, %esi, %edi, %ebp | 32(%esp)} + ;; expected {24(%esp) | 12(%esp), 4(%esp), (%esp), 8(%esp) | 32(%esp)} $LN16 at pypy_g_ll_@139: ; 1531 : OP_INT_ADD(l_v402, l_v421, l_v422); @@ -106,7 +106,7 @@ push edi call _pypy_g_mallocstr__Signed - ;; expected {28(%esp) | %ebx, %esi, %edi, %ebp | 36(%esp)} + ;; expected {28(%esp) | 16(%esp), 8(%esp), 4(%esp), 12(%esp) | 36(%esp)} ; 1486 : l_v405 = (void*)l_items_2; @@ -182,7 +182,7 @@ jl SHORT $LN10 at pypy_g_ll_@139 $LN9 at pypy_g_ll_@139: call _RPyAbort - ;; expected {24(%esp) | %ebx, %esi, %edi, %ebp | } + ;; expected {24(%esp) | 12(%esp), 4(%esp), (%esp), 8(%esp) | } $LN10 at pypy_g_ll_@139: ; 1517 : l_v413 = l_v412; @@ -194,7 +194,7 @@ test edi, edi jne SHORT $LN11 at pypy_g_ll_@139 call _RPyAbort - ;; expected {24(%esp) | %ebx, %esi, %edi, %ebp | } + ;; expected {24(%esp) | 12(%esp), 4(%esp), (%esp), 8(%esp) | } $LN11 at pypy_g_ll_@139: mov edi, DWORD PTR [edi+8] @@ -207,7 +207,7 @@ jl SHORT $LN13 at pypy_g_ll_@139 $LN12 at pypy_g_ll_@139: call _RPyAbort - ;; expected {24(%esp) | %ebx, %esi, %edi, %ebp | } + ;; expected {24(%esp) | 12(%esp), 4(%esp), (%esp), 8(%esp) | } $LN13 at pypy_g_ll_@139: ; 1520 : pypy_g_copy_string_contents__rpy_stringPtr_rpy_stringPt(l_v415, l_result_2, 0L, l_res_index_0, l_v414); @@ -235,7 +235,7 @@ $block0$80678: $block1$80679: call _memcpy - ;; expected {36(%esp) | %ebx, %esi, %edi, %ebp | } + ;; expected {36(%esp) | 24(%esp), 16(%esp), 12(%esp), 20(%esp) | } add esp, 12 ; 0000000cH ; 1521 : OP_INT_ADD(l_res_index_0, l_v414, l_v417); Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track8.s ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track8.s (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track8.s Thu Oct 22 13:13:22 2009 @@ -94,7 +94,7 @@ jle SHORT $LN17 at pypy_g_foo $LN16 at pypy_g_foo: call _LL_stack_too_big_slowpath - ;; expected {24(%esp) | %ebx, %esi, %edi, %ebp | %ebx, %edi, %ebp, 44(%esp), 48(%esp), 52(%esp)} + ;; expected {24(%esp) | 12(%esp), (%esp), 4(%esp), 8(%esp) | %ebx, %edi, %ebp, 44(%esp), 48(%esp), 52(%esp)} test eax, eax jne $LN71 at pypy_g_foo $LN17 at pypy_g_foo: @@ -147,7 +147,7 @@ push OFFSET _pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC $block11$34394: call _pypy_g_SemiSpaceGC_obtain_free_space - ;; expected {32(%esp) | %ebx, %esi, %edi, %ebp | %ebx, %edi, %ebp, 52(%esp), 56(%esp), 60(%esp)} + ;; expected {32(%esp) | 20(%esp), 8(%esp), 12(%esp), 16(%esp) | %ebx, %edi, %ebp, 52(%esp), 56(%esp), 60(%esp)} add esp, 8 ; 1192 : l_v425 = RPyField((&pypy_g_ExcData), ed_exc_type); @@ -293,7 +293,7 @@ $block7$34426: mov DWORD PTR [esi+4], OFFSET _pypy_g_src8_A_vtable call _pypy_g_foo - ;; expected {52(%esp) | %ebx, %esi, %edi, %ebp | %ebx, %esi, %edi, %ebp, 72(%esp), 76(%esp), 80(%esp)} + ;; expected {52(%esp) | 40(%esp), 28(%esp), 32(%esp), 36(%esp) | %ebx, %esi, %edi, %ebp, 72(%esp), 76(%esp), 80(%esp)} add esp, 28 ; 0000001cH ; 1125 : l_v376 = (void*)l_a2_1; @@ -389,7 +389,7 @@ push eax $block8$34437: call _pypy_g_foo - ;; expected {52(%esp) | %ebx, %esi, %edi, %ebp | %ebx, %edi, %ebp, 72(%esp), 76(%esp), 80(%esp)} + ;; expected {52(%esp) | 40(%esp), 28(%esp), 32(%esp), 36(%esp) | %ebx, %edi, %ebp, 72(%esp), 76(%esp), 80(%esp)} add esp, 28 ; 0000001cH ; 1156 : l_v401 = (void*)l_a2_1; Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Thu Oct 22 13:13:22 2009 @@ -34,7 +34,6 @@ r_jmp_source = re.compile(r"\d*[(](%[\w]+)[,)]") r_jmptable_item = re.compile(r"\t.long\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") r_jmptable_end = re.compile(r"\t.text|\t.section\s+.text|\t\.align|"+LABEL) -r_bottom_marker = re.compile(r"\t/[*] GC_STACK_BOTTOM [*]/") class FunctionGcRootTracker(object): skip = 0 @@ -91,7 +90,7 @@ shape = [retaddr] # the first gcroots are always the ones corresponding to # the callee-saved registers - for reg in CALLEE_SAVE_REGISTERS: + for reg in self.CALLEE_SAVE_REGISTERS: shape.append(LOC_NOWHERE) gcroots = [] for localvar, tag in insn.gcroots.items(): @@ -99,17 +98,17 @@ loc = localvar.getlocation(insn.framesize, self.uses_frame_pointer) else: - assert localvar in REG2LOC, "%s: %s" % (self.funcname, - localvar) - loc = REG2LOC[localvar] + assert localvar in self.REG2LOC, "%s: %s" % (self.funcname, + localvar) + loc = self.REG2LOC[localvar] assert isinstance(loc, int) if tag is None: gcroots.append(loc) else: - regindex = CALLEE_SAVE_REGISTERS.index(tag) + regindex = self.CALLEE_SAVE_REGISTERS.index(tag) shape[1 + regindex] = loc if LOC_NOWHERE in shape and not self.is_stack_bottom: - reg = CALLEE_SAVE_REGISTERS[shape.index(LOC_NOWHERE) - 1] + reg = self.CALLEE_SAVE_REGISTERS[shape.index(LOC_NOWHERE) - 1] raise AssertionError("cannot track where register %s is saved" % (reg,)) gcroots.sort() @@ -139,7 +138,7 @@ lst.append(previnsn) def parse_instructions(self): - self.insns = [InsnFunctionStart()] + self.insns = [InsnFunctionStart(self.CALLEE_SAVE_REGISTERS)] ignore_insns = False for lineno, line in enumerate(self.lines): if lineno < self.skip: @@ -147,7 +146,10 @@ self.currentlineno = lineno insn = [] match = r_insn.match(line) - if match: + + if self.r_bottom_marker.match(line): + self.is_stack_bottom = True + elif match: if not ignore_insns: opname = match.group(1) try: @@ -159,8 +161,6 @@ insn = meth(line) elif self.r_gcroot_marker.match(line): insn = self._visit_gcroot_marker(line) - elif r_bottom_marker.match(line): - self.is_stack_bottom = True elif line == '\t/* ignore_in_trackgcroot */\n': ignore_insns = True elif line == '\t/* end_ignore_in_trackgcroot */\n': @@ -176,7 +176,7 @@ else: self.append_instruction(insn) - del self.currentlineno + del self.currentlineno @classmethod def find_missing_visit_method(cls, opname): @@ -524,7 +524,7 @@ return self._visit_epilogue() + self._visit_pop(self.EBP) def visit_ret(self, line): - return InsnRet() + return InsnRet(self.CALLEE_SAVE_REGISTERS) def visit_jmp(self, line): tablelabels = [] @@ -579,7 +579,7 @@ if self.r_unaryinsn_star.match(line): # that looks like an indirect tail-call. # tail-calls are equivalent to RET for us - return InsnRet() + return InsnRet(self.CALLEE_SAVE_REGISTERS) try: self.conditional_jump(line) except KeyError: @@ -588,7 +588,7 @@ target = match.group(1) assert not target.startswith('.') # tail-calls are equivalent to RET for us - return InsnRet() + return InsnRet(self.CALLEE_SAVE_REGISTERS) return InsnStop() def register_jump_to(self, label): @@ -669,6 +669,9 @@ ESP = '%esp' EBP = '%ebp' EAX = '%eax' + CALLEE_SAVE_REGISTERS = ['%ebx', '%esi', '%edi', '%ebp'] + REG2LOC = dict((_reg, LOC_REG | (_i<<2)) + for _i, _reg in enumerate(CALLEE_SAVE_REGISTERS)) OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' TOP_OF_STACK = '0(%esp)' @@ -679,10 +682,12 @@ LOCALVARFP = LOCALVAR + r"|-?\d*[(]%ebp[)]" r_localvarnofp = re.compile(LOCALVAR) r_localvarfp = re.compile(LOCALVARFP) - r_gcroot_marker = re.compile(r"\t/[*] GCROOT ("+LOCALVARFP+") [*]/") r_localvar_esp = re.compile(r"(\d*)[(]%esp[)]") r_localvar_ebp = re.compile(r"(-?\d*)[(]%ebp[)]") + r_gcroot_marker = re.compile(r"\t/[*] GCROOT ("+LOCALVARFP+") [*]/") + r_bottom_marker = re.compile(r"\t/[*] GC_STACK_BOTTOM [*]/") + def __init__(self, lines, filetag=0): match = self.r_functionstart.match(lines[0]) funcname = match.group(1) @@ -718,6 +723,9 @@ ESP = 'esp' EBP = 'ebp' EAX = 'eax' + CALLEE_SAVE_REGISTERS = ['ebx', 'esi', 'edi', 'ebp'] + REG2LOC = dict((_reg, LOC_REG | (_i<<2)) + for _i, _reg in enumerate(CALLEE_SAVE_REGISTERS)) TOP_OF_STACK = 'DWORD PTR [esp]' OPERAND = r'(?:(:?WORD|DWORD|BYTE) PTR |OFFSET )?[_\w?:@$]*(?:[-+0-9]+)?(:?\[[-+*\w0-9]+\])?' @@ -728,14 +736,16 @@ r_functionend = re.compile(LABEL+r"\s+ENDP\s*$") r_symboldefine = re.compile(r"([_a-z0-9]+\$) = ([-0-9]+)\s*;.+\n") - LOCALVAR = r"eax|edx|ecx|ebx|esi|edi|ebp|DWORD PTR [-+]?\d*\[esp[-+]\d+\]" + LOCALVAR = r"eax|edx|ecx|ebx|esi|edi|ebp|DWORD PTR [-+]?\d*\[esp[-+]?\d*\]" LOCALVARFP = LOCALVAR + r"|DWORD PTR -?\d*\[ebp\]" r_localvarnofp = re.compile(LOCALVAR) r_localvarfp = re.compile(LOCALVARFP) - r_gcroot_marker = re.compile(r";.+ = pypy_asm_gcroot\(") r_localvar_esp = re.compile(r"DWORD PTR ([-+]?\d+)?\[esp([-+]?\d+)?\]") r_localvar_ebp = re.compile(r"DWORD PTR ([-+]?\d+)?\[ebp([-+]?\d+)?\]") + r_gcroot_marker = re.compile(r";.+ = pypy_asm_gcroot\(") + r_bottom_marker = re.compile(r"\tcall\t_pypy_asm_stack_bottom\s*") + @classmethod def init_regexp(cls): super(MsvcFunctionGcRootTracker, cls).init_regexp() @@ -1096,8 +1106,6 @@ mingw32='\t.text', msvc='_TEXT\tSEGMENT') - _globl('__gcrootanchor') - _globl('pypy_asm_stackwalk') _variant(elf='.type pypy_asm_stackwalk, @function', darwin='', @@ -1160,12 +1168,13 @@ _comment("A circular doubly-linked list of all") _comment("the ASM_FRAMEDATAs currently alive") if self.format == 'msvc': - print >> output, '%s:' % _globalname("__gcrootanchor") - print >> output, '\tDD FLAT:___gcrootanchor ; prev' + _globl('__gcrootanchor') + print >> output, '%s\tDD FLAT:___gcrootanchor ; prev' % _globalname("__gcrootanchor") print >> output, '\tDD FLAT:___gcrootanchor ; next' else: print >> output, '\t.data' print >> output, '\t.align\t4' + _globl('__gcrootanchor') _label('__gcrootanchor') print >> output, """\ .long\t__gcrootanchor /* prev */ @@ -1174,7 +1183,7 @@ _globl('__gcmapstart') if self.format == 'msvc': - print >> output, '%s:' % _globalname('__gcmapstart') + print >> output, '%s' % _globalname('__gcmapstart'), else: _label('__gcmapstart') for label, state, is_range in self.gcmaptable: @@ -1202,7 +1211,10 @@ print >> output, '\t.long\t%d' % (n,) _globl('__gcmapend') - _label('__gcmapend') + if self.format == 'msvc': + print >> output, '%s DD ?' % _globalname('__gcmapend') + else: + _label('__gcmapend') _variant(elf='.section\t.rodata', darwin='.const', mingw32='', @@ -1210,7 +1222,7 @@ _globl('__gccallshapes') if self.format == 'msvc': - print >> output, '%s:' % _globalname('__gccallshapes') + print >> output, _globalname('__gccallshapes'), else: _label('__gccallshapes') output.writelines(shapelines) @@ -1272,7 +1284,7 @@ elif kind == LOC_REG: reg = loc >> 2 assert 0 <= reg <= 3 - return CALLEE_SAVE_REGISTERS[reg] + return ElfFunctionGcRootTracker.CALLEE_SAVE_REGISTERS[reg] else: if kind == LOC_EBP_BASED: result = '(%ebp)' Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/src/mem.h (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/src/mem.h Thu Oct 22 13:13:22 2009 @@ -47,7 +47,7 @@ return _r1; } #define pypy_asm_keepalive(v) __asm { } -#define pypy_asm_stack_bottom() /* GC_STACK_BOTTOM */ +static __declspec(noinline) void* pypy_asm_stack_bottom() { } #endif From afa at codespeak.net Thu Oct 22 13:15:03 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Oct 2009 13:15:03 +0200 (CEST) Subject: [pypy-svn] r68711 - in pypy/branch/msvc-asmgcroot/pypy: rpython/memory/gctransform translator/c/src Message-ID: <20091022111503.AF776168010@codespeak.net> Author: afa Date: Thu Oct 22 13:15:03 2009 New Revision: 68711 Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gctransform/asmgcroot.py pypy/branch/msvc-asmgcroot/pypy/translator/c/src/char.h Log: On windows, a table of labels contains indirect addresses to real code. Do the transformation at runtime, just before we sort the gcmap table. This allows many tests to pass! Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gctransform/asmgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gctransform/asmgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gctransform/asmgcroot.py Thu Oct 22 13:15:03 2009 @@ -8,6 +8,7 @@ from pypy.objspace.flow.model import SpaceOperation from pypy.translator.unsimplify import copyvar from pypy.rlib.debug import ll_assert +import sys # @@ -296,6 +297,7 @@ return # the item may have been not found because the main array was # not sorted. Sort it and try again. + win32_follow_gcmap_jmp(gcmapstart, gcmapend) sort_gcmap(gcmapstart, gcmapend) item = search_in_gcmap(gcmapstart, gcmapend, retaddr) if item: @@ -384,6 +386,22 @@ rffi.cast(rffi.SIZE_T, arrayitemsize), llhelper(QSORT_CALLBACK_PTR, _compare_gcmap_entries)) +if sys.platform == 'win32': + def win32_follow_gcmap_jmp(start, end): + # The initial gcmap table contains addresses to a JMP + # instruction that jumps indirectly to the real code. + # Replace them with the target addresses. + while start < end: + code = rffi.cast(rffi.CCHARP, start.address[0])[0] + if code == '\xe9': # jmp + rel32 = rffi.cast(rffi.LONGP, start.address[0]+1)[0] + target = start.address[0] + (rel32 + 5) + start.address[0] = target + start += arrayitemsize +else: + def win32_follow_gcmap_jmp(start, end): + pass + def _compare_gcmap_entries(addr1, addr2): key1 = addr1.address[0] key2 = addr2.address[0] Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/src/char.h ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/src/char.h (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/src/char.h Thu Oct 22 13:15:03 2009 @@ -6,8 +6,8 @@ /*** binary operations ***/ -#define OP_CHAR_EQ(x,y,r) r = ((x) == (y)) -#define OP_CHAR_NE(x,y,r) r = ((x) != (y)) +#define OP_CHAR_EQ(x,y,r) r = ((unsigned char)(x) == (unsigned char)(y)) +#define OP_CHAR_NE(x,y,r) r = ((unsigned char)(x) != (unsigned char)(y)) #define OP_CHAR_LE(x,y,r) r = ((unsigned char)(x) <= (unsigned char)(y)) #define OP_CHAR_GT(x,y,r) r = ((unsigned char)(x) > (unsigned char)(y)) #define OP_CHAR_LT(x,y,r) r = ((unsigned char)(x) < (unsigned char)(y)) From afa at codespeak.net Thu Oct 22 14:29:54 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Oct 2009 14:29:54 +0200 (CEST) Subject: [pypy-svn] r68712 - in pypy/branch/msvc-asmgcroot/pypy/translator/c: . src Message-ID: <20091022122954.A1B6A16800D@codespeak.net> Author: afa Date: Thu Oct 22 14:29:53 2009 New Revision: 68712 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py pypy/branch/msvc-asmgcroot/pypy/translator/c/src/mem.h Log: Tweak compiler options. Now all tests but one pass! test_callback_simple loops forever. Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py Thu Oct 22 14:29:53 2009 @@ -503,15 +503,18 @@ lblofiles = ['%s.lbl.obj' % (cfile[:-2],) for cfile in mk.cfiles] mk.definition('ASMLBLOBJFILES', lblofiles) mk.definition('OBJECTS', 'gcmaptable.obj $(ASMLBLOBJFILES)') + # almost all optimizations implied by /O2, except /Og + mk.definition('ASM_CFLAGS', '$(CFLAGS) /Od /Oi /Ot /Oy /Ob2 /GF /Gy') mk.rule('.SUFFIXES', '.s', []) mk.rule('.s.obj', '', 'cmd /c $(MASM) /nologo /Cx /Cp /Zm /coff /Fo$@ /c $< $(INCLUDEDIRS)') mk.rule('.c.gcmap', '', - ['$(CC) /nologo $(CFLAGS) /c /FAs /Fa$*.s $< $(INCLUDEDIRS)', + ['$(CC) /nologo $(ASM_CFLAGS) /c /FAs /Fa$*.s $< $(INCLUDEDIRS)', 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc -t $*.s > $@'] ) mk.rule('gcmaptable.s', '$(GCMAPFILES)', 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc $(GCMAPFILES) > $@') + else: mk.definition('OBJECTS', '$(ASMLBLFILES) gcmaptable.s') mk.rule('%.s', '%.c', '$(CC) $(CFLAGS) -frandom-seed=$< -o $@ -S $< $(INCLUDEDIRS)') Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/src/mem.h (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/src/mem.h Thu Oct 22 14:29:53 2009 @@ -47,7 +47,7 @@ return _r1; } #define pypy_asm_keepalive(v) __asm { } -static __declspec(noinline) void* pypy_asm_stack_bottom() { } +static __declspec(noinline) void pypy_asm_stack_bottom() { } #endif From afa at codespeak.net Thu Oct 22 18:26:52 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 22 Oct 2009 18:26:52 +0200 (CEST) Subject: [pypy-svn] r68714 - in pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc: . test/msvc Message-ID: <20091022162652.B0E3D16800D@codespeak.net> Author: afa Date: Thu Oct 22 18:26:51 2009 New Revision: 68714 Added: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track1.s pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track2.s pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track_switch0.s Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Log: More fixes found during pypy translation: - update the list of non-returning functions - implement the famous table-based switches... - handle some functions with the __fastcall calling convention (Can we do it without this ugly list?) pypy-c-jit almost compiles, still some problems remaining Added: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track1.s ============================================================================== --- (empty file) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track1.s Thu Oct 22 18:26:51 2009 @@ -0,0 +1,31 @@ +; Function compile flags: /Odtp +_TEXT SEGMENT +_pypy_g_frameworkgc_setup PROC + +; 46 : void pypy_g_frameworkgc_setup(void) { + + push ebp + mov ebp, esp +$block0$37400: + +; 47 : +; 48 : block0: +; 49 : pypy_g_SemiSpaceGC_setup((&pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC)); + + push OFFSET _pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC + call _pypy_g_SemiSpaceGC_setup + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } + add esp, 4 +$block1$37401: + +; 50 : goto block1; +; 51 : +; 52 : block1: +; 53 : RPY_DEBUG_RETURN(); +; 54 : return /* nothing */; +; 55 : } + + pop ebp + ret 0 +_pypy_g_frameworkgc_setup ENDP +_TEXT ENDS Added: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track2.s ============================================================================== --- (empty file) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track2.s Thu Oct 22 18:26:51 2009 @@ -0,0 +1,747 @@ +; Function compile flags: /Odtpy +; COMDAT _pypy_g__inplace_divrem1 +_TEXT SEGMENT +tv158 = -292 ; size = 4 +tv152 = -288 ; size = 4 +tv72 = -284 ; size = 4 +tv65 = -280 ; size = 4 +_l_v30733$ = -273 ; size = 1 +_l_v30737$ = -272 ; size = 8 +_l_v30740$ = -257 ; size = 1 +_l_v30744$ = -256 ; size = 4 +_l_v30753$ = -252 ; size = 4 +_l_v30748$ = -248 ; size = 8 +_l_v30709$ = -233 ; size = 1 +_l_v30710$ = -232 ; size = 1 +_l_v30714$ = -231 ; size = 1 +_l_v30718$ = -230 ; size = 1 +_l_v30729$ = -229 ; size = 1 +_l_v30725$ = -228 ; size = 4 +_l_v30705$ = -224 ; size = 8 +_l_evalue_70$ = -216 ; size = 4 +_l_index_219$ = -212 ; size = 4 +_l_v30738$ = -205 ; size = 1 +_l_v30730$ = -204 ; size = 4 +_l_v30734$ = -200 ; size = 4 +_l_length_82$ = -196 ; size = 4 +_l_v30745$ = -189 ; size = 1 +_l_v30752$ = -188 ; size = 4 +_l_v30749$ = -184 ; size = 8 +_l_l_100$ = -172 ; size = 4 +_l_l_101$ = -168 ; size = 4 +_l_length_83$ = -164 ; size = 4 +_l_v30704$ = -160 ; size = 4 +_l_v30722$ = -156 ; size = 4 +_l_v30715$ = -152 ; size = 8 +_l_v30726$ = -144 ; size = 8 +_l_v30711$ = -132 ; size = 4 +_l_x_97$ = -128 ; size = 4 +_l_v30739$ = -121 ; size = 1 +_l_v30731$ = -120 ; size = 8 +_l_v30735$ = -112 ; size = 8 +_l_v30742$ = -97 ; size = 1 +_l_v30751$ = -96 ; size = 4 +_l_v30707$ = -90 ; size = 1 +_l_v30723$ = -89 ; size = 1 +_l_v30712$ = -88 ; size = 4 +_l_v30716$ = -84 ; size = 4 +_l_v30703$ = -80 ; size = 8 +_l_v30727$ = -72 ; size = 8 +_l_v30732$ = -64 ; size = 8 +_l_v30736$ = -56 ; size = 8 +_l_index_218$ = -44 ; size = 4 +_l_v30750$ = -40 ; size = 4 +_l_v30754$ = -36 ; size = 4 +_l_v30717$ = -30 ; size = 1 +_l_v30720$ = -29 ; size = 1 +_l_v30713$ = -28 ; size = 4 +_l_v30702$ = -24 ; size = 8 +_l_v30706$ = -16 ; size = 8 +_l_v30728$ = -8 ; size = 8 +_l_self_3688$ = 8 ; size = 4 +_l_pin_1$ = 12 ; size = 4 +_l_n_38$ = 16 ; size = 8 +_l_size_53$ = 24 ; size = 4 +_pypy_g__inplace_divrem1 PROC ; COMDAT + +; 16550: long pypy_g__inplace_divrem1(struct pypy_pypy_rlib_rbigint_rbigint0 *l_self_3688, struct pypy_pypy_rlib_rbigint_rbigint0 *l_pin_1, long long l_n_38, long l_size_53) { + + push ebp + mov ebp, esp + sub esp, 292 ; 00000124H +$block0$210880: + +; 16551: struct pypy_object0 *l_evalue_70; long l_index_218; long l_index_219; +; 16552: struct pypy_array5 *l_l_100; struct pypy_array5 *l_l_101; +; 16553: long l_length_82; long l_length_83; bool_t l_v30707; bool_t l_v30709; +; 16554: bool_t l_v30710; bool_t l_v30714; bool_t l_v30717; bool_t l_v30718; +; 16555: bool_t l_v30720; bool_t l_v30723; bool_t l_v30729; bool_t l_v30733; +; 16556: bool_t l_v30738; bool_t l_v30739; bool_t l_v30740; bool_t l_v30742; +; 16557: bool_t l_v30745; long l_v30704; long l_v30712; long l_v30713; +; 16558: long l_v30716; long l_v30722; long l_v30725; long l_v30730; +; 16559: long l_v30734; long l_v30744; long l_v30750; long l_v30751; +; 16560: long l_v30752; long l_v30753; long l_v30754; long long l_v30702; +; 16561: long long l_v30703; long long l_v30705; long long l_v30706; +; 16562: long long l_v30715; long long l_v30726; long long l_v30727; +; 16563: long long l_v30728; long long l_v30731; long long l_v30732; +; 16564: long long l_v30735; long long l_v30736; long long l_v30737; +; 16565: long long l_v30748; long long l_v30749; struct pypy_array5 *l_v30711; +; 16566: long l_x_97; +; 16567: +; 16568: block0: +; 16569: OP_LLONG_GT(l_n_38, 0LL, l_v30707); + + cmp DWORD PTR _l_n_38$[ebp+4], 0 + jl SHORT $LN11 at pypy_g__in + jg SHORT $LN19 at pypy_g__in + cmp DWORD PTR _l_n_38$[ebp], 0 + jbe SHORT $LN11 at pypy_g__in +$LN19 at pypy_g__in: + mov DWORD PTR tv65[ebp], 1 + jmp SHORT $LN12 at pypy_g__in +$LN11 at pypy_g__in: + mov DWORD PTR tv65[ebp], 0 +$LN12 at pypy_g__in: + mov al, BYTE PTR tv65[ebp] + mov BYTE PTR _l_v30707$[ebp], al + +; 16570: if (l_v30707) { + + movzx ecx, BYTE PTR _l_v30707$[ebp] + test ecx, ecx + je SHORT $LN8 at pypy_g__in + +; 16571: goto block3; + + jmp SHORT $block3$210882 +$LN8 at pypy_g__in: + +; 16572: } +; 16573: l_evalue_70 = (&pypy_g_exceptions_AssertionError.ae_super.se_super.e_super); + + mov DWORD PTR _l_evalue_70$[ebp], OFFSET _pypy_g_exceptions_AssertionError +$block1$210883: + +; 16574: goto block1; +; 16575: +; 16576: block1: +; 16577: pypy_g_RPyRaiseException((&pypy_g_exceptions_AssertionError_vtable.ae_super.se_super.e_super), l_evalue_70); + + mov edx, DWORD PTR _l_evalue_70$[ebp] + push edx + push OFFSET _pypy_g_exceptions_AssertionError_vtable + call _pypy_g_RPyRaiseException + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } + add esp, 8 + +; 16578: l_v30753 = -1L; + + mov DWORD PTR _l_v30753$[ebp], -1 +$block2$210884: + +; 16579: goto block2; +; 16580: +; 16581: block2: +; 16582: RPY_DEBUG_RETURN(); +; 16583: return l_v30753; + + mov eax, DWORD PTR _l_v30753$[ebp] + jmp $LN9 at pypy_g__in +$block3$210882: + +; 16584: +; 16585: block3: +; 16586: OP_LLONG_LE(l_n_38, 2147483647LL, l_v30709); + + cmp DWORD PTR _l_n_38$[ebp+4], 0 + jg SHORT $LN13 at pypy_g__in + jl SHORT $LN20 at pypy_g__in + cmp DWORD PTR _l_n_38$[ebp], 2147483647 ; 7fffffffH + ja SHORT $LN13 at pypy_g__in +$LN20 at pypy_g__in: + mov DWORD PTR tv72[ebp], 1 + jmp SHORT $LN14 at pypy_g__in +$LN13 at pypy_g__in: + mov DWORD PTR tv72[ebp], 0 +$LN14 at pypy_g__in: + mov al, BYTE PTR tv72[ebp] + mov BYTE PTR _l_v30709$[ebp], al + +; 16587: if (l_v30709) { + + movzx ecx, BYTE PTR _l_v30709$[ebp] + test ecx, ecx + je SHORT $LN7 at pypy_g__in + +; 16588: goto block4; + + jmp SHORT $block4$210886 +$LN7 at pypy_g__in: + +; 16589: } +; 16590: l_evalue_70 = (&pypy_g_exceptions_AssertionError.ae_super.se_super.e_super); + + mov DWORD PTR _l_evalue_70$[ebp], OFFSET _pypy_g_exceptions_AssertionError + +; 16591: goto block1; + + jmp SHORT $block1$210883 +$block4$210886: + +; 16592: +; 16593: block4: +; 16594: OP_INT_IS_TRUE(l_size_53, l_v30710); + + xor edx, edx + cmp DWORD PTR _l_size_53$[ebp], 0 + setne dl + mov BYTE PTR _l_v30710$[ebp], dl + +; 16595: if (l_v30710) { + + movzx eax, BYTE PTR _l_v30710$[ebp] + test eax, eax + je SHORT $block5$210889 + +; 16596: l_v30754 = l_size_53; + + mov ecx, DWORD PTR _l_size_53$[ebp] + mov DWORD PTR _l_v30754$[ebp], ecx + +; 16597: goto block6; + + jmp SHORT $block6$210888 +$block5$210889: + +; 16598: } +; 16599: goto block5; +; 16600: +; 16601: block5: +; 16602: l_v30711 = RPyField(l_pin_1, prrr_inst_digits); + + mov edx, DWORD PTR _l_pin_1$[ebp] + mov eax, DWORD PTR [edx+8] + mov DWORD PTR _l_v30711$[ebp], eax + +; 16603: l_v30712 = l_v30711->length; + + mov ecx, DWORD PTR _l_v30711$[ebp] + mov edx, DWORD PTR [ecx+4] + mov DWORD PTR _l_v30712$[ebp], edx + +; 16604: l_v30754 = l_v30712; + + mov eax, DWORD PTR _l_v30712$[ebp] + mov DWORD PTR _l_v30754$[ebp], eax +$block6$210888: + +; 16605: goto block6; +; 16606: +; 16607: block6: +; 16608: OP_INT_SUB(l_v30754, 1L, l_v30713); + + mov ecx, DWORD PTR _l_v30754$[ebp] + sub ecx, 1 + mov DWORD PTR _l_v30713$[ebp], ecx + +; 16609: l_v30702 = 0LL; + + mov DWORD PTR _l_v30702$[ebp], 0 + mov DWORD PTR _l_v30702$[ebp+4], 0 + +; 16610: l_x_97 = l_v30713; + + mov edx, DWORD PTR _l_v30713$[ebp] + mov DWORD PTR _l_x_97$[ebp], edx +$block7$210890: + +; 16611: goto block7; +; 16612: +; 16613: block7: +; 16614: OP_INT_GE(l_x_97, 0L, l_v30714); + + xor eax, eax + cmp DWORD PTR _l_x_97$[ebp], 0 + setge al + mov BYTE PTR _l_v30714$[ebp], al +$LN5 at pypy_g__in: + +; 16615: while (l_v30714) { + + movzx ecx, BYTE PTR _l_v30714$[ebp] + test ecx, ecx + je SHORT $block8$210896 + +; 16616: goto block9; + + jmp SHORT $block9$210894 +$block7_back$210895: + +; 16617: block7_back: ; +; 16618: OP_INT_GE(l_x_97, 0L, l_v30714); + + xor edx, edx + cmp DWORD PTR _l_x_97$[ebp], 0 + setge dl + mov BYTE PTR _l_v30714$[ebp], dl + +; 16619: } + + jmp SHORT $LN5 at pypy_g__in +$block8$210896: + +; 16620: goto block8; +; 16621: +; 16622: block8: +; 16623: OP_LLONG_AND(l_v30702, 2147483647LL, l_v30715); + + mov eax, DWORD PTR _l_v30702$[ebp] + and eax, 2147483647 ; 7fffffffH + mov ecx, DWORD PTR _l_v30702$[ebp+4] + and ecx, 0 + mov DWORD PTR _l_v30715$[ebp], eax + mov DWORD PTR _l_v30715$[ebp+4], ecx + +; 16624: OP_TRUNCATE_LONGLONG_TO_INT(l_v30715, l_v30716); + + mov edx, DWORD PTR _l_v30715$[ebp] + mov DWORD PTR _l_v30716$[ebp], edx + +; 16625: l_v30753 = l_v30716; + + mov eax, DWORD PTR _l_v30716$[ebp] + mov DWORD PTR _l_v30753$[ebp], eax + +; 16626: goto block2; + + jmp $block2$210884 +$block9$210894: + +; 16627: +; 16628: block9: +; 16629: OP_LLONG_LSHIFT(l_v30702, 31LL, l_v30703); + + mov eax, DWORD PTR _l_v30702$[ebp] + mov edx, DWORD PTR _l_v30702$[ebp+4] + mov cl, 31 ; 0000001fH + call __allshl + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } + mov DWORD PTR _l_v30703$[ebp], eax + mov DWORD PTR _l_v30703$[ebp+4], edx + +; 16630: l_l_100 = RPyField(l_pin_1, prrr_inst_digits); + + mov ecx, DWORD PTR _l_pin_1$[ebp] + mov edx, DWORD PTR [ecx+8] + mov DWORD PTR _l_l_100$[ebp], edx + +; 16631: l_length_83 = l_l_100->length; + + mov eax, DWORD PTR _l_l_100$[ebp] + mov ecx, DWORD PTR [eax+4] + mov DWORD PTR _l_length_83$[ebp], ecx + +; 16632: OP_INT_LT(l_x_97, 0L, l_v30717); + + xor edx, edx + cmp DWORD PTR _l_x_97$[ebp], 0 + setl dl + mov BYTE PTR _l_v30717$[ebp], dl + +; 16633: if (l_v30717) { + + movzx eax, BYTE PTR _l_v30717$[ebp] + test eax, eax + je SHORT $LN3 at pypy_g__in + +; 16634: goto block14; + + jmp $block14$210899 +$LN3 at pypy_g__in: + +; 16635: } +; 16636: l_index_218 = l_x_97; + + mov ecx, DWORD PTR _l_x_97$[ebp] + mov DWORD PTR _l_index_218$[ebp], ecx +$block10$210900: + +; 16637: goto block10; +; 16638: +; 16639: block10: +; 16640: OP_INT_GE(l_index_218, 0L, l_v30718); + + xor edx, edx + cmp DWORD PTR _l_index_218$[ebp], 0 + setge dl + mov BYTE PTR _l_v30718$[ebp], dl + +; 16641: RPyAssert(l_v30718, "negative list getitem index out of bound"); +; 16642: OP_INT_LT(l_index_218, l_length_83, l_v30720); + + mov eax, DWORD PTR _l_index_218$[ebp] + xor ecx, ecx + cmp eax, DWORD PTR _l_length_83$[ebp] + setl cl + mov BYTE PTR _l_v30720$[ebp], cl + +; 16643: RPyAssert(l_v30720, "list getitem index out of bound"); +; 16644: l_v30722 = l_l_100->length; + + mov edx, DWORD PTR _l_l_100$[ebp] + mov eax, DWORD PTR [edx+4] + mov DWORD PTR _l_v30722$[ebp], eax + +; 16645: OP_INT_LT(l_index_218, l_v30722, l_v30723); + + mov ecx, DWORD PTR _l_index_218$[ebp] + xor edx, edx + cmp ecx, DWORD PTR _l_v30722$[ebp] + setl dl + mov BYTE PTR _l_v30723$[ebp], dl + +; 16646: RPyAssert(l_v30723, "fixed getitem out of bounds"); +; 16647: l_v30725 = RPyItem(l_l_100, l_index_218); + + mov eax, DWORD PTR _l_index_218$[ebp] + mov ecx, DWORD PTR _l_l_100$[ebp] + mov edx, DWORD PTR [ecx+eax*4+8] + mov DWORD PTR _l_v30725$[ebp], edx + +; 16648: OP_CAST_INT_TO_LONGLONG(l_v30725, l_v30726); + + mov eax, DWORD PTR _l_v30725$[ebp] + cdq + mov DWORD PTR _l_v30726$[ebp], eax + mov DWORD PTR _l_v30726$[ebp+4], edx + +; 16649: OP_LLONG_ADD(l_v30703, l_v30726, l_v30706); + + mov eax, DWORD PTR _l_v30703$[ebp] + add eax, DWORD PTR _l_v30726$[ebp] + mov ecx, DWORD PTR _l_v30703$[ebp+4] + adc ecx, DWORD PTR _l_v30726$[ebp+4] + mov DWORD PTR _l_v30706$[ebp], eax + mov DWORD PTR _l_v30706$[ebp+4], ecx + +; 16650: OP_LLONG_FLOORDIV(l_v30706, l_n_38, l_v30727); + + mov edx, DWORD PTR _l_n_38$[ebp+4] + push edx + mov eax, DWORD PTR _l_n_38$[ebp] + push eax + mov ecx, DWORD PTR _l_v30706$[ebp+4] + push ecx + mov edx, DWORD PTR _l_v30706$[ebp] + push edx + call __alldiv + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } + mov DWORD PTR _l_v30727$[ebp], eax + mov DWORD PTR _l_v30727$[ebp+4], edx + +; 16651: OP_LLONG_XOR(l_v30706, l_n_38, l_v30728); + + mov eax, DWORD PTR _l_v30706$[ebp] + xor eax, DWORD PTR _l_n_38$[ebp] + mov ecx, DWORD PTR _l_v30706$[ebp+4] + xor ecx, DWORD PTR _l_n_38$[ebp+4] + mov DWORD PTR _l_v30728$[ebp], eax + mov DWORD PTR _l_v30728$[ebp+4], ecx + +; 16652: OP_LLONG_LE(l_v30728, 0LL, l_v30729); + + jg SHORT $LN15 at pypy_g__in + jl SHORT $LN21 at pypy_g__in + cmp DWORD PTR _l_v30728$[ebp], 0 + ja SHORT $LN15 at pypy_g__in +$LN21 at pypy_g__in: + mov DWORD PTR tv152[ebp], 1 + jmp SHORT $LN16 at pypy_g__in +$LN15 at pypy_g__in: + mov DWORD PTR tv152[ebp], 0 +$LN16 at pypy_g__in: + mov dl, BYTE PTR tv152[ebp] + mov BYTE PTR _l_v30729$[ebp], dl + +; 16653: OP_CAST_BOOL_TO_INT(l_v30729, l_v30730); + + movzx eax, BYTE PTR _l_v30729$[ebp] + mov DWORD PTR _l_v30730$[ebp], eax + +; 16654: OP_CAST_INT_TO_LONGLONG(l_v30730, l_v30731); + + mov eax, DWORD PTR _l_v30730$[ebp] + cdq + mov DWORD PTR _l_v30731$[ebp], eax + mov DWORD PTR _l_v30731$[ebp+4], edx + +; 16655: OP_LLONG_MOD(l_v30706, l_n_38, l_v30732); + + mov ecx, DWORD PTR _l_n_38$[ebp+4] + push ecx + mov edx, DWORD PTR _l_n_38$[ebp] + push edx + mov eax, DWORD PTR _l_v30706$[ebp+4] + push eax + mov ecx, DWORD PTR _l_v30706$[ebp] + push ecx + call __allrem + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } + mov DWORD PTR _l_v30732$[ebp], eax + mov DWORD PTR _l_v30732$[ebp+4], edx + +; 16656: OP_LLONG_NE(l_v30732, 0LL, l_v30733); + + mov edx, DWORD PTR _l_v30732$[ebp] + or edx, DWORD PTR _l_v30732$[ebp+4] + je SHORT $LN17 at pypy_g__in + mov DWORD PTR tv158[ebp], 1 + jmp SHORT $LN18 at pypy_g__in +$LN17 at pypy_g__in: + mov DWORD PTR tv158[ebp], 0 +$LN18 at pypy_g__in: + mov al, BYTE PTR tv158[ebp] + mov BYTE PTR _l_v30733$[ebp], al + +; 16657: OP_CAST_BOOL_TO_INT(l_v30733, l_v30734); + + movzx ecx, BYTE PTR _l_v30733$[ebp] + mov DWORD PTR _l_v30734$[ebp], ecx + +; 16658: OP_CAST_INT_TO_LONGLONG(l_v30734, l_v30735); + + mov eax, DWORD PTR _l_v30734$[ebp] + cdq + mov DWORD PTR _l_v30735$[ebp], eax + mov DWORD PTR _l_v30735$[ebp+4], edx + +; 16659: OP_LLONG_AND(l_v30731, l_v30735, l_v30736); + + mov edx, DWORD PTR _l_v30731$[ebp] + and edx, DWORD PTR _l_v30735$[ebp] + mov eax, DWORD PTR _l_v30731$[ebp+4] + and eax, DWORD PTR _l_v30735$[ebp+4] + mov DWORD PTR _l_v30736$[ebp], edx + mov DWORD PTR _l_v30736$[ebp+4], eax + +; 16660: OP_LLONG_SUB(l_v30727, l_v30736, l_v30705); + + mov ecx, DWORD PTR _l_v30727$[ebp] + sub ecx, DWORD PTR _l_v30736$[ebp] + mov edx, DWORD PTR _l_v30727$[ebp+4] + sbb edx, DWORD PTR _l_v30736$[ebp+4] + mov DWORD PTR _l_v30705$[ebp], ecx + mov DWORD PTR _l_v30705$[ebp+4], edx + +; 16661: OP_LLONG_AND(l_v30705, 2147483647LL, l_v30737); + + mov eax, DWORD PTR _l_v30705$[ebp] + and eax, 2147483647 ; 7fffffffH + mov ecx, DWORD PTR _l_v30705$[ebp+4] + and ecx, 0 + mov DWORD PTR _l_v30737$[ebp], eax + mov DWORD PTR _l_v30737$[ebp+4], ecx + +; 16662: OP_TRUNCATE_LONGLONG_TO_INT(l_v30737, l_v30704); + + mov edx, DWORD PTR _l_v30737$[ebp] + mov DWORD PTR _l_v30704$[ebp], edx + +; 16663: OP_INT_GE(l_v30704, 0L, l_v30738); + + xor eax, eax + cmp DWORD PTR _l_v30704$[ebp], 0 + setge al + mov BYTE PTR _l_v30738$[ebp], al + +; 16664: if (l_v30738) { + + movzx ecx, BYTE PTR _l_v30738$[ebp] + test ecx, ecx + je SHORT $LN2 at pypy_g__in + +; 16665: goto block11; + + jmp SHORT $block11$210908 +$LN2 at pypy_g__in: + +; 16666: } +; 16667: l_evalue_70 = (&pypy_g_exceptions_AssertionError.ae_super.se_super.e_super); + + mov DWORD PTR _l_evalue_70$[ebp], OFFSET _pypy_g_exceptions_AssertionError + +; 16668: goto block1; + + jmp $block1$210883 +$block11$210908: + +; 16669: +; 16670: block11: +; 16671: l_l_101 = RPyField(l_self_3688, prrr_inst_digits); + + mov edx, DWORD PTR _l_self_3688$[ebp] + mov eax, DWORD PTR [edx+8] + mov DWORD PTR _l_l_101$[ebp], eax + +; 16672: l_length_82 = l_l_101->length; + + mov ecx, DWORD PTR _l_l_101$[ebp] + mov edx, DWORD PTR [ecx+4] + mov DWORD PTR _l_length_82$[ebp], edx + +; 16673: OP_INT_LT(l_x_97, 0L, l_v30739); + + xor eax, eax + cmp DWORD PTR _l_x_97$[ebp], 0 + setl al + mov BYTE PTR _l_v30739$[ebp], al + +; 16674: if (l_v30739) { + + movzx ecx, BYTE PTR _l_v30739$[ebp] + test ecx, ecx + je SHORT $LN1 at pypy_g__in + +; 16675: goto block13; + + jmp $block13$210910 +$LN1 at pypy_g__in: + +; 16676: } +; 16677: l_index_219 = l_x_97; + + mov edx, DWORD PTR _l_x_97$[ebp] + mov DWORD PTR _l_index_219$[ebp], edx +$block12$210911: + +; 16678: goto block12; +; 16679: +; 16680: block12: +; 16681: OP_INT_GE(l_index_219, 0L, l_v30740); + + xor eax, eax + cmp DWORD PTR _l_index_219$[ebp], 0 + setge al + mov BYTE PTR _l_v30740$[ebp], al + +; 16682: RPyAssert(l_v30740, "negative list setitem index out of bound"); +; 16683: OP_INT_LT(l_index_219, l_length_82, l_v30742); + + mov ecx, DWORD PTR _l_index_219$[ebp] + xor edx, edx + cmp ecx, DWORD PTR _l_length_82$[ebp] + setl dl + mov BYTE PTR _l_v30742$[ebp], dl + +; 16684: RPyAssert(l_v30742, "list setitem index out of bound"); +; 16685: l_v30744 = l_l_101->length; + + mov eax, DWORD PTR _l_l_101$[ebp] + mov ecx, DWORD PTR [eax+4] + mov DWORD PTR _l_v30744$[ebp], ecx + +; 16686: OP_INT_LT(l_index_219, l_v30744, l_v30745); + + mov edx, DWORD PTR _l_index_219$[ebp] + xor eax, eax + cmp edx, DWORD PTR _l_v30744$[ebp] + setl al + mov BYTE PTR _l_v30745$[ebp], al + +; 16687: RPyAssert(l_v30745, "fixed setitem out of bounds"); +; 16688: RPyItem(l_l_101, l_index_219) = l_v30704; + + mov ecx, DWORD PTR _l_index_219$[ebp] + mov edx, DWORD PTR _l_l_101$[ebp] + mov eax, DWORD PTR _l_v30704$[ebp] + mov DWORD PTR [edx+ecx*4+8], eax + +; 16689: OP_LLONG_MUL(l_v30705, l_n_38, l_v30748); + + mov ecx, DWORD PTR _l_n_38$[ebp+4] + push ecx + mov edx, DWORD PTR _l_n_38$[ebp] + push edx + mov eax, DWORD PTR _l_v30705$[ebp+4] + push eax + mov ecx, DWORD PTR _l_v30705$[ebp] + push ecx + call __allmul + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } + mov DWORD PTR _l_v30748$[ebp], eax + mov DWORD PTR _l_v30748$[ebp+4], edx + +; 16690: OP_LLONG_SUB(l_v30706, l_v30748, l_v30749); + + mov edx, DWORD PTR _l_v30706$[ebp] + sub edx, DWORD PTR _l_v30748$[ebp] + mov eax, DWORD PTR _l_v30706$[ebp+4] + sbb eax, DWORD PTR _l_v30748$[ebp+4] + mov DWORD PTR _l_v30749$[ebp], edx + mov DWORD PTR _l_v30749$[ebp+4], eax + +; 16691: OP_INT_SUB(l_x_97, 1L, l_v30750); + + mov ecx, DWORD PTR _l_x_97$[ebp] + sub ecx, 1 + mov DWORD PTR _l_v30750$[ebp], ecx + +; 16692: l_v30702 = l_v30749; + + mov edx, DWORD PTR _l_v30749$[ebp] + mov DWORD PTR _l_v30702$[ebp], edx + mov eax, DWORD PTR _l_v30749$[ebp+4] + mov DWORD PTR _l_v30702$[ebp+4], eax + +; 16693: l_x_97 = l_v30750; + + mov ecx, DWORD PTR _l_v30750$[ebp] + mov DWORD PTR _l_x_97$[ebp], ecx + +; 16694: goto block7_back; + + jmp $block7_back$210895 +$block13$210910: + +; 16695: +; 16696: block13: +; 16697: OP_INT_ADD(l_x_97, l_length_82, l_v30751); + + mov edx, DWORD PTR _l_x_97$[ebp] + add edx, DWORD PTR _l_length_82$[ebp] + mov DWORD PTR _l_v30751$[ebp], edx + +; 16698: l_index_219 = l_v30751; + + mov eax, DWORD PTR _l_v30751$[ebp] + mov DWORD PTR _l_index_219$[ebp], eax + +; 16699: goto block12; + + jmp $block12$210911 +$block14$210899: + +; 16700: +; 16701: block14: +; 16702: OP_INT_ADD(l_x_97, l_length_83, l_v30752); + + mov ecx, DWORD PTR _l_x_97$[ebp] + add ecx, DWORD PTR _l_length_83$[ebp] + mov DWORD PTR _l_v30752$[ebp], ecx + +; 16703: l_index_218 = l_v30752; + + mov edx, DWORD PTR _l_v30752$[ebp] + mov DWORD PTR _l_index_218$[ebp], edx + +; 16704: goto block10; + + jmp $block10$210900 +$LN9 at pypy_g__in: + +; 16705: } + + mov esp, ebp + pop ebp + ret 0 +_pypy_g__inplace_divrem1 ENDP Added: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track_switch0.s ============================================================================== --- (empty file) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track_switch0.s Thu Oct 22 18:26:51 2009 @@ -0,0 +1,419 @@ +; Function compile flags: /Odtpy +; COMDAT _pypy_g_BuiltinActivation_UwS_UCD_ObjSpace_W_Root_W_Root +_TEXT SEGMENT +tv138 = -116 ; size = 4 +_l_v271568$ = -109 ; size = 1 +_l_v271567$ = -108 ; size = 4 +_l_v271575$ = -104 ; size = 4 +_l_v271551$ = -97 ; size = 1 +_l_v271579$ = -96 ; size = 4 +_l_v271576$ = -89 ; size = 1 +_l_v271572$ = -88 ; size = 4 +_l_v271583$ = -84 ; size = 4 +_l_v271556$ = -80 ; size = 4 +_l_v271559$ = -76 ; size = 4 +_l_v271544$ = -72 ; size = 4 +_l_v271545$ = -68 ; size = 4 +_l_v271580$ = -64 ; size = 4 +_l_v271557$ = -60 ; size = 4 +_l_v271581$ = -56 ; size = 4 +_l_v271553$ = -52 ; size = 4 +_l_v271570$ = -48 ; size = 4 +_l_v271554$ = -42 ; size = 1 +_l_v271565$ = -41 ; size = 1 +_l_v271558$ = -40 ; size = 4 +_l_v271562$ = -33 ; size = 1 +_l_v271561$ = -32 ; size = 4 +_l_v271547$ = -28 ; size = 4 +_l_v271548$ = -24 ; size = 4 +_l_v271573$ = -18 ; size = 1 +_l_v271546$ = -17 ; size = 1 +_l_v271582$ = -16 ; size = 4 +_l_v271550$ = -12 ; size = 4 +_l_v271564$ = -8 ; size = 4 +_l_v271578$ = -4 ; size = 4 +_l_self_596$ = 8 ; size = 4 +_l_scope_w_259$ = 12 ; size = 4 +_pypy_g_BuiltinActivation_UwS_UCD_ObjSpace_W_Root_W_Root PROC ; COMDAT + +; 15629: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *pypy_g_BuiltinActivation_UwS_UCD_ObjSpace_W_Root_W_Root(struct pypy_pypy_interpreter_gateway_BuiltinActivation0 *l_self_596, struct pypy_array1 *l_scope_w_259) { + + push ebp + mov ebp, esp + sub esp, 116 ; 00000074H +$block0$211591: + +; 15630: bool_t l_v271551; bool_t l_v271554; bool_t l_v271562; +; 15631: bool_t l_v271565; bool_t l_v271568; bool_t l_v271573; +; 15632: bool_t l_v271576; char l_v271546; long l_v271550; long l_v271553; +; 15633: long l_v271564; long l_v271567; long l_v271572; long l_v271575; +; 15634: struct pypy_object0 *l_v271556; struct pypy_object0 *l_v271570; +; 15635: struct pypy_object0 *l_v271578; +; 15636: struct pypy_object_vtable0 *l_v271561; +; 15637: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271544; +; 15638: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271545; +; 15639: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271557; +; 15640: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271579; +; 15641: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271580; +; 15642: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271581; +; 15643: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271582; +; 15644: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271583; +; 15645: struct pypy_pypy_interpreter_gateway_BuiltinActivation_UwS_UCD0 *l_v271548; +; 15646: struct pypy_pypy_module_unicodedata_interp_ucd_UCD0 *l_v271547; +; 15647: void* l_v271558; void* l_v271559; +; 15648: +; 15649: block0: +; 15650: l_v271548 = (struct pypy_pypy_interpreter_gateway_BuiltinActivation_UwS_UCD0 *)l_self_596; + + mov eax, DWORD PTR _l_self_596$[ebp] + mov DWORD PTR _l_v271548$[ebp], eax + +; 15651: l_v271546 = RPyField(l_v271548, bausucdoswrwr_inst_behavior); + + mov ecx, DWORD PTR _l_v271548$[ebp] + mov dl, BYTE PTR [ecx+8] + mov BYTE PTR _l_v271546$[ebp], dl + +; 15652: RPyAssert(1, "unexpectedly negative list getitem index"); +; 15653: l_v271550 = l_scope_w_259->length; + + mov eax, DWORD PTR _l_scope_w_259$[ebp] + mov ecx, DWORD PTR [eax+4] + mov DWORD PTR _l_v271550$[ebp], ecx + +; 15654: OP_INT_LT(0L, l_v271550, l_v271551); + + xor edx, edx + cmp DWORD PTR _l_v271550$[ebp], 0 + setg dl + mov BYTE PTR _l_v271551$[ebp], dl + +; 15655: RPyAssert(l_v271551, "list getitem index out of bound"); +; 15656: l_v271553 = l_scope_w_259->length; + + mov eax, DWORD PTR _l_scope_w_259$[ebp] + mov ecx, DWORD PTR [eax+4] + mov DWORD PTR _l_v271553$[ebp], ecx + +; 15657: OP_INT_LT(0L, l_v271553, l_v271554); + + xor edx, edx + cmp DWORD PTR _l_v271553$[ebp], 0 + setg dl + mov BYTE PTR _l_v271554$[ebp], dl + +; 15658: RPyAssert(l_v271554, "fixed getitem out of bounds"); +; 15659: l_v271556 = RPyItem(l_scope_w_259, 0L); + + mov eax, DWORD PTR _l_scope_w_259$[ebp] + mov ecx, DWORD PTR [eax+8] + mov DWORD PTR _l_v271556$[ebp], ecx + +; 15660: l_v271557 = (struct pypy_pypy_interpreter_baseobjspace_W_Root0 *)l_v271556; + + mov edx, DWORD PTR _l_v271556$[ebp] + mov DWORD PTR _l_v271557$[ebp], edx + +; 15661: l_v271547 = pypy_g_interp_w__UCD(l_v271557, 0); + + push 0 + mov eax, DWORD PTR _l_v271557$[ebp] + push eax + call _pypy_g_interp_w__UCD + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | 12(%ebp)} + add esp, 8 + mov DWORD PTR _l_v271547$[ebp], eax + +; 15662: l_v271558 = (void*)l_scope_w_259; + + mov ecx, DWORD PTR _l_scope_w_259$[ebp] + mov DWORD PTR _l_v271558$[ebp], ecx + +; 15663: l_v271559 = pypy_asm_gcroot(l_v271558); + + test DWORD PTR _l_v271558$[ebp], 0 + mov edx, DWORD PTR _l_v271558$[ebp] + mov DWORD PTR _l_v271559$[ebp], edx + +; 15664: l_scope_w_259 = l_v271559; /* for moving GCs */ + + mov eax, DWORD PTR _l_v271559$[ebp] + mov DWORD PTR _l_scope_w_259$[ebp], eax + +; 15665: l_v271561 = RPyField((&pypy_g_ExcData), ed_exc_type); + + mov ecx, DWORD PTR _pypy_g_ExcData + mov DWORD PTR _l_v271561$[ebp], ecx + +; 15666: l_v271562 = (l_v271561 == NULL); + + xor edx, edx + cmp DWORD PTR _l_v271561$[ebp], 0 + sete dl + mov BYTE PTR _l_v271562$[ebp], dl + +; 15667: if (!l_v271562) { + + movzx eax, BYTE PTR _l_v271562$[ebp] + test eax, eax + jne SHORT $block1$211600 + +; 15668: l_v271583 = ((struct pypy_pypy_interpreter_baseobjspace_W_Root0 *) NULL); + + mov DWORD PTR _l_v271583$[ebp], 0 + +; 15669: goto block3; + + jmp $block3$211599 +$block1$211600: + +; 15670: } +; 15671: goto block1; +; 15672: +; 15673: block1: +; 15674: RPyAssert(1, "unexpectedly negative list getitem index"); +; 15675: l_v271564 = l_scope_w_259->length; + + mov ecx, DWORD PTR _l_scope_w_259$[ebp] + mov edx, DWORD PTR [ecx+4] + mov DWORD PTR _l_v271564$[ebp], edx + +; 15676: OP_INT_LT(1L, l_v271564, l_v271565); + + xor eax, eax + cmp DWORD PTR _l_v271564$[ebp], 1 + setg al + mov BYTE PTR _l_v271565$[ebp], al + +; 15677: RPyAssert(l_v271565, "list getitem index out of bound"); +; 15678: l_v271567 = l_scope_w_259->length; + + mov ecx, DWORD PTR _l_scope_w_259$[ebp] + mov edx, DWORD PTR [ecx+4] + mov DWORD PTR _l_v271567$[ebp], edx + +; 15679: OP_INT_LT(1L, l_v271567, l_v271568); + + xor eax, eax + cmp DWORD PTR _l_v271567$[ebp], 1 + setg al + mov BYTE PTR _l_v271568$[ebp], al + +; 15680: RPyAssert(l_v271568, "fixed getitem out of bounds"); +; 15681: l_v271570 = RPyItem(l_scope_w_259, 1L); + + mov ecx, DWORD PTR _l_scope_w_259$[ebp] + mov edx, DWORD PTR [ecx+12] + mov DWORD PTR _l_v271570$[ebp], edx + +; 15682: l_v271544 = (struct pypy_pypy_interpreter_baseobjspace_W_Root0 *)l_v271570; + + mov eax, DWORD PTR _l_v271570$[ebp] + mov DWORD PTR _l_v271544$[ebp], eax + +; 15683: RPyAssert(1, "unexpectedly negative list getitem index"); +; 15684: l_v271572 = l_scope_w_259->length; + + mov ecx, DWORD PTR _l_scope_w_259$[ebp] + mov edx, DWORD PTR [ecx+4] + mov DWORD PTR _l_v271572$[ebp], edx + +; 15685: OP_INT_LT(2L, l_v271572, l_v271573); + + xor eax, eax + cmp DWORD PTR _l_v271572$[ebp], 2 + setg al + mov BYTE PTR _l_v271573$[ebp], al + +; 15686: RPyAssert(l_v271573, "list getitem index out of bound"); +; 15687: l_v271575 = l_scope_w_259->length; + + mov ecx, DWORD PTR _l_scope_w_259$[ebp] + mov edx, DWORD PTR [ecx+4] + mov DWORD PTR _l_v271575$[ebp], edx + +; 15688: OP_INT_LT(2L, l_v271575, l_v271576); + + xor eax, eax + cmp DWORD PTR _l_v271575$[ebp], 2 + setg al + mov BYTE PTR _l_v271576$[ebp], al + +; 15689: RPyAssert(l_v271576, "fixed getitem out of bounds"); +; 15690: l_v271578 = RPyItem(l_scope_w_259, 2L); + + mov ecx, DWORD PTR _l_scope_w_259$[ebp] + mov edx, DWORD PTR [ecx+16] + mov DWORD PTR _l_v271578$[ebp], edx + +; 15691: l_v271545 = (struct pypy_pypy_interpreter_baseobjspace_W_Root0 *)l_v271578; + + mov eax, DWORD PTR _l_v271578$[ebp] + mov DWORD PTR _l_v271545$[ebp], eax + +; 15692: switch (l_v271546) { + + movsx ecx, BYTE PTR _l_v271546$[ebp] + mov DWORD PTR tv138[ebp], ecx + cmp DWORD PTR tv138[ebp], 3 + ja SHORT $LN1 at pypy_g_Bui@2 + mov edx, DWORD PTR tv138[ebp] + jmp DWORD PTR $LN14 at pypy_g_Bui@2[edx*4] +$LN5 at pypy_g_Bui@2: + +; 15693: case 0: +; 15694: goto block2; + + jmp SHORT $block2$211608 +$LN4 at pypy_g_Bui@2: + +; 15695: case 1: +; 15696: goto block4; + + jmp SHORT $block4$211610 +$LN3 at pypy_g_Bui@2: + +; 15697: case 2: +; 15698: goto block5; + + jmp SHORT $block5$211612 +$LN2 at pypy_g_Bui@2: + +; 15699: case 3: +; 15700: goto block6; + + jmp $block6$211614 +$LN1 at pypy_g_Bui@2: + +; 15701: default: +; 15702: assert(!"bad switch!!"); + + mov eax, OFFSET ??_C at _0N@PGLFNKFI at bad?5switch?$CB?$CB?$AA@ + test eax, eax + je SHORT $block2$211608 + push 15702 ; 00003d56H + push OFFSET ??_C at _1BO@DMBFIACJ@?$AAi?$AAm?$AAp?$AAl?$AAe?$AAm?$AAe?$AAn?$AAt?$AA_?$AA1?$AA1?$AA?4?$AAc?$AA?$AA@ + push OFFSET ??_C at _1CA@EIJBLFPJ@?$AA?$CB?$AA?$CC?$AAb?$AAa?$AAd?$AA?5?$AAs?$AAw?$AAi?$AAt?$AAc?$AAh?$AA?$CB?$AA?$CB?$AA?$CC?$AA?$AA@ + call DWORD PTR __imp___wassert + add esp, 12 ; 0000000cH +$block2$211608: + +; 15703: } +; 15704: +; 15705: block2: +; 15706: l_v271579 = pypy_g_UCD_digit(l_v271547, l_v271544, l_v271545); + + mov edx, DWORD PTR _l_v271545$[ebp] + push edx + mov eax, DWORD PTR _l_v271544$[ebp] + push eax + mov ecx, DWORD PTR _l_v271547$[ebp] + push ecx + call _pypy_g_UCD_digit + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } + add esp, 12 ; 0000000cH + mov DWORD PTR _l_v271579$[ebp], eax + +; 15707: l_v271583 = l_v271579; + + mov edx, DWORD PTR _l_v271579$[ebp] + mov DWORD PTR _l_v271583$[ebp], edx +$block3$211599: + +; 15708: goto block3; +; 15709: +; 15710: block3: +; 15711: RPY_DEBUG_RETURN(); +; 15712: return l_v271583; + + mov eax, DWORD PTR _l_v271583$[ebp] + jmp SHORT $LN9 at pypy_g_Bui@2 +$block4$211610: + +; 15713: +; 15714: block4: +; 15715: l_v271580 = pypy_g_UCD_name(l_v271547, l_v271544, l_v271545); + + mov eax, DWORD PTR _l_v271545$[ebp] + push eax + mov ecx, DWORD PTR _l_v271544$[ebp] + push ecx + mov edx, DWORD PTR _l_v271547$[ebp] + push edx + call _pypy_g_UCD_name + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } + add esp, 12 ; 0000000cH + mov DWORD PTR _l_v271580$[ebp], eax + +; 15716: l_v271583 = l_v271580; + + mov eax, DWORD PTR _l_v271580$[ebp] + mov DWORD PTR _l_v271583$[ebp], eax + +; 15717: goto block3; + + jmp SHORT $block3$211599 +$block5$211612: + +; 15718: +; 15719: block5: +; 15720: l_v271581 = pypy_g_UCD_decimal(l_v271547, l_v271544, l_v271545); + + mov ecx, DWORD PTR _l_v271545$[ebp] + push ecx + mov edx, DWORD PTR _l_v271544$[ebp] + push edx + mov eax, DWORD PTR _l_v271547$[ebp] + push eax + call _pypy_g_UCD_decimal + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } + add esp, 12 ; 0000000cH + mov DWORD PTR _l_v271581$[ebp], eax + +; 15721: l_v271583 = l_v271581; + + mov ecx, DWORD PTR _l_v271581$[ebp] + mov DWORD PTR _l_v271583$[ebp], ecx + +; 15722: goto block3; + + jmp SHORT $block3$211599 +$block6$211614: + +; 15723: +; 15724: block6: +; 15725: l_v271582 = pypy_g_UCD_numeric(l_v271547, l_v271544, l_v271545); + + mov edx, DWORD PTR _l_v271545$[ebp] + push edx + mov eax, DWORD PTR _l_v271544$[ebp] + push eax + mov ecx, DWORD PTR _l_v271547$[ebp] + push ecx + call _pypy_g_UCD_numeric + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } + add esp, 12 ; 0000000cH + mov DWORD PTR _l_v271582$[ebp], eax + +; 15726: l_v271583 = l_v271582; + + mov edx, DWORD PTR _l_v271582$[ebp] + mov DWORD PTR _l_v271583$[ebp], edx + +; 15727: goto block3; + + jmp SHORT $block3$211599 +$LN9 at pypy_g_Bui@2: + +; 15728: } + + mov esp, ebp + pop ebp + ret 0 + npad 3 +$LN14 at pypy_g_Bui@2: + DD $LN5 at pypy_g_Bui@2 + DD $LN4 at pypy_g_Bui@2 + DD $LN3 at pypy_g_Bui@2 + DD $LN2 at pypy_g_Bui@2 +_pypy_g_BuiltinActivation_UwS_UCD_ObjSpace_W_Root_W_Root ENDP Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Thu Oct 22 18:26:51 2009 @@ -32,8 +32,6 @@ r_insn = re.compile(r"\t([a-z]\w*)\s") r_jmp_switch = re.compile(r"\tjmp\t[*]"+LABEL+"[(]") r_jmp_source = re.compile(r"\d*[(](%[\w]+)[,)]") -r_jmptable_item = re.compile(r"\t.long\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") -r_jmptable_end = re.compile(r"\t.text|\t.section\s+.text|\t\.align|"+LABEL) class FunctionGcRootTracker(object): skip = 0 @@ -41,7 +39,6 @@ @classmethod def init_regexp(cls): cls.r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+cls.OPERAND+")\s*$") - cls.r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+([*]"+cls.OPERAND+")\s*$") cls.r_binaryinsn = re.compile(r"\t[a-z]\w*\s+(?P"+cls.OPERAND+"),\s*(?P"+cls.OPERAND+")\s*$") cls.r_jump = re.compile(r"\tj\w+\s+"+cls.LABEL+"\s*$") @@ -539,7 +536,7 @@ # registry: # movl L9341(%eax), %eax # jmp *%eax - operand = self.r_unaryinsn_star.match(line).group(1)[1:] + operand = self.r_unaryinsn_star.match(line).group(1) def walker(insn, locs): sources = [] for loc in locs: @@ -567,10 +564,10 @@ assert len(tablelabels) <= 1 if tablelabels: tablelin = self.labels[tablelabels[0]].lineno + 1 - while not r_jmptable_end.match(self.lines[tablelin]): - match = r_jmptable_item.match(self.lines[tablelin]) + while not self.r_jmptable_end.match(self.lines[tablelin]): + match = self.r_jmptable_item.match(self.lines[tablelin]) if not match: - raise NoPatternMatch(self.lines[tablelin]) + raise NoPatternMatch(repr(self.lines[tablelin])) label = match.group(1) if label != '0': self.register_jump_to(label) @@ -660,6 +657,10 @@ # Function name is decorated with "@N" where N is the stack size if match and '@' in target: insns.append(InsnStackAdjust(int(target.rsplit('@', 1)[1]))) + # XXX some functions seem use the "fastcall" calling convention + # without their declaration, how can we guess the stack effect? + if match and target in ['__alldiv', '__allrem', '__allmul']: + insns.append(InsnStackAdjust(16)) return insns @@ -685,6 +686,10 @@ r_localvar_esp = re.compile(r"(\d*)[(]%esp[)]") r_localvar_ebp = re.compile(r"(-?\d*)[(]%ebp[)]") + r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+[*]("+OPERAND+")\s*$") + r_jmptable_item = re.compile(r"\t.long\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") + r_jmptable_end = re.compile(r"\t.text|\t.section\s+.text|\t\.align|"+LABEL) + r_gcroot_marker = re.compile(r"\t/[*] GCROOT ("+LOCALVARFP+") [*]/") r_bottom_marker = re.compile(r"\t/[*] GC_STACK_BOTTOM [*]/") @@ -746,6 +751,10 @@ r_gcroot_marker = re.compile(r";.+ = pypy_asm_gcroot\(") r_bottom_marker = re.compile(r"\tcall\t_pypy_asm_stack_bottom\s*") + r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+DWORD PTR ("+OPERAND+")\s*$") + r_jmptable_item = re.compile(r"\tDD\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") + r_jmptable_end = re.compile(r"[^\t]") + @classmethod def init_regexp(cls): super(MsvcFunctionGcRootTracker, cls).init_regexp() @@ -1264,10 +1273,13 @@ else: FUNCTIONS_NOT_RETURNING = { '_abort': None, - '__imp__abort': None, '__exit': None, '__assert': None, '__wassert': None, + '__imp__abort': None, + '__imp___wassert': None, + 'DWORD PTR __imp__abort': None, + 'DWORD PTR __imp___wassert': None, } # __________ debugging output __________ From benjamin at codespeak.net Fri Oct 23 02:23:03 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 23 Oct 2009 02:23:03 +0200 (CEST) Subject: [pypy-svn] r68715 - pypy/trunk/pypy/doc/jit Message-ID: <20091023002303.0C20E49843B@codespeak.net> Author: benjamin Date: Fri Oct 23 02:23:02 2009 New Revision: 68715 Modified: pypy/trunk/pypy/doc/jit/pyjitpl5.txt Log: edits; start writing more about optimization Modified: pypy/trunk/pypy/doc/jit/pyjitpl5.txt ============================================================================== --- pypy/trunk/pypy/doc/jit/pyjitpl5.txt (original) +++ pypy/trunk/pypy/doc/jit/pyjitpl5.txt Fri Oct 23 02:23:02 2009 @@ -11,7 +11,7 @@ The JIT's `theory`_ is great in principle, but actual code is a different story. This section tries to give a high level overview of how PyPy's JIT is implemented. It's helpful to have a basic understanding of the PyPy -`translation tool chain`_ before digging into the sources. +`translation tool chain`_ works before digging into the sources. Almost all JIT specific code is found in the two pypy/jit subdirectories, metainterp, and backend. The metainterp directory holds platform independent @@ -26,12 +26,12 @@ --------- To add a JIT to an interpreter, PyPy only requires that two hints be added to -the interpreter source. These are jit_merge_point and can_enter_jit. +the target interpreter. These are jit_merge_point and can_enter_jit. jit_merge_point is supposed to go at the start of opcode dispatch. It allows -the JIT to bail back to the interpreter in case assembler code fails at some -point. can_enter_jit goes at the close of a application level loop. In the -Python interpreter, this is the JUMP_ABSOLUTE bytecode. The Python interpreter -defines its hints in pypy/module/pypyjit/interp_jit.py. +the JIT to bail back to the interpreter in case running machine code is no +longer suitable. can_enter_jit goes at the end of a application level loop. In +the Python interpreter, this is the JUMP_ABSOLUTE bytecode. The Python +interpreter defines its hints in pypy/module/pypyjit/interp_jit.py. The interpreter wishing to use the PyPy's JIT must define a list of *green* variables and a list of *red* variables. The *green* variables are loop @@ -46,15 +46,16 @@ -------------- After the RTyping phase of translation, where high level Python operations are -turned into low-level ones for the backend, the translator calls apply_jit() in -metainterp/warmspot.py to add a JIT compiler to the currently translating -interpreter. apply_jit() decides what assembler backend to use then delegates -the rest of the work to the WarmRunnerDesc class. WarmRunnerDesc finds the two -JIT hints in the function graphs. It rewrites the graph containing the -jit_merge_point hint, called the portal graph, to be able to handle special JIT -exceptions, which indicate special operations to the interpreter. The location -of the can_enter_jit hint is changed to check if the current loop is "hot" and -should be compiled. +turned into low-level ones for the backend, the translation driver calls +apply_jit() in metainterp/warmspot.py to add a JIT compiler to the currently +translating interpreter. apply_jit() decides what assembler backend to use then +delegates the rest of the work to the WarmRunnerDesc class. WarmRunnerDesc +finds the two JIT hints in the function graphs. It rewrites the graph +containing the jit_merge_point hint, called the portal graph, to be able to +handle special JIT exceptions, which indicate special coditions to the +interpreter upon exiting from the JIT. The location of the can_enter_jit hint +is replaced with a call to a function, maybe_compile_and_run in warmstate.py, +that checks if current loop is "hot" and should be compiled. Next, starting with the portal graph, metainterp/codewriter.py converts the graphs of the interpreter into JIT bytecode. Since this bytecode is stored in @@ -63,7 +64,7 @@ part of the interpreter. In these cases, it simply inserts an opaque call. Finally, translation finishes, including the bytecode of the interpreter in the -final binary, and interpreter is ready to use the runtime component of the JIT! +final binary, and interpreter is ready to use the runtime component of the JIT. Tracing @@ -81,7 +82,7 @@ of the interpreter interpreting the application level code. This allows it to see the exact operations that make up the application level loop. Tracing is preformed by MetaInterp and MIFrame classes in metainterp/pyjitpl.py. -maybe_compile_and_run() creates a MetaInterp and calls the +maybe_compile_and_run() creates a MetaInterp and calls its compile_and_run_once() method. This initializes the MIFrame for the input arguments of the loop, the red and green variables passed from the jit_merge_point hint, and sets it to start interpreting the bytecode of the @@ -89,8 +90,8 @@ Before starting the interpretation, the loop input arguments are wrapped in a *box*. Boxes (defined in metainterp/history.py) wrap the value and type of a -variable in the program the JIT is interpreting. There are two main varieties -of boxes: constant boxes and normal boxes. Constant boxes are used for values +value in the program the JIT is interpreting. There are two main varieties of +boxes: constant boxes and normal boxes. Constant boxes are used for values assumed to be known during tracing. These are not necessarily compile time constants. All values which are "promoted", assumed to be constant by the JIT for optimization purposes, are also stored in constant boxes. Normal boxes @@ -99,14 +100,17 @@ boxes: ConstInt, ConstPtr, ConstFloat, and ConstAddr. The meta-interpreter starts interpreting the JIT bytecode. Each operation is -executed and then recorded in a list of operations and arguments called the -trace. All possible operations generated by tracing are listed in -metainterp/resoperation.py. When a (interpreter-level) call to a function the -JIT has bytecode for occurs in the bytecode, another frame is added to the stack -and the tracing continues with the same history. This flattens the list of -operations over calls. Most importantly, it unrolls the opcode dispatch loop. -Interpretation continues until the can_enter_jit hint is seen. At this point, a -whole interation of the application level loop has been seen and recorded. +executed and then recorded in a list of operations, called the trace. +Operations can have a list of boxes that operate on, arguments. Some operations +(like GETFIELD and GETARRAYITEM) also have special objects that describe how +their arguments are layed out in memory. All possible operations generated by +tracing are listed in metainterp/resoperation.py. When a (interpreter-level) +call to a function the JIT has bytecode for occurs during tracing, another +MIFrame is added to the stack and the tracing continues with the same history. +This flattens the list of operations over calls. Most importantly, it unrolls +the opcode dispatch loop. Interpretation continues until the can_enter_jit hint +is seen. At this point, a whole interation of the application level loop has +been seen and recorded. Because only one iteration has been recorded the JIT only knows about one codepath in the loop. For example, if there's a if statement construct like @@ -119,22 +123,39 @@ and ``x`` is true when the JIT does tracing, only the codepath ``do_something_exciting`` will be added to the trace. In future runs, to ensure -that it is picking the right path, a special operation called a *guard -operation* is added to the list. A guard is a small test that checks if -assumptions the JIT makes during tracing are still true. In the example above, -a GUARD_TRUE guard will be generated for ``x`` before running -``do_something_exciting``. - - -Optimization ------------- +that this path is still valid, a special operation called a *guard operation* is +added to the trace. A guard is a small test that checks if assumptions the JIT +makes during tracing are still true. In the example above, a GUARD_TRUE guard +will be generated for ``x`` before running ``do_something_exciting``. Once the meta-interpreter has verified that it has traced a loop, it decides how to compile what it has. There is an optional optimization phase between these actions which is covered future down this page. The backend converts the trace -operations into assembly for the particular machine. It then hands it back to -the frontend. The next time the loop is seen in application code, the optimized -version can be run instead of the normal intepreter! +operations into assembly for the particular machine. It then hands the compiled +loop back to the frontend. The next time the loop is seen in application code, +the optimized assembly can be run instead of the normal intepreter. + + +Optimizations +------------- + +The JIT employs several techniques, old and new, to make machine code run +faster. + +Virtuals and Virtualizables +*************************** + +A *virtual* value is an array, struct, or RPython level instance that is created +during the loop and does not escape from it via calls or longevity past the +loop. Since it is only used by the JIT, it be "optimized out"; the value +doesn't have to be allocated at all and its fields can be stored as first class +values instead of deferencing them in memory. Virtuals allow temporary objects +in the interpreter to be unwrapped. For example, a W_IntObject in the PyPy can +be unwrapped to just be its integer value as long as the object is known not to +escape the machine code. + +Most of the JIT's optimizer is contained 2 files optimizefindnodes.py and +optimizeopt.py. More resources @@ -145,7 +166,7 @@ * `Tracing the Meta-Level: PyPy's Tracing JIT Compiler`__ -.. __: http://codespeak.net/svn/pypy/extradoc/talk/icooolps2009/bolz-tracing-jit.pdf +.. __: http://codespeak.net/svn/pypy/extradoc/talk/ipcoolps2009/bolz-tracing-jit.pdf as well as the `blog posts with the JIT tag.`__ From fijal at codespeak.net Fri Oct 23 08:05:12 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 23 Oct 2009 08:05:12 +0200 (CEST) Subject: [pypy-svn] r68716 - pypy/trunk/pypy/doc/jit Message-ID: <20091023060512.E5DC949843B@codespeak.net> Author: fijal Date: Fri Oct 23 08:05:11 2009 New Revision: 68716 Modified: pypy/trunk/pypy/doc/jit/pyjitpl5.txt Log: Er, I'm missing something in this sentence, like this? Modified: pypy/trunk/pypy/doc/jit/pyjitpl5.txt ============================================================================== --- pypy/trunk/pypy/doc/jit/pyjitpl5.txt (original) +++ pypy/trunk/pypy/doc/jit/pyjitpl5.txt Fri Oct 23 08:05:11 2009 @@ -10,7 +10,7 @@ The JIT's `theory`_ is great in principle, but actual code is a different story. This section tries to give a high level overview of how PyPy's JIT is -implemented. It's helpful to have a basic understanding of the PyPy +implemented. It's helpful to have a basic understanding of how the PyPy `translation tool chain`_ works before digging into the sources. Almost all JIT specific code is found in the two pypy/jit subdirectories, From fijal at codespeak.net Fri Oct 23 10:31:48 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 23 Oct 2009 10:31:48 +0200 (CEST) Subject: [pypy-svn] r68717 - in pypy/trunk/pypy/jit/tool: . test Message-ID: <20091023083148.8D0E449843B@codespeak.net> Author: fijal Date: Fri Oct 23 10:31:43 2009 New Revision: 68717 Added: pypy/trunk/pypy/jit/tool/ pypy/trunk/pypy/jit/tool/__init__.py (contents, props changed) pypy/trunk/pypy/jit/tool/jitoutput.py (contents, props changed) pypy/trunk/pypy/jit/tool/test/ pypy/trunk/pypy/jit/tool/test/__init__.py (contents, props changed) pypy/trunk/pypy/jit/tool/test/test_jitoutput.py (contents, props changed) Log: Add a tool for parsing results of jit profiling Added: pypy/trunk/pypy/jit/tool/__init__.py ============================================================================== Added: pypy/trunk/pypy/jit/tool/jitoutput.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/jit/tool/jitoutput.py Fri Oct 23 10:31:43 2009 @@ -0,0 +1,75 @@ + +""" Helpers for parsing various outputs jit produces. +Notably: +1. Statistics of log.ops +2. Parsing what jitprof produces +""" + +import re + +REGEXES = [ + (('tracing_no', 'tracing_time'), '^Tracing:\s+([\d.]+)\s+([\d.]+)$'), + (('backend_no', 'backend_time'), '^Backend:\s+([\d.]+)\s+([\d.]+)$'), + (('asm_no', 'asm_time'), '^Running asm:\s+([\d.]+)\s+([\d.]+)$'), + (('blackhole_no', 'blackhole_time'), + '^Blackhole:\s+([\d.]+)\s+([\d.]+)$'), + (None, '^TOTAL.*$'), + (('ops.total',), '^ops:\s+(\d+)$'), + (('ops.calls',), '^\s+calls:\s+(\d+)$'), + (('ops.pure_calls',), '^\s+pure calls:\s+(\d+)$'), + (('recorded_ops.total',), '^recorded ops:\s+(\d+)$'), + (('recorded_ops.calls',), '^\s+calls:\s+(\d+)$'), + (('recorded_ops.pure_calls',), '^\s+pure calls:\s+(\d+)$'), + (('guards',), '^guards:\s+(\d+)$'), + (('blackholed_ops.total',), '^blackholed ops:\s+(\d+)$'), + (('blackholed_ops.pure_calls',), '^\s+pure calls:\s+(\d+)$'), + (('opt_ops',), '^opt ops:\s+(\d+)$'), + (('opt_guards',), '^opt guards:\s+(\d+)$'), + (('forcings',), '^forcings:\s+(\d+)$'), + (('trace_too_long',), '^trace too long:\s+(\d+)$'), + (('bridge_abort',), '^bridge abort:\s+(\d+)$'), + ] + +class Ops(object): + total = 0 + calls = 0 + pure_calls = 0 + +class OutputInfo(object): + tracing_no = 0 + tracing_time = 0.0 + backend_no = 0 + backend_time = 0.0 + asm_no = 0 + asm_time = 0.0 + guards = 0 + opt_ops = 0 + opt_guards = 0 + trace_too_long = 0 + bridge_abort = 0 + + def __init__(self): + self.ops = Ops() + self.recorded_ops = Ops() + self.blackholed_ops = Ops() + +def parse_prof(output): + lines = output.splitlines() + # assert len(lines) == len(REGEXES) + info = OutputInfo() + for (attrs, regexp), line in zip(REGEXES, lines): + m = re.match(regexp, line) + assert m is not None, "Error parsing line: %s" % line + if attrs: + for i, a in enumerate(attrs): + v = m.group(i + 1) + if '.' in v: + v = float(v) + else: + v = int(v) + if '.' in a: + before, after = a.split('.') + setattr(getattr(info, before), after, v) + else: + setattr(info, a, v) + return info Added: pypy/trunk/pypy/jit/tool/test/__init__.py ============================================================================== Added: pypy/trunk/pypy/jit/tool/test/test_jitoutput.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/jit/tool/test/test_jitoutput.py Fri Oct 23 10:31:43 2009 @@ -0,0 +1,96 @@ + +import py +from pypy.jit.metainterp.warmspot import ll_meta_interp +from pypy.rlib.jit import JitDriver, DEBUG_PROFILE +from pypy.jit.backend.llgraph import runner +from pypy.jit.metainterp.jitprof import Profiler, JITPROF_LINES +from pypy.jit.tool.jitoutput import parse_prof + +def test_really_run(): + """ This test checks whether output of jitprof did not change. + It'll explode when someone touches jitprof.py + """ + mydriver = JitDriver(reds = ['i', 'n'], greens = []) + def f(n): + i = 0 + while i < n: + mydriver.can_enter_jit(i=i, n=n) + mydriver.jit_merge_point(i=i, n=n) + i += 1 + + cap = py.io.StdCaptureFD() + try: + ll_meta_interp(f, [10], CPUClass=runner.LLtypeCPU, type_system='lltype', + ProfilerClass=Profiler, debug_level=DEBUG_PROFILE) + finally: + out, err = cap.reset() + err = "\n".join(err.splitlines()[-JITPROF_LINES:]) + print err + assert err.count("\n") == JITPROF_LINES - 1 + info = parse_prof(err) + # assert did not crash + # asserts below are a bit delicate, possibly they might be deleted + assert info.tracing_no == 1 + assert info.asm_no == 1 + assert info.blackhole_no == 1 + assert info.backend_no == 1 + assert info.ops.total == 2 + assert info.ops.calls == 0 + assert info.ops.pure_calls == 0 + assert info.recorded_ops.total == 2 + assert info.recorded_ops.calls == 0 + assert info.recorded_ops.pure_calls == 0 + assert info.guards == 1 + assert info.blackholed_ops.total == 0 + assert info.blackholed_ops.pure_calls == 0 + assert info.opt_ops == 6 + assert info.opt_guards == 1 + assert info.forcings == 0 + assert info.trace_too_long == 0 + assert info.bridge_abort == 0 + +DATA = '''Tracing: 1 0.006992 +Backend: 1 0.000525 +Running asm: 1 0.016957 +Blackhole: 1 0.000233 +TOTAL: 0.025532 +ops: 2 + calls: 1 + pure calls: 1 +recorded ops: 6 + calls: 3 + pure calls: 2 +guards: 1 +blackholed ops: 5 + pure calls: 3 +opt ops: 6 +opt guards: 1 +forcings: 1 +trace too long: 2 +bridge abort: 3 +''' + +def test_parse(): + info = parse_prof(DATA) + assert info.tracing_no == 1 + assert info.tracing_time == 0.006992 + assert info.asm_no == 1 + assert info.asm_time == 0.016957 + assert info.blackhole_no == 1 + assert info.blackhole_time == 0.000233 + assert info.backend_no == 1 + assert info.backend_time == 0.000525 + assert info.ops.total == 2 + assert info.ops.calls == 1 + assert info.ops.pure_calls == 1 + assert info.recorded_ops.total == 6 + assert info.recorded_ops.calls == 3 + assert info.recorded_ops.pure_calls == 2 + assert info.guards == 1 + assert info.blackholed_ops.total == 5 + assert info.blackholed_ops.pure_calls == 3 + assert info.opt_ops == 6 + assert info.opt_guards == 1 + assert info.forcings == 1 + assert info.trace_too_long == 2 + assert info.bridge_abort == 3 From fijal at codespeak.net Fri Oct 23 10:32:03 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 23 Oct 2009 10:32:03 +0200 (CEST) Subject: [pypy-svn] r68718 - in pypy/trunk/pypy/jit/tool: . test Message-ID: <20091023083203.BA5D649843D@codespeak.net> Author: fijal Date: Fri Oct 23 10:32:01 2009 New Revision: 68718 Modified: pypy/trunk/pypy/jit/tool/ (props changed) pypy/trunk/pypy/jit/tool/test/ (props changed) Log: fixeol From antocuni at codespeak.net Fri Oct 23 14:48:43 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 23 Oct 2009 14:48:43 +0200 (CEST) Subject: [pypy-svn] r68720 - in pypy/trunk/pypy/jit: backend/test metainterp/test Message-ID: <20091023124843.722F649843C@codespeak.net> Author: antocuni Date: Fri Oct 23 14:48:43 2009 New Revision: 68720 Modified: pypy/trunk/pypy/jit/backend/test/support.py pypy/trunk/pypy/jit/metainterp/test/test_loop.py Log: fix test_dump_storage for cli: - cli tests does not support functions that return void, so make it returning an int instead - be sure that JIT tests see the correct values for the various os.O_* flags Modified: pypy/trunk/pypy/jit/backend/test/support.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/support.py (original) +++ pypy/trunk/pypy/jit/backend/test/support.py Fri Oct 23 14:48:43 2009 @@ -25,6 +25,7 @@ for arg in args: assert isinstance(arg, int) + self.pre_translation_hook() t = self._get_TranslationContext() t.config.translation.type_system = self.type_system # force typesystem-specific options if listcomp: @@ -63,8 +64,15 @@ entry_point_graph = mixlevelann.getgraph(entry_point, [s_list_of_strings], annmodel.SomeInteger()) warmrunnerdesc.finish() + self.post_translation_hook() return self._compile_and_run(t, entry_point, entry_point_graph, args) + def pre_translation_hook(self): + pass + + def post_translation_hook(self): + pass + def check_loops(self, *args, **kwds): pass @@ -120,6 +128,14 @@ class CliCompiledMixin(BaseCompiledMixin): type_system = 'ootype' + def pre_translation_hook(self): + from pypy.translator.oosupport.support import patch_os + self.olddefs = patch_os() + + def post_translation_hook(self): + from pypy.translator.oosupport.support import unpatch_os + unpatch_os(self.olddefs) # restore original values + def _compile_and_run(self, t, entry_point, entry_point_graph, args): from pypy.translator.cli.test.runtest import compile_graph func = compile_graph(entry_point_graph, t, nowrap=True, standalone=True) Modified: pypy/trunk/pypy/jit/metainterp/test/test_loop.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_loop.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_loop.py Fri Oct 23 14:48:43 2009 @@ -608,7 +608,7 @@ jitdriver.can_enter_jit(n=n, i=i) jitdriver.jit_merge_point(n=n, i=i) i += 1 - + return i res = self.meta_interp(f, [10]) data = logfile.read() # assert did not crash finally: From antocuni at codespeak.net Fri Oct 23 16:49:39 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 23 Oct 2009 16:49:39 +0200 (CEST) Subject: [pypy-svn] r68721 - pypy/trunk/pypy/translator/jvm Message-ID: <20091023144939.D891E49843D@codespeak.net> Author: antocuni Date: Fri Oct 23 16:49:38 2009 New Revision: 68721 Modified: pypy/trunk/pypy/translator/jvm/genjvm.py Log: use an unique directory for each test to store the files Modified: pypy/trunk/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/trunk/pypy/translator/jvm/genjvm.py (original) +++ pypy/trunk/pypy/translator/jvm/genjvm.py Fri Oct 23 16:49:38 2009 @@ -9,6 +9,7 @@ from py.compat import subprocess from pypy.tool.udir import udir from pypy.translator.translator import TranslationContext +from pypy.translator.gensupp import uniquemodulename from pypy.translator.oosupport.genoo import GenOO from pypy.translator.backendopt.all import backend_optimizations from pypy.translator.backendopt.checkvirtual import check_virtual_methods @@ -225,8 +226,11 @@ backend_optimizations(t) main_graph = t.graphs[0] if getoption('view'): t.view() - if getoption('wd'): tmpdir = py.path.local('.') - else: tmpdir = udir + if getoption('wd'): + tmpdir = py.path.local('.') + else: + dirname = uniquemodulename('jvm_testing') + tmpdir = udir.ensure(dirname, dir=1) jvm = GenJvm(tmpdir, t, EntryPoint(main_graph, True, True)) return jvm.generate_source() From fijal at codespeak.net Fri Oct 23 18:49:20 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 23 Oct 2009 18:49:20 +0200 (CEST) Subject: [pypy-svn] r68722 - pypy/branch/gc-dump-heap Message-ID: <20091023164920.0FC5349843C@codespeak.net> Author: fijal Date: Fri Oct 23 18:49:20 2009 New Revision: 68722 Added: pypy/branch/gc-dump-heap/ - copied from r68721, pypy/trunk/ Log: A branch to experiment with dumping heap info to a file From verte at codespeak.net Sat Oct 24 05:18:04 2009 From: verte at codespeak.net (verte at codespeak.net) Date: Sat, 24 Oct 2009 05:18:04 +0200 (CEST) Subject: [pypy-svn] r68723 - in pypy/branch/effect-analysis/pypy/translator/backendopt/effect: . test Message-ID: <20091024031804.4C8ED168042@codespeak.net> Author: verte Date: Sat Oct 24 05:18:03 2009 New Revision: 68723 Added: pypy/branch/effect-analysis/pypy/translator/backendopt/effect/ pypy/branch/effect-analysis/pypy/translator/backendopt/effect/__init__.py pypy/branch/effect-analysis/pypy/translator/backendopt/effect/model.py pypy/branch/effect-analysis/pypy/translator/backendopt/effect/test/ pypy/branch/effect-analysis/pypy/translator/backendopt/effect/test/__init__.py pypy/branch/effect-analysis/pypy/translator/backendopt/effect/test/test_model.py Log: Initial commit, mostly just failing tests. Added: pypy/branch/effect-analysis/pypy/translator/backendopt/effect/__init__.py ============================================================================== Added: pypy/branch/effect-analysis/pypy/translator/backendopt/effect/model.py ============================================================================== --- (empty file) +++ pypy/branch/effect-analysis/pypy/translator/backendopt/effect/model.py Sat Oct 24 05:18:03 2009 @@ -0,0 +1,252 @@ +from collections import defaultdict + +ITEM = object() + +def namer(): + from itertools import count + num = count() + def make_name(): + for i in num: + return '%%r%x' % (i,) + return make_name + +make_name = namer() + + +def index_name_maker(context, name): + def aggregate_factory(): + return AggregateRegion(context, '%s[]' % (name,)) + return aggregate_factory + + +class Region(object): + def __init__(self, context=None, name=None): + self.context = context + if name is not None: + self.name = name + else: + self.name = make_name() + self.indexes = defaultdict(index_name_maker(context, self.name)) + + def get_item(self, index=ITEM): + return self.indexes[index] + + def set_item(self, value, index=ITEM): + self.indexes[index].add(value) + + def is_concrete(self, context): + return True + + def is_abstract(self): + return False + + def contents(self): + return () + + def __repr__(self): + return self.name + + +class AbstractRegion(Region): + def inside(self, context): + # if the context is a child of the one this is on, we are already in + # that context + value = context.lookup_abstract_value(self) + + if value is not None: + return value + + return self + + def is_concrete(self, context): + value = context.lookup_abstract_value(self) + return value is not None + + def contents(self): + if self.context is not None: + return [self.context.get_abstract_value(self)] + return () + + + +class ConcreteRegion(Region): + """A region refering to a known constant. + + Occasionally we know beforehand the entire value that the region + will take, in the case of global constants and what not. + """ + + +class SemiConcreteRegion(Region): + """A region referring to an object created on the domain of + interest. + + A semi-concrete region is one that we know most of its secrets. + Typically, a semi-concrete region will refer to all instances of + an object created at a certain point. We then use the call + context to distinguish various paths an object can take. + + In this sense a region is semi-concrete; that together with a + context, it can stand on its own and be absolutely disambiguated. + """ + + +class AggregateRegion(Region): + def __init__(self, *args, **kwargs): + self.regions = set() + super(AggregateRegion, self).__init__(*args, **kwargs) + + def is_concrete(self, context): + return False + + def contents(self): + return self.regions + + def add(self, region): + if region in self.regions: + return + self.regions.add(region) + for index, value in self.indexes.iteritems(): + value.add(region.get_item(index)) + + def get_item(self, index=ITEM): + return super(AggregateRegion, self).get_item(index) + + def set_item(self, value, index=ITEM): + super(AggregateRegion, self).set_item(value, index) + + +def distinct(x, y): + return False + + +# XXX: use .contents() instead of an ugly type check +def separate_aggregates(regions): + direct = set([r for r in regions if not isinstance(r, AggregateRegion)]) + refs = set([r for r in regions if isinstance(r, AggregateRegion)]) + return direct, refs + + +def flatten(regions): + """Flatten a set of regions. + + Dataflow graphs typically generate AggregateRegions that refer to more + AggregateRegions. flatten takes a set of regions and snaps pointers to + others, and removes self-references. + """ + + # Unfortunately suboptimal in the case of higher-order cycles. + # To be addressed. + + for region in regions: + if not isinstance(region, AggregateRegion): + continue + if not any(isinstance(r, AggregateRegion) for r in region.regions): + continue + + print '\nRegion:', region + # region.regions will be built as a set containing only non- + # aggregate references. + # + # refs_working is a set of references we have yet to deal with. + # + # refs_inlined is the set of references that have already been + # dealt with. + # + # self_refs is a subset of refs_inlined that contains those that + # are equal to this set. + + region.regions, refs_working = separate_aggregates(region.regions) + refs_inlined = set([region]) + self_refs = set([region]) + + while True: + reference = refs_working.pop() + refs_inlined.add(reference) + new_regions, new_refs = separate_aggregates(reference.regions) + if self_refs & reference.regions: + # these sets contain each other => they are equal + self_refs.add(reference) + reference.regions = region.regions + region.regions.update(new_regions) + refs_working.update(new_refs - refs_inlined) + if not refs_working: + break + + + +class RegionContext(object): + """Distinguish concrete regions across call boundaries. + + if we had: + + def newlist(): + return [] + + we would assign a semi concrete region to the new list so we could + identify its origin. However, that would not distinguish: + + def f(): + x = newlist() + y = newlist() + x.append(someregion) + return y + + The region of the return value of f would be seen to be modified. + So, we imbue function calls with a 'context', which will + disabmiguate between these cases. + + Contexts can also distinguish usage across function call and loop + boundaries, tightening the analysis further. However, we do not + track function applications yet and don't handle loops specially. + """ + def __init__(self, parent_context=None): + self.parent = parent_context + self.bindings = {} + + def become(self, region, value): + self.bindings[region] = value + + def get_abstract_value(self, abstract_region): + return self.bindings[abstract_region] + + def lookup_abstract_value(self, abstract_region): + while self is not None: + try: + return self.get_abstract_value(abstract_region) + except KeyError: + self = self.parent + + def aggregate_concreteness(self, aggregate_region): + def visitor(region): + if region.is_abstract(): + raise VisitException + + try: + walk(aggregate_region, visitor) + except VisitException: + return False + return True + + +class VisitException(Exception): + pass + + +def walk(root, visitor): + seen = set([root]) + working = set([root]) + + while working: + region = working.pop() + remaining = seen.difference(region.contents()) + working.update(remaining) + seen.update(remaining) + visitor(region) + + +context = RegionContext +aggregate = AggregateRegion +concrete = ConcreteRegion +semiconcrete = SemiConcreteRegion +abstract = AbstractRegion Added: pypy/branch/effect-analysis/pypy/translator/backendopt/effect/test/__init__.py ============================================================================== Added: pypy/branch/effect-analysis/pypy/translator/backendopt/effect/test/test_model.py ============================================================================== --- (empty file) +++ pypy/branch/effect-analysis/pypy/translator/backendopt/effect/test/test_model.py Sat Oct 24 05:18:03 2009 @@ -0,0 +1,141 @@ +from pypy.translator.backendopt.effect.model import distinct, context, \ + abstract, semiconcrete, concrete, flatten, aggregate + +class TestCanRaise(object): + + def test_semiconcrete_distinct(self): + "unique semiconcrete regions are distinct" + self.semiconcrete_distinct(semiconcrete(), semiconcrete()) + + def semiconcrete_distinct(self, x, y): + assert distinct(x, y) + assert not distinct(x, x) + + def test_semiconcrete_distinct_contexts(self): + "unique semiconcrete regions are distinct in their context" + self.semiconcrete_distinct_contexts(semiconcrete(), semiconcrete()) + + def semiconcrete_distinct_contexts(self, x, y): + assert distinct(x, y) + assert not distinct(x, x) + + context0 = context() + context1 = context() + assert distinct(x.inside(context0), x.inside(context1)) + assert distinct(x.inside(context0), y.inside(context0)) + assert not distinct(x.inside(context0), x.inside(context0)) + + def test_abstract_nondistinct(self): + "unless they have different type, abstract regions are not distinct" + x = abstract() + y = abstract() + + assert not distinct(x, y) + assert not distinct(x, x) + + def test_abstract_become(self): + "abstract regions may become concrete or semiconcrete" + x = abstract() + y = abstract() + + specialisation_context = context() + + specialisation_context.become(x, semiconcrete()) + + assert not distinct(x.inside(specialisation_context), y) + + specialisation_context.become(x, semiconcrete()) + + assert distinct(x.inside(specialisation_context), + y.inside(specialisation_context)) + + specialisation_context = context() + some_concrete = semiconcrete() + + specialisation_context.become(x, some_concrete) + assert not distinct(x.inside(specialisation_context), y) + + specialisation_context.become(y, some_concrete) + assert not distinct(x.inside(specialisation_context), + y.inside(specialisation_context)) + + def test_abstract_semiconcrete(self): + """The distinctness of semiconcrete and abstract may be + glorked from context. + + That is to say, the regions that are abstract and concrete in a + given context are distinct. This is because a region cannot both + come from inside and outside the function call. + + Actually, this is not true, the concrete region could have + been passed to some other function, and the abstract region is + the retun value of the function. + + shrug. + """ + + x = abstract() + specialisation_context = context() + y = semiconcrete(context=specialisation_context) + + assert distinct(x, y) + + assert not distinct(x.inside(specialisation_context), + y.inside(specialisation_context)) + + + def test_index(self): + r1 = concrete() + r2 = concrete() + + w, x, y, z = [concrete() for _ in xrange(4)] + r1.set_item(w) + r1.set_item(x) + r2.set_item(y) + r2.set_item(z) + + assert distinct(r1.get_item(), r2.get_item()) + + r1.set_item(y) + + assert not distinct(r1.get_item(), r2.get_item()) + + def test_attr(self): + r1 = concrete() + r2 = concrete() + + w, x, y, z = [concrete() for _ in xrange(4)] + r1.set_item(w, 'w') + r1.set_item(x, 'w') + r2.set_item(w, 'w') + r2.set_item(x, 'x') + + assert distinct(r2.get_item('x'), r2.get_item('w')) + assert distinct(r1.get_item('x'), r2.get_item('x')) + assert not distinct(r1.get_item('w'), r2.get_item('w')) + + def test_aggregates(self): + x = aggregate() + y = aggregate([x]) + z = aggregate([y]) + x.add(z) + + flatten([x]) + + assert distinct(x, abstract()) + + c = concrete() + x = aggregate([c]) + y = aggregate([x]) + z = aggregate([y]) + x.add(z) + + flatten([x]) + + assert not distinct(x, c) + assert x.regions == y.regions == z.regions + + c = concrete() + y.add(c) + + assert len(x.regions) == 2 From fijal at codespeak.net Sat Oct 24 11:59:07 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 24 Oct 2009 11:59:07 +0200 (CEST) Subject: [pypy-svn] r68724 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091024095907.2184E16804F@codespeak.net> Author: fijal Date: Sat Oct 24 11:59:00 2009 New Revision: 68724 Modified: pypy/trunk/pypy/jit/metainterp/jitprof.py Log: Oops, missing checkin Modified: pypy/trunk/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/jitprof.py (original) +++ pypy/trunk/pypy/jit/metainterp/jitprof.py Sat Oct 24 11:59:00 2009 @@ -30,6 +30,8 @@ ncounters = len(names) _setup() +JITPROF_LINES = ncounters + 1 + 5 # one for TOTAL, 5 for calls, update if needed + class BaseProfiler(object): pass From benjamin at codespeak.net Sat Oct 24 16:20:58 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 24 Oct 2009 16:20:58 +0200 (CEST) Subject: [pypy-svn] r68725 - pypy/trunk/pypy/interpreter/pyparser Message-ID: <20091024142058.2DBA416804F@codespeak.net> Author: benjamin Date: Sat Oct 24 16:20:57 2009 New Revision: 68725 Modified: pypy/trunk/pypy/interpreter/pyparser/future.py Log: remove pointless else Modified: pypy/trunk/pypy/interpreter/pyparser/future.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/future.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/future.py Sat Oct 24 16:20:57 2009 @@ -207,9 +207,7 @@ self.col_offset = col_offset self.lineno = line self.consume_empty_line() - else: - return - + def consume_mandatory_whitespace(self): if self.getc() not in whitespace + '\\': raise DoneException From benjamin at codespeak.net Sat Oct 24 16:22:03 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 24 Oct 2009 16:22:03 +0200 (CEST) Subject: [pypy-svn] r68726 - pypy/trunk/pypy/rpython Message-ID: <20091024142203.B6BD916804F@codespeak.net> Author: benjamin Date: Sat Oct 24 16:22:03 2009 New Revision: 68726 Modified: pypy/trunk/pypy/rpython/rvirtualizable2.py Log: remove unused py import Modified: pypy/trunk/pypy/rpython/rvirtualizable2.py ============================================================================== --- pypy/trunk/pypy/rpython/rvirtualizable2.py (original) +++ pypy/trunk/pypy/rpython/rvirtualizable2.py Sat Oct 24 16:22:03 2009 @@ -1,4 +1,3 @@ -import py from pypy.rpython.rmodel import inputconst, log from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype From benjamin at codespeak.net Sat Oct 24 16:35:40 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 24 Oct 2009 16:35:40 +0200 (CEST) Subject: [pypy-svn] r68727 - in pypy/trunk/pypy: interpreter/pyparser module/__builtin__/test Message-ID: <20091024143540.6D51C168050@codespeak.net> Author: benjamin Date: Sat Oct 24 16:35:39 2009 New Revision: 68727 Modified: pypy/trunk/pypy/interpreter/pyparser/future.py pypy/trunk/pypy/module/__builtin__/test/test_builtin.py Log: convince __future__ statements with no ending newlines to work Modified: pypy/trunk/pypy/interpreter/pyparser/future.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/future.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/future.py Sat Oct 24 16:35:39 2009 @@ -240,13 +240,19 @@ if self.getc() not in letters: raise DoneException p = self.pos - while 1: - self.pos += 1 - if self.getc() not in alphanumerics: - break - name = self.s[p:self.pos] - self.consume_whitespace() - return name + try: + while 1: + self.pos += 1 + if self.getc() not in alphanumerics: + break + except DoneException: + # If there's any name at all, we want to call self.set_flag(). + # Something else while get the DoneException again. + if self.pos == p: + raise + else: + self.consume_whitespace() + return self.s[p:self.pos] def get_more(self, paren_list=False): if paren_list and self.getc() == ')': Modified: pypy/trunk/pypy/module/__builtin__/test/test_builtin.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/test/test_builtin.py (original) +++ pypy/trunk/pypy/module/__builtin__/test/test_builtin.py Sat Oct 24 16:35:39 2009 @@ -397,6 +397,7 @@ def test_compile(self): co = compile('1+2', '?', 'eval') assert eval(co) == 3 + compile("from __future__ import with_statement", "", "exec") raises(SyntaxError, compile, '-', '?', 'eval') raises(ValueError, compile, '"\\xt"', '?', 'eval') raises(ValueError, compile, '1+2', '?', 'maybenot') From benjamin at codespeak.net Sat Oct 24 16:39:37 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 24 Oct 2009 16:39:37 +0200 (CEST) Subject: [pypy-svn] r68728 - pypy/trunk/pypy/doc/jit Message-ID: <20091024143937.6C424168050@codespeak.net> Author: benjamin Date: Sat Oct 24 16:39:36 2009 New Revision: 68728 Modified: pypy/trunk/pypy/doc/jit/pyjitpl5.txt Log: start of a paragraph Modified: pypy/trunk/pypy/doc/jit/pyjitpl5.txt ============================================================================== --- pypy/trunk/pypy/doc/jit/pyjitpl5.txt (original) +++ pypy/trunk/pypy/doc/jit/pyjitpl5.txt Sat Oct 24 16:39:36 2009 @@ -154,6 +154,10 @@ be unwrapped to just be its integer value as long as the object is known not to escape the machine code. +A *virtualizable* is similar to a virtual in that its structure is optimized out +in the machine code. Virtualizables, howerver, can escape from JIT controlled +code. + Most of the JIT's optimizer is contained 2 files optimizefindnodes.py and optimizeopt.py. From benjamin at codespeak.net Sat Oct 24 16:47:29 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 24 Oct 2009 16:47:29 +0200 (CEST) Subject: [pypy-svn] r68729 - pypy/trunk/pypy/interpreter/astcompiler Message-ID: <20091024144729.56468168050@codespeak.net> Author: benjamin Date: Sat Oct 24 16:47:28 2009 New Revision: 68729 Modified: pypy/trunk/pypy/interpreter/astcompiler/codegen.py Log: CRITICAL BUG FIX: from __future__ import braces didn't "work" Modified: pypy/trunk/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/codegen.py Sat Oct 24 16:47:28 2009 @@ -615,6 +615,8 @@ for alias in imp.names: assert isinstance(alias, ast.alias) if alias.name not in future.futureFlags_2_5.compiler_features: + if alias.name == "braces": + self.error("not a chance", imp) self.error("future feature %s is not defined" % (alias.name,), imp) if imp.level == 0 and \ From fijal at codespeak.net Sat Oct 24 17:28:53 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 24 Oct 2009 17:28:53 +0200 (CEST) Subject: [pypy-svn] r68730 - pypy/trunk/pypy/objspace/std Message-ID: <20091024152853.9D69916804E@codespeak.net> Author: fijal Date: Sat Oct 24 17:28:53 2009 New Revision: 68730 Modified: pypy/trunk/pypy/objspace/std/celldict.py Log: avoid unnecessary indirection Modified: pypy/trunk/pypy/objspace/std/celldict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/celldict.py (original) +++ pypy/trunk/pypy/objspace/std/celldict.py Sat Oct 24 17:28:53 2009 @@ -235,7 +235,7 @@ name = f.space.str_w(f.getname_w(nameindex)) implementation = getimplementation(f.w_globals) if isinstance(implementation, ModuleDictImplementation): - cell = find_cell_from_dict(implementation, name) + cell = implementation.getcell(name, False) if cell is None: builtin_impl = getimplementation(f.get_builtin().getdict()) cell = find_cell_from_dict(builtin_impl, name) From arigo at codespeak.net Sun Oct 25 11:17:45 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 25 Oct 2009 11:17:45 +0100 (CET) Subject: [pypy-svn] r68733 - pypy/trunk/pypy/translator/jvm Message-ID: <20091025101745.3F681168010@codespeak.net> Author: arigo Date: Sun Oct 25 11:17:44 2009 New Revision: 68733 Modified: pypy/trunk/pypy/translator/jvm/genjvm.py Log: Revert r68721. It is after all not the cause of the problems we see in the nightly test run. Modified: pypy/trunk/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/trunk/pypy/translator/jvm/genjvm.py (original) +++ pypy/trunk/pypy/translator/jvm/genjvm.py Sun Oct 25 11:17:44 2009 @@ -9,7 +9,6 @@ from py.compat import subprocess from pypy.tool.udir import udir from pypy.translator.translator import TranslationContext -from pypy.translator.gensupp import uniquemodulename from pypy.translator.oosupport.genoo import GenOO from pypy.translator.backendopt.all import backend_optimizations from pypy.translator.backendopt.checkvirtual import check_virtual_methods @@ -226,11 +225,8 @@ backend_optimizations(t) main_graph = t.graphs[0] if getoption('view'): t.view() - if getoption('wd'): - tmpdir = py.path.local('.') - else: - dirname = uniquemodulename('jvm_testing') - tmpdir = udir.ensure(dirname, dir=1) + if getoption('wd'): tmpdir = py.path.local('.') + else: tmpdir = udir jvm = GenJvm(tmpdir, t, EntryPoint(main_graph, True, True)) return jvm.generate_source() From arigo at codespeak.net Sun Oct 25 12:52:38 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 25 Oct 2009 12:52:38 +0100 (CET) Subject: [pypy-svn] r68734 - pypy/trunk/pypy/translator/jvm Message-ID: <20091025115238.38885168013@codespeak.net> Author: arigo Date: Sun Oct 25 12:52:37 2009 New Revision: 68734 Modified: pypy/trunk/pypy/translator/jvm/genjvm.py Log: Partly kill the caching logic, which was not working anyway and which probably caused the random test failures on wyvern. Modified: pypy/trunk/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/trunk/pypy/translator/jvm/genjvm.py (original) +++ pypy/trunk/pypy/translator/jvm/genjvm.py Sun Oct 25 12:52:37 2009 @@ -68,6 +68,7 @@ The following attributes also exist to find the state of the sources: compiled --- True once the sources have been compiled successfully """ + _cached = None def __init__(self, tmpdir, package): """ @@ -117,29 +118,20 @@ def _compile_helper(self): # HACK: compile the Java helper classes. Should eventually # use rte.py + if JvmGeneratedSource._cached == self.classdir: + return + log.red('Compiling java classes') javafiles = self.srcdir.listdir('*.java') - classfiles = self.srcdir.listdir('*.class') - - recompile = True - if len(classfiles) == len(javafiles): - last_modified_java = max([java.mtime() for java in javafiles]) - last_modified_class = max([cls.mtime() for cls in classfiles]) - if last_modified_java < last_modified_class: - recompile = False - - if recompile: - log.red('Compiling java classes') - javasrcs = [str(jf) for jf in javafiles] - self._invoke([getoption('javac'), - '-nowarn', - '-d', str(self.rootdir), - '-classpath', str(self.jnajar) - ] + javasrcs, - True) - - # copy .class files to classdir - for classfile in self.srcdir.listdir('*.class'): - classfile.copy(self.classdir.join('pypy')) + javasrcs = [str(jf) for jf in javafiles] + self._invoke([getoption('javac'), + '-nowarn', + '-d', str(self.classdir), + '-classpath', str(self.jnajar), + ] + javasrcs, + True) + # NOTE if you are trying to add more caching: some .java files + # compile to several .class files of various names. + JvmGeneratedSource._cached = self.classdir def compile(self): """ From benjamin at codespeak.net Sun Oct 25 15:31:22 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 25 Oct 2009 15:31:22 +0100 (CET) Subject: [pypy-svn] r68735 - pypy/trunk/pypy/interpreter/pyparser Message-ID: <20091025143122.1B487168013@codespeak.net> Author: benjamin Date: Sun Oct 25 15:31:18 2009 New Revision: 68735 Modified: pypy/trunk/pypy/interpreter/pyparser/future.py Log: make sure to capture the name correctly even if there is whitespace afterwards Modified: pypy/trunk/pypy/interpreter/pyparser/future.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/future.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/future.py Sun Oct 25 15:31:18 2009 @@ -241,18 +241,18 @@ raise DoneException p = self.pos try: - while 1: + while self.getc() in alphanumerics: self.pos += 1 - if self.getc() not in alphanumerics: - break except DoneException: # If there's any name at all, we want to call self.set_flag(). # Something else while get the DoneException again. if self.pos == p: raise + end = self.pos else: + end = self.pos self.consume_whitespace() - return self.s[p:self.pos] + return self.s[p:end] def get_more(self, paren_list=False): if paren_list and self.getc() == ')': From fijal at codespeak.net Sun Oct 25 16:28:01 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Oct 2009 16:28:01 +0100 (CET) Subject: [pypy-svn] r68736 - pypy/branch/no-trace-if-frame-escapes Message-ID: <20091025152801.4AB51168013@codespeak.net> Author: fijal Date: Sun Oct 25 16:28:00 2009 New Revision: 68736 Added: pypy/branch/no-trace-if-frame-escapes/ - copied from r68735, pypy/trunk/ Log: Create a branch to experiment with further limiting tracing From fijal at codespeak.net Sun Oct 25 16:33:11 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Oct 2009 16:33:11 +0100 (CET) Subject: [pypy-svn] r68737 - in pypy/branch/gc-dump-heap/pypy/rpython: . lltypesystem memory/gc memory/gctransform memory/test Message-ID: <20091025153311.C5042168013@codespeak.net> Author: fijal Date: Sun Oct 25 16:33:11 2009 New Revision: 68737 Modified: pypy/branch/gc-dump-heap/pypy/rpython/llinterp.py pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/lloperation.py pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py Log: in-progress a dump heap operation Modified: pypy/branch/gc-dump-heap/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/llinterp.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/llinterp.py Sun Oct 25 16:33:11 2009 @@ -824,6 +824,9 @@ def op_gc_assume_young_pointers(self, addr): raise NotImplementedError + def op_gc_dump_heap(self, fd): + raise NotImplementedError # impossible + def op_gc_obtain_free_space(self, size): raise NotImplementedError Modified: pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/lloperation.py Sun Oct 25 16:33:11 2009 @@ -459,6 +459,7 @@ 'gc_thread_run' : LLOp(), 'gc_thread_die' : LLOp(), 'gc_assume_young_pointers': LLOp(), + 'gc_dump_heap' : LLOp(), # ------- JIT & GC interaction, only for some GCs ---------- Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py Sun Oct 25 16:33:11 2009 @@ -24,6 +24,8 @@ memoryError = MemoryError() +ARRAY_OF_SIGNED = lltype.Array(lltype.Signed) + class SemiSpaceGC(MovingGCBase): _alloc_flavor_ = "raw" @@ -636,3 +638,25 @@ hdr.tid |= GCFLAG_HASHTAKEN # return llmemory.cast_adr_to_int(obj) # direct case + + def _dump_heap_extraarg(self, addr, ignored): + self._dump_heap(addr) + + def _dump_heap(self, addr): + os.write(self._fd_dump, "X") + self.trace(addr, self._dump_heap_extraarg, None) + + def dump_heap(self, fd): + ll_typeid_usage = lltype.nullptr(ARRAY_OF_SIGNED) + self._fd_dump = fd + try: + ll_typeid_usage = lltype.malloc(ARRAY_OF_SIGNED, + self.root_walker.gcdata.max_type_id, flavor='raw') + self.root_walker.walk_roots( + SemiSpaceGC._dump_heap, # stack roots + SemiSpaceGC._dump_heap, # static in prebuilt non-gc structures + SemiSpaceGC._dump_heap) # static in prebuilt gc objects + finally: + if ll_typeid_usage: + lltype.free(ll_typeid_usage, flavor='raw') + self._fd_dump = -1 Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py Sun Oct 25 16:33:11 2009 @@ -24,7 +24,6 @@ TYPE_ID = rffi.USHORT - class CollectAnalyzer(graphanalyze.BoolGraphAnalyzer): def analyze_direct_call(self, graph, seen=None): @@ -152,6 +151,7 @@ gcdata.static_root_start = a_random_address # patched in finish() gcdata.static_root_nongcend = a_random_address # patched in finish() gcdata.static_root_end = a_random_address # patched in finish() + gcdata.max_type_id = 13 # patched in finish() self.gcdata = gcdata self.malloc_fnptr_cache = {} @@ -188,6 +188,9 @@ data_classdef.generalize_attr( 'static_root_end', annmodel.SomeAddress()) + data_classdef.generalize_attr( + 'max_type_id', + annmodel.SomeInteger()) annhelper = annlowlevel.MixLevelHelperAnnotator(self.translator.rtyper) @@ -267,6 +270,12 @@ [s_gc, annmodel.SomeAddress()], annmodel.s_None) + if hasattr(GCClass, 'dump_heap'): + self.dump_heap_ptr = getfn( + GCClass.dump_heap.im_func, + [s_gc, annmodel.SomeInteger()], + annmodel.s_None) + # in some GCs we can inline the common case of # malloc_fixedsize(typeid, size, True, False, False) if getattr(GCClass, 'inline_simple_malloc', False): @@ -497,14 +506,16 @@ ll_static_roots_inside = lltype.malloc(lltype.Array(llmemory.Address), len(addresses_of_static_ptrs), immortal=True) + for i in range(len(addresses_of_static_ptrs)): ll_static_roots_inside[i] = addresses_of_static_ptrs[i] ll_instance.inst_static_root_start = llmemory.cast_ptr_to_adr(ll_static_roots_inside) + llmemory.ArrayItemsOffset(lltype.Array(llmemory.Address)) ll_instance.inst_static_root_nongcend = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(self.layoutbuilder.addresses_of_static_ptrs_in_nongc) ll_instance.inst_static_root_end = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(addresses_of_static_ptrs) - newgcdependencies = [] newgcdependencies.append(ll_static_roots_inside) + ll_instance.inst_max_type_id = len(group.members) + self.write_typeid_list() return newgcdependencies @@ -634,6 +645,12 @@ hop.genop("direct_call", [self.assume_young_pointers_ptr, self.c_const_gc, v_addr]) + def gct_gc_dump_heap(self, hop): + op = hop.spaceop + v_fd = op.args[0] + hop.genop("direct_call", [self.dump_heap_ptr, + self.c_const_gc, v_fd]) + def gct_gc_adr_of_nursery_free(self, hop): if getattr(self.gcdata.gc, 'nursery_free', None) is None: raise NotImplementedError("gc_adr_of_nursery_free only for generational gcs") Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py Sun Oct 25 16:33:11 2009 @@ -796,6 +796,26 @@ run = self.runner("do_malloc_operations_in_call") run([]) + def define_gc_dump_heap(cls): + S = lltype.GcStruct('S', ('x', lltype.Signed)) + + def f(fd, ign): + l = [] + for i in range(10): + l.append(lltype.malloc(S)) + rgc.dump_heap(fd) + return 0 + return f + + def test_gc_dump_heap(self): + from pypy.tool.udir import udir + f = udir.join("gcdump.log") + handle = open(str(f), "w") + run = self.runner("gc_dump_heap") + run([handle.fileno(), 0]) + handle.close() + assert f.read() == 'xxx' + # ________________________________________________________________ class TestMarkSweepGC(GenericGCTests): From fijal at codespeak.net Sun Oct 25 21:00:09 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 25 Oct 2009 21:00:09 +0100 (CET) Subject: [pypy-svn] r68739 - in pypy/branch/gc-dump-heap/pypy: rlib rpython rpython/lltypesystem rpython/memory/gc rpython/memory/gctransform rpython/memory/test Message-ID: <20091025200009.E8F5349845C@codespeak.net> Author: fijal Date: Sun Oct 25 21:00:07 2009 New Revision: 68739 Modified: pypy/branch/gc-dump-heap/pypy/rlib/rgc.py pypy/branch/gc-dump-heap/pypy/rpython/llinterp.py pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/lloperation.py pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/opimpl.py pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/base.py pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_gc.py pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py Log: Implement heap dumping for a semispace gc. fails with generation and hybrid for now. Also explodes on C backend. Introduce a new operation llop.get_member_index which returns an index in memberoffset Modified: pypy/branch/gc-dump-heap/pypy/rlib/rgc.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rlib/rgc.py (original) +++ pypy/branch/gc-dump-heap/pypy/rlib/rgc.py Sun Oct 25 21:00:07 2009 @@ -184,6 +184,41 @@ hop.exception_cannot_occur() return hop.genop('gc_can_move', hop.args_v, resulttype=hop.r_result) +def dump_heap(fd): + import os + + tb = _dump_heap() + for i in range(len(tb)): + next = tb[i] + os.write(fd, str(next.count) + " " + ",".join([ + str(next.links[j]) for j in range(len(tb))]) + "\n") + _clear_dump_heap(tb) + +def _clear_dump_heap(tb): + from pypy.rpython.lltypesystem import lltype + for i in range(len(tb)): + lltype.free(tb[i].links, flavor='raw') + lltype.free(tb, flavor='raw') + +def _dump_heap(): + raise NotImplementedError # can't be run directly + +class DumpHeapEntry(ExtRegistryEntry): + _about_ = _dump_heap + + def compute_result_annotation(self): + from pypy.annotation import model as annmodel + from pypy.rpython.memory.gc.base import ARRAY_TYPEID_MAP + from pypy.rpython.lltypesystem import lltype + return annmodel.SomePtr(lltype.Ptr(ARRAY_TYPEID_MAP)) + + def specialize_call(self, hop): + from pypy.rpython.lltypesystem import lltype + from pypy.rpython.memory.gc.base import ARRAY_TYPEID_MAP + from pypy.rpython.lltypesystem import lltype + hop.exception_cannot_occur() + return hop.genop('gc_dump_heap', [], resulttype=hop.r_result) + def malloc_nonmovable(TP, n=None, zero=False): """ Allocate a non-moving buffer or return nullptr. When running directly, will pretend that gc is always Modified: pypy/branch/gc-dump-heap/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/llinterp.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/llinterp.py Sun Oct 25 21:00:07 2009 @@ -824,8 +824,8 @@ def op_gc_assume_young_pointers(self, addr): raise NotImplementedError - def op_gc_dump_heap(self, fd): - raise NotImplementedError # impossible + def op_gc_dump_heap(self): + raise NotImplementedError def op_gc_obtain_free_space(self, size): raise NotImplementedError Modified: pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/lloperation.py Sun Oct 25 21:00:07 2009 @@ -422,6 +422,7 @@ 'extract_ushort': LLOp(canfold=True), 'combine_ushort': LLOp(canfold=True), 'gc_gettypeptr_group': LLOp(canfold=True), + 'get_member_index': LLOp(canfold=True), # __________ used by the JIT ________ Modified: pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/opimpl.py Sun Oct 25 21:00:07 2009 @@ -429,6 +429,11 @@ return lltype.cast_pointer(TYPE, member) op_get_group_member.need_result_type = True +def op_get_member_index(memberoffset): + from pypy.rpython.lltypesystem import llgroup + assert isinstance(memberoffset, llgroup.GroupMemberOffset) + return memberoffset.index + def op_get_next_group_member(TYPE, grpptr, memberoffset, skipoffset): from pypy.rpython.lltypesystem import llgroup assert isinstance(memberoffset, llgroup.GroupMemberOffset) Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/base.py Sun Oct 25 21:00:07 2009 @@ -6,6 +6,10 @@ from pypy.rpython.memory.support import AddressDict from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage +TYPEID_MAP = lltype.Struct('TYPEID_MAP', ('count', lltype.Signed), + ('links', lltype.Array(lltype.Signed))) +ARRAY_TYPEID_MAP = lltype.Array(lltype.Ptr(TYPEID_MAP)) + class GCBase(object): _alloc_flavor_ = "raw" moving_gc = False Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py Sun Oct 25 21:00:07 2009 @@ -9,7 +9,8 @@ from pypy.rlib.debug import ll_assert from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck -from pypy.rpython.memory.gc.base import MovingGCBase +from pypy.rpython.memory.gc.base import MovingGCBase, ARRAY_TYPEID_MAP,\ + TYPEID_MAP import sys, os, time @@ -24,9 +25,6 @@ memoryError = MemoryError() -ARRAY_OF_SIGNED = lltype.Array(lltype.Signed) - - class SemiSpaceGC(MovingGCBase): _alloc_flavor_ = "raw" inline_simple_malloc = True @@ -639,24 +637,38 @@ # return llmemory.cast_adr_to_int(obj) # direct case - def _dump_heap_extraarg(self, addr, ignored): + def _dump_heap_extraarg(self, addr, parent): + parent_idx = llop.get_member_index(lltype.Signed, + self.get_type_id(parent)) + idx = llop.get_member_index(lltype.Signed, + self.get_type_id(addr.address[0])) + self._ll_typeid_map[parent_idx].links[idx] += 1 self._dump_heap(addr) - def _dump_heap(self, addr): - os.write(self._fd_dump, "X") - self.trace(addr, self._dump_heap_extraarg, None) - - def dump_heap(self, fd): - ll_typeid_usage = lltype.nullptr(ARRAY_OF_SIGNED) - self._fd_dump = fd - try: - ll_typeid_usage = lltype.malloc(ARRAY_OF_SIGNED, - self.root_walker.gcdata.max_type_id, flavor='raw') - self.root_walker.walk_roots( - SemiSpaceGC._dump_heap, # stack roots - SemiSpaceGC._dump_heap, # static in prebuilt non-gc structures - SemiSpaceGC._dump_heap) # static in prebuilt gc objects - finally: - if ll_typeid_usage: - lltype.free(ll_typeid_usage, flavor='raw') - self._fd_dump = -1 + def _dump_heap(self, root): + adr = root.address[0] + idx = llop.get_member_index(lltype.Signed, self.get_type_id(adr)) + self._ll_typeid_map[idx].count += 1 + self.trace(adr, self._dump_heap_extraarg, adr) + + def dump_heap(self): + max_tid = self.root_walker.gcdata.max_type_id + ll_typeid_map = lltype.malloc(ARRAY_TYPEID_MAP, max_tid, + flavor='raw') + i = 0 + while i < max_tid: + ll_typeid_map[i] = lltype.malloc(TYPEID_MAP, max_tid, + flavor='raw') + ll_typeid_map[i].count = 0 + j = 0 + while j < max_tid: + ll_typeid_map[i].links[j] = 0 + j += 1 + i += 1 + self._ll_typeid_map = ll_typeid_map + self.root_walker.walk_roots( + SemiSpaceGC._dump_heap, # stack roots + SemiSpaceGC._dump_heap, # static in prebuilt non-gc structures + SemiSpaceGC._dump_heap) # static in prebuilt gc objects + self._ll_typeid_map = lltype.nullptr(ARRAY_TYPEID_MAP) + return ll_typeid_map Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py Sun Oct 25 21:00:07 2009 @@ -116,6 +116,7 @@ def __init__(self, translator): from pypy.rpython.memory.gc.base import choose_gc_from_config + from pypy.rpython.memory.gc.base import ARRAY_TYPEID_MAP super(FrameworkGCTransformer, self).__init__(translator, inline=True) if hasattr(self, 'GC_PARAMS'): # for tests: the GC choice can be specified as class attributes @@ -271,10 +272,8 @@ annmodel.s_None) if hasattr(GCClass, 'dump_heap'): - self.dump_heap_ptr = getfn( - GCClass.dump_heap.im_func, - [s_gc, annmodel.SomeInteger()], - annmodel.s_None) + self.dump_heap_ptr = getfn(GCClass.dump_heap.im_func, + [s_gc], annmodel.SomePtr(lltype.Ptr(ARRAY_TYPEID_MAP))) # in some GCs we can inline the common case of # malloc_fixedsize(typeid, size, True, False, False) @@ -647,9 +646,8 @@ def gct_gc_dump_heap(self, hop): op = hop.spaceop - v_fd = op.args[0] - hop.genop("direct_call", [self.dump_heap_ptr, - self.c_const_gc, v_fd]) + hop.genop("direct_call", [self.dump_heap_ptr, self.c_const_gc], + resultvar=op.result) def gct_gc_adr_of_nursery_free(self, hop): if getattr(self.gcdata.gc, 'nursery_free', None) is None: Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_gc.py Sun Oct 25 21:00:07 2009 @@ -552,6 +552,27 @@ assert res == 111 + def test_gc_dump_heap(self): + if getattr(self.GCClass, 'dump_heap', None) is None: + py.test.skip("unsupported gc") + S = lltype.GcStruct('S', ('x', lltype.Signed)) + + def fun(fd): + l = [] + for i in range(10): + l.append(lltype.malloc(S)) + rgc.dump_heap(fd) + keepalive_until_here(l) + return 0 + + from pypy.tool.udir import udir + f = udir.join("gcdump_direct.log") + handle = open(str(f), "w") + run = self.interpret(fun, [handle.fileno()]) + handle.close() + assert f.read() == 'xxx' + + from pypy.rlib.objectmodel import UnboxedValue class TaggedBase(object): Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py Sun Oct 25 21:00:07 2009 @@ -13,6 +13,7 @@ from pypy.rlib import rgc from pypy import conftest from pypy.rlib.rstring import StringBuilder +from pypy.rlib.objectmodel import keepalive_until_here INT_SIZE = struct.calcsize("i") # only for estimates @@ -798,23 +799,34 @@ def define_gc_dump_heap(cls): S = lltype.GcStruct('S', ('x', lltype.Signed)) + l = [] - def f(fd, ign): - l = [] + def f(): for i in range(10): l.append(lltype.malloc(S)) - rgc.dump_heap(fd) - return 0 + # We cheat here and only read the table which we later on + # process ourselves, otherwise this test takes ages + tb = rgc._dump_heap() + a = 0 + nr = 0 + b = 0 + c = 0 + for i in range(len(tb)): + if tb[i].count == 10: + a += 1 + nr = i + for i in range(len(tb)): + if tb[i].count == 1: + b += 1 + c = tb[i].links[nr] + rgc._clear_dump_heap(tb) + return c * 100 + b * 10 + a return f def test_gc_dump_heap(self): - from pypy.tool.udir import udir - f = udir.join("gcdump.log") - handle = open(str(f), "w") run = self.runner("gc_dump_heap") - run([handle.fileno(), 0]) - handle.close() - assert f.read() == 'xxx' + res = run([]) + assert res == 1011 # ________________________________________________________________ From fijal at codespeak.net Mon Oct 26 00:21:33 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Oct 2009 00:21:33 +0100 (CET) Subject: [pypy-svn] r68740 - in pypy/branch/gc-dump-heap/pypy/rpython: lltypesystem memory memory/gc memory/gctransform Message-ID: <20091025232133.B2D981683D1@codespeak.net> Author: fijal Date: Mon Oct 26 00:21:32 2009 New Revision: 68740 Modified: pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/opimpl.py pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/base.py pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py pypy/branch/gc-dump-heap/pypy/rpython/memory/gctypelayout.py Log: Hack differently. Since we have a couple of bits left, encode the member num in infobits. Modified: pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/opimpl.py Mon Oct 26 00:21:32 2009 @@ -429,11 +429,6 @@ return lltype.cast_pointer(TYPE, member) op_get_group_member.need_result_type = True -def op_get_member_index(memberoffset): - from pypy.rpython.lltypesystem import llgroup - assert isinstance(memberoffset, llgroup.GroupMemberOffset) - return memberoffset.index - def op_get_next_group_member(TYPE, grpptr, memberoffset, skipoffset): from pypy.rpython.lltypesystem import llgroup assert isinstance(memberoffset, llgroup.GroupMemberOffset) Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/base.py Mon Oct 26 00:21:32 2009 @@ -52,7 +52,8 @@ varsize_offset_to_variable_part, varsize_offset_to_length, varsize_offsets_to_gcpointers_in_var_part, - weakpointer_offset): + weakpointer_offset, + member_index): self.getfinalizer = getfinalizer self.is_varsize = is_varsize self.has_gcptr_in_varsize = has_gcptr_in_varsize @@ -64,6 +65,10 @@ self.varsize_offset_to_length = varsize_offset_to_length self.varsize_offsets_to_gcpointers_in_var_part = varsize_offsets_to_gcpointers_in_var_part self.weakpointer_offset = weakpointer_offset + self.member_index = member_index + + def get_member_index(self, type_id): + return self.member_index(type_id) def set_root_walker(self, root_walker): self.root_walker = root_walker Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py Mon Oct 26 00:21:32 2009 @@ -274,6 +274,10 @@ if hasattr(GCClass, 'dump_heap'): self.dump_heap_ptr = getfn(GCClass.dump_heap.im_func, [s_gc], annmodel.SomePtr(lltype.Ptr(ARRAY_TYPEID_MAP))) + self.get_member_index_ptr = getfn( + GCClass.get_member_index.im_func, + [s_gc, annmodel.SomeInteger(knowntype=rffi.r_ushort)], + annmodel.SomeInteger()) # in some GCs we can inline the common case of # malloc_fixedsize(typeid, size, True, False, False) @@ -514,7 +518,6 @@ newgcdependencies = [] newgcdependencies.append(ll_static_roots_inside) ll_instance.inst_max_type_id = len(group.members) - self.write_typeid_list() return newgcdependencies @@ -649,6 +652,12 @@ hop.genop("direct_call", [self.dump_heap_ptr, self.c_const_gc], resultvar=op.result) + def gct_get_member_index(self, hop): + op = hop.spaceop + v_typeid = op.args[0] + hop.genop("direct_call", [self.get_member_index_ptr, self.c_const_gc, + v_typeid], resultvar=op.result) + def gct_gc_adr_of_nursery_free(self, hop): if getattr(self.gcdata.gc, 'nursery_free', None) is None: raise NotImplementedError("gc_adr_of_nursery_free only for generational gcs") Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gctypelayout.py Mon Oct 26 00:21:32 2009 @@ -92,6 +92,10 @@ return weakptr_offset return -1 + def q_member_index(self, typeid): + infobits = self.get(typeid).infobits + return infobits & T_MEMBER_INDEX + def set_query_functions(self, gc): gc.set_query_functions( self.q_is_varsize, @@ -104,23 +108,26 @@ self.q_varsize_offset_to_variable_part, self.q_varsize_offset_to_length, self.q_varsize_offsets_to_gcpointers_in_var_part, - self.q_weakpointer_offset) + self.q_weakpointer_offset, + self.q_member_index) -T_IS_VARSIZE = 0x01 -T_HAS_GCPTR_IN_VARSIZE = 0x02 -T_IS_GCARRAY_OF_GCPTR = 0x04 -T_IS_WEAKREF = 0x08 +# the lowest 16bits are used to store group member index +T_MEMBER_INDEX = 0xffff +T_IS_VARSIZE = 0x10000 +T_HAS_GCPTR_IN_VARSIZE = 0x20000 +T_IS_GCARRAY_OF_GCPTR = 0x40000 +T_IS_WEAKREF = 0x80000 def _check_typeid(typeid): ll_assert(llop.is_group_member_nonzero(lltype.Bool, typeid), "invalid type_id") -def encode_type_shape(builder, info, TYPE): +def encode_type_shape(builder, info, TYPE, index): """Encode the shape of the TYPE into the TYPE_INFO structure 'info'.""" offsets = offsets_to_gc_pointers(TYPE) - infobits = 0 + infobits = index info.ofstoptrs = builder.offsets2table(offsets, TYPE) info.finalizer = builder.make_finalizer_funcptr_for_type(TYPE) if not TYPE._is_varsize(): @@ -211,12 +218,12 @@ fullinfo = lltype.malloc(GCData.VARSIZE_TYPE_INFO, immortal=True, zero=True) info = fullinfo.header + type_id = self.type_info_group.add_member(fullinfo) if self.can_encode_type_shape: - encode_type_shape(self, info, TYPE) + encode_type_shape(self, info, TYPE, type_id.index) else: - self._pending_type_shapes.append((info, TYPE)) + self._pending_type_shapes.append((info, TYPE, type_id.index)) # store it - type_id = self.type_info_group.add_member(fullinfo) self.id_of_type[TYPE] = type_id # store the vtable of the type (if any) immediately thereafter # (note that if gcconfig.removetypeptr is False, lltype2vtable @@ -245,8 +252,8 @@ def encode_type_shapes_now(self): if not self.can_encode_type_shape: self.can_encode_type_shape = True - for info, TYPE in self._pending_type_shapes: - encode_type_shape(self, info, TYPE) + for info, TYPE, index in self._pending_type_shapes: + encode_type_shape(self, info, TYPE, index) del self._pending_type_shapes def delay_encoding(self): From fijal at codespeak.net Mon Oct 26 09:58:29 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Oct 2009 09:58:29 +0100 (CET) Subject: [pypy-svn] r68741 - in pypy/branch/gc-dump-heap/pypy: rlib rpython/lltypesystem rpython/memory/gc rpython/memory/gctransform rpython/memory/test Message-ID: <20091026085829.927591683D3@codespeak.net> Author: fijal Date: Mon Oct 26 09:58:27 2009 New Revision: 68741 Modified: pypy/branch/gc-dump-heap/pypy/rlib/rgc.py pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/lloperation.py pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/base.py pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py Log: Kill ugly raw malloc & free, instead say that dump_heap can unwind the gc (which should not be a problem, usually you want to collect first anyway). Modified: pypy/branch/gc-dump-heap/pypy/rlib/rgc.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rlib/rgc.py (original) +++ pypy/branch/gc-dump-heap/pypy/rlib/rgc.py Mon Oct 26 09:58:27 2009 @@ -192,13 +192,6 @@ next = tb[i] os.write(fd, str(next.count) + " " + ",".join([ str(next.links[j]) for j in range(len(tb))]) + "\n") - _clear_dump_heap(tb) - -def _clear_dump_heap(tb): - from pypy.rpython.lltypesystem import lltype - for i in range(len(tb)): - lltype.free(tb[i].links, flavor='raw') - lltype.free(tb, flavor='raw') def _dump_heap(): raise NotImplementedError # can't be run directly @@ -216,7 +209,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.memory.gc.base import ARRAY_TYPEID_MAP from pypy.rpython.lltypesystem import lltype - hop.exception_cannot_occur() + hop.exception_is_here() return hop.genop('gc_dump_heap', [], resulttype=hop.r_result) def malloc_nonmovable(TP, n=None, zero=False): Modified: pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/lloperation.py Mon Oct 26 09:58:27 2009 @@ -460,7 +460,7 @@ 'gc_thread_run' : LLOp(), 'gc_thread_die' : LLOp(), 'gc_assume_young_pointers': LLOp(), - 'gc_dump_heap' : LLOp(), + 'gc_dump_heap' : LLOp(canunwindgc=True), # ------- JIT & GC interaction, only for some GCs ---------- Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/base.py Mon Oct 26 09:58:27 2009 @@ -6,9 +6,9 @@ from pypy.rpython.memory.support import AddressDict from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage -TYPEID_MAP = lltype.Struct('TYPEID_MAP', ('count', lltype.Signed), +TYPEID_MAP = lltype.GcStruct('TYPEID_MAP', ('count', lltype.Signed), ('links', lltype.Array(lltype.Signed))) -ARRAY_TYPEID_MAP = lltype.Array(lltype.Ptr(TYPEID_MAP)) +ARRAY_TYPEID_MAP = lltype.GcArray(lltype.Ptr(TYPEID_MAP)) class GCBase(object): _alloc_flavor_ = "raw" Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py Mon Oct 26 09:58:27 2009 @@ -653,18 +653,9 @@ def dump_heap(self): max_tid = self.root_walker.gcdata.max_type_id - ll_typeid_map = lltype.malloc(ARRAY_TYPEID_MAP, max_tid, - flavor='raw') - i = 0 - while i < max_tid: - ll_typeid_map[i] = lltype.malloc(TYPEID_MAP, max_tid, - flavor='raw') - ll_typeid_map[i].count = 0 - j = 0 - while j < max_tid: - ll_typeid_map[i].links[j] = 0 - j += 1 - i += 1 + ll_typeid_map = lltype.malloc(ARRAY_TYPEID_MAP, max_tid, zero=True) + for i in range(max_tid): + ll_typeid_map[i] = lltype.malloc(TYPEID_MAP, max_tid, zero=True) self._ll_typeid_map = ll_typeid_map self.root_walker.walk_roots( SemiSpaceGC._dump_heap, # stack roots Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py Mon Oct 26 09:58:27 2009 @@ -273,7 +273,8 @@ if hasattr(GCClass, 'dump_heap'): self.dump_heap_ptr = getfn(GCClass.dump_heap.im_func, - [s_gc], annmodel.SomePtr(lltype.Ptr(ARRAY_TYPEID_MAP))) + [s_gc], annmodel.SomePtr(lltype.Ptr(ARRAY_TYPEID_MAP)), + minimal_transform=False) self.get_member_index_ptr = getfn( GCClass.get_member_index.im_func, [s_gc, annmodel.SomeInteger(knowntype=rffi.r_ushort)], Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py Mon Oct 26 09:58:27 2009 @@ -819,7 +819,6 @@ if tb[i].count == 1: b += 1 c = tb[i].links[nr] - rgc._clear_dump_heap(tb) return c * 100 + b * 10 + a return f From fijal at codespeak.net Mon Oct 26 10:09:29 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Oct 2009 10:09:29 +0100 (CET) Subject: [pypy-svn] r68742 - pypy/branch/gc-dump-heap/pypy/translator/c/test Message-ID: <20091026090929.14AA71683D3@codespeak.net> Author: fijal Date: Mon Oct 26 10:09:28 2009 New Revision: 68742 Modified: pypy/branch/gc-dump-heap/pypy/translator/c/test/test_newgc.py Log: Write a test that works on C backend (also only for semispace gc so far) Modified: pypy/branch/gc-dump-heap/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/gc-dump-heap/pypy/translator/c/test/test_newgc.py Mon Oct 26 10:09:28 2009 @@ -899,7 +899,39 @@ def test_gc_set_max_heap_size(self): res = self.run('gc_set_max_heap_size') assert res == 2 + + def define_gc_dump_heap(cls): + S = lltype.GcStruct('S', ('x', lltype.Signed)) + l = [] + def f(): + fd = os.open("/tmp/x.log", os.O_WRONLY | os.O_CREAT, 0777) + for i in range(10): + l.append(lltype.malloc(S)) + rgc.dump_heap(fd) + os.close(fd) + tb = rgc._dump_heap() + a = 0 + nr = 0 + b = 0 + c = 0 + for i in range(len(tb)): + if tb[i].count == 10: + a += 1 + nr = i + for i in range(len(tb)): + if tb[i].count == 1: + b += 1 + c += tb[i].links[nr] + # we don't count b here since there can be more singletons, + # important one is c, a is for check + return c * 10 + a + return f + + def test_gc_dump_heap(self): + res = self.run("gc_dump_heap") + assert res == 101 + def definestr_string_builder(cls): def fn(_): s = StringBuilder() From arigo at codespeak.net Mon Oct 26 11:06:39 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 26 Oct 2009 11:06:39 +0100 (CET) Subject: [pypy-svn] r68744 - pypy/branch/logging Message-ID: <20091026100639.336881683D4@codespeak.net> Author: arigo Date: Mon Oct 26 11:06:37 2009 New Revision: 68744 Added: pypy/branch/logging/ - copied from r68743, pypy/trunk/ Log: A branch for general RPython-level logging support. From cfbolz at codespeak.net Mon Oct 26 11:14:21 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 26 Oct 2009 11:14:21 +0100 (CET) Subject: [pypy-svn] r68745 - pypy/trunk/pypy/translator/backendopt Message-ID: <20091026101421.185B11683D5@codespeak.net> Author: cfbolz Date: Mon Oct 26 11:14:21 2009 New Revision: 68745 Modified: pypy/trunk/pypy/translator/backendopt/stat.py Log: make the statistics printing show the number of mallocs Modified: pypy/trunk/pypy/translator/backendopt/stat.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/stat.py (original) +++ pypy/trunk/pypy/translator/backendopt/stat.py Mon Oct 26 11:14:21 2009 @@ -7,6 +7,7 @@ num_graphs = 0 num_blocks = 0 num_ops = 0 + num_mallocs = 0 per_graph = {} while stack: graph = stack.pop() @@ -16,6 +17,7 @@ num_graphs += 1 old_num_blocks = num_blocks old_num_ops = num_ops + old_num_mallocs = num_mallocs for block in graph.iterblocks(): num_blocks += 1 for op in block.operations: @@ -30,29 +32,34 @@ called_graphs = op.args[-1].value if called_graphs is not None: stack.extend(called_graphs) + elif op.opname.startswith("malloc"): + num_mallocs += 1 num_ops += 1 - per_graph[graph] = (num_blocks-old_num_blocks, num_ops-old_num_ops) + per_graph[graph] = (num_blocks-old_num_blocks, num_ops-old_num_ops, num_mallocs-old_num_mallocs) if save_per_graph_details: details = [] - for graph, (nblocks, nops) in per_graph.iteritems(): + for graph, (nblocks, nops, nmallocs) in per_graph.iteritems(): try: code = graph.func.func_code.co_code except AttributeError: code = "None" hash = md5(code).hexdigest() - details.append((hash, graph.name, nblocks, nops)) + details.append((hash, graph.name, nblocks, nops, nmallocs)) details.sort() f = open(save_per_graph_details, "w") try: - for hash, name, nblocks, nops in details: - print >>f, hash, name, nblocks, nops + for hash, name, nblocks, nops, nmallocs in details: + print >>f, hash, name, nblocks, nops, nmallocs finally: f.close() - return num_graphs, num_blocks, num_ops + return num_graphs, num_blocks, num_ops, num_mallocs def print_statistics(graph, translator, save_per_graph_details=None, ignore_stack_checks=False): - num_graphs, num_blocks, num_ops = get_statistics(graph, translator, save_per_graph_details, - ignore_stack_checks=ignore_stack_checks) + num_graphs, num_blocks, num_ops, num_mallocs = get_statistics( + graph, translator, save_per_graph_details, + ignore_stack_checks=ignore_stack_checks) print ("Statistics:\nnumber of graphs %s\n" "number of blocks %s\n" - "number of operations %s\n") % (num_graphs, num_blocks, num_ops) + "number of operations %s\n" + "number of mallocs %s\n" + ) % (num_graphs, num_blocks, num_ops, num_mallocs) From cfbolz at codespeak.net Mon Oct 26 11:25:34 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 26 Oct 2009 11:25:34 +0100 (CET) Subject: [pypy-svn] r68746 - in pypy/trunk/pypy/translator/backendopt: . test Message-ID: <20091026102534.34E3B1683D5@codespeak.net> Author: cfbolz Date: Mon Oct 26 11:25:33 2009 New Revision: 68746 Modified: pypy/trunk/pypy/translator/backendopt/all.py pypy/trunk/pypy/translator/backendopt/escape.py pypy/trunk/pypy/translator/backendopt/test/test_escape.py Log: Cleanup escape analysis: - kill the useless heap2stack transformation - don't track changes to objects, that's a bit useless - instead track which objects are returned - in turn, make returning not escape an object - add a helper that makes it possible to find functions that are "like mallocs", i.e. make an object and return it Modified: pypy/trunk/pypy/translator/backendopt/all.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/all.py (original) +++ pypy/trunk/pypy/translator/backendopt/all.py Mon Oct 26 11:25:33 2009 @@ -6,7 +6,6 @@ from pypy.translator.backendopt.stat import print_statistics from pypy.translator.backendopt.merge_if_blocks import merge_if_blocks from pypy.translator import simplify -from pypy.translator.backendopt.escape import malloc_to_stack from pypy.translator.backendopt import mallocprediction from pypy.translator.backendopt.removeassert import remove_asserts from pypy.translator.backendopt.support import log @@ -123,10 +122,6 @@ call_count_pred=call_count_pred) constfold(config, graphs) - if config.heap2stack: - assert graphs is translator.graphs # XXX for now - malloc_to_stack(translator) - if config.merge_if_blocks: log.mergeifblocks("starting to merge if blocks") for graph in graphs: Modified: pypy/trunk/pypy/translator/backendopt/escape.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/escape.py (original) +++ pypy/trunk/pypy/translator/backendopt/escape.py Mon Oct 26 11:25:33 2009 @@ -1,4 +1,3 @@ -from pypy.annotation.model import setunion from pypy.objspace.flow.model import Variable, Constant from pypy.rpython.lltypesystem import lltype from pypy.translator.simplify import get_graph @@ -7,37 +6,31 @@ from pypy.tool.uid import uid class CreationPoint(object): - def __init__(self, creation_method, lltype): - self.changes = False + def __init__(self, creation_method, TYPE, op=None): self.escapes = False + self.returns = False self.creation_method = creation_method if creation_method == "constant": - self.changes = True self.escapes = True - self.malloced = False - self.lltype = lltype + self.TYPE = TYPE + self.op = op def __repr__(self): - return ("CreationPoint(<0x%x>, %r, %s, esc=%s, cha=%s)" % - (uid(self), self.lltype, self.creation_method, self.escapes, self.changes)) + return ("CreationPoint(<0x%x>, %r, %s, esc=%s)" % + (uid(self), self.TYPE, self.creation_method, self.escapes)) class VarState(object): - def __init__(self, crep=None): - self.creation_points = {} - if crep is not None: - self.creation_points[crep] = True + def __init__(self, *creps): + self.creation_points = set() + for crep in creps: + self.creation_points.add(crep) def contains(self, other): - for crep in other.creation_points: - if crep not in self.creation_points: - return False - return True + return other.creation_points.issubset(self.creation_points) def merge(self, other): - creation_points = setunion(self.creation_points, other.creation_points) - newstate = VarState() - newstate.creation_points = creation_points - return newstate + creation_points = self.creation_points.union(other.creation_points) + return VarState(*creation_points) def setescapes(self): changed = [] @@ -47,29 +40,28 @@ crep.escapes = True return changed - def setchanges(self): + def setreturns(self): changed = [] for crep in self.creation_points: - if not crep.changes: + if not crep.returns: changed.append(crep) - crep.changes = True + crep.returns = True return changed - + def does_escape(self): for crep in self.creation_points: if crep.escapes: return True return False - def does_change(self): + def does_return(self): for crep in self.creation_points: - if crep.changes: + if crep.returns: return True return False - + def __repr__(self): - crepsrepr = (", ".join([repr(crep) for crep in self.creation_points]), ) - return "VarState({%s})" % crepsrepr + return "" % (self.creation_points, ) class AbstractDataFlowInterpreter(object): def __init__(self, translation_context): @@ -108,15 +100,14 @@ def setstate(self, var, state): self.varstates[var] = state - def get_creationpoint(self, var, method="?"): + def get_creationpoint(self, var, method="?", op=None): if var in self.creationpoints: return self.creationpoints[var] - crep = CreationPoint(method, var.concretetype) + crep = CreationPoint(method, var.concretetype, op) self.creationpoints[var] = crep return crep def schedule_function(self, graph): - #print "scheduling function:", graph.name startblock = graph.startblock if graph in self.functionargs: args = self.functionargs[graph] @@ -136,54 +127,39 @@ return resultstate, args def flow_block(self, block, graph): - #print "flowing in block %s of function %s" % (block, graph.name) self.flown_blocks[block] = True if block is graph.returnblock: if isonheap(block.inputargs[0]): - changed = self.getstate(block.inputargs[0]).setescapes() - self.handle_changed(changed) + self.returns(self.getstate(block.inputargs[0])) return if block is graph.exceptblock: if isonheap(block.inputargs[0]): - changed = self.getstate(block.inputargs[0]).setescapes() - self.handle_changed(changed) + self.escapes(self.getstate(block.inputargs[0])) if isonheap(block.inputargs[1]): - changed = self.getstate(block.inputargs[1]).setescapes() - self.handle_changed(changed) + self.escapes(self.getstate(block.inputargs[1])) return self.curr_block = block self.curr_graph = graph - #print "inputargs", self.getstates(block.inputargs) for op in block.operations: self.flow_operation(op) - #print "checking exits..." for exit in block.exits: - #print "exit", exit args = self.getstates(exit.args) targetargs = self.getstates(exit.target.inputargs) - #print " newargs", args - #print " targetargs", targetargs - # flow every block at least once: + # flow every block at least once if (multicontains(targetargs, args) and exit.target in self.flown_blocks): - #print " not necessary" continue - #else: - #print " scheduling for flowin" for prevstate, origstate, var in zip(args, targetargs, exit.target.inputargs): if not isonheap(var): continue newstate = prevstate.merge(origstate) self.setstate(var, newstate) - #print " args", self.getstates(exit.target.inputargs) self.scheduled[exit.target] = graph def flow_operation(self, op): - #print "handling", op args = self.getstates(op.args) - #print "args:", args opimpl = getattr(self, 'op_'+op.opname, None) if opimpl is not None: res = opimpl(op, *args) @@ -194,18 +170,21 @@ if isonheap(op.result) or filter(None, args): for arg in args: if arg is not None: - changed = arg.setchanges() - self.handle_changed(changed) - changed = arg.setescapes() - self.handle_changed(changed) - #raise NotImplementedError("can't handle %s" % (op.opname, )) - #print "assuming that '%s' is irrelevant" % op + self.escapes(arg) def complete(self): while self.scheduled: block, graph = self.scheduled.popitem() self.flow_block(block, graph) + def escapes(self, arg): + changed = arg.setescapes() + self.handle_changed(changed) + + def returns(self, arg): + changed = arg.setreturns() + self.handle_changed(changed) + def handle_changed(self, changed): for crep in changed: if crep not in self.dependencies: @@ -222,13 +201,10 @@ def register_state_dependency(self, state1, state2): "state1 depends on state2: if state2 does escape/change, so does state1" # change state1 according to how state2 is now - #print "registering dependency of %s on %s" % (state1, state2) if state2.does_escape(): - changed = state1.setescapes() # mark all crep's as escaping - self.handle_changed(changed) - if state2.does_change(): - changed = state1.setchanges() # mark all crep's as changing - self.handle_changed(changed) + self.escapes(state1) + if state2.does_return(): + self.returns(state1) # register a dependency of the current block on state2: # that means that if state2 changes the current block will be reflown # triggering this function again and thus updating state1 @@ -242,14 +218,14 @@ flags = op.args[1].value if flags != {'flavor': 'gc'}: return NotImplemented - return VarState(self.get_creationpoint(op.result, "malloc")) + return VarState(self.get_creationpoint(op.result, "malloc", op)) def op_malloc_varsize(self, op, typestate, flagsstate, lengthstate): assert flagsstate is None flags = op.args[1].value if flags != {'flavor': 'gc'}: return NotImplemented - return VarState(self.get_creationpoint(op.result, "malloc_varsize")) + return VarState(self.get_creationpoint(op.result, "malloc_varsize", op)) def op_keepalive(self, op, state): return None @@ -258,49 +234,26 @@ return state def op_setfield(self, op, objstate, fieldname, valuestate): - changed = objstate.setchanges() - self.handle_changed(changed) if valuestate is not None: # be pessimistic for now: - # everything that gets stored into a structure escapes and changes - self.handle_changed(changed) - changed = valuestate.setchanges() - self.handle_changed(changed) - changed = valuestate.setescapes() - self.handle_changed(changed) + # everything that gets stored into a structure escapes + self.escapes(valuestate) return None def op_setarrayitem(self, op, objstate, indexstate, valuestate): - changed = objstate.setchanges() - self.handle_changed(changed) if valuestate is not None: - # everything that gets stored into a structure escapes and changes - self.handle_changed(changed) - changed = valuestate.setchanges() - self.handle_changed(changed) - changed = valuestate.setescapes() - self.handle_changed(changed) + # everything that gets stored into a structure escapes + self.escapes(valuestate) return None def op_getarrayitem(self, op, objstate, indexstate): if isonheap(op.result): - return VarState(self.get_creationpoint(op.result, "getarrayitem")) + return VarState(self.get_creationpoint(op.result, "getarrayitem", op)) def op_getfield(self, op, objstate, fieldname): if isonheap(op.result): # assume that getfield creates a new value - return VarState(self.get_creationpoint(op.result, "getfield")) - - def op_getsubstruct(self, op, objstate, fieldname): - # since this is really an embedded struct, it has the same - # state, the same creationpoints, etc. - return objstate - - def op_getarraysubstruct(self, op, arraystate, indexstate): - # since this is really a struct embedded somewhere in the array it has - # the same state, creationpoints, etc. in most cases the resulting - # pointer should not be used much anyway - return arraystate + return VarState(self.get_creationpoint(op.result, "getfield", op)) def op_getarraysize(self, op, arraystate): pass @@ -311,9 +264,8 @@ for arg in args: if arg is None: continue - # an external function can change every parameter: - changed = arg.setchanges() - self.handle_changed(changed) + # an external function can escape every parameter: + self.escapes(arg) funcargs = [None] * len(args) else: result, funcargs = self.schedule_function(graph) @@ -326,7 +278,7 @@ self.register_state_dependency(localarg, funcarg) if isonheap(op.result): # assume that a call creates a new value - return VarState(self.get_creationpoint(op.result, "direct_call")) + return VarState(self.get_creationpoint(op.result, "direct_call", op)) def op_indirect_call(self, op, function, *args): graphs = op.args[-1].value @@ -335,10 +287,7 @@ for localarg in args: if localarg is None: continue - changed = localarg.setescapes() - self.handle_changed(changed) - changed = localarg.setchanges() - self.handle_changed(changed) + self.escapes(localarg) else: for graph in graphs: result, funcargs = self.schedule_function(graph) @@ -350,7 +299,7 @@ self.register_state_dependency(localarg, funcarg) if isonheap(op.result): # assume that a call creates a new value - return VarState(self.get_creationpoint(op.result, "indirect_call")) + return VarState(self.get_creationpoint(op.result, "indirect_call", op)) def op_ptr_iszero(self, op, ptrstate): return None @@ -377,35 +326,38 @@ return False return True -def malloc_to_stack(t): - adi = AbstractDataFlowInterpreter(t) - for graph in t.graphs: - if graph.startblock not in adi.flown_blocks: - adi.schedule_function(graph) - adi.complete() - for graph in t.graphs: - loop_blocks = support.find_loop_blocks(graph) - for block, op in graph.iterblockops(): - if op.opname != 'malloc': - continue - STRUCT = op.args[0].value - # must not remove mallocs of structures that have a RTTI with a destructor - try: - destr_ptr = lltype.getRuntimeTypeInfo(STRUCT)._obj.destructor_funcptr - if destr_ptr: - continue - except (ValueError, AttributeError), e: - pass - varstate = adi.getstate(op.result) - assert len(varstate.creation_points) == 1 - crep = varstate.creation_points.keys()[0] - if not crep.escapes: - if block not in loop_blocks: - print "moving object from heap to stack %s in %s" % (op, graph.name) - flags = op.args[1].value - assert flags == {'flavor': 'gc'} - op.args[1] = Constant({'flavor': 'stack'}, lltype.Void) - else: - print "%s in %s is a non-escaping malloc in a loop" % (op, graph.name) + +def is_malloc_like(adi, graph, seen): + if graph in seen: + return seen[graph] + return_state = adi.getstate(graph.getreturnvar()) + if return_state is None or len(return_state.creation_points) != 1: + seen[graph] = False + return False + crep, = return_state.creation_points + if crep.escapes: + seen[graph] = False + return False + if crep.creation_method in ["malloc", "malloc_varsize"]: + assert crep.returns + seen[graph] = True + return True + if crep.creation_method == "direct_call": + subgraph = get_graph(crep.op.args[0], adi.translation_context) + if subgraph is None: + seen[graph] = False + return False + res = is_malloc_like(adi, subgraph, seen) + seen[graph] = res + return res + seen[graph] = False + return False + + +def malloc_like_graphs(adi): + seen = {} + return [graph for graph in adi.seen_graphs() + if is_malloc_like(adi, graph, seen)] + Modified: pypy/trunk/pypy/translator/backendopt/test/test_escape.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/test/test_escape.py (original) +++ pypy/trunk/pypy/translator/backendopt/test/test_escape.py Mon Oct 26 11:25:33 2009 @@ -1,41 +1,22 @@ -from pypy.translator.translator import TranslationContext, graphof -from pypy.translator.backendopt.escape import AbstractDataFlowInterpreter, malloc_to_stack -from pypy.translator.backendopt.support import find_backedges, find_loop_blocks -from pypy.rpython.llinterp import LLInterpreter +from pypy.translator.interactive import Translation +from pypy.translator.translator import graphof +from pypy.translator.backendopt.escape import AbstractDataFlowInterpreter +from pypy.translator.backendopt.escape import malloc_like_graphs from pypy.rlib.objectmodel import instantiate from pypy import conftest import py def build_adi(function, types): - t = TranslationContext() - t.buildannotator().build_types(function, types) - t.buildrtyper().specialize() + t = Translation(function) + t.rtype(types) if conftest.option.view: t.view() - adi = AbstractDataFlowInterpreter(t) - graph = graphof(t, function) + adi = AbstractDataFlowInterpreter(t.context) + graph = graphof(t.context, function) adi.schedule_function(graph) adi.complete() - return t, adi, graph - -def check_malloc_removal(function, types, args, expected_result, must_remove=True): - t = TranslationContext() - t.buildannotator().build_types(function, types) - t.buildrtyper().specialize() - interp = LLInterpreter(t.rtyper) - graph = graphof(t, function) - res = interp.eval_graph(graph, args) - assert res == expected_result - malloc_to_stack(t) - if must_remove: - for block in graph.iterblocks(): - for op in block.operations: - if op.opname == "malloc": - assert op.args[1].value['flavor'] == 'stack' - res = interp.eval_graph(graph, args) - assert res == expected_result - return t + return t.context, adi, graph def test_simple(): class A(object): @@ -47,9 +28,7 @@ t, adi, graph = build_adi(f, []) avar = graph.startblock.operations[0].result state = adi.getstate(avar) - assert len(state.creation_points) == 1 - crep = state.creation_points.keys()[0] - assert crep.changes + crep, = state.creation_points assert not crep.escapes def test_branch(): @@ -66,9 +45,7 @@ t, adi, graph = build_adi(fn2, [int, int]) tvar = graph.startblock.operations[0].result state = adi.getstate(tvar) - assert len(state.creation_points) == 1 - crep = state.creation_points.keys()[0] - assert crep.changes + crep, = state.creation_points assert not crep.escapes def test_loop(): @@ -85,9 +62,7 @@ t, adi, graph = build_adi(f, []) avar = graph.startblock.operations[0].result state = adi.getstate(avar) - assert len(state.creation_points) == 1 - crep = state.creation_points.keys()[0] - assert crep.changes + crep, = state.creation_points assert not crep.escapes avarinloop = graph.startblock.exits[0].target.inputargs[1] state1 = adi.getstate(avarinloop) @@ -106,8 +81,7 @@ avar = graph.startblock.operations[0].result state = adi.getstate(avar) assert len(state.creation_points) == 1 - crep = state.creation_points.keys()[0] - assert crep.changes + crep, = state.creation_points assert crep.escapes def test_classattrs(): @@ -121,9 +95,7 @@ t, adi, graph = build_adi(fn5, []) bvar = graph.startblock.operations[0].result state = adi.getstate(bvar) - assert len(state.creation_points) == 1 - crep = state.creation_points.keys()[0] - assert crep.changes + crep, = state.creation_points assert not crep.escapes def test_aliasing(): @@ -144,8 +116,7 @@ avar = graph.startblock.exits[0].target.inputargs[1] state = adi.getstate(avar) assert len(state.creation_points) == 2 - for crep in state.creation_points.keys(): - assert crep.changes + for crep in state.creation_points: assert not crep.escapes def test_call(): @@ -161,15 +132,11 @@ g_graph = graphof(t, g) bvar = g_graph.startblock.inputargs[0] bstate = adi.getstate(bvar) - assert len(bstate.creation_points) == 1 - bcrep = bstate.creation_points.keys()[0] - assert not bcrep.changes + bcrep, = bstate.creation_points assert not bcrep.escapes avar = graph.startblock.operations[0].result astate = adi.getstate(avar) - assert len(astate.creation_points) == 1 - acrep = astate.creation_points.keys()[0] - assert acrep.changes + acrep, = astate.creation_points assert not acrep.escapes def test_dependencies(): @@ -197,10 +164,6 @@ reallygraph = resizegraph.startblock.exits[0].target.operations[0].args[0].value._obj.graph reallyarg0 = reallygraph.startblock.inputargs[0] reallystate = adi.getstate(reallyarg0) - assert reallystate.does_change() - assert resizestate.does_change() - assert appendstate.does_change() - assert astate.does_change() def test_substruct(): class A(object): @@ -221,14 +184,10 @@ b0var = graph.startblock.operations[3].result a0state = adi.getstate(a0var) b0state = adi.getstate(b0var) - assert len(a0state.creation_points) == 1 - a0crep = a0state.creation_points.keys()[0] + a0crep, = a0state.creation_points assert not a0crep.escapes - assert a0crep.changes - assert len(b0state.creation_points) == 1 - b0crep = b0state.creation_points.keys()[0] + b0crep, = b0state.creation_points assert b0crep.escapes - assert b0crep.changes def test_multiple_calls(): class A(object): @@ -251,14 +210,11 @@ a1state = adi.getstate(a1var) a2state = adi.getstate(a2var) a3state = adi.getstate(a3var) - assert len(a1state.creation_points) == 1 - assert len(a2state.creation_points) == 1 - assert len(a3state.creation_points) == 1 - a1crep = a1state.creation_points.keys()[0] - a2crep = a2state.creation_points.keys()[0] - a3crep = a3state.creation_points.keys()[0] - assert a1crep.changes and a2crep.changes and a3crep.changes - assert not a1crep.escapes and a2crep.escapes and a3crep.escapes + a1crep, = a1state.creation_points + a2crep, = a2state.creation_points + a3crep, = a3state.creation_points + assert not a1crep.escapes and not a2crep.escapes and not a3crep.escapes + assert not a1crep.returns and a2crep.returns and a3crep.returns def test_indirect_call(): class A(object): @@ -293,12 +249,9 @@ a2var = graph.startblock.operations[3].result a1state = adi.getstate(a1var) a2state = adi.getstate(a2var) - assert len(a1state.creation_points) == 1 - assert len(a2state.creation_points) == 1 - a1crep = a1state.creation_points.keys()[0] - a2crep = a2state.creation_points.keys()[0] - assert a1crep.changes and a2crep.changes - assert not a1crep.escapes and a2crep.escapes + a1crep, = a1state.creation_points + a2crep, = a2state.creation_points + assert not a1crep.escapes and a2crep.returns def test_indirect_call_unknown_graphs(): class A: @@ -324,9 +277,7 @@ t, adi, graph = build_adi(f, []) avar = graph.startblock.operations[0].result state = adi.getstate(avar) - assert len(state.creation_points) == 1 - crep = state.creation_points.keys()[0] - assert crep.changes + crep, = state.creation_points assert crep.escapes def test_flow_blocksonce(): @@ -367,7 +318,6 @@ t, adi, graph = build_adi(createdict, [int, int]) dvar = graph.startblock.operations[0].result dstate = adi.getstate(dvar) - assert dstate.does_change() assert not dstate.does_escape() def test_raise_escapes(): @@ -378,7 +328,18 @@ avar = graph.startblock.operations[0].result state = adi.getstate(avar) assert state.does_escape() - assert state.does_change() + +def test_return(): + class A(object): + pass + def f(): + a = A() + return a + t, adi, graph = build_adi(f, []) + avar = graph.startblock.operations[0].result + state = adi.getstate(avar) + assert not state.does_escape() + assert state.does_return() def test_big(): @@ -387,95 +348,23 @@ # does not crash t, adi, graph = build_adi(entrypoint, [int]) -def test_extfunc_onheaparg(): - py.test.skip("not a valid test anymore") - import os - def f(i): - s = str(i) - os.write(2, s) - return len(s) - t, adi, graph = build_adi(f, [int]) - svar = graph.startblock.operations[0].result - state = adi.getstate(svar) - assert not state.does_escape() - assert state.does_change() - -def test_extfunc_resultonheap(): - py.test.skip("not a valid test anymore") - import os - def f(i): - s = str(i) - return len(s) - t, adi, graph = build_adi(f, [float]) - svar = graph.startblock.operations[0].result - state = adi.getstate(svar) - assert not state.does_escape() - - - -#__________________________________________________________ -# malloc removal tests - -def test_remove_simple(): - class A(object): - pass - def f(): - a = A() - a.x = 1 - return a.x - check_malloc_removal(f, [], [], 1) - -def test_remove_aliasing(): - class A: - pass - def fn6(n): - a1 = A() - a1.x = 5 - a2 = A() - a2.x = 6 - if n > 0: - a = a1 - else: - a = a2 - a.x = 12 - return a1.x - t = check_malloc_removal(fn6, [int], [2], 12) - -def test_remove_call(): +def test_find_malloc_like_graphs(): class A(object): pass - def g(b): - return b.i + 2 - def f(): + def f(x): a = A() - a.i = 2 - return g(a) - t = check_malloc_removal(f, [], [], 4) + a.x = x + return a -def test_dont_alloca_in_loops(): - class A(object): - pass - def f(x): - result = 0 - for i in range(x): - a = A() - a.i = i - result += a.i - return result - t = check_malloc_removal(f, [int], [3], 3, must_remove=False) - graph = graphof(t, f) - assert graph.startblock.exits[0].target.exits[0].target.operations[0].opname == "malloc" + def g(a): + return a -def test_dont_remove_del_objects(): - class A(object): - def __del__(self): - pass - def f(): - a = A() - a.i = 1 - return a.i - t = check_malloc_removal(f, [], [], 1, must_remove=False) - graph = graphof(t, f) - assert graph.startblock.operations[0].opname == "malloc" - + def h(x): + return f(x + 1) + def main(x): + return f(x).x + g(h(x)).x + + t, adi, graph = build_adi(main, [int]) + graphs = malloc_like_graphs(adi) + assert [g.name for g in graphs] == ["f", "h"] From arigo at codespeak.net Mon Oct 26 11:43:56 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 26 Oct 2009 11:43:56 +0100 (CET) Subject: [pypy-svn] r68747 - pypy/trunk/lib-python/modified-2.5.2/test Message-ID: <20091026104356.7CBD91683D5@codespeak.net> Author: arigo Date: Mon Oct 26 11:43:55 2009 New Revision: 68747 Added: pypy/trunk/lib-python/modified-2.5.2/test/test_strptime.py - copied, changed from r68734, pypy/trunk/lib-python/2.5.2/test/test_strptime.py Log: Bogus usage of id() in a test. Failed once on pypy-c because the old object was dead, and the id() returned for the new object was the same integer, completely by chance. Copied: pypy/trunk/lib-python/modified-2.5.2/test/test_strptime.py (from r68734, pypy/trunk/lib-python/2.5.2/test/test_strptime.py) ============================================================================== --- pypy/trunk/lib-python/2.5.2/test/test_strptime.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/test/test_strptime.py Mon Oct 26 11:43:55 2009 @@ -523,16 +523,16 @@ return try: _strptime.strptime('10', '%d') - # Get id of current cache object. - first_time_re_id = id(_strptime._TimeRE_cache) + # Get the current cache object. + first_time_re = _strptime._TimeRE_cache try: # Change the locale and force a recreation of the cache. locale.setlocale(locale.LC_TIME, ('de_DE', 'UTF8')) _strptime.strptime('10', '%d') - # Get the new cache object's id. - second_time_re_id = id(_strptime._TimeRE_cache) - # They should not be equal. - self.failIfEqual(first_time_re_id, second_time_re_id) + # Get the new cache object. + second_time_re = _strptime._TimeRE_cache + # They should not be identical. + self.assert_(first_time_re is not second_time_re) # Possible test locale is not supported while initial locale is. # If this is the case just suppress the exception and fall-through # to the reseting to the original locale. From cfbolz at codespeak.net Mon Oct 26 11:55:12 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 26 Oct 2009 11:55:12 +0100 (CET) Subject: [pypy-svn] r68748 - in pypy/trunk/pypy/translator/backendopt: . test Message-ID: <20091026105512.8113F1683D5@codespeak.net> Author: cfbolz Date: Mon Oct 26 11:55:12 2009 New Revision: 68748 Modified: pypy/trunk/pypy/translator/backendopt/mallocprediction.py pypy/trunk/pypy/translator/backendopt/test/test_mallocprediction.py Log: Improve malloc prediction to also consider malloc-like calls to be mallocs, works rather well, but not quite well enough. - Modified: pypy/trunk/pypy/translator/backendopt/mallocprediction.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/mallocprediction.py (original) +++ pypy/trunk/pypy/translator/backendopt/mallocprediction.py Mon Oct 26 11:55:12 2009 @@ -1,15 +1,16 @@ from pypy.translator.backendopt.escape import AbstractDataFlowInterpreter +from pypy.translator.backendopt.escape import malloc_like_graphs from pypy.translator.backendopt.all import remove_mallocs from pypy.translator.backendopt import inline from pypy.rpython.lltypesystem import lltype -from pypy.translator import simplify +from pypy.translator.simplify import get_graph from pypy.translator.backendopt import removenoops from pypy.translator.backendopt.support import log SMALL_THRESHOLD = 15 BIG_THRESHOLD = 50 -def find_malloc_creps(graph, adi, translator): +def find_malloc_creps(graph, adi, translator, malloc_graphs): # mapping from malloc creation point to graphs that it flows into malloc_creps = {} # find all mallocs that don't escape @@ -29,8 +30,17 @@ pass varstate = adi.getstate(op.result) assert len(varstate.creation_points) == 1 - crep = varstate.creation_points.keys()[0] - if not crep.escapes: + crep, = varstate.creation_points + if not crep.escapes and not crep.returns: + malloc_creps[crep] = {} + if op.opname == 'direct_call': + called_graph = get_graph(op.args[0], translator) + if called_graph not in malloc_graphs: + continue + varstate = adi.getstate(op.result) + assert len(varstate.creation_points) == 1 + crep, = varstate.creation_points + if not crep.escapes and not crep.returns: malloc_creps[crep] = {} return malloc_creps @@ -64,16 +74,19 @@ del interesting_creps[crep] elif op.opname == "direct_call": #print op, interesting_creps - called_graph = simplify.get_graph(op.args[0], translator) + called_graph = get_graph(op.args[0], translator) interesting = {} - for i, var in enumerate(op.args[1:]): - #print i, var, + if called_graph is None: + graphvars = [None] * len(op.args) + else: + graphvars = called_graph.getargs() + [called_graph.getreturnvar()] + for var, graphvar in zip(op.args[1:] + [op.result], graphvars): varstate = adi.getstate(var) if varstate is None: #print "no varstate" continue if len(varstate.creation_points) == 1: - crep = varstate.creation_points.keys()[0] + crep, = varstate.creation_points if crep not in interesting_creps: #print "not interesting" continue @@ -81,16 +94,14 @@ del interesting_creps[crep] #print "graph not found" continue - if (called_graph, i) in seen: - seen[(called_graph, i)][graph] = True + if called_graph in seen: + seen[called_graph][graph] = True #print "seen already" else: #print "taking", crep - seen[(called_graph, i)] = {graph: True} - arg = called_graph.startblock.inputargs[i] - argstate = adi.getstate(arg) - argcrep = [c for c in argstate.creation_points - if c.creation_method == "arg"][0] + seen[called_graph] = {graph: True} + argstate = adi.getstate(graphvar) + argcrep, = argstate.creation_points interesting[argcrep] = True #print interesting if interesting: @@ -104,21 +115,21 @@ if graph.startblock not in adi.flown_blocks: adi.schedule_function(graph) adi.complete() + malloc_graphs = malloc_like_graphs(adi) targetset = dict.fromkeys(graphs) caller_candidates = {} seen = {} for graph in adi.seen_graphs(): - creps = find_malloc_creps(graph, adi, t) - #print "malloc creps", creps + creps = find_malloc_creps(graph, adi, t, malloc_graphs) if creps: find_calls_where_creps_go(creps, graph, adi, t, seen) if creps: if graph in targetset: caller_candidates[graph] = True callgraph = [] - for (called_graph, i), callers in seen.iteritems(): + for called_graph, callers in seen.iteritems(): for caller in callers: - if caller in targetset: + if caller in targetset and called_graph in targetset: callgraph.append((caller, called_graph)) else: log.inlineandremove.WARNING("would like to inline into" Modified: pypy/trunk/pypy/translator/backendopt/test/test_mallocprediction.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/test/test_mallocprediction.py (original) +++ pypy/trunk/pypy/translator/backendopt/test/test_mallocprediction.py Mon Oct 26 11:55:12 2009 @@ -54,7 +54,7 @@ assert caller_candidates == {graph: True} assert len(callgraph) == 1 ggraph = graphof(t, g) - assert callgraph[graph] == {ggraph: True} + assert callgraph == {graph: {ggraph: True}} def test_multiple_calls(): class A: @@ -81,9 +81,48 @@ assert len(callgraph) == 1 g1graph = graphof(t, g1) g2graph = graphof(t, g2) - assert callgraph[graph] == {g1graph: True} + assert callgraph == {graph: {g1graph: True}} callgraph, caller_candidates = check_inlining(t, graph, [0], 3 * 42) - assert callgraph[graph] == {g2graph: True} + assert callgraph == {graph: {g2graph: True}} + +def test_malloc_returns(): + class A: + pass + def g(a): + return a.x + def h(x): + return x + 42 + def make_a(x): + a = A() + a.x = x + return a + def fn(i): + a = make_a(h(i)) + return g(a) + t, graph = rtype(fn, [int]) + callgraph, caller_candidates = check_inlining(t, graph, [0], 42) + assert caller_candidates == {graph: True} + assert len(callgraph) == 1 + ggraph = graphof(t, g) + makegraph = graphof(t, make_a) + assert callgraph == {graph: {ggraph: True, makegraph: True}} + +def test_tuple(): + def f(x, y): + return h(x + 1, x * y) + def h(x, y): + return x, y + + def g(x): + a, b = f(x, x*5) + return a + b + t, graph = rtype(g, [int]) + callgraph, caller_candidates = check_inlining(t, graph, [2], 23) + assert caller_candidates == {graph: True} + assert len(callgraph) == 2 + fgraph = graphof(t, f) + hgraph = graphof(t, h) + assert callgraph == {graph: {fgraph: True}, fgraph: {hgraph: True}} def test_indirect_call(): class A(object): @@ -133,9 +172,21 @@ assert total0 + total == 10 def test_richards(): - py.test.skip("Unsure if this makes any sense any more") from pypy.translator.goal.richards import entry_point t, graph = rtype(entry_point, [int]) total0 = preparation(t, t.graphs) total = clever_inlining_and_malloc_removal(t) assert total0 + total == 9 + +def test_loop(): + l = [10, 12, 15, 1] + def f(x): + res = 0 + for i in range(x): + res += i + for i in l: + res += i + return res + t, graph = rtype(f, [int]) + total = clever_inlining_and_malloc_removal(t) + assert total == 3 # range, two iterators From cfbolz at codespeak.net Mon Oct 26 14:46:32 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 26 Oct 2009 14:46:32 +0100 (CET) Subject: [pypy-svn] r68749 - in pypy/branch/shrink-multidict/pypy: interpreter module/__builtin__ module/__builtin__/test objspace/std objspace/std/test Message-ID: <20091026134632.F18F91683D7@codespeak.net> Author: cfbolz Date: Mon Oct 26 14:46:32 2009 New Revision: 68749 Removed: pypy/branch/shrink-multidict/pypy/objspace/std/dictbucket.py pypy/branch/shrink-multidict/pypy/objspace/std/test/test_dictbucket.py Modified: pypy/branch/shrink-multidict/pypy/interpreter/baseobjspace.py pypy/branch/shrink-multidict/pypy/interpreter/typedef.py pypy/branch/shrink-multidict/pypy/module/__builtin__/interp_classobj.py pypy/branch/shrink-multidict/pypy/module/__builtin__/test/test_classobj.py pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py pypy/branch/shrink-multidict/pypy/objspace/std/dicttype.py pypy/branch/shrink-multidict/pypy/objspace/std/marshal_impl.py pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py pypy/branch/shrink-multidict/pypy/objspace/std/test/test_celldict.py pypy/branch/shrink-multidict/pypy/objspace/std/test/test_dictmultiobject.py pypy/branch/shrink-multidict/pypy/objspace/std/test/test_shadowtracking.py pypy/branch/shrink-multidict/pypy/objspace/std/typeobject.py Log: Refactor the multidict to use one level of indirection less. Instead of having an indirection, all W_DictMultiObjects have an attribute r_dict_content that is usually None. When the dict devolves, it is set to an r_dict and used instead of the normal attributes. Modified: pypy/branch/shrink-multidict/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/shrink-multidict/pypy/interpreter/baseobjspace.py Mon Oct 26 14:46:32 2009 @@ -20,6 +20,7 @@ in a 'normal' object space like StdObjSpace.""" __slots__ = () _settled_ = True + user_overridden_class = False def getdict(self): return None Modified: pypy/branch/shrink-multidict/pypy/interpreter/typedef.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/interpreter/typedef.py (original) +++ pypy/branch/shrink-multidict/pypy/interpreter/typedef.py Mon Oct 26 14:46:32 2009 @@ -190,6 +190,8 @@ if "user" in features: # generic feature needed by all subcls class Proto(object): + user_overridden_class = True + def getclass(self, space): return hint(self.w__class__, promote=True) @@ -253,30 +255,20 @@ def user_setup(self, space, w_subtype): self.space = space self.w__class__ = w_subtype - if space.config.objspace.std.withsharingdict: - from pypy.objspace.std import dictmultiobject - self.w__dict__ = dictmultiobject.W_DictMultiObject(space, - sharing=True) - elif space.config.objspace.std.withshadowtracking: - from pypy.objspace.std import dictmultiobject - self.w__dict__ = dictmultiobject.W_DictMultiObject(space) - self.w__dict__.implementation = \ - dictmultiobject.ShadowDetectingDictImplementation( - space, w_subtype) - else: - self.w__dict__ = space.newdict() + self.w__dict__ = space.newdict( + instance=True, classofinstance=w_subtype) self.user_setup_slots(w_subtype.nslots) def setclass(self, space, w_subtype): # only used by descr_set___class__ self.w__class__ = w_subtype if space.config.objspace.std.withshadowtracking: - self.w__dict__.implementation.set_shadows_anything() + self.w__dict__.set_shadows_anything() def getdictvalue_attr_is_in_class(self, space, w_name): w_dict = self.w__dict__ if space.config.objspace.std.withshadowtracking: - if not w_dict.implementation.shadows_anything(): + if not w_dict.shadows_anything(): return None return space.finditem(w_dict, w_name) Modified: pypy/branch/shrink-multidict/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/shrink-multidict/pypy/module/__builtin__/interp_classobj.py Mon Oct 26 14:46:32 2009 @@ -304,12 +304,7 @@ class W_InstanceObject(Wrappable): def __init__(self, space, w_class, w_dict=None): if w_dict is None: - if space.config.objspace.std.withsharingdict: - from pypy.objspace.std import dictmultiobject - w_dict = dictmultiobject.W_DictMultiObject(space, - sharing=True) - else: - w_dict = space.newdict() + w_dict = space.newdict(instance=True) assert isinstance(w_class, W_ClassObject) self.w_class = w_class self.w_dict = w_dict Modified: pypy/branch/shrink-multidict/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/shrink-multidict/pypy/module/__builtin__/test/test_classobj.py Mon Oct 26 14:46:32 2009 @@ -772,8 +772,7 @@ def is_sharing(space, w_inst): from pypy.objspace.std.sharingdict import SharedDictImplementation, W_DictMultiObject w_d = w_inst.getdict() - return space.wrap(isinstance(w_d, W_DictMultiObject) and - isinstance(w_d.implementation, SharedDictImplementation)) + return space.wrap(isinstance(w_d, SharedDictImplementation)) cls.w_is_sharing = cls.space.wrap(gateway.interp2app(is_sharing)) Modified: pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py Mon Oct 26 14:46:32 2009 @@ -1,4 +1,3 @@ -from pypy.objspace.std.dictmultiobject import DictImplementation from pypy.objspace.std.dictmultiobject import IteratorImplementation from pypy.objspace.std.dictmultiobject import W_DictMultiObject, _is_sane_hash from pypy.rlib import jit @@ -15,7 +14,7 @@ def __repr__(self): return "" % (self.w_value, ) -class ModuleDictImplementation(DictImplementation): +class ModuleDictImplementation(W_DictMultiObject): def __init__(self, space): self.space = space self.content = {} @@ -44,14 +43,14 @@ w_value = cell.invalidate() cell = impl.content[name] = ModuleCell(w_value) - def setitem(self, w_key, w_value): + def impl_setitem(self, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_str): - return self.setitem_str(w_key, w_value) + self.impl_setitem_str(w_key, w_value) else: - return self._as_rdict().setitem(w_key, w_value) + self._as_rdict().setitem(w_key, w_value) - def setitem_str(self, w_key, w_value, shadows_type=True): + def impl_setitem_str(self, w_key, w_value, shadows_type=True): name = self.space.str_w(w_key) self.getcell(name).w_value = w_value @@ -59,9 +58,7 @@ self.invalidate_unshadowed_builtin(name) del self.unshadowed_builtins[name] - return self - - def delitem(self, w_key): + def impl_delitem(self, w_key): space = self.space w_key_type = space.type(w_key) if space.is_w(w_key_type, space.w_str): @@ -71,16 +68,15 @@ raise KeyError cell.invalidate() del self.content[key] - return self elif _is_sane_hash(space, w_key_type): raise KeyError else: - return self._as_rdict().delitem(w_key) + self._as_rdict().delitem(w_key) - def length(self): + def impl_length(self): return len(self.content) - def get(self, w_lookup): + def impl_getitem(self, w_lookup): space = self.space w_lookup_type = space.type(w_lookup) if space.is_w(w_lookup_type, space.w_str): @@ -91,64 +87,45 @@ elif _is_sane_hash(space, w_lookup_type): return None else: - return self._as_rdict().get(w_lookup) - - def iteritems(self): - return ModuleDictItemIteratorImplementation(self.space, self) - - def iterkeys(self): - return ModuleDictKeyIteratorImplementation(self.space, self) + return self._as_rdict().getitem(w_lookup) - def itervalues(self): - return ModuleDictValueIteratorImplementation(self.space, self) + def impl_iter(self): + return ModuleDictIteratorImplementation(self.space, self) - def keys(self): + def impl_keys(self): space = self.space return [space.wrap(key) for key in self.content.iterkeys()] - def values(self): + def impl_values(self): return [cell.w_value for cell in self.content.itervalues()] - def items(self): + def impl_items(self): space = self.space return [space.newtuple([space.wrap(key), cell.w_value]) for (key, cell) in self.content.iteritems()] - def _as_rdict(self): - newimpl = self.space.DefaultDictImpl(self.space) + def impl_clear(self): + # inefficient, but who cares for k, cell in self.content.iteritems(): - newimpl.setitem(self.space.wrap(k), cell.w_value) cell.invalidate() for k in self.unshadowed_builtins: self.invalidate_unshadowed_builtin(k) - return newimpl - -# grrrrr. just a copy-paste from StrKeyIteratorImplementation in dictmultiobject -class ModuleDictKeyIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.iterkeys() + self.content.clear() + self.unshadowed_builtins.clear() - def next_entry(self): - # note that this 'for' loop only runs once, at most - for key in self.iterator: - return self.space.wrap(key) - else: - return None -class ModuleDictValueIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.itervalues() - - def next_entry(self): - # note that this 'for' loop only runs once, at most - for cell in self.iterator: - return cell.w_value - else: - return None + def _as_rdict(self): + r_dict_content = self.initialize_as_rdict() + for k, cell in self.content.iteritems(): + r_dict_content[self.space.wrap(k)] = cell.w_value + cell.invalidate() + for k in self.unshadowed_builtins: + self.invalidate_unshadowed_builtin(k) + self.content = None + self.unshadowed_builtins = None + return self -class ModuleDictItemIteratorImplementation(IteratorImplementation): +class ModuleDictIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): IteratorImplementation.__init__(self, space, dictimplementation) self.iterator = dictimplementation.content.iteritems() @@ -156,14 +133,9 @@ def next_entry(self): # note that this 'for' loop only runs once, at most for key, cell in self.iterator: - return self.space.newtuple([self.space.wrap(key), cell.w_value]) + return (key, cell.w_value) else: - return None - - - - - + return None, None class State(object): @@ -211,8 +183,8 @@ return holder.getcache(space, code, w_globals) def getimplementation(w_dict): - if type(w_dict) is W_DictMultiObject: - return w_dict.implementation + if type(w_dict) is ModuleDictImplementation and w_dict.r_dict_content is None: + return w_dict else: return None Modified: pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py Mon Oct 26 14:46:32 2009 @@ -22,92 +22,222 @@ space.is_w(w_lookup_type, space.w_float) ) +class W_DictMultiObject(W_Object): + from pypy.objspace.std.dicttype import dict_typedef as typedef -# DictImplementation lattice + r_dict_content = None -# a dictionary starts with an EmptyDictImplementation, and moves down -# in this list: -# -# EmptyDictImplementation -# / \ -# | | -# StrDictImplementation | -# \ / -# RDictImplementation -# -# (in addition, any dictionary can go back to EmptyDictImplementation) + @staticmethod + def allocate_and_init_instance(space, w_type=None, module=False, + instance=False, classofinstance=None, + from_strdict_shared=None): + if from_strdict_shared is not None: + assert w_type is None + assert not module and not instance and classofinstance is None + w_self = StrDictImplementation(space) + w_self.content = from_strdict_shared + return w_self + if space.config.objspace.std.withcelldict and module: + from pypy.objspace.std.celldict import ModuleDictImplementation + assert w_type is None + return ModuleDictImplementation(space) + elif space.config.objspace.opcodes.CALL_LIKELY_BUILTIN and module: + assert w_type is None + return WaryDictImplementation(space) + elif space.config.objspace.std.withdictmeasurement: + assert w_type is None + return MeasuringDictImplementation(space) + elif space.config.objspace.std.withsharingdict and instance: + from pypy.objspace.std.sharingdict import SharedDictImplementation + assert w_type is None + return SharedDictImplementation(space) + elif (space.config.objspace.std.withshadowtracking and instance and + classofinstance is not None): + assert w_type is None + return ShadowDetectingDictImplementation(space, classofinstance) + elif instance: + assert w_type is None + return StrDictImplementation(space) + else: + if w_type is None: + w_type = space.w_dict + w_self = space.allocate_instance(W_DictMultiObject, w_type) + W_DictMultiObject.__init__(w_self, space) + w_self.initialize_as_rdict() + return w_self -class DictImplementation(object): - - def get(self, w_lookup): + def __init__(self, space): + self.space = space + + def initialize_as_rdict(self): + assert self.r_dict_content is None + self.r_dict_content = r_dict(self.space.eq_w, self.space.hash_w) + return self.r_dict_content + + + def initialize_content(w_self, list_pairs_w): + for w_k, w_v in list_pairs_w: + w_self.setitem(w_k, w_v) + + def __repr__(w_self): + """ representation for debugging purposes """ + return "%s()" % (w_self.__class__.__name__, ) + + def unwrap(w_dict, space): + result = {} + items = w_dict.items() + for w_pair in items: + key, val = space.unwrap(w_pair) + result[key] = val + return result + + def missing_method(w_dict, space, w_key): + if not space.is_w(space.type(w_dict), space.w_dict): + w_missing = space.lookup(w_dict, "__missing__") + if w_missing is None: + return None + return space.call_function(w_missing, w_dict, w_key) + else: + return None + + def set_str_keyed_item(w_dict, w_key, w_value, shadows_type=True): + w_dict.setitem_str(w_key, w_value, shadows_type) + + # _________________________________________________________________ + # implementation methods + def impl_getitem(self, w_key): #return w_value or None raise NotImplementedError("abstract base class") - def setitem_str(self, w_key, w_value, shadows_type=True): + def impl_setitem_str(self, w_key, w_value, shadows_type=True): #return implementation raise NotImplementedError("abstract base class") - def setitem(self, w_key, w_value): + def impl_setitem(self, w_key, w_value): #return implementation raise NotImplementedError("abstract base class") - def delitem(self, w_key): + def impl_delitem(self, w_key): #return implementation raise NotImplementedError("abstract base class") - def length(self): + def impl_length(self): raise NotImplementedError("abstract base class") - def iteritems(self): - raise NotImplementedError("abstract base class") - def iterkeys(self): - raise NotImplementedError("abstract base class") - def itervalues(self): + def impl_iter(self): raise NotImplementedError("abstract base class") - - def keys(self): - iterator = self.iterkeys() + def impl_keys(self): + iterator = self.impl_iter() result = [] while 1: - w_key = iterator.next() + w_key, w_value = iterator.next() if w_key is not None: result.append(w_key) else: return result - def values(self): - iterator = self.itervalues() + def impl_values(self): + iterator = self.impl_iter() result = [] while 1: - w_value = iterator.next() + w_key, w_value = iterator.next() if w_value is not None: result.append(w_value) else: return result - def items(self): - iterator = self.iteritems() + def impl_items(self): + iterator = self.impl_iter() result = [] while 1: - w_item = iterator.next() + w_key, w_value = iterator.next() if w_item is not None: - result.append(w_item) + result.append(self.space.newtuple([w_key, w_value])) else: return result -# the following method only makes sense when the option to use the -# CALL_LIKELY_BUILTIN opcode is set. Otherwise it won't even be seen -# by the annotator - def get_builtin_indexed(self, i): + # the following method only makes sense when the option to use the + # CALL_LIKELY_BUILTIN opcode is set. Otherwise it won't even be seen + # by the annotator + def impl_get_builtin_indexed(self, i): w_key = self.space.wrap(OPTIMIZED_BUILTINS[i]) - return self.get(w_key) + return self.getitem(w_key) -# this method will only be seen whan a certain config option is used - def shadows_anything(self): + # this method will only be seen whan a certain config option is used + def impl_shadows_anything(self): return True - def set_shadows_anything(self): + def impl_set_shadows_anything(self): pass + # _________________________________________________________________ + # fallback implementation methods + + def impl_fallback_setitem(self, w_key, w_value): + self.r_dict_content[w_key] = w_value + + def impl_fallback_setitem_str(self, w_key, w_value, shadows_type=True): + return self.impl_fallback_setitem(w_key, w_value) + + def impl_fallback_delitem(self, w_key): + del self.r_dict_content[w_key] + + def impl_fallback_length(self): + return len(self.r_dict_content) + + def impl_fallback_getitem(self, w_key): + return self.r_dict_content.get(w_key, None) + + def impl_fallback_iter(self): + return RDictIteratorImplementation(self.space, self) + + def impl_fallback_keys(self): + return self.r_dict_content.keys() + def impl_fallback_values(self): + return self.r_dict_content.values() + def impl_fallback_items(self): + return [self.space.newtuple([w_key, w_val]) + for w_key, w_val in self.r_dict_content.iteritems()] + def impl_fallback_clear(self): + self.r_dict_content.clear() + +implementation_methods = [ + "getitem", + "length", + "setitem_str", + "setitem", + "delitem", + "iter", + "items", + "values", + "keys", + "clear", + "get_builtin_indexed", + "shadows_anything", + "set_shadows_anything", +] + + +def _make_method(name, fallback): + def implementation_method(self, *args): + if self.r_dict_content is not None: + if not hasattr(self, fallback): + return getattr(W_DictMultiObject, name)(self, *args) + return getattr(self, fallback)(*args) + return getattr(self, name)(*args) + return implementation_method + +def _install_methods(): + for name in implementation_methods: + implname = "impl_" + name + fallbackname = "impl_fallback_" + name + func = _make_method(implname, fallbackname) + setattr(W_DictMultiObject, name, func) +_install_methods() + +registerimplementation(W_DictMultiObject) + +# DictImplementation lattice +# XXX fix me # Iterator Implementation base classes @@ -120,19 +250,19 @@ def next(self): if self.dictimplementation is None: - return None + return None, None if self.len != self.dictimplementation.length(): self.len = -1 # Make this error state sticky raise OperationError(self.space.w_RuntimeError, self.space.wrap("dictionary changed size during iteration")) # look for the next entry - w_result = self.next_entry() - if w_result is not None: + if self.pos < self.len: + result = self.next_entry() self.pos += 1 - return w_result + return result # no more entries self.dictimplementation = None - return None + return None, None def next_entry(self): """ Purely abstract method @@ -148,170 +278,86 @@ # concrete subclasses of the above -class EmptyDictImplementation(DictImplementation): - def __init__(self, space): - self.space = space - - def get(self, w_lookup): - space = self.space - if not _is_str(space, w_lookup) and not _is_sane_hash(space, - space.type(w_lookup)): - # give hash a chance to raise an exception - space.hash(w_lookup) - return None - - def setitem(self, w_key, w_value): - space = self.space - if _is_str(space, w_key): - return StrDictImplementation(space).setitem_str(w_key, w_value) - else: - return space.DefaultDictImpl(space).setitem(w_key, w_value) - def setitem_str(self, w_key, w_value, shadows_type=True): - return StrDictImplementation(self.space).setitem_str(w_key, w_value) - #return SmallStrDictImplementation(self.space, w_key, w_value) - - def delitem(self, w_key): - space = self.space - if not _is_str(space, w_key) and not _is_sane_hash(space, - space.type(w_key)): - # count hash - space.hash(w_key) - raise KeyError - - def length(self): - return 0 - - def iteritems(self): - return EmptyIteratorImplementation(self.space, self) - def iterkeys(self): - return EmptyIteratorImplementation(self.space, self) - def itervalues(self): - return EmptyIteratorImplementation(self.space, self) - - def keys(self): - return [] - def values(self): - return [] - def items(self): - return [] - -class EmptyIteratorImplementation(IteratorImplementation): - def next_entry(self): - return None - - -class StrDictImplementation(DictImplementation): +class StrDictImplementation(W_DictMultiObject): def __init__(self, space): self.space = space self.content = {} - def setitem(self, w_key, w_value): + def impl_setitem(self, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_str): - return self.setitem_str(w_key, w_value) + self.impl_setitem_str(w_key, w_value) else: - return self._as_rdict().setitem(w_key, w_value) + self._as_rdict().setitem(w_key, w_value) - def setitem_str(self, w_key, w_value, shadows_type=True): + def impl_setitem_str(self, w_key, w_value, shadows_type=True): self.content[self.space.str_w(w_key)] = w_value - return self - def delitem(self, w_key): + def impl_delitem(self, w_key): space = self.space w_key_type = space.type(w_key) if space.is_w(w_key_type, space.w_str): del self.content[space.str_w(w_key)] - if self.content: - return self - else: - return space.emptydictimpl + return elif _is_sane_hash(space, w_key_type): raise KeyError else: - return self._as_rdict().delitem(w_key) + self._as_rdict().delitem(w_key) - def length(self): + def impl_length(self): return len(self.content) - def get(self, w_lookup): + def impl_getitem(self, w_key): space = self.space # -- This is called extremely often. Hack for performance -- - if type(w_lookup) is space.StringObjectCls: - return self.content.get(w_lookup.unwrap(space), None) + if type(w_key) is space.StringObjectCls: + return self.content.get(w_key.unwrap(space), None) # -- End of performance hack -- - w_lookup_type = space.type(w_lookup) + w_lookup_type = space.type(w_key) if space.is_w(w_lookup_type, space.w_str): - return self.content.get(space.str_w(w_lookup), None) + return self.content.get(space.str_w(w_key), None) elif _is_sane_hash(space, w_lookup_type): return None else: - return self._as_rdict().get(w_lookup) + return self._as_rdict().getitem(w_key) - def iteritems(self): - return StrItemIteratorImplementation(self.space, self) + def impl_iter(self): + return StrIteratorImplementation(self.space, self) - def iterkeys(self): - return StrKeyIteratorImplementation(self.space, self) - - def itervalues(self): - return StrValueIteratorImplementation(self.space, self) - - def keys(self): + def impl_keys(self): space = self.space return [space.wrap(key) for key in self.content.iterkeys()] - def values(self): + def impl_values(self): return self.content.values() - def items(self): + def impl_items(self): space = self.space return [space.newtuple([space.wrap(key), w_value]) for (key, w_value) in self.content.iteritems()] + def impl_clear(self): + self.content.clear() + def _as_rdict(self): - newimpl = self.space.DefaultDictImpl(self.space) + r_dict_content = self.initialize_as_rdict() for k, w_v in self.content.items(): - newimpl.setitem(self.space.wrap(k), w_v) - return newimpl - -# the following are very close copies of the base classes above - -class StrKeyIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.iterkeys() - - def next_entry(self): - # note that this 'for' loop only runs once, at most - for key in self.iterator: - return self.space.wrap(key) - else: - return None - -class StrValueIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.itervalues() - - def next_entry(self): - # note that this 'for' loop only runs once, at most - for w_value in self.iterator: - return w_value - else: - return None + r_dict_content[self.space.wrap(k)] = w_v + self.content = None + return self -class StrItemIteratorImplementation(IteratorImplementation): +class StrIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): IteratorImplementation.__init__(self, space, dictimplementation) self.iterator = dictimplementation.content.iteritems() def next_entry(self): # note that this 'for' loop only runs once, at most - for key, w_value in self.iterator: - return self.space.newtuple([self.space.wrap(key), w_value]) + for item in self.iterator: + return item else: - return None + return None, None class ShadowDetectingDictImplementation(StrDictImplementation): @@ -324,29 +370,29 @@ else: self._shadows_anything = False - def setitem_str(self, w_key, w_value, shadows_type=True): + def impl_setitem_str(self, w_key, w_value, shadows_type=True): if shadows_type: self._shadows_anything = True - return StrDictImplementation.setitem_str( + StrDictImplementation.setitem_str( self, w_key, w_value, shadows_type) - def setitem(self, w_key, w_value): + def impl_setitem(self, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_str): if not self._shadows_anything: w_obj = self.w_type.lookup(space.str_w(w_key)) if w_obj is not None: self._shadows_anything = True - return StrDictImplementation.setitem_str( + StrDictImplementation.setitem_str( self, w_key, w_value, False) else: - return self._as_rdict().setitem(w_key, w_value) + self._as_rdict().setitem(w_key, w_value) - def shadows_anything(self): + def impl_shadows_anything(self): return (self._shadows_anything or self.w_type.version_tag is not self.original_version_tag) - def set_shadows_anything(self): + def impl_set_shadows_anything(self): self._shadows_anything = True class WaryDictImplementation(StrDictImplementation): @@ -354,15 +400,14 @@ StrDictImplementation.__init__(self, space) self.shadowed = [None] * len(BUILTIN_TO_INDEX) - def setitem_str(self, w_key, w_value, shadows_type=True): + def impl_setitem_str(self, w_key, w_value, shadows_type=True): key = self.space.str_w(w_key) i = BUILTIN_TO_INDEX.get(key, -1) if i != -1: self.shadowed[i] = w_value self.content[key] = w_value - return self - def delitem(self, w_key): + def impl_delitem(self, w_key): space = self.space w_key_type = space.type(w_key) if space.is_w(w_key_type, space.w_str): @@ -371,96 +416,30 @@ i = BUILTIN_TO_INDEX.get(key, -1) if i != -1: self.shadowed[i] = None - return self elif _is_sane_hash(space, w_key_type): raise KeyError else: - return self._as_rdict().delitem(w_key) + self._as_rdict().delitem(w_key) - def get_builtin_indexed(self, i): + def impl_get_builtin_indexed(self, i): return self.shadowed[i] -class RDictImplementation(DictImplementation): - def __init__(self, space): - self.space = space - self.content = r_dict(space.eq_w, space.hash_w) - - def __repr__(self): - return "%s<%s>" % (self.__class__.__name__, self.content) - - def setitem(self, w_key, w_value): - self.content[w_key] = w_value - return self - - def setitem_str(self, w_key, w_value, shadows_type=True): - return self.setitem(w_key, w_value) - - def delitem(self, w_key): - del self.content[w_key] - if self.content: - return self - else: - return self.space.emptydictimpl - - def length(self): - return len(self.content) - def get(self, w_lookup): - return self.content.get(w_lookup, None) - - def iteritems(self): - return RDictItemIteratorImplementation(self.space, self) - def iterkeys(self): - return RDictKeyIteratorImplementation(self.space, self) - def itervalues(self): - return RDictValueIteratorImplementation(self.space, self) - - def keys(self): - return self.content.keys() - def values(self): - return self.content.values() - def items(self): - return [self.space.newtuple([w_key, w_val]) - for w_key, w_val in self.content.iteritems()] - -class RDictKeyIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.iterkeys() - - def next_entry(self): - # note that this 'for' loop only runs once, at most - for w_key in self.iterator: - return w_key - else: - return None - -class RDictValueIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.itervalues() - - def next_entry(self): - # note that this 'for' loop only runs once, at most - for w_value in self.iterator: - return w_value - else: - return None - -class RDictItemIteratorImplementation(IteratorImplementation): +class RDictIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.iteritems() + self.iterator = dictimplementation.r_dict_content.iteritems() def next_entry(self): # note that this 'for' loop only runs once, at most - for w_key, w_value in self.iterator: - return self.space.newtuple([w_key, w_value]) + for item in self.iterator: + return item else: - return None + return None, None +# XXX fix this thing import time, py class DictInfo(object): @@ -518,7 +497,7 @@ def __del__(self): self.info.lifetime = time.time() - self.info.createtime -class MeasuringDictImplementation(DictImplementation): +class MeasuringDictImplementation(W_DictMultiObject): def __init__(self, space): self.space = space self.content = r_dict(space.eq_w, space.hash_w) @@ -544,7 +523,7 @@ else: self.info.misses += 1 - def setitem(self, w_key, w_value): + def impl_setitem(self, w_key, w_value): if not self.info.seen_non_string_in_write and not self._is_str(w_key): self.info.seen_non_string_in_write = True self.info.size_on_non_string_seen_in_write = len(self.content) @@ -552,11 +531,10 @@ self.info.writes += 1 self.content[w_key] = w_value self.info.maxcontents = max(self.info.maxcontents, len(self.content)) - return self - def setitem_str(self, w_key, w_value, shadows_type=True): + def impl_setitem_str(self, w_key, w_value, shadows_type=True): self.info.setitem_strs += 1 - return self.setitem(w_key, w_value) - def delitem(self, w_key): + self.impl_setitem(w_key, w_value) + def impl_delitem(self, w_key): if not self.info.seen_non_string_in_write \ and not self.info.seen_non_string_in_read_first \ and not self._is_str(w_key): @@ -565,38 +543,37 @@ self.info.delitems += 1 self.info.writes += 1 del self.content[w_key] - return self - def length(self): + def impl_length(self): self.info.lengths += 1 return len(self.content) - def get(self, w_lookup): + def impl_getitem(self, w_key): self.info.gets += 1 - self._read(w_lookup) - return self.content.get(w_lookup, None) + self._read(w_key) + return self.content.get(w_key, None) - def iteritems(self): + def impl_iteritems(self): self.info.iteritems += 1 self.info.iterations += 1 return RDictItemIteratorImplementation(self.space, self) - def iterkeys(self): + def impl_iterkeys(self): self.info.iterkeys += 1 self.info.iterations += 1 return RDictKeyIteratorImplementation(self.space, self) - def itervalues(self): + def impl_itervalues(self): self.info.itervalues += 1 self.info.iterations += 1 return RDictValueIteratorImplementation(self.space, self) - def keys(self): + def impl_keys(self): self.info.keys += 1 self.info.listings += 1 return self.content.keys() - def values(self): + def impl_values(self): self.info.values += 1 self.info.listings += 1 return self.content.values() - def items(self): + def impl_items(self): self.info.items += 1 self.info.listings += 1 return [self.space.newtuple([w_key, w_val]) @@ -630,71 +607,6 @@ os.close(fd) os.write(2, "Reporting done.\n") -class W_DictMultiObject(W_Object): - from pypy.objspace.std.dicttype import dict_typedef as typedef - - def __init__(w_self, space, module=False, sharing=False): - if space.config.objspace.std.withcelldict and module: - from pypy.objspace.std.celldict import ModuleDictImplementation - w_self.implementation = ModuleDictImplementation(space) - elif space.config.objspace.opcodes.CALL_LIKELY_BUILTIN and module: - w_self.implementation = WaryDictImplementation(space) - elif space.config.objspace.std.withdictmeasurement: - w_self.implementation = MeasuringDictImplementation(space) - elif space.config.objspace.std.withsharingdict and sharing: - from pypy.objspace.std.sharingdict import SharedDictImplementation - w_self.implementation = SharedDictImplementation(space) - else: - w_self.implementation = space.emptydictimpl - w_self.space = space - - def initialize_content(w_self, list_pairs_w): - impl = w_self.implementation - for w_k, w_v in list_pairs_w: - impl = impl.setitem(w_k, w_v) - w_self.implementation = impl - - def initialize_from_strdict_shared(w_self, strdict): - impl = StrDictImplementation(w_self.space) - impl.content = strdict - w_self.implementation = impl - - def __repr__(w_self): - """ representation for debugging purposes """ - return "%s(%s)" % (w_self.__class__.__name__, w_self.implementation) - - def unwrap(w_dict, space): - result = {} - items = w_dict.implementation.items() - for w_pair in items: - key, val = space.unwrap(w_pair) - result[key] = val - return result - - def missing_method(w_dict, space, w_key): - if not space.is_w(space.type(w_dict), space.w_dict): - w_missing = space.lookup(w_dict, "__missing__") - if w_missing is None: - return None - return space.call_function(w_missing, w_dict, w_key) - else: - return None - - def len(w_self): - return w_self.implementation.length() - - def get(w_dict, w_key, w_default): - w_value = w_dict.implementation.get(w_key) - if w_value is not None: - return w_value - else: - return w_default - - def set_str_keyed_item(w_dict, w_key, w_value, shadows_type=True): - w_dict.implementation = w_dict.implementation.setitem_str( - w_key, w_value, shadows_type) - -registerimplementation(W_DictMultiObject) init_signature = Signature(['seq_or_map'], None, 'kwargs') @@ -715,7 +627,7 @@ raise OperationError(space.w_ValueError, space.wrap("dict() takes a sequence of pairs")) w_k, w_v = pair - w_dict.implementation = w_dict.implementation.setitem(w_k, w_v) + w_dict.setitem(w_k, w_v) else: if space.is_true(w_src): from pypy.objspace.std.dicttype import update1 @@ -724,71 +636,67 @@ from pypy.objspace.std.dicttype import update1 update1(space, w_dict, w_kwds) -def getitem__DictMulti_ANY(space, w_dict, w_lookup): - w_value = w_dict.implementation.get(w_lookup) +def getitem__DictMulti_ANY(space, w_dict, w_key): + w_value = w_dict.getitem(w_key) if w_value is not None: return w_value - w_missing_item = w_dict.missing_method(space, w_lookup) + w_missing_item = w_dict.missing_method(space, w_key) if w_missing_item is not None: return w_missing_item - space.raise_key_error(w_lookup) + space.raise_key_error(w_key) def setitem__DictMulti_ANY_ANY(space, w_dict, w_newkey, w_newvalue): - w_dict.implementation = w_dict.implementation.setitem(w_newkey, w_newvalue) + w_dict.setitem(w_newkey, w_newvalue) -def delitem__DictMulti_ANY(space, w_dict, w_lookup): +def delitem__DictMulti_ANY(space, w_dict, w_key): try: - w_dict.implementation = w_dict.implementation.delitem(w_lookup) + w_dict.delitem(w_key) except KeyError: - space.raise_key_error(w_lookup) + space.raise_key_error(w_key) def len__DictMulti(space, w_dict): - return space.wrap(w_dict.implementation.length()) + return space.wrap(w_dict.length()) -def contains__DictMulti_ANY(space, w_dict, w_lookup): - return space.newbool(w_dict.implementation.get(w_lookup) is not None) +def contains__DictMulti_ANY(space, w_dict, w_key): + return space.newbool(w_dict.getitem(w_key) is not None) dict_has_key__DictMulti_ANY = contains__DictMulti_ANY def iter__DictMulti(space, w_dict): - return W_DictMultiIterObject(space, w_dict.implementation.iterkeys()) + return W_DictMultiIterObject(space, w_dict.iter(), KEYSITER) def eq__DictMulti_DictMulti(space, w_left, w_right): if space.is_w(w_left, w_right): return space.w_True - if w_left.implementation.length() != w_right.implementation.length(): + if w_left.length() != w_right.length(): return space.w_False - iteratorimplementation = w_left.implementation.iteritems() + iteratorimplementation = w_left.iter() while 1: - w_item = iteratorimplementation.next() - if w_item is None: + w_key, w_val = iteratorimplementation.next() + if w_key is None: break - w_key = space.getitem(w_item, space.wrap(0)) - w_val = space.getitem(w_item, space.wrap(1)) - w_rightval = w_right.implementation.get(w_key) + w_rightval = w_right.getitem(w_key) if w_rightval is None: return space.w_False if not space.eq_w(w_val, w_rightval): return space.w_False return space.w_True -def characterize(space, aimpl, bimpl): +def characterize(space, w_a, w_b): """ (similar to CPython) returns the smallest key in acontent for which b's value is different or absent and this value """ w_smallest_diff_a_key = None w_its_value = None - iteratorimplementation = aimpl.iteritems() + iteratorimplementation = w_a.iter() while 1: - w_item = iteratorimplementation.next() - if w_item is None: + w_key, w_val = iteratorimplementation.next() + if w_key is None: break - w_key = space.getitem(w_item, space.wrap(0)) - w_val = space.getitem(w_item, space.wrap(1)) if w_smallest_diff_a_key is None or space.is_true(space.lt(w_key, w_smallest_diff_a_key)): - w_bvalue = bimpl.get(w_key) + w_bvalue = w_b.getitem(w_key) if w_bvalue is None: w_its_value = w_val w_smallest_diff_a_key = w_key @@ -800,18 +708,16 @@ def lt__DictMulti_DictMulti(space, w_left, w_right): # Different sizes, no problem - leftimpl = w_left.implementation - rightimpl = w_right.implementation - if leftimpl.length() < rightimpl.length(): + if w_left.length() < w_right.length(): return space.w_True - if leftimpl.length() > rightimpl.length(): + if w_left.length() > w_right.length(): return space.w_False # Same size - w_leftdiff, w_leftval = characterize(space, leftimpl, rightimpl) + w_leftdiff, w_leftval = characterize(space, w_left, w_right) if w_leftdiff is None: return space.w_False - w_rightdiff, w_rightval = characterize(space, rightimpl, leftimpl) + w_rightdiff, w_rightval = characterize(space, w_right, w_left) if w_rightdiff is None: # w_leftdiff is not None, w_rightdiff is None return space.w_True @@ -824,47 +730,51 @@ def dict_copy__DictMulti(space, w_self): from pypy.objspace.std.dicttype import update1 - w_new = W_DictMultiObject(space) + w_new = W_DictMultiObject.allocate_and_init_instance(space) update1(space, w_new, w_self) return w_new def dict_items__DictMulti(space, w_self): - return space.newlist(w_self.implementation.items()) + return space.newlist(w_self.items()) def dict_keys__DictMulti(space, w_self): - return space.newlist(w_self.implementation.keys()) + return space.newlist(w_self.keys()) def dict_values__DictMulti(space, w_self): - return space.newlist(w_self.implementation.values()) + return space.newlist(w_self.values()) def dict_iteritems__DictMulti(space, w_self): - return W_DictMultiIterObject(space, w_self.implementation.iteritems()) + return W_DictMultiIterObject(space, w_self.iter(), ITEMSITER) def dict_iterkeys__DictMulti(space, w_self): - return W_DictMultiIterObject(space, w_self.implementation.iterkeys()) + return W_DictMultiIterObject(space, w_self.iter(), KEYSITER) def dict_itervalues__DictMulti(space, w_self): - return W_DictMultiIterObject(space, w_self.implementation.itervalues()) + return W_DictMultiIterObject(space, w_self.iter(), VALUESITER) def dict_clear__DictMulti(space, w_self): - w_self.implementation = space.emptydictimpl + w_self.clear() -def dict_get__DictMulti_ANY_ANY(space, w_dict, w_lookup, w_default): - return w_dict.get(w_lookup, w_default) +def dict_get__DictMulti_ANY_ANY(space, w_dict, w_key, w_default): + w_value = w_dict.getitem(w_key) + if w_value is not None: + return w_value + else: + return w_default def dict_pop__DictMulti_ANY(space, w_dict, w_key, w_defaults): defaults = space.unpackiterable(w_defaults) len_defaults = len(defaults) if len_defaults > 1: raise OperationError(space.w_TypeError, space.wrap("pop expected at most 2 arguments, got %d" % (1 + len_defaults, ))) - w_item = w_dict.implementation.get(w_key) + w_item = w_dict.getitem(w_key) if w_item is None: if len_defaults > 0: return defaults[0] else: space.raise_key_error(w_key) else: - w_dict.implementation.delitem(w_key) + w_dict.delitem(w_key) return w_item app = gateway.applevel(''' @@ -895,7 +805,7 @@ dictrepr = app.interphook("dictrepr") def repr__DictMulti(space, w_dict): - if w_dict.implementation.length() == 0: + if w_dict.length() == 0: return space.wrap('{}') ec = space.getexecutioncontext() w_currently_in_repr = ec._py_repr @@ -908,12 +818,17 @@ # Iteration +KEYSITER = 0 +ITEMSITER = 1 +VALUESITER = 2 + class W_DictMultiIterObject(W_Object): from pypy.objspace.std.dicttype import dictiter_typedef as typedef - def __init__(w_self, space, iteratorimplementation): + def __init__(w_self, space, iteratorimplementation, itertype): w_self.space = space w_self.iteratorimplementation = iteratorimplementation + w_self.itertype = itertype registerimplementation(W_DictMultiIterObject) @@ -922,16 +837,19 @@ def next__DictMultiIterObject(space, w_dictiter): iteratorimplementation = w_dictiter.iteratorimplementation - w_result = iteratorimplementation.next() - if w_result is not None: - return w_result + w_key, w_value = iteratorimplementation.next() + if w_key is not None: + itertype = w_dictiter.itertype + if itertype == KEYSITER: + return w_key + elif itertype == VALUESITER: + return w_value + elif itertype == ITEMSITER: + return space.newtuple([w_key, w_value]) + else: + assert 0, "should be unreachable" raise OperationError(space.w_StopIteration, space.w_None) -# XXX __length_hint__() -##def len__DictMultiIterObject(space, w_dictiter): -## iteratorimplementation = w_dictiter.iteratorimplementation -## return space.wrap(iteratorimplementation.length()) - # ____________________________________________________________ from pypy.objspace.std import dicttype Modified: pypy/branch/shrink-multidict/pypy/objspace/std/dicttype.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/dicttype.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/dicttype.py Mon Oct 26 14:46:32 2009 @@ -141,8 +141,8 @@ # ____________________________________________________________ def descr__new__(space, w_dicttype, __args__): - w_obj = space.allocate_instance(space.DictObjectCls, w_dicttype) - space.DictObjectCls.__init__(w_obj, space) + from pypy.objspace.std.dictmultiobject import W_DictMultiObject + w_obj = W_DictMultiObject.allocate_and_init_instance(space, w_dicttype) return w_obj # ____________________________________________________________ Modified: pypy/branch/shrink-multidict/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/marshal_impl.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/marshal_impl.py Mon Oct 26 14:46:32 2009 @@ -355,7 +355,7 @@ def marshal_w__DictMulti(space, w_dict, m): m.start(TYPE_DICT) - for w_tuple in w_dict.implementation.items(): + for w_tuple in w_dict.items(): w_key, w_value = space.viewiterable(w_tuple, 2) m.put_w_obj(w_key) m.put_w_obj(w_value) Modified: pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py Mon Oct 26 14:46:32 2009 @@ -140,13 +140,13 @@ w_globals = f.w_globals num = oparg >> 8 assert isinstance(w_globals, W_DictMultiObject) - w_value = w_globals.implementation.get_builtin_indexed(num) + w_value = w_globals.get_builtin_indexed(num) if w_value is None: builtins = f.get_builtin() assert isinstance(builtins, Module) w_builtin_dict = builtins.w_dict assert isinstance(w_builtin_dict, W_DictMultiObject) - w_value = w_builtin_dict.implementation.get_builtin_indexed(num) + w_value = w_builtin_dict.get_builtin_indexed(num) ## if w_value is not None: ## print "CALL_LIKELY_BUILTIN fast" if w_value is None: @@ -240,16 +240,9 @@ self.FrameClass = StdObjSpaceFrame - # XXX store the dict class on the space to access it in various places + # store the dict class on the space to access it in various places from pypy.objspace.std import dictmultiobject self.DictObjectCls = dictmultiobject.W_DictMultiObject - self.emptydictimpl = dictmultiobject.EmptyDictImplementation(self) - if self.config.objspace.std.withbucketdict: - from pypy.objspace.std import dictbucket - self.DefaultDictImpl = dictbucket.BucketDictImplementation - else: - self.DefaultDictImpl = dictmultiobject.RDictImplementation - assert self.DictObjectCls in self.model.typeorder from pypy.objspace.std import tupleobject self.TupleObjectCls = tupleobject.W_TupleObject @@ -577,11 +570,13 @@ from pypy.objspace.std.listobject import W_ListObject return W_ListObject(list_w) - def newdict(self, module=False): + def newdict(self, module=False, instance=False, classofinstance=None, + from_strdict_shared=None): from pypy.objspace.std.dictmultiobject import W_DictMultiObject - if module: - return W_DictMultiObject(self, module=True) - return W_DictMultiObject(self) + return W_DictMultiObject.allocate_and_init_instance( + self, module=module, instance=instance, + classofinstance=classofinstance, + from_strdict_shared=from_strdict_shared) def newslice(self, w_start, w_end, w_step): return W_SliceObject(w_start, w_end, w_step) @@ -672,7 +667,6 @@ return self.int_w(l_w[0]), self.int_w(l_w[1]), self.int_w(l_w[2]) def is_(self, w_one, w_two): - # XXX a bit of hacking to gain more speed if w_one is w_two: return self.w_True return self.w_False @@ -735,13 +729,15 @@ def finditem(self, w_obj, w_key): # performance shortcut to avoid creating the OperationError(KeyError) - if type(w_obj) is self.DictObjectCls: - return w_obj.get(w_key, None) + if (isinstance(w_obj, self.DictObjectCls) and + not w_obj.user_overridden_class): + return w_obj.getitem(w_key) return ObjSpace.finditem(self, w_obj, w_key) def set_str_keyed_item(self, w_obj, w_key, w_value, shadows_type=True): # performance shortcut to avoid creating the OperationError(KeyError) - if type(w_obj) is self.DictObjectCls: + if (isinstance(w_obj, self.DictObjectCls) and + not w_obj.user_overridden_class): w_obj.set_str_keyed_item(w_key, w_value, shadows_type) else: self.setitem(w_obj, w_key, w_value) Modified: pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py Mon Oct 26 14:46:32 2009 @@ -1,4 +1,3 @@ -from pypy.objspace.std.dictmultiobject import DictImplementation, StrDictImplementation from pypy.objspace.std.dictmultiobject import IteratorImplementation from pypy.objspace.std.dictmultiobject import W_DictMultiObject, _is_sane_hash from pypy.rlib.jit import purefunction, hint, we_are_jitted, unroll_safe @@ -75,14 +74,14 @@ self.emptylist = [] -class SharedDictImplementation(DictImplementation): +class SharedDictImplementation(W_DictMultiObject): def __init__(self, space): self.space = space self.structure = space.fromcache(State).empty_structure self.entries = space.fromcache(State).emptylist - def get(self, w_lookup): + def impl_getitem(self, w_lookup): space = self.space w_lookup_type = space.type(w_lookup) if space.is_w(w_lookup_type, space.w_str): @@ -96,20 +95,19 @@ else: return self._as_rdict().get(w_lookup) - def setitem(self, w_key, w_value): + def impl_setitem(self, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_str): - return self.setitem_str(w_key, w_value) + self.impl_setitem_str(w_key, w_value) else: - return self._as_rdict().setitem(w_key, w_value) + self._as_rdict().setitem(w_key, w_value) @unroll_safe - def setitem_str(self, w_key, w_value, shadows_type=True): + def impl_setitem_str(self, w_key, w_value, shadows_type=True): key = self.space.str_w(w_key) i = self.structure.lookup_position(key) if i != -1: self.entries[i] = w_value - return self new_structure = self.structure.get_next_structure(key) if new_structure.length > len(self.entries): new_entries = [None] * new_structure.size_estimate() @@ -120,9 +118,8 @@ self.entries[new_structure.length - 1] = w_value assert self.structure.length + 1 == new_structure.length self.structure = new_structure - return self - def delitem(self, w_key): + def impl_delitem(self, w_key): space = self.space w_key_type = space.type(w_key) if space.is_w(w_key_type, space.w_str): @@ -131,64 +128,45 @@ key == self.structure.last_key): self.entries[self.structure.length - 1] = None self.structure = self.structure.back_struct - return self - return self._as_rdict().delitem(w_key) + return + self._as_rdict().delitem(w_key) elif _is_sane_hash(space, w_key_type): raise KeyError else: - return self._as_rdict().delitem(w_key) + self._as_rdict().delitem(w_key) - def length(self): + def impl_length(self): return self.structure.length - def iteritems(self): - return SharedItemIteratorImplementation(self.space, self) - - def iterkeys(self): - return SharedKeyIteratorImplementation(self.space, self) - - def itervalues(self): - return SharedValueIteratorImplementation(self.space, self) + def impl_iter(self): + return SharedIteratorImplementation(self.space, self) - def keys(self): + def impl_keys(self): space = self.space return [space.wrap(key) - for (key, item) in self.structure.keys.iteritems() - if item >= 0] + for (key, item) in self.structure.keys.iteritems()] - def values(self): + def impl_values(self): return self.entries[:self.structure.length] - def items(self): + def impl_items(self): space = self.space return [space.newtuple([space.wrap(key), self.entries[item]]) - for (key, item) in self.structure.keys.iteritems() - if item >= 0] - - def _as_rdict(self, as_strdict=False): - if as_strdict: - newimpl = StrDictImplementation(self.space) - else: - newimpl = self.space.DefaultDictImpl(self.space) - for k, i in self.structure.keys.items(): - if i >= 0: - newimpl.setitem_str(self.space.wrap(k), self.entries[i]) - return newimpl + for (key, item) in self.structure.keys.iteritems()] + def impl_clear(self): + SharedDictImplementation.__init__(self, self.space) -class SharedValueIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.values = dictimplementation.entries + def _as_rdict(self): + r_dict_content = self.initialize_as_rdict() + for k, i in self.structure.keys.items(): + r_dict_content[self.space.wrap(k)] = self.entries[i] + self.structure = None + self.entries = None + return self - def next(self): - if self.pos < self.len: - return self.values[self.pos] - else: - self.values = None - return None -class SharedItemIteratorImplementation(IteratorImplementation): +class SharedIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): IteratorImplementation.__init__(self, space, dictimplementation) self.iterator = dictimplementation.structure.keys.iteritems() @@ -196,21 +174,8 @@ def next_entry(self): implementation = self.dictimplementation assert isinstance(implementation, SharedDictImplementation) - for key, index in self.iterator: + for w_key, index in self.iterator: w_value = implementation.entries[index] - return self.space.newtuple([self.space.wrap(key), w_value]) + return w_key, w_value else: - return None - -class SharedKeyIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.structure.keys.iteritems() - - def next_entry(self): - implementation = self.dictimplementation - assert isinstance(implementation, SharedDictImplementation) - for key, index in self.iterator: - return self.space.wrap(key) - else: - return None + return None, None Modified: pypy/branch/shrink-multidict/pypy/objspace/std/test/test_celldict.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/test/test_celldict.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/test/test_celldict.py Mon Oct 26 14:46:32 2009 @@ -26,7 +26,7 @@ def rescue_builtins(space): w_dict = space.builtin.getdict() content = {} - for key, cell in w_dict.implementation.content.iteritems(): + for key, cell in w_dict.content.iteritems(): newcell = ModuleCell() newcell.w_value = cell.w_value content[key] = newcell @@ -35,9 +35,9 @@ cls.w_rescue_builtins = cls.space.wrap(rescue_builtins) def restore_builtins(space): w_dict = space.builtin.getdict() - if not isinstance(w_dict.implementation, ModuleDictImplementation): - w_dict.implementation = ModuleDictImplementation(space) - w_dict.implementation.content = stored_builtins.pop() + assert isinstance(w_dict, ModuleDictImplementation) + w_dict.content = stored_builtins.pop() + w_dict.fallback = None restore_builtins = gateway.interp2app(restore_builtins) cls.w_restore_builtins = cls.space.wrap(restore_builtins) Modified: pypy/branch/shrink-multidict/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/test/test_dictmultiobject.py Mon Oct 26 14:46:32 2009 @@ -1,10 +1,10 @@ from pypy.interpreter.error import OperationError from pypy.objspace.std.dictmultiobject import \ W_DictMultiObject, setitem__DictMulti_ANY_ANY, getitem__DictMulti_ANY, \ - EmptyDictImplementation, RDictImplementation, StrDictImplementation, \ - MeasuringDictImplementation + StrDictImplementation from pypy.objspace.std.celldict import ModuleDictImplementation +from pypy.objspace.std.sharingdict import SharedDictImplementation from pypy.conftest import gettestobjspace @@ -15,13 +15,13 @@ def test_empty(self): space = self.space - d = self.space.DictObjectCls(space) + d = self.space.newdict() assert not self.space.is_true(d) def test_nonempty(self): space = self.space wNone = space.w_None - d = self.space.DictObjectCls(space) + d = self.space.newdict() d.initialize_content([(wNone, wNone)]) assert space.is_true(d) i = space.getitem(d, wNone) @@ -32,7 +32,7 @@ space = self.space wk1 = space.wrap('key') wone = space.wrap(1) - d = self.space.DictObjectCls(space) + d = self.space.newdict() d.initialize_content([(space.wrap('zero'),space.wrap(0))]) space.setitem(d,wk1,wone) wback = space.getitem(d,wk1) @@ -41,7 +41,7 @@ def test_delitem(self): space = self.space wk1 = space.wrap('key') - d = self.space.DictObjectCls(space) + d = self.space.newdict() d.initialize_content( [(space.wrap('zero'),space.wrap(0)), (space.wrap('one'),space.wrap(1)), (space.wrap('two'),space.wrap(2))]) @@ -52,7 +52,7 @@ space.getitem,d,space.wrap('one')) def test_wrap_dict(self): - assert isinstance(self.space.wrap({}), self.space.DictObjectCls) + assert isinstance(self.space.wrap({}), W_DictMultiObject) def test_dict_compare(self): @@ -134,8 +134,7 @@ space = self.space w = space.wrap d = {"a": w(1), "b": w(2)} - w_d = space.DictObjectCls(space) - w_d.initialize_from_strdict_shared(d) + w_d = space.newdict(from_strdict_shared=d) assert self.space.eq_w(space.getitem(w_d, w("a")), w(1)) assert self.space.eq_w(space.getitem(w_d, w("b")), w(2)) @@ -143,8 +142,7 @@ space = self.space w = space.wrap d = {"a": w(1), "b": w(2)} - w_d = space.DictObjectCls(space) - w_d.initialize_from_strdict_shared(d) + w_d = space.newdict(from_strdict_shared=d) assert self.space.eq_w(space.getitem(w_d, w("a")), w(1)) assert self.space.eq_w(space.getitem(w_d, w("b")), w(2)) d["c"] = w(41) @@ -587,9 +585,24 @@ def newtuple(self, l): return tuple(l) + def newdict(self, module=False, instance=False, classofinstance=None, + from_strdict_shared=None): + return W_DictMultiObject.allocate_and_init_instance( + self, module=module, instance=instance, + classofinstance=classofinstance, + from_strdict_shared=from_strdict_shared) + + def allocate_instance(self, cls, type): + return object.__new__(cls) + + def fromcache(self, cls): + return cls(self) + w_StopIteration = StopIteration w_None = None StringObjectCls = None # xxx untested: shortcut in StrDictImpl.getitem + w_dict = None + iter = iter class Config: @@ -599,6 +612,7 @@ withsharingdict = False withsmalldicts = False withcelldict = False + withshadowtracking = False class opcodes: CALL_LIKELY_BUILTIN = False @@ -608,143 +622,140 @@ class TestDictImplementation: def setup_method(self,method): self.space = FakeSpace() - self.space.emptydictimpl = EmptyDictImplementation(self.space) - self.space.DefaultDictImpl = RDictImplementation def test_stressdict(self): from random import randint - d = self.space.DictObjectCls(self.space) + d = self.space.newdict() N = 10000 pydict = {} for i in range(N): x = randint(-N, N) setitem__DictMulti_ANY_ANY(self.space, d, x, i) pydict[x] = i - for x in pydict: - assert pydict[x] == getitem__DictMulti_ANY(self.space, d, x) + for key, value in pydict.iteritems(): + assert value == getitem__DictMulti_ANY(self.space, d, key) -class TestRDictImplementation: - ImplementionClass = RDictImplementation - DevolvedClass = RDictImplementation - EmptyClass = EmptyDictImplementation - DefaultDictImpl = RDictImplementation +class BaseTestRDictImplementation: def setup_method(self,method): - self.space = FakeSpace() - self.space.emptydictimpl = EmptyDictImplementation(self.space) - self.space.DefaultDictImpl = self.DefaultDictImpl - self.string = self.space.wrap("fish") - self.string2 = self.space.wrap("fish2") + self.fakespace = FakeSpace() + self.string = self.fakespace.wrap("fish") + self.string2 = self.fakespace.wrap("fish2") self.impl = self.get_impl() def get_impl(self): - "Needs to be empty, or one entry with key self.string" - return self.ImplementionClass(self.space) + return self.ImplementionClass(self.fakespace) + + def fill_impl(self): + self.impl.setitem(self.string, 1000) + self.impl.setitem(self.string2, 2000) + + def check_not_devolved(self): + assert self.impl.r_dict_content is None def test_setitem(self): - assert self.impl.setitem(self.string, 1000) is self.impl + self.impl.setitem(self.string, 1000) assert self.impl.length() == 1 - assert self.impl.get(self.string) == 1000 + assert self.impl.getitem(self.string) == 1000 + self.check_not_devolved() def test_setitem_str(self): - assert self.impl.setitem_str(self.space.str_w(self.string), 1000) is self.impl + self.impl.setitem_str(self.fakespace.str_w(self.string), 1000) assert self.impl.length() == 1 - assert self.impl.get(self.string) == 1000 + assert self.impl.getitem(self.string) == 1000 + self.check_not_devolved() def test_delitem(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) + self.fill_impl() assert self.impl.length() == 2 - newimpl = self.impl.delitem(self.string) + self.impl.delitem(self.string2) assert self.impl.length() == 1 - assert newimpl is self.impl - newimpl = self.impl.delitem(self.string2) + self.impl.delitem(self.string) assert self.impl.length() == 0 - assert isinstance(newimpl, self.EmptyClass) + self.check_not_devolved() def test_keys(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) + self.fill_impl() keys = self.impl.keys() keys.sort() assert keys == [self.string, self.string2] + self.check_not_devolved() def test_values(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) + self.fill_impl() values = self.impl.values() values.sort() assert values == [1000, 2000] + self.check_not_devolved() def test_items(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) + self.fill_impl() items = self.impl.items() items.sort() assert items == zip([self.string, self.string2], [1000, 2000]) + self.check_not_devolved() - def test_iterkeys(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) - iteratorimplementation = self.impl.iterkeys() - keys = [] - while 1: - key = iteratorimplementation.next() - if key is None: - break - keys.append(key) - keys.sort() - assert keys == [self.string, self.string2] - - def test_itervalues(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) - iteratorimplementation = self.impl.itervalues() - values = [] - while 1: - value = iteratorimplementation.next() - if value is None: - break - values.append(value) - values.sort() - assert values == [1000, 2000] - - def test_iteritems(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) - iteratorimplementation = self.impl.iteritems() + def test_iter(self): + self.fill_impl() + iteratorimplementation = self.impl.iter() items = [] while 1: item = iteratorimplementation.next() - if item is None: + if item == (None, None): break items.append(item) items.sort() assert items == zip([self.string, self.string2], [1000, 2000]) + self.check_not_devolved() def test_devolve(self): impl = self.impl for x in xrange(100): - impl = impl.setitem(self.space.str_w(str(x)), x) - impl = impl.setitem(x, x) - assert isinstance(impl, self.DevolvedClass) + impl.setitem(self.fakespace.str_w(str(x)), x) + impl.setitem(x, x) + assert impl.r_dict_content is not None -class TestStrDictImplementation(TestRDictImplementation): +class TestStrDictImplementation(BaseTestRDictImplementation): ImplementionClass = StrDictImplementation -class TestMeasuringDictImplementation(TestRDictImplementation): - ImplementionClass = MeasuringDictImplementation - DevolvedClass = MeasuringDictImplementation - EmptyClass = MeasuringDictImplementation +## class TestMeasuringDictImplementation(BaseTestRDictImplementation): +## ImplementionClass = MeasuringDictImplementation +## DevolvedClass = MeasuringDictImplementation + +class TestModuleDictImplementation(BaseTestRDictImplementation): + ImplementionClass = ModuleDictImplementation + +class TestModuleDictImplementationWithBuiltinNames(BaseTestRDictImplementation): + ImplementionClass = ModuleDictImplementation + + string = "int" + string2 = "isinstance" + +class TestSharedDictImplementation(BaseTestRDictImplementation): + ImplementionClass = SharedDictImplementation + -class TestModuleDictImplementation(TestRDictImplementation): + +class BaseTestDevolvedDictImplementation(BaseTestRDictImplementation): + def fill_impl(self): + BaseTestRDictImplementation.fill_impl(self) + self.impl._as_rdict() + + def check_not_devolved(self): + pass + +class TestDevolvedStrDictImplementation(BaseTestDevolvedDictImplementation): + ImplementionClass = StrDictImplementation + +class TestDevolvedModuleDictImplementation(BaseTestDevolvedDictImplementation): ImplementionClass = ModuleDictImplementation - EmptyClass = ModuleDictImplementation -class TestModuleDictImplementationWithBuiltinNames(TestRDictImplementation): +class TestDevolvedModuleDictImplementationWithBuiltinNames(BaseTestDevolvedDictImplementation): ImplementionClass = ModuleDictImplementation - EmptyClass = ModuleDictImplementation string = "int" string2 = "isinstance" +class TestDevolvedSharedDictImplementation(BaseTestDevolvedDictImplementation): + ImplementionClass = SharedDictImplementation + Modified: pypy/branch/shrink-multidict/pypy/objspace/std/test/test_shadowtracking.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/test/test_shadowtracking.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/test/test_shadowtracking.py Mon Oct 26 14:46:32 2009 @@ -13,15 +13,15 @@ a = A() return a """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.w__dict__.shadows_anything() space.appexec([w_inst], """(a): a.g = "foo" """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.w__dict__.shadows_anything() space.appexec([w_inst], """(a): a.f = "foo" """) - assert w_inst.w__dict__.implementation.shadows_anything() + assert w_inst.w__dict__.shadows_anything() def test_shadowing_via__dict__(self): space = self.space @@ -32,15 +32,15 @@ a = A() return a """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.w__dict__.shadows_anything() space.appexec([w_inst], """(a): a.__dict__["g"] = "foo" """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.w__dict__.shadows_anything() space.appexec([w_inst], """(a): a.__dict__["f"] = "foo" """) - assert w_inst.w__dict__.implementation.shadows_anything() + assert w_inst.w__dict__.shadows_anything() def test_changing__dict__(self): space = self.space @@ -51,11 +51,11 @@ a = A() return a """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.w__dict__.shadows_anything() space.appexec([w_inst], """(a): a.__dict__ = {} """) - assert w_inst.w__dict__.implementation.shadows_anything() + assert w_inst.w__dict__.shadows_anything() def test_changing__class__(self): space = self.space @@ -66,14 +66,14 @@ a = A() return a """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.w__dict__.shadows_anything() space.appexec([w_inst], """(a): class B(object): def g(self): return 42 a.__class__ = B """) - assert w_inst.w__dict__.implementation.shadows_anything() + assert w_inst.w__dict__.shadows_anything() def test_changing_the_type(self): space = self.space @@ -84,13 +84,13 @@ a.x = 72 return a """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.w__dict__.shadows_anything() w_x = space.appexec([w_inst], """(a): a.__class__.x = 42 return a.x """) assert space.unwrap(w_x) == 72 - assert w_inst.w__dict__.implementation.shadows_anything() + assert w_inst.w__dict__.shadows_anything() class AppTestShadowTracking(object): def setup_class(cls): Modified: pypy/branch/shrink-multidict/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/typeobject.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/typeobject.py Mon Oct 26 14:46:32 2009 @@ -281,7 +281,7 @@ if w_self.lazyloaders: w_self._freeze_() # force un-lazification space = w_self.space - newdic = space.DictObjectCls(space) + newdic = space.newdict() newdic.initialize_from_strdict_shared(w_self.dict_w) return W_DictProxyObject(newdic) From fijal at codespeak.net Mon Oct 26 15:15:34 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Oct 2009 15:15:34 +0100 (CET) Subject: [pypy-svn] r68750 - pypy/branch/gc-dump-heap/pypy/rlib Message-ID: <20091026141534.BB39D1683D7@codespeak.net> Author: fijal Date: Mon Oct 26 15:15:34 2009 New Revision: 68750 Modified: pypy/branch/gc-dump-heap/pypy/rlib/rgc.py Log: Kill this helper. If not for anything else, it has no test. Modified: pypy/branch/gc-dump-heap/pypy/rlib/rgc.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rlib/rgc.py (original) +++ pypy/branch/gc-dump-heap/pypy/rlib/rgc.py Mon Oct 26 15:15:34 2009 @@ -184,15 +184,6 @@ hop.exception_cannot_occur() return hop.genop('gc_can_move', hop.args_v, resulttype=hop.r_result) -def dump_heap(fd): - import os - - tb = _dump_heap() - for i in range(len(tb)): - next = tb[i] - os.write(fd, str(next.count) + " " + ",".join([ - str(next.links[j]) for j in range(len(tb))]) + "\n") - def _dump_heap(): raise NotImplementedError # can't be run directly From cfbolz at codespeak.net Mon Oct 26 15:28:39 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 26 Oct 2009 15:28:39 +0100 (CET) Subject: [pypy-svn] r68751 - pypy/branch/shrink-multidict/pypy/objspace/std Message-ID: <20091026142839.73A991683D7@codespeak.net> Author: cfbolz Date: Mon Oct 26 15:28:38 2009 New Revision: 68751 Modified: pypy/branch/shrink-multidict/pypy/objspace/std/objectobject.py Log: don't use * import Modified: pypy/branch/shrink-multidict/pypy/objspace/std/objectobject.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/objectobject.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/objectobject.py Mon Oct 26 15:28:38 2009 @@ -1,4 +1,4 @@ -from pypy.objspace.std.objspace import * +from pypy.objspace.std.objspace import W_Object, register_all class W_ObjectObject(W_Object): From cfbolz at codespeak.net Mon Oct 26 15:34:37 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 26 Oct 2009 15:34:37 +0100 (CET) Subject: [pypy-svn] r68752 - pypy/branch/shrink-multidict/pypy/objspace/std/test Message-ID: <20091026143437.9A6B11683D4@codespeak.net> Author: cfbolz Date: Mon Oct 26 15:34:37 2009 New Revision: 68752 Modified: pypy/branch/shrink-multidict/pypy/objspace/std/test/test_obj.py Log: uh? Modified: pypy/branch/shrink-multidict/pypy/objspace/std/test/test_obj.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/test/test_obj.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/test/test_obj.py Mon Oct 26 15:34:37 2009 @@ -1,4 +1,3 @@ -# -*- coding: iso-8859-1 -*- from pypy.conftest import option class AppTestObject: From cfbolz at codespeak.net Mon Oct 26 15:34:59 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 26 Oct 2009 15:34:59 +0100 (CET) Subject: [pypy-svn] r68753 - pypy/branch/shrink-multidict/pypy/objspace/std Message-ID: <20091026143459.749BD1683D4@codespeak.net> Author: cfbolz Date: Mon Oct 26 15:34:58 2009 New Revision: 68753 Modified: pypy/branch/shrink-multidict/pypy/objspace/std/objecttype.py Log: remove old commented out code Modified: pypy/branch/shrink-multidict/pypy/objspace/std/objecttype.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/objecttype.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/objecttype.py Mon Oct 26 15:34:58 2009 @@ -63,7 +63,6 @@ space.wrap("default __new__ takes " "no parameters")) w_obj = space.allocate_instance(W_ObjectObject, w_type) - #W_ObjectObject.__init__(w_obj) return w_obj def descr__init__(space, w_obj, __args__): From cfbolz at codespeak.net Mon Oct 26 15:43:39 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 26 Oct 2009 15:43:39 +0100 (CET) Subject: [pypy-svn] r68754 - in pypy/branch/shrink-multidict/pypy/objspace/std: . test Message-ID: <20091026144339.9FA5B1683D4@codespeak.net> Author: cfbolz Date: Mon Oct 26 15:43:38 2009 New Revision: 68754 Modified: pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py pypy/branch/shrink-multidict/pypy/objspace/std/test/test_dictmultiobject.py pypy/branch/shrink-multidict/pypy/objspace/std/typeobject.py Log: A number of fixes. Modified: pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py Mon Oct 26 15:43:38 2009 @@ -133,7 +133,7 @@ def next_entry(self): # note that this 'for' loop only runs once, at most for key, cell in self.iterator: - return (key, cell.w_value) + return (self.space.wrap(key), cell.w_value) else: return None, None Modified: pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py Mon Oct 26 15:43:38 2009 @@ -110,15 +110,12 @@ raise NotImplementedError("abstract base class") def impl_setitem_str(self, w_key, w_value, shadows_type=True): - #return implementation raise NotImplementedError("abstract base class") def impl_setitem(self, w_key, w_value): - #return implementation raise NotImplementedError("abstract base class") def impl_delitem(self, w_key): - #return implementation raise NotImplementedError("abstract base class") def impl_length(self): @@ -127,6 +124,9 @@ def impl_iter(self): raise NotImplementedError("abstract base class") + def impl_clear(self): + raise NotImplementedError("abstract base class") + def impl_keys(self): iterator = self.impl_iter() result = [] @@ -150,7 +150,7 @@ result = [] while 1: w_key, w_value = iterator.next() - if w_item is not None: + if w_key is not None: result.append(self.space.newtuple([w_key, w_value])) else: return result @@ -160,7 +160,7 @@ # by the annotator def impl_get_builtin_indexed(self, i): w_key = self.space.wrap(OPTIMIZED_BUILTINS[i]) - return self.getitem(w_key) + return self.impl_getitem(w_key) # this method will only be seen whan a certain config option is used def impl_shadows_anything(self): @@ -197,40 +197,55 @@ def impl_fallback_items(self): return [self.space.newtuple([w_key, w_val]) for w_key, w_val in self.r_dict_content.iteritems()] + def impl_fallback_clear(self): self.r_dict_content.clear() + def impl_fallback_get_builtin_indexed(self, i): + w_key = self.space.wrap(OPTIMIZED_BUILTINS[i]) + return self.impl_fallback_getitem(w_key) + + def impl_fallback_shadows_anything(self): + return True + + def impl_fallback_set_shadows_anything(self): + pass + + implementation_methods = [ - "getitem", - "length", - "setitem_str", - "setitem", - "delitem", - "iter", - "items", - "values", - "keys", - "clear", - "get_builtin_indexed", - "shadows_anything", - "set_shadows_anything", + ("getitem", 1), + ("length", 0), + ("setitem_str", 3), + ("setitem", 2), + ("delitem", 1), + ("iter", 0), + ("items", 0), + ("values", 0), + ("keys", 0), + ("clear", 0), + ("get_builtin_indexed", 1), + ("shadows_anything", 0), + ("set_shadows_anything", 0), ] -def _make_method(name, fallback): - def implementation_method(self, *args): +def _make_method(name, implname, fallback, numargs): + args = ", ".join(["a" + str(i) for i in range(numargs)]) + code = """def %s(self, %s): if self.r_dict_content is not None: - if not hasattr(self, fallback): - return getattr(W_DictMultiObject, name)(self, *args) - return getattr(self, fallback)(*args) - return getattr(self, name)(*args) + return self.%s(%s) + return self.%s(%s)""" % (name, args, fallback, args, implname, args) + d = {} + exec py.code.Source(code).compile() in d + implementation_method = d[name] + implementation_method.func_defaults = getattr(W_DictMultiObject, implname).func_defaults return implementation_method def _install_methods(): - for name in implementation_methods: + for name, numargs in implementation_methods: implname = "impl_" + name fallbackname = "impl_fallback_" + name - func = _make_method(implname, fallbackname) + func = _make_method(name, implname, fallbackname, numargs) setattr(W_DictMultiObject, name, func) _install_methods() @@ -354,8 +369,8 @@ def next_entry(self): # note that this 'for' loop only runs once, at most - for item in self.iterator: - return item + for str, w_value in self.iterator: + return self.space.wrap(str), w_value else: return None, None @@ -373,7 +388,7 @@ def impl_setitem_str(self, w_key, w_value, shadows_type=True): if shadows_type: self._shadows_anything = True - StrDictImplementation.setitem_str( + StrDictImplementation.impl_setitem_str( self, w_key, w_value, shadows_type) def impl_setitem(self, w_key, w_value): @@ -383,7 +398,7 @@ w_obj = self.w_type.lookup(space.str_w(w_key)) if w_obj is not None: self._shadows_anything = True - StrDictImplementation.setitem_str( + StrDictImplementation.impl_setitem_str( self, w_key, w_value, False) else: self._as_rdict().setitem(w_key, w_value) Modified: pypy/branch/shrink-multidict/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/test/test_dictmultiobject.py Mon Oct 26 15:43:38 2009 @@ -603,6 +603,7 @@ StringObjectCls = None # xxx untested: shortcut in StrDictImpl.getitem w_dict = None iter = iter + viewiterable = list class Config: Modified: pypy/branch/shrink-multidict/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/typeobject.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/typeobject.py Mon Oct 26 15:43:38 2009 @@ -281,8 +281,7 @@ if w_self.lazyloaders: w_self._freeze_() # force un-lazification space = w_self.space - newdic = space.newdict() - newdic.initialize_from_strdict_shared(w_self.dict_w) + newdic = space.newdict(from_strdict_shared=w_self.dict_w) return W_DictProxyObject(newdic) def unwrap(w_self, space): From arigo at codespeak.net Mon Oct 26 15:58:25 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 26 Oct 2009 15:58:25 +0100 (CET) Subject: [pypy-svn] r68755 - in pypy/branch/logging/pypy/rlib: . test Message-ID: <20091026145825.711B81683D7@codespeak.net> Author: arigo Date: Mon Oct 26 15:58:24 2009 New Revision: 68755 Added: pypy/branch/logging/pypy/rlib/rlog.py (contents, props changed) pypy/branch/logging/pypy/rlib/test/test_rlog.py (contents, props changed) Log: First draft of rlib/rlog.py. Added: pypy/branch/logging/pypy/rlib/rlog.py ============================================================================== --- (empty file) +++ pypy/branch/logging/pypy/rlib/rlog.py Mon Oct 26 15:58:24 2009 @@ -0,0 +1,165 @@ +import py, os, time +from pypy.tool.ansi_print import ansi_log +from pypy.rlib.unroll import unrolling_iterable +from pypy.rpython.extregistry import ExtRegistryEntry + +_log = py.log.Producer("rlog") +py.log.setconsumer("rlog", ansi_log) + +# ____________________________________________________________ + + +def debug_log(_category, _message, **_kwds): + getattr(_log, _category)(_message % _kwds) + +# ____________________________________________________________ + + +class DebugLogEntry(ExtRegistryEntry): + _about_ = debug_log + + def compute_result_annotation(self, s_category, s_message, **kwds_s): + from pypy.annotation import model as annmodel + assert s_category.is_constant() + assert s_message.is_constant() + translator = self.bookkeeper.annotator.translator + try: + logcategories = translator._logcategories + except AttributeError: + logcategories = translator._logcategories = {} + try: + cat = logcategories[s_category.const] + except KeyError: + num = len(logcategories) + 1 + logcategories[s_category.const] = LogCategory(s_category.const, + s_message.const, + num) + else: + assert cat.message == s_message.const, ( + "log category %r is used with different messages:\n\t%s\n\t%s" + % (s_category.const, cat.message, s_message.const)) + return annmodel.s_None + + def specialize_call(self, hop, **kwds_i): + from pypy.rpython.lltypesystem import lltype + translator = self.bookkeeper.annotator.translator + logcategories = translator._logcategories + cat = logcategories[hop.args_s[0].const] + args_v = [] + for name, typechar in cat.entries: + assert typechar == 'd' + args_v.append(hop.inputarg(lltype.Signed, arg=kwds_i[name])) + hop.exception_cannot_occur() + hop.gendirectcall(cat.call, *args_v) + +# ____________________________________________________________ + +import re + +r_entry = re.compile(r"%\((\w+)\)([sd])") + + +class LogCategory(object): + + def __init__(self, category, message, index): + self.category = category + self.message = message + self.index = index + self.entries = [] + seen = {} + for (name, typechar) in r_entry.findall(message): + assert name not in seen, ( + "duplicate name %r in the log message %r" % (name, message)) + seen[name] = True + self.entries.append((name, typechar)) + + def get_call(self, logwriter): + types = [typechar for name, typechar in self.entries] + types = unrolling_iterable(types) + # + def call(*args): + if logwriter.enabled: + logwriter.add_entry(self) + i = 0 + for typechar in types: + methname = 'add_subentry_' + typechar + getattr(logwriter, methname)(args[i]) + i = i + 1 + call._always_inline_ = True + return call + + def _freeze_(self): + return True + + +class AbstractLogWriter(object): + BUFSIZE = 8192 + + def __init__(self): + self.enabled = True + self.initialized_file = False + self.initialized_index = {} + self.fd = -1 + + def get_filename(self): + return os.environ.get('PYPYLOG') + + def open_file(self): + from pypy.rpython.lltypesystem import lltype, rffi + filename = self.get_filename() + if filename: + self.fd = os.open(filename, + os.O_WRONLY | os.O_CREAT | os.O_TRUNC, + 0666) + self.enabled = self.fd >= 0 + # write the header + if self.enabled: + self.create_buffer() + for c in 'RLog\n': + self.write_int(ord(c)) + self.initialized_file = True + + def define_new_category(self, cat): + if not self.initialized_file: + self.open_file() + # write the category definition line + if self.enabled: + self.write_int(0) + self.write_int(cat.index) + self.write_str(cat.message) + self.initialized_index[cat.index] = None + + def add_entry(self, cat): + if cat.index not in self.initialized_index: + self.define_new_category(cat) + if self.enabled: + self.write_int(cat.index) + add_entry._dont_inline_ = True + + def add_subentry_d(self, num): + if self.enabled: + self.write_int(num) + add_subentry_d._dont_inline_ = True + + def add_subentry_s(self, str): + if self.enabled: + self.write_str(str) + add_subentry_d._dont_inline_ = True + + +class LLLogWriter(AbstractLogWriter): + + def create_buffer(self): + self.buffer = lltype.malloc(rffi.CCHARP.TO, LogWriter.BUFSIZE, + flavor='raw') + self.buffer_position = 0 + + def write_int(self, n): + yyy + + def write_str(self, s): + zzz + + def flush(self): + if self.initialized_file: + xxx Added: pypy/branch/logging/pypy/rlib/test/test_rlog.py ============================================================================== --- (empty file) +++ pypy/branch/logging/pypy/rlib/test/test_rlog.py Mon Oct 26 15:58:24 2009 @@ -0,0 +1,76 @@ +from pypy.rlib import rlog +from pypy.tool.udir import udir + + +def test_log_direct(): + messages = [] + class MyLog: + def Aa(self, msg): + messages.append(msg) + previous = rlog._log + try: + rlog._log = MyLog() + rlog.debug_log("Aa", "hello %(foo)d %(bar)d", foo=5, bar=7) + assert messages == ["hello 5 7"] + finally: + rlog._log = previous + +def test_logcategory(): + message = "abc%(foo)ddef%(bar)sghi" + cat = rlog.LogCategory("Aa", message, 17) + assert cat.category == "Aa" + assert cat.message == message + assert cat.index == 17 + assert cat.entries == [('foo', 'd'), ('bar', 's')] + + +class MyLogWriter(rlog.AbstractLogWriter): + _path = udir.join('test_rlog.logwriter') + + def get_filename(self): + return str(self._path) + def create_buffer(self): + self.content = [] + def write_int(self, n): + assert isinstance(n, int) + self.content.append(n) + def write_str(self, s): + assert isinstance(s, str) + self.content.append(s) + +def test_logwriter(): + class FakeCategory: + def __init__(self, index, message): + self.index = index + self.message = message + # + logwriter = MyLogWriter() + cat5 = FakeCategory(5, "foobar") + cat7 = FakeCategory(7, "baz") + logwriter.add_entry(cat5) + logwriter.add_entry(cat5) + logwriter.add_entry(cat7) + logwriter.add_entry(cat5) + # + assert logwriter.content == [ + ord('R'), ord('L'), ord('o'), ord('g'), ord('\n'), + 0, 5, "foobar", + 5, + 5, + 0, 7, "baz", + 7, + 5] + +def test_logcategory_call(): + message = "abc%(foo)ddef%(bar)sghi" + cat = rlog.LogCategory("Aa", message, 17) + logwriter = MyLogWriter() + call = cat.get_call(logwriter) + call(515, "hellooo") + call(2873, "woooooorld") + # + assert logwriter.content == [ + ord('R'), ord('L'), ord('o'), ord('g'), ord('\n'), + 0, 17, message, + 17, 515, "hellooo", + 17, 2873, "woooooorld"] From fijal at codespeak.net Mon Oct 26 16:12:47 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Oct 2009 16:12:47 +0100 (CET) Subject: [pypy-svn] r68756 - in pypy/branch/gc-dump-heap/pypy/rpython/memory: gc test Message-ID: <20091026151247.1127F1683D7@codespeak.net> Author: fijal Date: Mon Oct 26 16:12:46 2009 New Revision: 68756 Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/generation.py pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py Log: Fix generation gc, improve test and fix so we don't track the same object twice (although we do track links multiple times, but that's unavoidable) Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/generation.py Mon Oct 26 16:12:46 2009 @@ -517,6 +517,17 @@ def _id_grow_older(self, obj, id, ignored): self.objects_with_id.setitem(obj, id) + def dump_heap_walk_roots(self): + self.last_generation_root_objects.foreach( + self._track_heap_ext, None) + self.root_walker.walk_roots( + SemiSpaceGC._track_heap_root, + SemiSpaceGC._track_heap_root, + SemiSpaceGC._track_heap_root) + + def _track_heap_ext(self, adr, ignored): + self.trace(adr, self.track_heap_parent, adr) + def debug_check_object(self, obj): """Check the invariants about 'obj' that should be true between collections.""" @@ -570,6 +581,7 @@ else: SemiSpaceGC.debug_check_can_copy(self, obj) + # ____________________________________________________________ import os Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py Mon Oct 26 16:12:46 2009 @@ -637,29 +637,40 @@ # return llmemory.cast_adr_to_int(obj) # direct case - def _dump_heap_extraarg(self, addr, parent): + def track_heap_parent(self, obj, parent): + addr = obj.address[0] parent_idx = llop.get_member_index(lltype.Signed, self.get_type_id(parent)) - idx = llop.get_member_index(lltype.Signed, - self.get_type_id(addr.address[0])) + idx = llop.get_member_index(lltype.Signed, self.get_type_id(addr)) self._ll_typeid_map[parent_idx].links[idx] += 1 - self._dump_heap(addr) + self.track_heap(addr) - def _dump_heap(self, root): - adr = root.address[0] + def track_heap(self, adr): + if self._tracked_dict.contains(adr): + return + llop.debug_print(lltype.Void, adr) + self._tracked_dict.add(adr) idx = llop.get_member_index(lltype.Signed, self.get_type_id(adr)) self._ll_typeid_map[idx].count += 1 - self.trace(adr, self._dump_heap_extraarg, adr) + self.trace(adr, self.track_heap_parent, adr) + def _track_heap_root(self, root): + self.track_heap(root.address[0]) + + def dump_heap_walk_roots(self): + self.root_walker.walk_roots( + SemiSpaceGC._track_heap_root, + SemiSpaceGC._track_heap_root, + SemiSpaceGC._track_heap_root) + def dump_heap(self): + self._tracked_dict = self.AddressDict() max_tid = self.root_walker.gcdata.max_type_id ll_typeid_map = lltype.malloc(ARRAY_TYPEID_MAP, max_tid, zero=True) for i in range(max_tid): ll_typeid_map[i] = lltype.malloc(TYPEID_MAP, max_tid, zero=True) self._ll_typeid_map = ll_typeid_map - self.root_walker.walk_roots( - SemiSpaceGC._dump_heap, # stack roots - SemiSpaceGC._dump_heap, # static in prebuilt non-gc structures - SemiSpaceGC._dump_heap) # static in prebuilt gc objects + self.dump_heap_walk_roots() self._ll_typeid_map = lltype.nullptr(ARRAY_TYPEID_MAP) + self._tracked_dict.delete() return ll_typeid_map Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py Mon Oct 26 16:12:46 2009 @@ -799,13 +799,21 @@ def define_gc_dump_heap(cls): S = lltype.GcStruct('S', ('x', lltype.Signed)) - l = [] + l1 = [] + l2 = [] + l3 = [] + l4 = [] def f(): for i in range(10): - l.append(lltype.malloc(S)) + s = lltype.malloc(S) + l1.append(s) + l2.append(s) + l3.append(s) + l4.append(s) # We cheat here and only read the table which we later on # process ourselves, otherwise this test takes ages + llop.gc__collect(lltype.Void) tb = rgc._dump_heap() a = 0 nr = 0 @@ -816,16 +824,16 @@ a += 1 nr = i for i in range(len(tb)): - if tb[i].count == 1: + if tb[i].count == 4: b += 1 - c = tb[i].links[nr] + c += tb[i].links[nr] return c * 100 + b * 10 + a return f def test_gc_dump_heap(self): run = self.runner("gc_dump_heap") res = run([]) - assert res == 1011 + assert res == 4011 # ________________________________________________________________ From fijal at codespeak.net Mon Oct 26 16:24:42 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Oct 2009 16:24:42 +0100 (CET) Subject: [pypy-svn] r68757 - pypy/branch/gc-dump-heap/pypy/translator/c/test Message-ID: <20091026152442.8B5D71683D4@codespeak.net> Author: fijal Date: Mon Oct 26 16:24:42 2009 New Revision: 68757 Modified: pypy/branch/gc-dump-heap/pypy/translator/c/test/test_newgc.py Log: fix the test Modified: pypy/branch/gc-dump-heap/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/gc-dump-heap/pypy/translator/c/test/test_newgc.py Mon Oct 26 16:24:42 2009 @@ -902,14 +902,16 @@ def define_gc_dump_heap(cls): S = lltype.GcStruct('S', ('x', lltype.Signed)) - l = [] + l1 = [] + l2 = [] + l3 = [] def f(): - fd = os.open("/tmp/x.log", os.O_WRONLY | os.O_CREAT, 0777) for i in range(10): - l.append(lltype.malloc(S)) - rgc.dump_heap(fd) - os.close(fd) + s = lltype.malloc(S) + l1.append(s) + l2.append(s) + l3.append(s) tb = rgc._dump_heap() a = 0 nr = 0 @@ -920,17 +922,17 @@ a += 1 nr = i for i in range(len(tb)): - if tb[i].count == 1: + if tb[i].count == 3: b += 1 c += tb[i].links[nr] # we don't count b here since there can be more singletons, # important one is c, a is for check - return c * 10 + a + return c * 100 + b * 10 + a return f def test_gc_dump_heap(self): res = self.run("gc_dump_heap") - assert res == 101 + assert res == 3011 def definestr_string_builder(cls): def fn(_): From fijal at codespeak.net Mon Oct 26 16:34:47 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Oct 2009 16:34:47 +0100 (CET) Subject: [pypy-svn] r68758 - pypy/branch/gc-dump-heap/pypy/rpython/memory/gc Message-ID: <20091026153447.ED6111683D4@codespeak.net> Author: fijal Date: Mon Oct 26 16:34:47 2009 New Revision: 68758 Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/base.py pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py Log: Store also the size of an object and kill a debug print Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/base.py Mon Oct 26 16:34:47 2009 @@ -7,7 +7,8 @@ from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage TYPEID_MAP = lltype.GcStruct('TYPEID_MAP', ('count', lltype.Signed), - ('links', lltype.Array(lltype.Signed))) + ('size', lltype.Signed), + ('links', lltype.Array(lltype.Signed))) ARRAY_TYPEID_MAP = lltype.GcArray(lltype.Ptr(TYPEID_MAP)) class GCBase(object): Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py Mon Oct 26 16:34:47 2009 @@ -648,10 +648,10 @@ def track_heap(self, adr): if self._tracked_dict.contains(adr): return - llop.debug_print(lltype.Void, adr) self._tracked_dict.add(adr) idx = llop.get_member_index(lltype.Signed, self.get_type_id(adr)) self._ll_typeid_map[idx].count += 1 + self._ll_typeid_map[idx].size = self.get_size(adr) self.trace(adr, self.track_heap_parent, adr) def _track_heap_root(self, root): From cfbolz at codespeak.net Mon Oct 26 16:54:50 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 26 Oct 2009 16:54:50 +0100 (CET) Subject: [pypy-svn] r68759 - in pypy/branch/shrink-multidict/pypy: config interpreter interpreter/test objspace/std objspace/std/test Message-ID: <20091026155450.500EF1683D4@codespeak.net> Author: cfbolz Date: Mon Oct 26 16:54:49 2009 New Revision: 68759 Added: pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py (contents, props changed) pypy/branch/shrink-multidict/pypy/objspace/std/test/test_inlinedict.py (contents, props changed) Modified: pypy/branch/shrink-multidict/pypy/config/pypyoption.py pypy/branch/shrink-multidict/pypy/interpreter/test/test_typedef.py pypy/branch/shrink-multidict/pypy/interpreter/typedef.py pypy/branch/shrink-multidict/pypy/objspace/std/model.py pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py pypy/branch/shrink-multidict/pypy/objspace/std/test/test_shadowtracking.py Log: Make it possible for objects to not have an applevel dict at all: - the W_UserDictObjectObject gets the attributes of a StrDictImplementation inlined. As long as nobody accesses the applevel .__dict__ attribute, only those are used - if the __dict__ is requested, we create a dict that forwards all requests to the instance - if the __dict__ devolves (e.g. a non-string-key is added), the content is stored into the r_dict_content of the forwarding dict instance. In this case the information on the object is invalid. Modified: pypy/branch/shrink-multidict/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/config/pypyoption.py (original) +++ pypy/branch/shrink-multidict/pypy/config/pypyoption.py Mon Oct 26 16:54:49 2009 @@ -236,10 +236,10 @@ "about dictionaries", default=False), - BoolOption("withbucketdict", - "use dictionaries with chained hash tables " - "(default is open addressing)", - default=False), + BoolOption("withinlineddict", + "make instances more compact by revoming a level of indirection", + default=False, + requires=[("objspace.std.withshadowtracking", False)]), BoolOption("withrangelist", "enable special range list implementation that does not " Modified: pypy/branch/shrink-multidict/pypy/interpreter/test/test_typedef.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/interpreter/test/test_typedef.py (original) +++ pypy/branch/shrink-multidict/pypy/interpreter/test/test_typedef.py Mon Oct 26 16:54:49 2009 @@ -125,7 +125,7 @@ checks[2], checks[3])) subclasses = {} for key, subcls in typedef._subclass_cache.items(): - cls = key[0] + cls = key[1] subclasses.setdefault(cls, {}) subclasses[cls][subcls] = True for cls, set in subclasses.items(): Modified: pypy/branch/shrink-multidict/pypy/interpreter/typedef.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/interpreter/typedef.py (original) +++ pypy/branch/shrink-multidict/pypy/interpreter/typedef.py Mon Oct 26 16:54:49 2009 @@ -97,23 +97,24 @@ # / \ # 5 6 -def get_unique_interplevel_subclass(cls, hasdict, wants_slots, needsdel=False, - weakrefable=False): +def get_unique_interplevel_subclass(config, cls, hasdict, wants_slots, + needsdel=False, weakrefable=False): "NOT_RPYTHON: initialization-time only" if hasattr(cls, '__del__') and getattr(cls, "handle_del_manually", False): needsdel = False assert cls.typedef.acceptable_as_base_class - key = cls, hasdict, wants_slots, needsdel, weakrefable + key = config, cls, hasdict, wants_slots, needsdel, weakrefable try: return _subclass_cache[key] except KeyError: - subcls = _getusercls(cls, hasdict, wants_slots, needsdel, weakrefable) + subcls = _getusercls(config, cls, hasdict, wants_slots, needsdel, + weakrefable) _subclass_cache[key] = subcls return subcls get_unique_interplevel_subclass._annspecialcase_ = "specialize:memo" _subclass_cache = {} -def enum_interplevel_subclasses(cls): +def enum_interplevel_subclasses(config, cls): """Return a list of all the extra interp-level subclasses of 'cls' that can be built by get_unique_interplevel_subclass().""" result = [] @@ -121,14 +122,13 @@ for flag2 in (False, True): for flag3 in (False, True): for flag4 in (False, True): - result.append(get_unique_interplevel_subclass(cls, flag1, - flag2, flag3, - flag4)) + result.append(get_unique_interplevel_subclass( + config, cls, flag1, flag2, flag3, flag4)) result = dict.fromkeys(result) assert len(result) <= 6 return result.keys() -def _getusercls(cls, wants_dict, wants_slots, wants_del, weakrefable): +def _getusercls(config, cls, wants_dict, wants_slots, wants_del, weakrefable): typedef = cls.typedef if wants_dict and typedef.hasdict: wants_dict = False @@ -136,47 +136,47 @@ if wants_del: if wants_dict: # case 5. Parent class is 3. - parentcls = get_unique_interplevel_subclass(cls, True, True, + parentcls = get_unique_interplevel_subclass(config, cls, True, True, False, True) else: # case 6. Parent class is 4. - parentcls = get_unique_interplevel_subclass(cls, False, True, + parentcls = get_unique_interplevel_subclass(config, cls, False, True, False, True) - return _usersubclswithfeature(parentcls, "del") + return _usersubclswithfeature(config, parentcls, "del") elif wants_dict: if wants_slots: # case 3. Parent class is 1. - parentcls = get_unique_interplevel_subclass(cls, True, False, + parentcls = get_unique_interplevel_subclass(config, cls, True, False, False, True) - return _usersubclswithfeature(parentcls, "slots") + return _usersubclswithfeature(config, parentcls, "slots") else: # case 1 (we need to add weakrefable unless it's already in 'cls') if not typedef.weakrefable: - return _usersubclswithfeature(cls, "user", "dict", "weakref") + return _usersubclswithfeature(config, cls, "user", "dict", "weakref") else: - return _usersubclswithfeature(cls, "user", "dict") + return _usersubclswithfeature(config, cls, "user", "dict") else: if weakrefable and not typedef.weakrefable: # case 4. Parent class is 2. - parentcls = get_unique_interplevel_subclass(cls, False, True, + parentcls = get_unique_interplevel_subclass(config, cls, False, True, False, False) - return _usersubclswithfeature(parentcls, "weakref") + return _usersubclswithfeature(config, parentcls, "weakref") else: # case 2 (if the base is already weakrefable, case 2 == case 4) - return _usersubclswithfeature(cls, "user", "slots") + return _usersubclswithfeature(config, cls, "user", "slots") -def _usersubclswithfeature(parentcls, *features): - key = parentcls, features +def _usersubclswithfeature(config, parentcls, *features): + key = config, parentcls, features try: return _usersubclswithfeature_cache[key] except KeyError: - subcls = _builduserclswithfeature(parentcls, *features) + subcls = _builduserclswithfeature(config, parentcls, *features) _usersubclswithfeature_cache[key] = subcls return subcls _usersubclswithfeature_cache = {} _allusersubcls_cache = {} -def _builduserclswithfeature(supercls, *features): +def _builduserclswithfeature(config, supercls, *features): "NOT_RPYTHON: initialization-time only" name = supercls.__name__ name += ''.join([name.capitalize() for name in features]) @@ -244,7 +244,16 @@ return self.slots_w[index] add(Proto) - if "dict" in features: + wantdict = "dict" in features + if wantdict and config.objspace.std.withinlineddict: + from pypy.objspace.std.objectobject import W_ObjectObject + from pypy.objspace.std.inlinedict import make_mixin + if supercls is W_ObjectObject: + Mixin = make_mixin(config) + add(Mixin) + wantdict = False + + if wantdict: class Proto(object): def getdict(self): return self.w__dict__ Added: pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py ============================================================================== --- (empty file) +++ pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py Mon Oct 26 16:54:49 2009 @@ -0,0 +1,420 @@ +import py +from pypy.interpreter.typedef import check_new_dictionary +from pypy.objspace.std.dictmultiobject import W_DictMultiObject +from pypy.objspace.std.dictmultiobject import StrDictImplementation +from pypy.objspace.std.dictmultiobject import IteratorImplementation +from pypy.objspace.std.dictmultiobject import implementation_methods +from pypy.tool.sourcetools import func_with_new_name + +def make_mixin(config): + if config.objspace.std.withsharingdict: + from pypy.objspace.std.sharingdict import SharedDictImplementation + return make_inlinedict_mixin(SharedDictImplementation, "structure") + else: + return make_inlinedict_mixin(StrDictImplementation, "content") + +def make_indirection_method(methname, numargs): + # *args don't work, the call normalization gets confused + args = ", ".join(["a" + str(i) for i in range(numargs)]) + code = """def f(self, %s): + return self.w_obj.%s(%s) +""" % (args, methname, args) + d = {} + exec py.code.Source(code).compile() in d + func = d["f"] + func.func_name = methname + "_indirect" + func.func_defaults = getattr(W_DictMultiObject, methname).func_defaults + return func + +def make_inlinedict_mixin(dictimplclass, attrname): + assert dictimplclass.__base__ is W_DictMultiObject + class IndirectionIterImplementation(IteratorImplementation): + def __init__(self, space, dictimpl, itemlist): + IteratorImplementation.__init__(self, space, dictimpl) + self.itemlist = itemlist + + def next_entry(self): + return self.itemlist[self.pos] + + class IndirectionDictImplementation(W_DictMultiObject): + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def impl_iter(self): + # XXX sucky + items = [] + for w_item in self.impl_items(): + w_key, w_value = self.space.viewiterable(w_item) + items.append((w_key, w_value)) + return IndirectionIterImplementation(self.space, self, items) + + IndirectionDictImplementation.__name__ = "IndirectionDictImplementation" + dictimplclass.__name__ + + for methname, numargs in implementation_methods: + implname = "impl_" + methname + if implname != "impl_iter": + setattr(IndirectionDictImplementation, implname, + make_indirection_method(implname, numargs)) + + init_dictattributes = func_with_new_name(dictimplclass.__init__.im_func, + "init_dictattributes") + make_rdict = func_with_new_name(dictimplclass._as_rdict.im_func, + "make_rdict") + + class InlineDictMixin(object): + + def user_setup(self, space, w_subtype): + self.space = space + self.w__class__ = w_subtype + self.w__dict__ = None + init_dictattributes(self, space) + assert getattr(self, attrname) is not None + self.user_setup_slots(w_subtype.nslots) + + def getdict(self): + w__dict__ = self.w__dict__ + if w__dict__ is None: + w__dict__ = IndirectionDictImplementation(self.space, self) + self.w__dict__ = w__dict__ + assert isinstance(w__dict__, W_DictMultiObject) + return w__dict__ + + def _inlined_dict_valid(self): + return getattr(self, attrname) is not None + + def getdictvalue_w(self, space, attr): + return self.getdictvalue(space, space.wrap(attr)) + + def getdictvalue(self, space, w_attr): + if self._inlined_dict_valid(): + return self.impl_getitem(w_attr) + w_dict = self.getdict() + return w_dict.getitem(w_attr) + + def getdictvalue_attr_is_in_class(self, space, w_attr): + return self.getdictvalue(space, w_attr) + + def setdictvalue(self, space, w_attr, w_value, shadows_type=True): + if self._inlined_dict_valid(): + # XXX don't ignore shadows_type + self.impl_setitem(w_attr, w_value) + return True + w_dict = self.getdict() + w_dict.setitem(w_attr, w_value) + return True + + def deldictvalue(self, space, w_attr): + if self._inlined_dict_valid(): + try: + self.impl_delitem(w_attr) + except KeyError: + return False + return True + w_dict = self.getdict() + try: + w_dict.delitem(w_attr) + except KeyError: + return False + return True + + def setdict(self, space, w_dict): + make_rdict(self) # invalidate attributes on self + self.w__dict__ = check_new_dictionary(space, w_dict) + + def _as_rdict(self): + make_rdict(self) + return self.getdict() + + def initialize_as_rdict(self): + return self.getdict().initialize_as_rdict() + + for methname, _ in implementation_methods: + implname = "impl_" + methname + meth = func_with_new_name(getattr(dictimplclass, implname).im_func, + implname) + if not hasattr(InlineDictMixin, implname): + setattr(InlineDictMixin, implname, meth) + return InlineDictMixin +import py +from pypy.interpreter.typedef import check_new_dictionary +from pypy.objspace.std.dictmultiobject import W_DictMultiObject +from pypy.objspace.std.dictmultiobject import StrDictImplementation +from pypy.objspace.std.dictmultiobject import IteratorImplementation +from pypy.objspace.std.dictmultiobject import implementation_methods +from pypy.tool.sourcetools import func_with_new_name + +def make_mixin(config): + if config.objspace.std.withsharingdict: + from pypy.objspace.std.sharingdict import SharedDictImplementation + return make_inlinedict_mixin(SharedDictImplementation, "structure") + else: + return make_inlinedict_mixin(StrDictImplementation, "content") + +def make_indirection_method(methname, numargs): + # *args don't work, the call normalization gets confused + args = ", ".join(["a" + str(i) for i in range(numargs)]) + code = """def f(self, %s): + return self.w_obj.%s(%s) +""" % (args, methname, args) + d = {} + exec py.code.Source(code).compile() in d + func = d["f"] + func.func_name = methname + "_indirect" + func.func_defaults = getattr(W_DictMultiObject, methname).func_defaults + return func + +def make_inlinedict_mixin(dictimplclass, attrname): + assert dictimplclass.__base__ is W_DictMultiObject + class IndirectionIterImplementation(IteratorImplementation): + def __init__(self, space, dictimpl, itemlist): + IteratorImplementation.__init__(self, space, dictimpl) + self.itemlist = itemlist + + def next_entry(self): + return self.itemlist[self.pos] + + class IndirectionDictImplementation(W_DictMultiObject): + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def impl_iter(self): + # XXX sucky + items = [] + for w_item in self.impl_items(): + w_key, w_value = self.space.viewiterable(w_item) + items.append((w_key, w_value)) + return IndirectionIterImplementation(self.space, self, items) + + IndirectionDictImplementation.__name__ = "IndirectionDictImplementation" + dictimplclass.__name__ + + for methname, numargs in implementation_methods: + implname = "impl_" + methname + if implname != "impl_iter": + setattr(IndirectionDictImplementation, implname, + make_indirection_method(implname, numargs)) + + init_dictattributes = func_with_new_name(dictimplclass.__init__.im_func, + "init_dictattributes") + make_rdict = func_with_new_name(dictimplclass._as_rdict.im_func, + "make_rdict") + + class InlineDictMixin(object): + + def user_setup(self, space, w_subtype): + self.space = space + self.w__class__ = w_subtype + self.w__dict__ = None + init_dictattributes(self, space) + assert getattr(self, attrname) is not None + self.user_setup_slots(w_subtype.nslots) + + def getdict(self): + w__dict__ = self.w__dict__ + if w__dict__ is None: + w__dict__ = IndirectionDictImplementation(self.space, self) + self.w__dict__ = w__dict__ + assert isinstance(w__dict__, W_DictMultiObject) + return w__dict__ + + def _inlined_dict_valid(self): + return getattr(self, attrname) is not None + + def getdictvalue_w(self, space, attr): + if self._inlined_dict_valid(): + return self.impl_getitem_str(attr) + w_dict = self.getdict() + return w_dict.getitem_str(attr) + + def getdictvalue(self, space, w_attr): + if self._inlined_dict_valid(): + return self.impl_getitem(w_attr) + w_dict = self.getdict() + return w_dict.getitem(w_attr) + + def getdictvalue_attr_is_in_class(self, space, attr): + return self.getdictvalue_w(space, attr) + + def setdictvalue(self, space, w_attr, w_value, shadows_type=True): + if self._inlined_dict_valid(): + # XXX don't ignore shadows_type + self.impl_setitem(w_attr, w_value) + return True + w_dict = self.getdict() + w_dict.setitem(w_attr, w_value) + return True + + def deldictvalue(self, space, w_attr): + if self._inlined_dict_valid(): + try: + self.impl_delitem(w_attr) + except KeyError: + return False + return True + w_dict = self.getdict() + try: + w_dict.delitem(w_attr) + except KeyError: + return False + return True + + def setdict(self, space, w_dict): + make_rdict(self) # invalidate attributes on self + self.w__dict__ = check_new_dictionary(space, w_dict) + + def _as_rdict(self): + make_rdict(self) + return self.getdict() + + def initialize_as_rdict(self): + return self.getdict().initialize_as_rdict() + + for methname, _ in implementation_methods: + implname = "impl_" + methname + meth = func_with_new_name(getattr(dictimplclass, implname).im_func, + implname) + if not hasattr(InlineDictMixin, implname): + setattr(InlineDictMixin, implname, meth) + return InlineDictMixin +import py +from pypy.interpreter.typedef import check_new_dictionary +from pypy.objspace.std.dictmultiobject import W_DictMultiObject +from pypy.objspace.std.dictmultiobject import StrDictImplementation +from pypy.objspace.std.dictmultiobject import IteratorImplementation +from pypy.objspace.std.dictmultiobject import implementation_methods +from pypy.tool.sourcetools import func_with_new_name + +def make_mixin(config): + if config.objspace.std.withsharingdict: + from pypy.objspace.std.sharingdict import SharedDictImplementation + return make_inlinedict_mixin(SharedDictImplementation, "structure") + else: + return make_inlinedict_mixin(StrDictImplementation, "content") + +def make_indirection_method(methname, numargs): + # *args don't work, the call normalization gets confused + args = ", ".join(["a" + str(i) for i in range(numargs)]) + code = """def f(self, %s): + return self.w_obj.%s(%s) +""" % (args, methname, args) + d = {} + exec py.code.Source(code).compile() in d + func = d["f"] + func.func_name = methname + "_indirect" + func.func_defaults = getattr(W_DictMultiObject, methname).func_defaults + return func + +def make_inlinedict_mixin(dictimplclass, attrname): + assert dictimplclass.__base__ is W_DictMultiObject + class IndirectionIterImplementation(IteratorImplementation): + def __init__(self, space, dictimpl, itemlist): + IteratorImplementation.__init__(self, space, dictimpl) + self.itemlist = itemlist + + def next_entry(self): + return self.itemlist[self.pos] + + class IndirectionDictImplementation(W_DictMultiObject): + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def impl_iter(self): + # XXX sucky + items = [] + for w_item in self.impl_items(): + w_key, w_value = self.space.viewiterable(w_item) + items.append((w_key, w_value)) + return IndirectionIterImplementation(self.space, self, items) + + IndirectionDictImplementation.__name__ = "IndirectionDictImplementation" + dictimplclass.__name__ + + for methname, numargs in implementation_methods: + implname = "impl_" + methname + if implname != "impl_iter": + setattr(IndirectionDictImplementation, implname, + make_indirection_method(implname, numargs)) + + init_dictattributes = func_with_new_name(dictimplclass.__init__.im_func, + "init_dictattributes") + make_rdict = func_with_new_name(dictimplclass._as_rdict.im_func, + "make_rdict") + + class InlineDictMixin(object): + + def user_setup(self, space, w_subtype): + self.space = space + self.w__class__ = w_subtype + self.w__dict__ = None + init_dictattributes(self, space) + assert getattr(self, attrname) is not None + self.user_setup_slots(w_subtype.nslots) + + def getdict(self): + w__dict__ = self.w__dict__ + if w__dict__ is None: + w__dict__ = IndirectionDictImplementation(self.space, self) + self.w__dict__ = w__dict__ + assert isinstance(w__dict__, W_DictMultiObject) + return w__dict__ + + def _inlined_dict_valid(self): + return getattr(self, attrname) is not None + + def getdictvalue_w(self, space, attr): + if self._inlined_dict_valid(): + return self.impl_getitem_str(attr) + w_dict = self.getdict() + return w_dict.getitem_str(attr) + + def getdictvalue(self, space, w_attr): + if self._inlined_dict_valid(): + return self.impl_getitem(w_attr) + w_dict = self.getdict() + return w_dict.getitem(w_attr) + + def getdictvalue_attr_is_in_class(self, space, attr): + return self.getdictvalue_w(space, attr) + + def setdictvalue(self, space, w_attr, w_value, shadows_type=True): + if self._inlined_dict_valid(): + # XXX don't ignore shadows_type + self.impl_setitem(w_attr, w_value) + return True + w_dict = self.getdict() + w_dict.setitem(w_attr, w_value) + return True + + def deldictvalue(self, space, w_attr): + if self._inlined_dict_valid(): + try: + self.impl_delitem(w_attr) + except KeyError: + return False + return True + w_dict = self.getdict() + try: + w_dict.delitem(w_attr) + except KeyError: + return False + return True + + def setdict(self, space, w_dict): + make_rdict(self) # invalidate attributes on self + self.w__dict__ = check_new_dictionary(space, w_dict) + + def _as_rdict(self): + make_rdict(self) + return self.getdict() + + def initialize_as_rdict(self): + return self.getdict().initialize_as_rdict() + + for methname, _ in implementation_methods: + implname = "impl_" + methname + meth = func_with_new_name(getattr(dictimplclass, implname).im_func, + implname) + if not hasattr(InlineDictMixin, implname): + setattr(InlineDictMixin, implname, meth) + return InlineDictMixin Modified: pypy/branch/shrink-multidict/pypy/objspace/std/model.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/model.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/model.py Mon Oct 26 16:54:49 2009 @@ -26,6 +26,7 @@ def __init__(self, config): """NOT_RPYTHON: inititialization only""" + self.config = config # All the Python types that we want to provide in this StdObjSpace class result: from pypy.objspace.std.objecttype import object_typedef @@ -250,7 +251,7 @@ for cls in self.typeorder: if (hasattr(cls, 'typedef') and cls.typedef is not None and cls.typedef.acceptable_as_base_class): - subclslist = enum_interplevel_subclasses(cls) + subclslist = enum_interplevel_subclasses(self.config, cls) for subcls in subclslist: if cls in subcls.__bases__: # only direct subclasses # for user subclasses we only accept "generic" Modified: pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py Mon Oct 26 16:54:49 2009 @@ -610,12 +610,14 @@ user-defined type, without actually __init__ializing the instance.""" w_type = self.gettypeobject(cls.typedef) if self.is_w(w_type, w_subtype): - instance = instantiate(cls) + instance = instantiate(cls) elif cls.typedef.acceptable_as_base_class: # the purpose of the above check is to avoid the code below # to be annotated at all for 'cls' if it is not necessary w_subtype = w_type.check_user_subclass(w_subtype) - subcls = get_unique_interplevel_subclass(cls, w_subtype.hasdict, w_subtype.nslots != 0, w_subtype.needsdel, w_subtype.weakrefable) + subcls = get_unique_interplevel_subclass( + self.config, cls, w_subtype.hasdict, w_subtype.nslots != 0, + w_subtype.needsdel, w_subtype.weakrefable) instance = instantiate(subcls) instance.user_setup(self, w_subtype) else: Modified: pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py Mon Oct 26 16:54:49 2009 @@ -108,6 +108,7 @@ i = self.structure.lookup_position(key) if i != -1: self.entries[i] = w_value + return new_structure = self.structure.get_next_structure(key) if new_structure.length > len(self.entries): new_entries = [None] * new_structure.size_estimate() @@ -174,8 +175,8 @@ def next_entry(self): implementation = self.dictimplementation assert isinstance(implementation, SharedDictImplementation) - for w_key, index in self.iterator: + for key, index in self.iterator: w_value = implementation.entries[index] - return w_key, w_value + return self.space.wrap(key), w_value else: return None, None Added: pypy/branch/shrink-multidict/pypy/objspace/std/test/test_inlinedict.py ============================================================================== --- (empty file) +++ pypy/branch/shrink-multidict/pypy/objspace/std/test/test_inlinedict.py Mon Oct 26 16:54:49 2009 @@ -0,0 +1,381 @@ +from pypy.conftest import gettestobjspace +from pypy.objspace.std.inlinedict import make_inlinedict_mixin +from pypy.objspace.std.dictmultiobject import StrDictImplementation +from pypy.objspace.std.test.test_dictmultiobject import FakeSpace +from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation +from pypy.objspace.std.sharingdict import SharedDictImplementation + +class FakeSubtype: + nslots = 0 + +class TestMixin(object): + Mixin = make_inlinedict_mixin(StrDictImplementation, "content") + class FakeObject(Mixin): + def user_setup_slots(self, nslots): + pass + + fakespace = FakeSpace() + + def make_obj(self): + obj = self.FakeObject() + obj.user_setup(self.fakespace, FakeSubtype) + obj.setdictvalue(self.fakespace, "hello", 1) + obj.setdictvalue(self.fakespace, "world", 2) + assert obj._inlined_dict_valid() + assert obj.w__dict__ is None + return obj + + def test_setgetdel_dictvalue(self): + obj = self.make_obj() + assert obj.getdictvalue(self.fakespace, "hello") == 1 + assert obj.getdictvalue(self.fakespace, "world") == 2 + assert obj.getdictvalue(self.fakespace, "bla") is None + assert not obj.deldictvalue(self.fakespace, "bla") + obj.deldictvalue(self.fakespace, "world") + assert obj.getdictvalue(self.fakespace, "world") is None + obj.deldictvalue(self.fakespace, "hello") + assert obj.getdictvalue(self.fakespace, "hello") is None + + + def test_getdict(self): + obj = self.make_obj() + w_dict = obj.getdict() + assert obj.getdict() or w_dict # always get the same dict + assert obj.w__dict__ is w_dict + + assert w_dict.getitem("hello") == 1 + assert w_dict.getitem("world") == 2 + w_dict.setitem("hello", 4) + w_dict.setitem("world", 5) + assert obj.getdictvalue(self.fakespace, "hello") == 4 + assert obj.getdictvalue(self.fakespace, "world") == 5 + + def test_dict_devolves_via_object(self): + obj = self.make_obj() + obj.setdictvalue(self.fakespace, 4, 1) + obj.setdictvalue(self.fakespace, 5, 2) + w_dict = obj.w__dict__ + assert w_dict is not None + assert dict(w_dict.r_dict_content) == {4: 1, 5: 2, "hello": 1, "world": 2} + assert obj.getdictvalue(self.fakespace, "hello") == 1 + assert obj.getdictvalue(self.fakespace, "world") == 2 + assert obj.getdictvalue(self.fakespace, 4) == 1 + assert obj.getdictvalue(self.fakespace, 5) == 2 + obj.deldictvalue(self.fakespace, "world") + assert obj.getdictvalue(self.fakespace, "world") is None + obj.deldictvalue(self.fakespace, "hello") + assert obj.getdictvalue(self.fakespace, "hello") is None + + def test_dict_devolves_via_dict(self): + obj = self.make_obj() + w_dict = obj.getdict() + w_dict.setitem(4, 1) + w_dict.setitem(5, 2) + assert dict(w_dict.r_dict_content) == {4: 1, 5: 2, "hello": 1, "world": 2} + assert obj.getdictvalue(self.fakespace, "hello") == 1 + assert obj.getdictvalue(self.fakespace, "world") == 2 + assert obj.getdictvalue(self.fakespace, 4) == 1 + assert obj.getdictvalue(self.fakespace, 5) == 2 + obj.deldictvalue(self.fakespace, "world") + assert obj.getdictvalue(self.fakespace, "world") is None + obj.deldictvalue(self.fakespace, "hello") + assert obj.getdictvalue(self.fakespace, "hello") is None + +class TestMixinShared(TestMixin): + Mixin = make_inlinedict_mixin(SharedDictImplementation, "structure") + class FakeObject(Mixin): + def user_setup_slots(self, nslots): + pass + +class TestIndirectDict(BaseTestRDictImplementation): + Mixin = make_inlinedict_mixin(StrDictImplementation, "content") + class FakeObject(Mixin): + def user_setup_slots(self, nslots): + pass + + def get_impl(self): + obj = self.FakeObject() + obj.user_setup(self.fakespace, FakeSubtype) + return obj.getdict() + + +class TestIndirectDictShared(TestIndirectDict): + Mixin = make_inlinedict_mixin(SharedDictImplementation, "structure") + class FakeObject(Mixin): + def user_setup_slots(self, nslots): + pass + + + + +class TestInlineDict(object): + def setup_class(cls): + cls.space = gettestobjspace(**{"objspace.std.withinlineddict": True}) + + def test_simple(self): + w_a = self.space.appexec([], """(): + class A(object): + pass + a = A() + a.x = 12 + a.y = 13 + return a + """) + assert w_a.w__dict__ is None + assert self.space.int_w(w_a.content['x']) == 12 + assert self.space.int_w(w_a.content['y']) == 13 + +from pypy.conftest import gettestobjspace +from pypy.objspace.std.inlinedict import make_inlinedict_mixin +from pypy.objspace.std.dictmultiobject import StrDictImplementation +from pypy.objspace.std.test.test_dictmultiobject import FakeSpace +from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation +from pypy.objspace.std.sharingdict import SharedDictImplementation + +class FakeSubtype: + nslots = 0 + +class TestMixin(object): + Mixin = make_inlinedict_mixin(StrDictImplementation, "content") + class FakeObject(Mixin): + def user_setup_slots(self, nslots): + pass + + fakespace = FakeSpace() + + def make_obj(self): + obj = self.FakeObject() + obj.user_setup(self.fakespace, FakeSubtype) + obj.setdictvalue(self.fakespace, "hello", 1) + obj.setdictvalue(self.fakespace, "world", 2) + assert obj._inlined_dict_valid() + assert obj.w__dict__ is None + return obj + + def test_setgetdel_dictvalue(self): + obj = self.make_obj() + assert obj.getdictvalue(self.fakespace, "hello") == 1 + assert obj.getdictvalue(self.fakespace, "world") == 2 + assert obj.getdictvalue(self.fakespace, "bla") is None + assert not obj.deldictvalue(self.fakespace, "bla") + obj.deldictvalue(self.fakespace, "world") + assert obj.getdictvalue(self.fakespace, "world") is None + obj.deldictvalue(self.fakespace, "hello") + assert obj.getdictvalue(self.fakespace, "hello") is None + + + def test_getdict(self): + obj = self.make_obj() + w_dict = obj.getdict() + assert obj.getdict() or w_dict # always get the same dict + assert obj.w__dict__ is w_dict + + assert w_dict.getitem("hello") == 1 + assert w_dict.getitem("world") == 2 + w_dict.setitem("hello", 4) + w_dict.setitem("world", 5) + assert obj.getdictvalue(self.fakespace, "hello") == 4 + assert obj.getdictvalue(self.fakespace, "world") == 5 + + def test_dict_devolves_via_object(self): + obj = self.make_obj() + obj.setdictvalue(self.fakespace, 4, 1) + obj.setdictvalue(self.fakespace, 5, 2) + w_dict = obj.w__dict__ + assert w_dict is not None + assert dict(w_dict.r_dict_content) == {4: 1, 5: 2, "hello": 1, "world": 2} + assert obj.getdictvalue(self.fakespace, "hello") == 1 + assert obj.getdictvalue(self.fakespace, "world") == 2 + assert obj.getdictvalue(self.fakespace, 4) == 1 + assert obj.getdictvalue(self.fakespace, 5) == 2 + obj.deldictvalue(self.fakespace, "world") + assert obj.getdictvalue(self.fakespace, "world") is None + obj.deldictvalue(self.fakespace, "hello") + assert obj.getdictvalue(self.fakespace, "hello") is None + + def test_dict_devolves_via_dict(self): + obj = self.make_obj() + w_dict = obj.getdict() + w_dict.setitem(4, 1) + w_dict.setitem(5, 2) + assert dict(w_dict.r_dict_content) == {4: 1, 5: 2, "hello": 1, "world": 2} + assert obj.getdictvalue(self.fakespace, "hello") == 1 + assert obj.getdictvalue(self.fakespace, "world") == 2 + assert obj.getdictvalue(self.fakespace, 4) == 1 + assert obj.getdictvalue(self.fakespace, 5) == 2 + obj.deldictvalue(self.fakespace, "world") + assert obj.getdictvalue(self.fakespace, "world") is None + obj.deldictvalue(self.fakespace, "hello") + assert obj.getdictvalue(self.fakespace, "hello") is None + +class TestMixinShared(TestMixin): + Mixin = make_inlinedict_mixin(SharedDictImplementation, "structure") + class FakeObject(Mixin): + def user_setup_slots(self, nslots): + pass + +class TestIndirectDict(BaseTestRDictImplementation): + Mixin = make_inlinedict_mixin(StrDictImplementation, "content") + class FakeObject(Mixin): + def user_setup_slots(self, nslots): + pass + + def get_impl(self): + obj = self.FakeObject() + obj.user_setup(self.fakespace, FakeSubtype) + return obj.getdict() + + +class TestIndirectDictShared(TestIndirectDict): + Mixin = make_inlinedict_mixin(SharedDictImplementation, "structure") + class FakeObject(Mixin): + def user_setup_slots(self, nslots): + pass + + + + +class TestInlineDict(object): + def setup_class(cls): + cls.space = gettestobjspace(**{"objspace.std.withinlineddict": True}) + + def test_simple(self): + w_a = self.space.appexec([], """(): + class A(object): + pass + a = A() + a.x = 12 + a.y = 13 + return a + """) + assert w_a.w__dict__ is None + assert self.space.int_w(w_a.content['x']) == 12 + assert self.space.int_w(w_a.content['y']) == 13 + +from pypy.conftest import gettestobjspace +from pypy.objspace.std.inlinedict import make_inlinedict_mixin +from pypy.objspace.std.dictmultiobject import StrDictImplementation +from pypy.objspace.std.test.test_dictmultiobject import FakeSpace +from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation +from pypy.objspace.std.sharingdict import SharedDictImplementation + +class FakeSubtype: + nslots = 0 + +class TestMixin(object): + Mixin = make_inlinedict_mixin(StrDictImplementation, "content") + class FakeObject(Mixin): + def user_setup_slots(self, nslots): + pass + + fakespace = FakeSpace() + + def make_obj(self): + obj = self.FakeObject() + obj.user_setup(self.fakespace, FakeSubtype) + obj.setdictvalue(self.fakespace, "hello", 1) + obj.setdictvalue(self.fakespace, "world", 2) + assert obj._inlined_dict_valid() + assert obj.w__dict__ is None + return obj + + def test_setgetdel_dictvalue(self): + obj = self.make_obj() + assert obj.getdictvalue(self.fakespace, "hello") == 1 + assert obj.getdictvalue(self.fakespace, "world") == 2 + assert obj.getdictvalue(self.fakespace, "bla") is None + assert not obj.deldictvalue(self.fakespace, "bla") + obj.deldictvalue(self.fakespace, "world") + assert obj.getdictvalue(self.fakespace, "world") is None + obj.deldictvalue(self.fakespace, "hello") + assert obj.getdictvalue(self.fakespace, "hello") is None + + + def test_getdict(self): + obj = self.make_obj() + w_dict = obj.getdict() + assert obj.getdict() or w_dict # always get the same dict + assert obj.w__dict__ is w_dict + + assert w_dict.getitem("hello") == 1 + assert w_dict.getitem("world") == 2 + w_dict.setitem("hello", 4) + w_dict.setitem("world", 5) + assert obj.getdictvalue(self.fakespace, "hello") == 4 + assert obj.getdictvalue(self.fakespace, "world") == 5 + + def test_dict_devolves_via_object(self): + obj = self.make_obj() + obj.setdictvalue(self.fakespace, 4, 1) + obj.setdictvalue(self.fakespace, 5, 2) + w_dict = obj.w__dict__ + assert w_dict is not None + assert dict(w_dict.r_dict_content) == {4: 1, 5: 2, "hello": 1, "world": 2} + assert obj.getdictvalue(self.fakespace, "hello") == 1 + assert obj.getdictvalue(self.fakespace, "world") == 2 + assert obj.getdictvalue(self.fakespace, 4) == 1 + assert obj.getdictvalue(self.fakespace, 5) == 2 + obj.deldictvalue(self.fakespace, "world") + assert obj.getdictvalue(self.fakespace, "world") is None + obj.deldictvalue(self.fakespace, "hello") + assert obj.getdictvalue(self.fakespace, "hello") is None + + def test_dict_devolves_via_dict(self): + obj = self.make_obj() + w_dict = obj.getdict() + w_dict.setitem(4, 1) + w_dict.setitem(5, 2) + assert dict(w_dict.r_dict_content) == {4: 1, 5: 2, "hello": 1, "world": 2} + assert obj.getdictvalue(self.fakespace, "hello") == 1 + assert obj.getdictvalue(self.fakespace, "world") == 2 + assert obj.getdictvalue(self.fakespace, 4) == 1 + assert obj.getdictvalue(self.fakespace, 5) == 2 + obj.deldictvalue(self.fakespace, "world") + assert obj.getdictvalue(self.fakespace, "world") is None + obj.deldictvalue(self.fakespace, "hello") + assert obj.getdictvalue(self.fakespace, "hello") is None + +class TestMixinShared(TestMixin): + Mixin = make_inlinedict_mixin(SharedDictImplementation, "structure") + class FakeObject(Mixin): + def user_setup_slots(self, nslots): + pass + +class TestIndirectDict(BaseTestRDictImplementation): + Mixin = make_inlinedict_mixin(StrDictImplementation, "content") + class FakeObject(Mixin): + def user_setup_slots(self, nslots): + pass + + def get_impl(self): + obj = self.FakeObject() + obj.user_setup(self.fakespace, FakeSubtype) + return obj.getdict() + + +class TestIndirectDictShared(TestIndirectDict): + Mixin = make_inlinedict_mixin(SharedDictImplementation, "structure") + class FakeObject(Mixin): + def user_setup_slots(self, nslots): + pass + + + + +class TestInlineDict(object): + def setup_class(cls): + cls.space = gettestobjspace(**{"objspace.std.withinlineddict": True}) + + def test_simple(self): + w_a = self.space.appexec([], """(): + class A(object): + pass + a = A() + a.x = 12 + a.y = 13 + return a + """) + assert w_a.w__dict__ is None + assert self.space.int_w(w_a.content['x']) == 12 + assert self.space.int_w(w_a.content['y']) == 13 + Modified: pypy/branch/shrink-multidict/pypy/objspace/std/test/test_shadowtracking.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/test/test_shadowtracking.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/test/test_shadowtracking.py Mon Oct 26 16:54:49 2009 @@ -13,15 +13,15 @@ a = A() return a """) - assert not w_inst.w__dict__.shadows_anything() + assert not w_inst.getdict().shadows_anything() space.appexec([w_inst], """(a): a.g = "foo" """) - assert not w_inst.w__dict__.shadows_anything() + assert not w_inst.getdict().shadows_anything() space.appexec([w_inst], """(a): a.f = "foo" """) - assert w_inst.w__dict__.shadows_anything() + assert w_inst.getdict().shadows_anything() def test_shadowing_via__dict__(self): space = self.space @@ -32,15 +32,15 @@ a = A() return a """) - assert not w_inst.w__dict__.shadows_anything() + assert not w_inst.getdict().shadows_anything() space.appexec([w_inst], """(a): a.__dict__["g"] = "foo" """) - assert not w_inst.w__dict__.shadows_anything() + assert not w_inst.getdict().shadows_anything() space.appexec([w_inst], """(a): a.__dict__["f"] = "foo" """) - assert w_inst.w__dict__.shadows_anything() + assert w_inst.getdict().shadows_anything() def test_changing__dict__(self): space = self.space @@ -51,11 +51,11 @@ a = A() return a """) - assert not w_inst.w__dict__.shadows_anything() + assert not w_inst.getdict().shadows_anything() space.appexec([w_inst], """(a): a.__dict__ = {} """) - assert w_inst.w__dict__.shadows_anything() + assert w_inst.getdict().shadows_anything() def test_changing__class__(self): space = self.space @@ -66,14 +66,14 @@ a = A() return a """) - assert not w_inst.w__dict__.shadows_anything() + assert not w_inst.getdict().shadows_anything() space.appexec([w_inst], """(a): class B(object): def g(self): return 42 a.__class__ = B """) - assert w_inst.w__dict__.shadows_anything() + assert w_inst.getdict().shadows_anything() def test_changing_the_type(self): space = self.space @@ -84,13 +84,13 @@ a.x = 72 return a """) - assert not w_inst.w__dict__.shadows_anything() + assert not w_inst.getdict().shadows_anything() w_x = space.appexec([w_inst], """(a): a.__class__.x = 42 return a.x """) assert space.unwrap(w_x) == 72 - assert w_inst.w__dict__.shadows_anything() + assert w_inst.getdict().shadows_anything() class AppTestShadowTracking(object): def setup_class(cls): From arigo at codespeak.net Mon Oct 26 16:59:28 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 26 Oct 2009 16:59:28 +0100 (CET) Subject: [pypy-svn] r68760 - in pypy/branch/logging/pypy: rpython rpython/test translator Message-ID: <20091026155928.5AE051683D4@codespeak.net> Author: arigo Date: Mon Oct 26 16:59:27 2009 New Revision: 68760 Modified: pypy/branch/logging/pypy/rpython/annlowlevel.py pypy/branch/logging/pypy/rpython/test/test_annlowlevel.py pypy/branch/logging/pypy/translator/unsimplify.py Log: An interface to allow specialize_call() to ask the rtyper to call a given function at the end of the RPython program. Based on unsimplify.call_final_function(), whacked a bit. Modified: pypy/branch/logging/pypy/rpython/annlowlevel.py ============================================================================== --- pypy/branch/logging/pypy/rpython/annlowlevel.py (original) +++ pypy/branch/logging/pypy/rpython/annlowlevel.py Mon Oct 26 16:59:27 2009 @@ -145,6 +145,7 @@ self.delayedconsts = [] self.delayedfuncs = [] self.newgraphs = {} + self.c_final_funcs = [] def getgraph(self, ll_function, args_s, s_result): # get the graph of the mix-level helper ll_function and prepare it for @@ -235,6 +236,10 @@ else: return repr.convert_const(obj) + def register_atexit(self, ll_function): + c_func = self.constfunc(ll_function, [], annmodel.s_None) + self.c_final_funcs.append(c_func) + def finish(self): self.finish_annotate() self.finish_rtype() @@ -288,9 +293,13 @@ assert FUNCTYPE == REAL p._become(real_p) rtyper.specialize_more_blocks() + for c_func in self.c_final_funcs: + from pypy.translator.unsimplify import write_call_to_final_function + write_call_to_final_function(rtyper.annotator.translator, c_func) self.delayedreprs.clear() del self.delayedconsts[:] del self.delayedfuncs[:] + del self.c_final_funcs[:] for graph in translator.graphs[original_graph_count:]: self.newgraphs[graph] = True Modified: pypy/branch/logging/pypy/rpython/test/test_annlowlevel.py ============================================================================== --- pypy/branch/logging/pypy/rpython/test/test_annlowlevel.py (original) +++ pypy/branch/logging/pypy/rpython/test/test_annlowlevel.py Mon Oct 26 16:59:27 2009 @@ -2,13 +2,45 @@ """ Few tests for annlowlevel helpers """ +import os +from pypy.tool.udir import udir from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin from pypy.rpython.lltypesystem.rstr import mallocstr, mallocunicode +from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype from pypy.rpython.annlowlevel import hlstr, llstr, oostr from pypy.rpython.annlowlevel import hlunicode, llunicode -class TestLLType(BaseRtypingTest, LLRtypeMixin): + +class AnnLowLevelTests(BaseRtypingTest): + + def test_register_atexit(self): + from pypy.annotation import model as annmodel + from pypy.rpython.extregistry import ExtRegistryEntry + def callback(): + pass + class CallbackEntry(ExtRegistryEntry): + _about_ = callback + def compute_result_annotation(self): + return annmodel.s_None + def specialize_call(self, hop): + annhelper = hop.rtyper.getannmixlevel() + annhelper.register_atexit(atexit) + return hop.inputconst(lltype.Void, None) + def f(n): + callback() + return n * 5 + def atexit(): + fd = os.open(tmpfn, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0666) + os.close(fd) + tmpfn = str(udir.join('%s.test_register_atexit' % + self.__class__.__name__)) + res = self.interpret(f, [5]) + assert res == 25 + assert os.path.exists(tmpfn) + + +class TestLLType(AnnLowLevelTests, LLRtypeMixin): def test_hlstr(self): s = mallocstr(3) s.chars[0] = "a" @@ -54,7 +86,7 @@ assert res == 3 -class TestOOType(BaseRtypingTest, OORtypeMixin): +class TestOOType(AnnLowLevelTests, OORtypeMixin): def test_hlstr(self): s = ootype.make_string("abc") assert hlstr(s) == "abc" Modified: pypy/branch/logging/pypy/translator/unsimplify.py ============================================================================== --- pypy/branch/logging/pypy/translator/unsimplify.py (original) +++ pypy/branch/logging/pypy/translator/unsimplify.py Mon Oct 26 16:59:27 2009 @@ -158,7 +158,6 @@ def call_final_function(translator, final_func, annhelper=None): """When the program finishes normally, call 'final_func()'.""" from pypy.annotation import model as annmodel - from pypy.rpython.lltypesystem import lltype from pypy.rpython.annlowlevel import MixLevelHelperAnnotator own_annhelper = (annhelper is None) @@ -167,7 +166,10 @@ c_final_func = annhelper.constfunc(final_func, [], annmodel.s_None) if own_annhelper: annhelper.finish() + write_call_to_final_function(translator, c_final_func) +def write_call_to_final_function(translator, c_final_func): + from pypy.rpython.lltypesystem import lltype entry_point = translator.graphs[0] v = copyvar(translator.annotator, entry_point.getreturnvar()) extrablock = Block([v]) From arigo at codespeak.net Mon Oct 26 17:51:41 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 26 Oct 2009 17:51:41 +0100 (CET) Subject: [pypy-svn] r68761 - in pypy/branch/logging/pypy/rlib: . test Message-ID: <20091026165141.771161683D4@codespeak.net> Author: arigo Date: Mon Oct 26 17:51:41 2009 New Revision: 68761 Modified: pypy/branch/logging/pypy/rlib/rlog.py pypy/branch/logging/pypy/rlib/test/test_rlog.py Log: Implement the log, for now only on lltype. Modified: pypy/branch/logging/pypy/rlib/rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog.py (original) +++ pypy/branch/logging/pypy/rlib/rlog.py Mon Oct 26 17:51:41 2009 @@ -1,6 +1,7 @@ import py, os, time from pypy.tool.ansi_print import ansi_log from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.rarithmetic import r_uint from pypy.rpython.extregistry import ExtRegistryEntry _log = py.log.Producer("rlog") @@ -27,6 +28,7 @@ logcategories = translator._logcategories except AttributeError: logcategories = translator._logcategories = {} + translator._logwriter = None try: cat = logcategories[s_category.const] except KeyError: @@ -43,14 +45,17 @@ def specialize_call(self, hop, **kwds_i): from pypy.rpython.lltypesystem import lltype translator = self.bookkeeper.annotator.translator - logcategories = translator._logcategories - cat = logcategories[hop.args_s[0].const] + logwriter = translator._logwriter + if logwriter is None: + logwriter = translator._logwriter = LLLogWriter() + logwriter._register(hop.rtyper) + cat = translator._logcategories[hop.args_s[0].const] args_v = [] for name, typechar in cat.entries: assert typechar == 'd' args_v.append(hop.inputarg(lltype.Signed, arg=kwds_i[name])) hop.exception_cannot_occur() - hop.gendirectcall(cat.call, *args_v) + hop.gendirectcall(cat.gen_call(logwriter), *args_v) # ____________________________________________________________ @@ -72,28 +77,33 @@ "duplicate name %r in the log message %r" % (name, message)) seen[name] = True self.entries.append((name, typechar)) + self.call = None - def get_call(self, logwriter): - types = [typechar for name, typechar in self.entries] - types = unrolling_iterable(types) - # - def call(*args): - if logwriter.enabled: - logwriter.add_entry(self) - i = 0 - for typechar in types: - methname = 'add_subentry_' + typechar - getattr(logwriter, methname)(args[i]) - i = i + 1 - call._always_inline_ = True - return call + def gen_call(self, logwriter): + if self.call is None: + self.logwriter = logwriter + types = [typechar for name, typechar in self.entries] + types = unrolling_iterable(types) + # + def call(*args): + if logwriter.enabled: + logwriter.add_entry(self) + i = 0 + for typechar in types: + methname = 'add_subentry_' + typechar + getattr(logwriter, methname)(args[i]) + i = i + 1 + call._always_inline_ = True + self.call = call + else: + assert self.logwriter is logwriter + return self.call def _freeze_(self): return True class AbstractLogWriter(object): - BUFSIZE = 8192 def __init__(self): self.enabled = True @@ -117,6 +127,8 @@ self.create_buffer() for c in 'RLog\n': self.write_int(ord(c)) + self.write_int(-1) + self.write_int(0) self.initialized_file = True def define_new_category(self, cat): @@ -126,6 +138,7 @@ if self.enabled: self.write_int(0) self.write_int(cat.index) + self.write_str(cat.category) self.write_str(cat.message) self.initialized_index[cat.index] = None @@ -134,32 +147,88 @@ self.define_new_category(cat) if self.enabled: self.write_int(cat.index) - add_entry._dont_inline_ = True def add_subentry_d(self, num): if self.enabled: self.write_int(num) - add_subentry_d._dont_inline_ = True def add_subentry_s(self, str): if self.enabled: self.write_str(str) - add_subentry_d._dont_inline_ = True + +# ____________________________________________________________ class LLLogWriter(AbstractLogWriter): + BUFSIZE = 8192 + + def do_write(self, fd, buf, size): + "NOT_RPYTHON" + l = [buf[i] for i in range(size)] + s = ''.join(l) + os.write(fd, s) + self.writecount += 1 + + def _register(self, rtyper): + from pypy.rpython.lltypesystem import rffi + from pypy.rpython.module.ll_os import underscore_on_windows + # attach 'do_write' to self, overwriting the NOT_RPYTHON method + def do_write(fd, buf, size): + os_write(rffi.cast(rffi.INT, fd), + buf, + rffi.cast(rffi.SIZE_T), size) + self.writecount += 1 + self.do_write = do_write + os_write = rffi.llexternal(underscore_on_windows+'write', + [rffi.INT, rffi.CHARP, rffi.SIZE_T], + rffi.SIZE_T) + # register flush() to be called at program exit + def flush_log_cache(): + if self.initialized_file: + self._flush() + annhelper = rtyper.getannmixlevel() + annhelper.register_atexit(flush_log_cache) def create_buffer(self): - self.buffer = lltype.malloc(rffi.CCHARP.TO, LogWriter.BUFSIZE, - flavor='raw') + from pypy.rpython.lltypesystem import lltype, rffi + self.buffer = lltype.malloc(rffi.CCHARP.TO, self.BUFSIZE, flavor='raw') self.buffer_position = 0 + self.writecount = 0 def write_int(self, n): - yyy + self._write_int_noflush(n) + if self.buffer_position > self.BUFSIZE-48: + self._flush() + + def _write_int_noflush(self, n): + p = self.buffer_position + buf = self.buffer + n = r_uint(n) + while n > 0x7F: + buf[p] = chr((n & 0x7F) | 0x80) + p += 1 + n >>= 7 + buf[p] = chr(n) + self.buffer_position = p + 1 def write_str(self, s): - zzz + self._write_int_noflush(len(s)) + p = self.buffer_position + if p + len(s) > self.BUFSIZE-24: + self._flush() + os.write(self.fd, s) + self.writecount += 1 + else: + buf = self.buffer + for i in range(len(s)): + buf[p + i] = s[i] + self.buffer_position = p + len(s) + + def _flush(self): + if self.buffer_position > 0: + self.do_write(self.fd, self.buffer, self.buffer_position) + self.buffer_position = 0 - def flush(self): - if self.initialized_file: - xxx + def _close(self): + self._flush() + os.close(self.fd) Modified: pypy/branch/logging/pypy/rlib/test/test_rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/test/test_rlog.py (original) +++ pypy/branch/logging/pypy/rlib/test/test_rlog.py Mon Oct 26 17:51:41 2009 @@ -1,4 +1,5 @@ from pypy.rlib import rlog +from pypy.rlib.rarithmetic import intmask from pypy.tool.udir import udir @@ -40,24 +41,25 @@ def test_logwriter(): class FakeCategory: - def __init__(self, index, message): + def __init__(self, index, category, message): self.index = index + self.category = category self.message = message # logwriter = MyLogWriter() - cat5 = FakeCategory(5, "foobar") - cat7 = FakeCategory(7, "baz") + cat5 = FakeCategory(5, "F5", "foobar") + cat7 = FakeCategory(7, "F7", "baz") logwriter.add_entry(cat5) logwriter.add_entry(cat5) logwriter.add_entry(cat7) logwriter.add_entry(cat5) # assert logwriter.content == [ - ord('R'), ord('L'), ord('o'), ord('g'), ord('\n'), - 0, 5, "foobar", + ord('R'), ord('L'), ord('o'), ord('g'), ord('\n'), -1, 0, + 0, 5, "F5", "foobar", 5, 5, - 0, 7, "baz", + 0, 7, "F7", "baz", 7, 5] @@ -65,12 +67,103 @@ message = "abc%(foo)ddef%(bar)sghi" cat = rlog.LogCategory("Aa", message, 17) logwriter = MyLogWriter() - call = cat.get_call(logwriter) + call = cat.gen_call(logwriter) call(515, "hellooo") call(2873, "woooooorld") # assert logwriter.content == [ - ord('R'), ord('L'), ord('o'), ord('g'), ord('\n'), - 0, 17, message, + ord('R'), ord('L'), ord('o'), ord('g'), ord('\n'), -1, 0, + 0, 17, "Aa", message, 17, 515, "hellooo", 17, 2873, "woooooorld"] + + +class TestLLLogWriter: + COUNTER = 0 + + def open(self): + path = udir.join('test_rlog.lllogwriter%d' % TestLLLogWriter.COUNTER) + self.path = path + TestLLLogWriter.COUNTER += 1 + # + class MyLLLogWriter(rlog.LLLogWriter): + def get_filename(self): + return str(path) + # + logwriter = MyLLLogWriter() + logwriter.open_file() + return logwriter + + def read_uint(self, f): + shift = 0 + result = 0 + lastbyte = ord(f.read(1)) + while lastbyte & 0x80: + result |= ((lastbyte & 0x7F) << shift) + shift += 7 + lastbyte = ord(f.read(1)) + result |= (lastbyte << shift) + return result + + def check(self, expected): + f = self.path.open('rb') + f.seek(0, 2) + totalsize = f.tell() + f.seek(0, 0) + header = f.read(5) + assert header == 'RLog\n' + for expect in [-1, 0] + expected: + if isinstance(expect, int): + result = self.read_uint(f) + assert intmask(result) == expect + elif isinstance(expect, str): + length = self.read_uint(f) + assert length < totalsize + got = f.read(length) + assert got == expect + else: + assert 0, expect + moredata = f.read(10) + assert not moredata + + def test_write_int(self): + logwriter = self.open() + for i in range(logwriter.BUFSIZE): + logwriter.write_int(i) + logwriter._close() + self.check(range(logwriter.BUFSIZE)) + assert logwriter.writecount <= 3 + + def test_write_str(self): + logwriter = self.open() + slist = map(str, range(logwriter.BUFSIZE)) + for s in slist: + logwriter.write_str(s) + logwriter._close() + self.check(slist) + assert logwriter.writecount <= 14 + + def test_write_mixed(self): + logwriter = self.open() + xlist = [] + for i in range(logwriter.BUFSIZE): + if i & 1: + i = str(i) + xlist.append(i) + for x in xlist: + if isinstance(x, int): + logwriter.write_int(x) + else: + logwriter.write_str(x) + logwriter._close() + self.check(xlist) + assert logwriter.writecount <= 7 + + def test_write_long_str(self): + logwriter = self.open() + slist = ['abcdefg' * n for n in [10, 100, 1000, 10000]] + for s in slist: + logwriter.write_str(s) + logwriter._close() + self.check(slist) + assert logwriter.writecount <= 9 From fijal at codespeak.net Mon Oct 26 17:56:12 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Oct 2009 17:56:12 +0100 (CET) Subject: [pypy-svn] r68762 - pypy/branch/gc-dump-heap/pypy/module/gc Message-ID: <20091026165612.90CD61683D6@codespeak.net> Author: fijal Date: Mon Oct 26 17:56:12 2009 New Revision: 68762 Modified: pypy/branch/gc-dump-heap/pypy/module/gc/__init__.py pypy/branch/gc-dump-heap/pypy/module/gc/interp_gc.py Log: A dump_heap operation, no test so far Modified: pypy/branch/gc-dump-heap/pypy/module/gc/__init__.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/module/gc/__init__.py (original) +++ pypy/branch/gc-dump-heap/pypy/module/gc/__init__.py Mon Oct 26 17:56:12 2009 @@ -12,4 +12,5 @@ 'disable_finalizers': 'interp_gc.disable_finalizers', 'estimate_heap_size': 'interp_gc.estimate_heap_size', 'garbage' : 'space.newlist([])', + 'dump_heap': 'interp_gc.dump_heap', } Modified: pypy/branch/gc-dump-heap/pypy/module/gc/interp_gc.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/module/gc/interp_gc.py (original) +++ pypy/branch/gc-dump-heap/pypy/module/gc/interp_gc.py Mon Oct 26 17:56:12 2009 @@ -1,6 +1,7 @@ from pypy.interpreter.gateway import ObjSpace from pypy.interpreter.error import OperationError from pypy.rlib import rgc +from pypy.rlib.streamio import open_file_as_stream def collect(space): "Run a full collection." @@ -52,3 +53,13 @@ raise OperationError(space.w_RuntimeError, space.wrap("can't estimate the heap size")) estimate_heap_size.unwrap_spec = [ObjSpace] + +def dump_heap(space, filename): + tb = rgc._dump_heap() + f = open_file_as_stream(filename, mode="w") + for i in range(len(tb)): + f.write("%d %d " % (tb[i].count, tb[i].size)) + f.write(",".join([str(tb[i].links[j]) for j in range(len(tb))]) + "\n") + f.close() + +dump_heap.unwrap_spec = [ObjSpace, str] From cfbolz at codespeak.net Mon Oct 26 18:42:39 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 26 Oct 2009 18:42:39 +0100 (CET) Subject: [pypy-svn] r68763 - in pypy/branch/shrink-multidict/pypy/interpreter: . test Message-ID: <20091026174239.086D61683D5@codespeak.net> Author: cfbolz Date: Mon Oct 26 18:42:39 2009 New Revision: 68763 Modified: pypy/branch/shrink-multidict/pypy/interpreter/function.py pypy/branch/shrink-multidict/pypy/interpreter/test/test_function.py Log: the repr of bound methods was wrong (and quite confusing) Modified: pypy/branch/shrink-multidict/pypy/interpreter/function.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/interpreter/function.py (original) +++ pypy/branch/shrink-multidict/pypy/interpreter/function.py Mon Oct 26 18:42:39 2009 @@ -478,9 +478,8 @@ return space.wrap(s) else: objrepr = space.str_w(space.repr(self.w_instance)) - info = 'bound method %s.%s of %s' % (typename, name, objrepr) - # info = "method %s of %s object" % (name, typename) - return self.w_instance.getrepr(self.space, info) + s = '' % (typename, name, objrepr) + return space.wrap(s) def descr_method_getattribute(self, w_attr): space = self.space Modified: pypy/branch/shrink-multidict/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/interpreter/test/test_function.py (original) +++ pypy/branch/shrink-multidict/pypy/interpreter/test/test_function.py Mon Oct 26 18:42:39 2009 @@ -346,11 +346,13 @@ pass assert repr(A.f) == "" assert repr(A().f).startswith(">") class B: def f(self): pass assert repr(B.f) == "" assert repr(B().f).startswith(">") def test_method_call(self): From fijal at codespeak.net Mon Oct 26 19:08:28 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Oct 2009 19:08:28 +0100 (CET) Subject: [pypy-svn] r68764 - pypy/branch/gc-dump-heap/pypy/module/gc/test Message-ID: <20091026180828.793A7168012@codespeak.net> Author: fijal Date: Mon Oct 26 19:08:27 2009 New Revision: 68764 Modified: pypy/branch/gc-dump-heap/pypy/module/gc/test/test_gc.py Log: A test for dump_heap. Not sure why it does not correctly work on compiled pypy Modified: pypy/branch/gc-dump-heap/pypy/module/gc/test/test_gc.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/module/gc/test/test_gc.py (original) +++ pypy/branch/gc-dump-heap/pypy/module/gc/test/test_gc.py Mon Oct 26 19:08:27 2009 @@ -1,3 +1,5 @@ +from pypy.conftest import gettestobjspace + class AppTestGC(object): def test_collect(self): import gc @@ -72,4 +74,35 @@ assert gc.isenabled() gc.enable() assert gc.isenabled() + +class AppTestGcDumpHeap(object): + def setup_class(cls): + from pypy.tool.udir import udir + from pypy.rlib import rgc + class X(object): + def __init__(self, count, size, links): + self.count = count + self.size = size + self.links = links + + def fake_dump_heap(): + return [X(1, 12, [0, 0]), X(2, 10, [10, 0])] + cls._dump_heap = rgc._dump_heap + rgc._dump_heap = fake_dump_heap + fname = udir.join('gcdump.log') + cls.space = gettestobjspace() + cls.w_fname = cls.space.wrap(str(fname)) + cls._fname = fname + + def teardown_class(cls): + import py + from pypy.rlib import rgc + + rgc._dump_heap = cls._dump_heap + assert py.path.local(cls._fname).read() == '1 12 0,0\n2 10 10,0\n' + + def test_gc_dump_heap(self): + import gc + gc.dump_heap(self.fname) + From arigo at codespeak.net Mon Oct 26 19:20:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 26 Oct 2009 19:20:42 +0100 (CET) Subject: [pypy-svn] r68765 - in pypy/branch/logging/pypy: rlib rlib/test rpython/lltypesystem Message-ID: <20091026182042.63B1C168012@codespeak.net> Author: arigo Date: Mon Oct 26 19:20:41 2009 New Revision: 68765 Modified: pypy/branch/logging/pypy/rlib/rlog.py pypy/branch/logging/pypy/rlib/test/test_rlog.py pypy/branch/logging/pypy/rpython/lltypesystem/rffi.py Log: Implement floats logging. Note that they are written as 4 bytes, i.e. C floats. Modified: pypy/branch/logging/pypy/rlib/rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog.py (original) +++ pypy/branch/logging/pypy/rlib/rlog.py Mon Oct 26 19:20:41 2009 @@ -1,7 +1,7 @@ -import py, os, time +import py, os, time, struct from pypy.tool.ansi_print import ansi_log from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib.rarithmetic import r_uint +from pypy.rlib.rarithmetic import r_uint, r_singlefloat from pypy.rpython.extregistry import ExtRegistryEntry _log = py.log.Producer("rlog") @@ -61,7 +61,7 @@ import re -r_entry = re.compile(r"%\((\w+)\)([sd])") +r_entry = re.compile(r"%\((\w+)\)([sdf])") class LogCategory(object): @@ -128,7 +128,7 @@ for c in 'RLog\n': self.write_int(ord(c)) self.write_int(-1) - self.write_int(0) + self.write_float(1.0) self.initialized_file = True def define_new_category(self, cat): @@ -161,9 +161,10 @@ class LLLogWriter(AbstractLogWriter): BUFSIZE = 8192 + SIZEOF_FLOAT = struct.calcsize("f") def do_write(self, fd, buf, size): - "NOT_RPYTHON" + "NOT_RPYTHON (too slow :-)" l = [buf[i] for i in range(size)] s = ''.join(l) os.write(fd, s) @@ -224,6 +225,15 @@ buf[p + i] = s[i] self.buffer_position = p + len(s) + def write_float(self, f): + from pypy.rpython.lltypesystem import rffi + p = self.buffer_position + ptr = rffi.cast(rffi.FLOATP, rffi.ptradd(self.buffer, p)) + ptr[0] = r_singlefloat(f) + self.buffer_position = p + self.SIZEOF_FLOAT + if self.buffer_position > self.BUFSIZE-48: + self._flush() + def _flush(self): if self.buffer_position > 0: self.do_write(self.fd, self.buffer, self.buffer_position) Modified: pypy/branch/logging/pypy/rlib/test/test_rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/test/test_rlog.py (original) +++ pypy/branch/logging/pypy/rlib/test/test_rlog.py Mon Oct 26 19:20:41 2009 @@ -1,9 +1,13 @@ +import struct from pypy.rlib import rlog from pypy.rlib.rarithmetic import intmask from pypy.tool.udir import udir def test_log_direct(): + # this should produce some output via py.log... + rlog.debug_log("Aa", "hello %(foo)d %(bar)d", foo=5, bar=7) + # let's replace py.log with our own class to check messages = [] class MyLog: def Aa(self, msg): @@ -38,6 +42,9 @@ def write_str(self, s): assert isinstance(s, str) self.content.append(s) + def write_float(self, f): + assert isinstance(f, float) + self.content.append(f) def test_logwriter(): class FakeCategory: @@ -55,7 +62,7 @@ logwriter.add_entry(cat5) # assert logwriter.content == [ - ord('R'), ord('L'), ord('o'), ord('g'), ord('\n'), -1, 0, + ord('R'), ord('L'), ord('o'), ord('g'), ord('\n'), -1, 1.0, 0, 5, "F5", "foobar", 5, 5, @@ -72,12 +79,15 @@ call(2873, "woooooorld") # assert logwriter.content == [ - ord('R'), ord('L'), ord('o'), ord('g'), ord('\n'), -1, 0, + ord('R'), ord('L'), ord('o'), ord('g'), ord('\n'), -1, 1.0, 0, 17, "Aa", message, 17, 515, "hellooo", 17, 2873, "woooooorld"] +TIMESTAMP = object() +SIZEOF_FLOAT = rlog.LLLogWriter.SIZEOF_FLOAT + class TestLLLogWriter: COUNTER = 0 @@ -105,6 +115,9 @@ result |= (lastbyte << shift) return result + def read_float(self, f): + return struct.unpack("f", f.read(SIZEOF_FLOAT))[0] + def check(self, expected): f = self.path.open('rb') f.seek(0, 2) @@ -112,8 +125,10 @@ f.seek(0, 0) header = f.read(5) assert header == 'RLog\n' - for expect in [-1, 0] + expected: - if isinstance(expect, int): + for expect in [-1, 1.0] + expected: + if expect is TIMESTAMP: + f.read(SIZEOF_FLOAT) # ignore result + elif isinstance(expect, int): result = self.read_uint(f) assert intmask(result) == expect elif isinstance(expect, str): @@ -121,6 +136,9 @@ assert length < totalsize got = f.read(length) assert got == expect + elif isinstance(expect, float): + result = self.read_float(f) + assert abs(result - expect) < 1E-6 else: assert 0, expect moredata = f.read(10) @@ -167,3 +185,38 @@ logwriter._close() self.check(slist) assert logwriter.writecount <= 9 + + def test_write_float(self): + import math + logwriter = self.open() + flist = [math.log(x+0.1) for x in range(logwriter.BUFSIZE)] + for f in flist: + logwriter.write_float(f) + logwriter._close() + self.check(flist) + assert logwriter.writecount <= 6 + + +class TestCompiled: + COUNTER = 0 + + def f(x): + rlog.debug_log("Aa", "hello %(foo)d %(bar)d", foo=x, bar=7) + rlog.debug_log("Ab", "<<%(baz)s>>", baz="hi there") + + def setup_method(self, _): + self.old_pypylog = os.environ.get('PYPYLOG') + self.pypylog = str(udir.join('test_rlog.TestCompiled%d' % + TestCompiled.COUNTER)) + TestCompiled.COUNTER += 1 + os.environ['PYPYLOG'] = self.pypylog + + def teardown_method(self, _): + if self.old_pypylog is None: + del os.environ['PYPYLOG'] + else: + os.environ['PYPYLOG'] = self.old_pypylog + + #def test_interpret(self): + # self.interpret(self.f.im_func, [132]) + # ... Modified: pypy/branch/logging/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/logging/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/logging/pypy/rpython/lltypesystem/rffi.py Mon Oct 26 19:20:41 2009 @@ -494,7 +494,8 @@ DOUBLE = lltype.Float # float - corresponds to pypy.rlib.rarithmetic.r_float, and supports no -# operation except rffi.cast() between FLOAT and DOUBLE +# operation except casts between FLOAT and DOUBLE, by using +# r_singlefloat(x) or float(x). FLOAT = lltype.SingleFloat r_singlefloat = rarithmetic.r_singlefloat From arigo at codespeak.net Mon Oct 26 19:28:58 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 26 Oct 2009 19:28:58 +0100 (CET) Subject: [pypy-svn] r68766 - in pypy/branch/logging/pypy/rlib: . test Message-ID: <20091026182858.0712516803B@codespeak.net> Author: arigo Date: Mon Oct 26 19:28:58 2009 New Revision: 68766 Modified: pypy/branch/logging/pypy/rlib/rlog.py pypy/branch/logging/pypy/rlib/test/test_rlog.py Log: Write the time at which each log entry is produced. Modified: pypy/branch/logging/pypy/rlib/rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog.py (original) +++ pypy/branch/logging/pypy/rlib/rlog.py Mon Oct 26 19:28:58 2009 @@ -104,12 +104,14 @@ class AbstractLogWriter(object): + get_time = time.time def __init__(self): self.enabled = True self.initialized_file = False self.initialized_index = {} self.fd = -1 + self.curtime = 0.0 def get_filename(self): return os.environ.get('PYPYLOG') @@ -146,7 +148,13 @@ if cat.index not in self.initialized_index: self.define_new_category(cat) if self.enabled: + now = self.get_time() + timestamp_delta = now - self.curtime + self.curtime = now self.write_int(cat.index) + self.write_float(timestamp_delta) + # NB. we store the delta since the last log entry to get a good + # precision even though it's encoded as a 4-bytes 'C float' def add_subentry_d(self, num): if self.enabled: Modified: pypy/branch/logging/pypy/rlib/test/test_rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/test/test_rlog.py (original) +++ pypy/branch/logging/pypy/rlib/test/test_rlog.py Mon Oct 26 19:28:58 2009 @@ -32,6 +32,8 @@ class MyLogWriter(rlog.AbstractLogWriter): _path = udir.join('test_rlog.logwriter') + def get_time(self): + return 123.0 def get_filename(self): return str(self._path) def create_buffer(self): @@ -64,11 +66,11 @@ assert logwriter.content == [ ord('R'), ord('L'), ord('o'), ord('g'), ord('\n'), -1, 1.0, 0, 5, "F5", "foobar", - 5, - 5, + 5, 123.0, + 5, 0.0, 0, 7, "F7", "baz", - 7, - 5] + 7, 0.0, + 5, 0.0] def test_logcategory_call(): message = "abc%(foo)ddef%(bar)sghi" @@ -81,8 +83,8 @@ assert logwriter.content == [ ord('R'), ord('L'), ord('o'), ord('g'), ord('\n'), -1, 1.0, 0, 17, "Aa", message, - 17, 515, "hellooo", - 17, 2873, "woooooorld"] + 17, 123.0, 515, "hellooo", + 17, 0.0, 2873, "woooooorld"] TIMESTAMP = object() @@ -201,7 +203,7 @@ COUNTER = 0 def f(x): - rlog.debug_log("Aa", "hello %(foo)d %(bar)d", foo=x, bar=7) + rlog.debug_log("Aa", "hello %(foo)d %(bar)f", foo=x, bar=-7.3) rlog.debug_log("Ab", "<<%(baz)s>>", baz="hi there") def setup_method(self, _): From fijal at codespeak.net Mon Oct 26 19:48:01 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Oct 2009 19:48:01 +0100 (CET) Subject: [pypy-svn] r68767 - pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform Message-ID: <20091026184801.195CF16803B@codespeak.net> Author: fijal Date: Mon Oct 26 19:48:00 2009 New Revision: 68767 Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py Log: push/pop roots around call to gc dump heap Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py Mon Oct 26 19:48:00 2009 @@ -650,8 +650,10 @@ def gct_gc_dump_heap(self, hop): op = hop.spaceop + livevars = self.push_roots(hop) hop.genop("direct_call", [self.dump_heap_ptr, self.c_const_gc], resultvar=op.result) + self.pop_roots(hop, livevars) def gct_get_member_index(self, hop): op = hop.spaceop From arigo at codespeak.net Mon Oct 26 20:10:58 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 26 Oct 2009 20:10:58 +0100 (CET) Subject: [pypy-svn] r68768 - in pypy/branch/logging/pypy/rlib: . test Message-ID: <20091026191058.6C0191683CC@codespeak.net> Author: arigo Date: Mon Oct 26 20:10:57 2009 New Revision: 68768 Added: pypy/branch/logging/pypy/rlib/rlog_parsing.py (contents, props changed) pypy/branch/logging/pypy/rlib/test/test_rlog_parsing.py (contents, props changed) Modified: pypy/branch/logging/pypy/rlib/rlog.py Log: Parsing of the generated log files. Modified: pypy/branch/logging/pypy/rlib/rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog.py (original) +++ pypy/branch/logging/pypy/rlib/rlog.py Mon Oct 26 20:10:57 2009 @@ -77,13 +77,13 @@ "duplicate name %r in the log message %r" % (name, message)) seen[name] = True self.entries.append((name, typechar)) + self.types = [typechar for name, typechar in self.entries] self.call = None def gen_call(self, logwriter): if self.call is None: self.logwriter = logwriter - types = [typechar for name, typechar in self.entries] - types = unrolling_iterable(types) + types = unrolling_iterable(self.types) # def call(*args): if logwriter.enabled: Added: pypy/branch/logging/pypy/rlib/rlog_parsing.py ============================================================================== --- (empty file) +++ pypy/branch/logging/pypy/rlib/rlog_parsing.py Mon Oct 26 20:10:57 2009 @@ -0,0 +1,73 @@ +import struct +from pypy.rlib.rarithmetic import intmask +from pypy.rlib import rlog + +SIZEOF_FLOAT = rlog.LLLogWriter.SIZEOF_FLOAT + + +class LogParser(object): + + def __init__(self, file, has_signature=True): + self.f = file + if has_signature: + signature = self.f.read(5) + assert signature == 'RLog\n' + extra1 = self.read_int() + assert extra1 == -1 + extra2 = self.read_float() + assert extra2 == 1.0 + self.startpos = self.f.tell() + + def read_int(self): + nextc = self.f.read(1) + if not nextc: + raise EOFError + shift = 0 + result = 0 + lastbyte = ord(nextc) + while lastbyte & 0x80: + result |= ((lastbyte & 0x7F) << shift) + shift += 7 + lastbyte = ord(self.f.read(1)) + result |= (lastbyte << shift) + return intmask(result) + + def read_str(self): + length = self.read_int() + assert length >= 0 + return self.f.read(length) + + def read_float(self): + return struct.unpack("f", self.f.read(SIZEOF_FLOAT))[0] + + def enum_entries(self): + self.f.seek(self.startpos) + categories = {} + readers = { + 'd': self.read_int, + 's': self.read_str, + 'f': self.read_float, + } + curtime = 0.0 + while 1: + try: + c = self.read_int() + except EOFError: + return + if c == 0: + # define_new_category + index = self.read_int() + category = self.read_str() + message = self.read_str() + assert index not in categories + categories[index] = rlog.LogCategory(category, message, index) + else: + curtime += self.read_float() + cat = categories[c] + entries = [readers[t]() for t in cat.types] + yield curtime, cat, entries + + +def parse_log(filename): + logparser = LogParser(open(filename, 'rb')) + return logparser.enum_entries() Added: pypy/branch/logging/pypy/rlib/test/test_rlog_parsing.py ============================================================================== --- (empty file) +++ pypy/branch/logging/pypy/rlib/test/test_rlog_parsing.py Mon Oct 26 20:10:57 2009 @@ -0,0 +1,48 @@ +import py, struct +from cStringIO import StringIO +from pypy.rlib.rlog_parsing import LogParser + +def test_read_int_str(): + s = (chr(42) + + chr(0x9A) + chr(15) + + chr(3) + 'Abc' + + chr(0x82) + chr(1) + 'x'*130 + + chr(0)) + logparser = LogParser(StringIO(s), has_signature=False) + x = logparser.read_int() + assert x == 42 + x = logparser.read_int() + assert x == (15 << 7) | 0x1A + x = logparser.read_str() + assert x == 'Abc' + x = logparser.read_str() + assert x == 'x'*130 + x = logparser.read_int() + assert x == 0 + py.test.raises(EOFError, logparser.read_int) + +def test_simple_parsing_int(): + s = ('\x00\x01\x02Aa\x0EHello %(foo)d.' + + '\x01' + struct.pack("f", 12.5) + chr(42) + + '\x01' + struct.pack("f", 2.5) + chr(61)) + logparser = LogParser(StringIO(s), has_signature=False) + entries = list(logparser.enum_entries()) + assert len(entries) == 2 + cat = entries[0][1] + assert entries == [ + (12.5, cat, [42]), + (15.0, cat, [61]), + ] + +def test_simple_parsing_float(): + s = ('\x00\x82\x05\x02Aa\x0EHello %(foo)f.' + + '\x82\x05' + struct.pack("ff", 12.5, -62.5) + + '\x82\x05' + struct.pack("ff", 2.5, -0.25)) + logparser = LogParser(StringIO(s), has_signature=False) + entries = list(logparser.enum_entries()) + assert len(entries) == 2 + cat = entries[0][1] + assert entries == [ + (12.5, cat, [-62.5]), + (15.0, cat, [-0.25]), + ] From arigo at codespeak.net Mon Oct 26 20:37:31 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 26 Oct 2009 20:37:31 +0100 (CET) Subject: [pypy-svn] r68769 - in pypy/branch/logging/pypy/rlib: . test Message-ID: <20091026193731.1ECB21683D7@codespeak.net> Author: arigo Date: Mon Oct 26 20:37:31 2009 New Revision: 68769 Modified: pypy/branch/logging/pypy/rlib/rlog.py pypy/branch/logging/pypy/rlib/test/test_rlog.py Log: Finish and test translation. Modified: pypy/branch/logging/pypy/rlib/rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog.py (original) +++ pypy/branch/logging/pypy/rlib/rlog.py Mon Oct 26 20:37:31 2009 @@ -2,7 +2,10 @@ from pypy.tool.ansi_print import ansi_log from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rarithmetic import r_uint, r_singlefloat +from pypy.rlib.objectmodel import we_are_translated +from pypy.tool.sourcetools import func_with_new_name from pypy.rpython.extregistry import ExtRegistryEntry +from pypy.rpython.annlowlevel import hlstr _log = py.log.Producer("rlog") py.log.setconsumer("rlog", ansi_log) @@ -43,19 +46,39 @@ return annmodel.s_None def specialize_call(self, hop, **kwds_i): + from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem import lltype - translator = self.bookkeeper.annotator.translator + translator = hop.rtyper.annotator.translator logwriter = translator._logwriter if logwriter is None: logwriter = translator._logwriter = LLLogWriter() logwriter._register(hop.rtyper) cat = translator._logcategories[hop.args_s[0].const] - args_v = [] + ann = { + 'd': annmodel.SomeInteger(), + 'f': annmodel.SomeFloat(), + 's': annmodel.SomeString(can_be_None=True), + } + annhelper = hop.rtyper.getannmixlevel() + args_s = [ann[t] for t in cat.types] + c_func = annhelper.constfunc(cat.gen_call(logwriter), args_s, + annmodel.s_None) + args_v = [c_func] for name, typechar in cat.entries: - assert typechar == 'd' - args_v.append(hop.inputarg(lltype.Signed, arg=kwds_i[name])) + arg = kwds_i['i_'+name] + if typechar == 'd': + v = hop.inputarg(lltype.Signed, arg=arg) + elif typechar == 'f': + v = hop.inputarg(lltype.Float, arg=arg) + elif typechar == 's': + v = hop.inputarg(hop.rtyper.type_system.rstr.string_repr, + arg=arg) + else: + assert 0, typechar + args_v.append(v) hop.exception_cannot_occur() - hop.gendirectcall(cat.gen_call(logwriter), *args_v) + hop.genop('direct_call', args_v) + return hop.inputconst(lltype.Void, None) # ____________________________________________________________ @@ -93,6 +116,7 @@ methname = 'add_subentry_' + typechar getattr(logwriter, methname)(args[i]) i = i + 1 + call = func_with_new_name(call, 'debug_log_' + self.category) call._always_inline_ = True self.call = call else: @@ -160,9 +184,17 @@ if self.enabled: self.write_int(num) - def add_subentry_s(self, str): + def add_subentry_s(self, llstr): + if self.enabled: + if llstr: + s = hlstr(llstr) + else: + s = '(null)' + self.write_str(s) + + def add_subentry_f(self, float): if self.enabled: - self.write_str(str) + self.write_float(float) # ____________________________________________________________ @@ -172,25 +204,23 @@ SIZEOF_FLOAT = struct.calcsize("f") def do_write(self, fd, buf, size): - "NOT_RPYTHON (too slow :-)" - l = [buf[i] for i in range(size)] - s = ''.join(l) - os.write(fd, s) + if we_are_translated(): + from pypy.rpython.lltypesystem import rffi + self._os_write(rffi.cast(rffi.INT, fd), + buf, + rffi.cast(rffi.SIZE_T, size)) + else: + l = [buf[i] for i in range(size)] + s = ''.join(l) + os.write(fd, s) self.writecount += 1 def _register(self, rtyper): from pypy.rpython.lltypesystem import rffi from pypy.rpython.module.ll_os import underscore_on_windows - # attach 'do_write' to self, overwriting the NOT_RPYTHON method - def do_write(fd, buf, size): - os_write(rffi.cast(rffi.INT, fd), - buf, - rffi.cast(rffi.SIZE_T), size) - self.writecount += 1 - self.do_write = do_write - os_write = rffi.llexternal(underscore_on_windows+'write', - [rffi.INT, rffi.CHARP, rffi.SIZE_T], - rffi.SIZE_T) + self._os_write = rffi.llexternal(underscore_on_windows+'write', + [rffi.INT, rffi.CCHARP, rffi.SIZE_T], + rffi.SIZE_T) # register flush() to be called at program exit def flush_log_cache(): if self.initialized_file: Modified: pypy/branch/logging/pypy/rlib/test/test_rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/test/test_rlog.py (original) +++ pypy/branch/logging/pypy/rlib/test/test_rlog.py Mon Oct 26 20:37:31 2009 @@ -1,7 +1,8 @@ -import struct -from pypy.rlib import rlog +import struct, os +from pypy.rlib import rlog, rlog_parsing from pypy.rlib.rarithmetic import intmask from pypy.tool.udir import udir +from pypy.rpython.test.test_llinterp import interpret def test_log_direct(): @@ -87,7 +88,6 @@ 17, 0.0, 2873, "woooooorld"] -TIMESTAMP = object() SIZEOF_FLOAT = rlog.LLLogWriter.SIZEOF_FLOAT class TestLLLogWriter: @@ -128,9 +128,7 @@ header = f.read(5) assert header == 'RLog\n' for expect in [-1, 1.0] + expected: - if expect is TIMESTAMP: - f.read(SIZEOF_FLOAT) # ignore result - elif isinstance(expect, int): + if isinstance(expect, int): result = self.read_uint(f) assert intmask(result) == expect elif isinstance(expect, str): @@ -199,6 +197,13 @@ assert logwriter.writecount <= 6 +class roughly(float): + def __eq__(self, other): + return abs(self - other) < 1E-6 + def __ne__(self, other): + return not self.__eq__(other) + + class TestCompiled: COUNTER = 0 @@ -219,6 +224,25 @@ else: os.environ['PYPYLOG'] = self.old_pypylog - #def test_interpret(self): - # self.interpret(self.f.im_func, [132]) - # ... + def check_result(self): + entries = list(rlog_parsing.parse_log(self.pypylog)) + assert len(entries) == 2 + # + assert isinstance(entries[0][0], float) + assert isinstance(entries[1][0], float) + # + Aa = entries[0][1] + Ab = entries[1][1] + assert Aa.category == 'Aa' + assert Aa.message == 'hello %(foo)d %(bar)f' + assert Aa.entries == [('foo', 'd'), ('bar', 'f')] + assert Ab.category == 'Ab' + assert Ab.message == '<<%(baz)s>>' + assert Ab.entries == [('baz', 's')] + # + assert entries[0][2] == [132, roughly(-7.3)] + assert entries[1][2] == ['hi there'] + + def test_interpret(self): + interpret(self.f.im_func, [132], malloc_check=False) + self.check_result() From arigo at codespeak.net Mon Oct 26 20:40:23 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 26 Oct 2009 20:40:23 +0100 (CET) Subject: [pypy-svn] r68770 - pypy/branch/logging/pypy/rlib/test Message-ID: <20091026194023.8A0951683D8@codespeak.net> Author: arigo Date: Mon Oct 26 20:40:22 2009 New Revision: 68770 Modified: pypy/branch/logging/pypy/rlib/test/test_rlog.py Log: Fix a test. Improve another test. Modified: pypy/branch/logging/pypy/rlib/test/test_rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/test/test_rlog.py (original) +++ pypy/branch/logging/pypy/rlib/test/test_rlog.py Mon Oct 26 20:40:22 2009 @@ -74,12 +74,13 @@ 5, 0.0] def test_logcategory_call(): + from pypy.rpython.annlowlevel import llstr message = "abc%(foo)ddef%(bar)sghi" cat = rlog.LogCategory("Aa", message, 17) logwriter = MyLogWriter() call = cat.gen_call(logwriter) - call(515, "hellooo") - call(2873, "woooooorld") + call(515, llstr("hellooo")) + call(2873, llstr("woooooorld")) # assert logwriter.content == [ ord('R'), ord('L'), ord('o'), ord('g'), ord('\n'), -1, 1.0, @@ -209,6 +210,7 @@ def f(x): rlog.debug_log("Aa", "hello %(foo)d %(bar)f", foo=x, bar=-7.3) + rlog.debug_log("Aa", "hello %(foo)d %(bar)f", foo=x+1, bar=x+0.5) rlog.debug_log("Ab", "<<%(baz)s>>", baz="hi there") def setup_method(self, _): @@ -226,13 +228,15 @@ def check_result(self): entries = list(rlog_parsing.parse_log(self.pypylog)) - assert len(entries) == 2 + assert len(entries) == 3 # assert isinstance(entries[0][0], float) assert isinstance(entries[1][0], float) + assert isinstance(entries[2][0], float) # Aa = entries[0][1] - Ab = entries[1][1] + Ab = entries[2][1] + assert entries[1][1] is Aa assert Aa.category == 'Aa' assert Aa.message == 'hello %(foo)d %(bar)f' assert Aa.entries == [('foo', 'd'), ('bar', 'f')] @@ -241,7 +245,8 @@ assert Ab.entries == [('baz', 's')] # assert entries[0][2] == [132, roughly(-7.3)] - assert entries[1][2] == ['hi there'] + assert entries[1][2] == [133, 132.5] + assert entries[2][2] == ['hi there'] def test_interpret(self): interpret(self.f.im_func, [132], malloc_check=False) From fijal at codespeak.net Mon Oct 26 21:26:40 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Oct 2009 21:26:40 +0100 (CET) Subject: [pypy-svn] r68771 - pypy/branch/gc-dump-heap/pypy/tool Message-ID: <20091026202640.85D651683CD@codespeak.net> Author: fijal Date: Mon Oct 26 21:26:38 2009 New Revision: 68771 Added: pypy/branch/gc-dump-heap/pypy/tool/gcdump.py (contents, props changed) Log: A tool to create kcachegrind compatible output Added: pypy/branch/gc-dump-heap/pypy/tool/gcdump.py ============================================================================== --- (empty file) +++ pypy/branch/gc-dump-heap/pypy/tool/gcdump.py Mon Oct 26 21:26:38 2009 @@ -0,0 +1,74 @@ +#!/usr/bin/env python +""" Usage: gcdump.py gcdump typeids [outfile] +""" + +from __future__ import division +import re +import sys + +class GcDump(object): + def __init__(self, count, size, links): + self.count = count + self.size = size + self.links = links + +def read_gcdump(f): + lines = f.readlines() + r = [None] * len(lines) + for i, line in enumerate(lines): + count, size, rest = line.split(" ") + r[i] = GcDump(int(count), int(size), + [int(j) for j in rest.split(",")]) + return r + +def read_typeids(f): + res = [] + for line in f.readlines(): + member, name = re.split("\s+", line, 1) + assert member == "member%d" % len(res) + res.append(name.strip("\n")) + return res + +def getname(name, _cache = {}): + try: + return _cache[name] + except KeyError: + no = len(_cache) + _cache[name] = '(%d)' % len(_cache) + return '(%d) %s' % (no, name) + +def process(f, gcdump, typeids): + f.write("events: B\n\n") + for tid, name in enumerate(typeids): + if not tid % 100: + sys.stderr.write("%d%%.." % (tid / len(typeids) * 100)) + f.write("fn=%s\n" % getname(name)) + f.write("0 %d\n" % (gcdump[tid].count * gcdump[tid].size)) + for subtid, no in enumerate(gcdump[tid].links): + if no != 0: + f.write("cfn=%s\n" % getname(typeids[subtid])) + f.write("calls=0 %d\n" % no) + f.write("0 %d\n" % (gcdump[subtid].count * gcdump[subtid].size)) + f.write("\n") + sys.stderr.write("100%\n") + +def main(gcdump_f, typeids_f, outfile): + gcdump = read_gcdump(gcdump_f) + gcdump_f.close() + typeids = read_typeids(typeids_f) + typeids_f.close() + process(outfile, gcdump, typeids) + +if __name__ == '__main__': + if len(sys.argv) == 4: + outfile = open(sys.argv[3], "w") + elif len(sys.argv) == 3: + outfile = sys.stdout + else: + print __doc__ + sys.exit(1) + gcdump = open(sys.argv[1]) + typeids = open(sys.argv[2]) + main(gcdump, typeids, outfile) + if len(sys.argv) == 4: + outfile.close() From arigo at codespeak.net Mon Oct 26 21:34:27 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 26 Oct 2009 21:34:27 +0100 (CET) Subject: [pypy-svn] r68772 - pypy/branch/logging/pypy/rlib Message-ID: <20091026203427.95B8D1683D7@codespeak.net> Author: arigo Date: Mon Oct 26 21:34:26 2009 New Revision: 68772 Modified: pypy/branch/logging/pypy/rlib/rlog.py Log: Minor cleanups. Modified: pypy/branch/logging/pypy/rlib/rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog.py (original) +++ pypy/branch/logging/pypy/rlib/rlog.py Mon Oct 26 21:34:26 2009 @@ -109,13 +109,15 @@ types = unrolling_iterable(self.types) # def call(*args): - if logwriter.enabled: - logwriter.add_entry(self) - i = 0 - for typechar in types: - methname = 'add_subentry_' + typechar - getattr(logwriter, methname)(args[i]) - i = i + 1 + if not logwriter.enabled: + return + if not logwriter.add_entry(self): + return + i = 0 + for typechar in types: + methname = 'add_subentry_' + typechar + getattr(logwriter, methname)(args[i]) + i = i + 1 call = func_with_new_name(call, 'debug_log_' + self.category) call._always_inline_ = True self.call = call @@ -153,6 +155,9 @@ self.create_buffer() for c in 'RLog\n': self.write_int(ord(c)) + # Write two numbers at the start, to ensure that the log is + # considered invalid on machines with different endianness + # or word size. They also play the role of version numbers. self.write_int(-1) self.write_float(1.0) self.initialized_file = True @@ -166,35 +171,34 @@ self.write_int(cat.index) self.write_str(cat.category) self.write_str(cat.message) - self.initialized_index[cat.index] = None + self.initialized_index[cat.index] = None def add_entry(self, cat): if cat.index not in self.initialized_index: self.define_new_category(cat) - if self.enabled: - now = self.get_time() - timestamp_delta = now - self.curtime - self.curtime = now - self.write_int(cat.index) - self.write_float(timestamp_delta) - # NB. we store the delta since the last log entry to get a good - # precision even though it's encoded as a 4-bytes 'C float' + if not self.enabled: + return False + now = self.get_time() + timestamp_delta = now - self.curtime + self.curtime = now + self.write_int(cat.index) + self.write_float(timestamp_delta) + # NB. we store the time delta since the previous log entry to get a + # good precision even though it's encoded as a 4-bytes 'C float' + return True def add_subentry_d(self, num): - if self.enabled: - self.write_int(num) + self.write_int(num) def add_subentry_s(self, llstr): - if self.enabled: - if llstr: - s = hlstr(llstr) - else: - s = '(null)' - self.write_str(s) + if llstr: + s = hlstr(llstr) + else: + s = '(null)' + self.write_str(s) def add_subentry_f(self, float): - if self.enabled: - self.write_float(float) + self.write_float(float) # ____________________________________________________________ From fijal at codespeak.net Mon Oct 26 21:41:42 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 26 Oct 2009 21:41:42 +0100 (CET) Subject: [pypy-svn] r68773 - in pypy/branch/gc-dump-heap/pypy/rpython/memory: gc test Message-ID: <20091026204142.E3F5C1683D7@codespeak.net> Author: fijal Date: Mon Oct 26 21:41:41 2009 New Revision: 68773 Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py Log: Improve the test and also not count objects used for storage Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py Mon Oct 26 21:41:41 2009 @@ -670,6 +670,11 @@ for i in range(max_tid): ll_typeid_map[i] = lltype.malloc(TYPEID_MAP, max_tid, zero=True) self._ll_typeid_map = ll_typeid_map + self._tracked_dict.add(llmemory.cast_ptr_to_adr(ll_typeid_map)) + i = 0 + while i < max_tid: + self._tracked_dict.add(llmemory.cast_ptr_to_adr(ll_typeid_map[i])) + i += 1 self.dump_heap_walk_roots() self._ll_typeid_map = lltype.nullptr(ARRAY_TYPEID_MAP) self._tracked_dict.delete() Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py Mon Oct 26 21:41:41 2009 @@ -819,15 +819,18 @@ nr = 0 b = 0 c = 0 + d = 0 for i in range(len(tb)): if tb[i].count == 10: a += 1 nr = i + if tb[i].count > 50: + d += 1 for i in range(len(tb)): if tb[i].count == 4: b += 1 c += tb[i].links[nr] - return c * 100 + b * 10 + a + return d * 1000 + c * 100 + b * 10 + a return f def test_gc_dump_heap(self): From arigo at codespeak.net Mon Oct 26 21:56:30 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 26 Oct 2009 21:56:30 +0100 (CET) Subject: [pypy-svn] r68774 - in pypy/branch/logging/pypy/rlib: . test Message-ID: <20091026205630.87D991683D7@codespeak.net> Author: arigo Date: Mon Oct 26 21:56:28 2009 New Revision: 68774 Modified: pypy/branch/logging/pypy/rlib/rlog.py pypy/branch/logging/pypy/rlib/test/test_rlog.py Log: Add the has_log() function. Modified: pypy/branch/logging/pypy/rlib/rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog.py (original) +++ pypy/branch/logging/pypy/rlib/rlog.py Mon Oct 26 21:56:28 2009 @@ -13,12 +13,33 @@ # ____________________________________________________________ +def has_log(): + return True + def debug_log(_category, _message, **_kwds): getattr(_log, _category)(_message % _kwds) # ____________________________________________________________ +class HasLogEntry(ExtRegistryEntry): + _about_ = has_log + + def compute_result_annotation(self): + from pypy.annotation import model as annmodel + return annmodel.s_Bool + + def specialize_call(self, hop): + from pypy.annotation import model as annmodel + from pypy.rpython.lltypesystem import lltype + logwriter = get_logwriter(hop.rtyper) + annhelper = hop.rtyper.getannmixlevel() + c_func = annhelper.constfunc(logwriter.has_log, [], + annmodel.s_Bool) + hop.exception_cannot_occur() + return hop.genop('direct_call', [c_func], resulttype=lltype.Bool) + + class DebugLogEntry(ExtRegistryEntry): _about_ = debug_log @@ -31,7 +52,6 @@ logcategories = translator._logcategories except AttributeError: logcategories = translator._logcategories = {} - translator._logwriter = None try: cat = logcategories[s_category.const] except KeyError: @@ -48,11 +68,8 @@ def specialize_call(self, hop, **kwds_i): from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem import lltype + logwriter = get_logwriter(hop.rtyper) translator = hop.rtyper.annotator.translator - logwriter = translator._logwriter - if logwriter is None: - logwriter = translator._logwriter = LLLogWriter() - logwriter._register(hop.rtyper) cat = translator._logcategories[hop.args_s[0].const] ann = { 'd': annmodel.SomeInteger(), @@ -80,6 +97,15 @@ hop.genop('direct_call', args_v) return hop.inputconst(lltype.Void, None) +def get_logwriter(rtyper): + try: + return rtyper.annotator.translator._logwriter + except AttributeError: + logwriter = LLLogWriter() + logwriter._register(rtyper) + rtyper.annotator.translator._logwriter = logwriter + return logwriter + # ____________________________________________________________ import re @@ -138,6 +164,12 @@ self.initialized_index = {} self.fd = -1 self.curtime = 0.0 + # + def has_log(): + if not self.initialized_file: + self.open_file() + return self.enabled + self.has_log = has_log def get_filename(self): return os.environ.get('PYPYLOG') Modified: pypy/branch/logging/pypy/rlib/test/test_rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/test/test_rlog.py (original) +++ pypy/branch/logging/pypy/rlib/test/test_rlog.py Mon Oct 26 21:56:28 2009 @@ -209,9 +209,11 @@ COUNTER = 0 def f(x): + assert rlog.has_log() rlog.debug_log("Aa", "hello %(foo)d %(bar)f", foo=x, bar=-7.3) rlog.debug_log("Aa", "hello %(foo)d %(bar)f", foo=x+1, bar=x+0.5) rlog.debug_log("Ab", "<<%(baz)s>>", baz="hi there") + assert rlog.has_log() def setup_method(self, _): self.old_pypylog = os.environ.get('PYPYLOG') From cfbolz at codespeak.net Tue Oct 27 10:01:25 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 27 Oct 2009 10:01:25 +0100 (CET) Subject: [pypy-svn] r68776 - in pypy/branch/shrink-multidict/pypy: interpreter objspace objspace/std objspace/std/test Message-ID: <20091027090125.BA9671683DA@codespeak.net> Author: cfbolz Date: Tue Oct 27 10:01:24 2009 New Revision: 68776 Modified: pypy/branch/shrink-multidict/pypy/interpreter/baseobjspace.py pypy/branch/shrink-multidict/pypy/interpreter/mixedmodule.py pypy/branch/shrink-multidict/pypy/interpreter/typedef.py pypy/branch/shrink-multidict/pypy/objspace/descroperation.py pypy/branch/shrink-multidict/pypy/objspace/std/callmethod.py pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py pypy/branch/shrink-multidict/pypy/objspace/std/proxyobject.py pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py pypy/branch/shrink-multidict/pypy/objspace/std/test/test_dictmultiobject.py pypy/branch/shrink-multidict/pypy/objspace/std/test/test_inlinedict.py Log: A new shortcut in the objspace: finditem_str that takes an unwrapped string as the item to find. Please don't ask me how I managed to triplicate the content of inlinedict.py and test_inlinedict.py Modified: pypy/branch/shrink-multidict/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/shrink-multidict/pypy/interpreter/baseobjspace.py Tue Oct 27 10:01:24 2009 @@ -26,7 +26,10 @@ return None def getdictvalue_w(self, space, attr): - return self.getdictvalue(space, space.wrap(attr)) + w_dict = self.getdict() + if w_dict is not None: + return space.finditem_str(w_dict, attr) + return None def getdictvalue(self, space, w_attr): w_dict = self.getdict() @@ -34,8 +37,8 @@ return space.finditem(w_dict, w_attr) return None - def getdictvalue_attr_is_in_class(self, space, w_attr): - return self.getdictvalue(space, w_attr) + def getdictvalue_attr_is_in_class(self, space, attr): + return self.getdictvalue_w(space, attr) def setdictvalue(self, space, w_attr, w_value, shadows_type=True): w_dict = self.getdict() @@ -551,12 +554,6 @@ # that can be defined in term of more primitive ones. Subclasses # may also override specific functions for performance. - #def is_(self, w_x, w_y): -- not really useful. Must be subclassed - # "'x is y'." - # w_id_x = self.id(w_x) - # w_id_y = self.id(w_y) - # return self.eq(w_id_x, w_id_y) - def not_(self, w_obj): return self.wrap(not self.is_true(w_obj)) @@ -575,6 +572,9 @@ def set_str_keyed_item(self, w_obj, w_key, w_value, shadows_type=True): return self.setitem(w_obj, w_key, w_value) + def finditem_str(self, w_obj, key): + return self.finditem(w_obj, self.wrap(key)) + def finditem(self, w_obj, w_key): try: return self.getitem(w_obj, w_key) Modified: pypy/branch/shrink-multidict/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/interpreter/mixedmodule.py (original) +++ pypy/branch/shrink-multidict/pypy/interpreter/mixedmodule.py Tue Oct 27 10:01:24 2009 @@ -41,33 +41,43 @@ w_builtin = self.get(name) return self.space.call_function(w_builtin, *args_w) + def getdictvalue_w(self, space, name): + w_value = space.finditem_str(self.w_dict, name) + if self.lazy and w_value is None: + return self._load_lazily(space, name) + return w_value + def getdictvalue(self, space, w_name): w_value = space.finditem(self.w_dict, w_name) if self.lazy and w_value is None: - name = space.str_w(w_name) - w_name = space.new_interned_w_str(w_name) - try: - loader = self.loaders[name] - except KeyError: - return None - else: - #print "trying to load", name - w_value = loader(space) - #print "loaded", w_value - # obscure - func = space.interpclass_w(w_value) - if type(func) is Function: - try: - bltin = func._builtinversion_ - except AttributeError: - bltin = BuiltinFunction(func) - bltin.w_module = self.w_name - func._builtinversion_ = bltin - bltin.name = name - w_value = space.wrap(bltin) - space.setitem(self.w_dict, w_name, w_value) + return self._load_lazily(space, space.str_w(w_name)) return w_value + def _load_lazily(self, space, name): + w_name = space.new_interned_str(name) + try: + loader = self.loaders[name] + except KeyError: + return None + else: + #print "trying to load", name + w_value = loader(space) + #print "loaded", w_value + # obscure + func = space.interpclass_w(w_value) + if type(func) is Function: + try: + bltin = func._builtinversion_ + except AttributeError: + bltin = BuiltinFunction(func) + bltin.w_module = self.w_name + func._builtinversion_ = bltin + bltin.name = name + w_value = space.wrap(bltin) + space.setitem(self.w_dict, w_name, w_value) + return w_value + + def getdict(self): if self.lazy: space = self.space Modified: pypy/branch/shrink-multidict/pypy/interpreter/typedef.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/interpreter/typedef.py (original) +++ pypy/branch/shrink-multidict/pypy/interpreter/typedef.py Tue Oct 27 10:01:24 2009 @@ -274,12 +274,12 @@ if space.config.objspace.std.withshadowtracking: self.w__dict__.set_shadows_anything() - def getdictvalue_attr_is_in_class(self, space, w_name): + def getdictvalue_attr_is_in_class(self, space, name): w_dict = self.w__dict__ if space.config.objspace.std.withshadowtracking: if not w_dict.shadows_anything(): return None - return space.finditem(w_dict, w_name) + return space.finditem_str(w_dict, name) add(Proto) Modified: pypy/branch/shrink-multidict/pypy/objspace/descroperation.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/descroperation.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/descroperation.py Tue Oct 27 10:01:24 2009 @@ -36,9 +36,9 @@ if w_descr is not None: if space.is_data_descr(w_descr): return space.get(w_descr, w_obj) - w_value = w_obj.getdictvalue_attr_is_in_class(space, w_name) + w_value = w_obj.getdictvalue_attr_is_in_class(space, name) else: - w_value = w_obj.getdictvalue(space, w_name) + w_value = w_obj.getdictvalue_w(space, name) if w_value is not None: return w_value if w_descr is not None: Modified: pypy/branch/shrink-multidict/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/callmethod.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/callmethod.py Tue Oct 27 10:01:24 2009 @@ -54,9 +54,9 @@ if w_descr is None: # this handles directly the common case # module.function(args..) - w_value = w_obj.getdictvalue(space, w_name) + w_value = w_obj.getdictvalue_w(space, name) elif type(w_descr) is function.Function: - w_value = w_obj.getdictvalue_attr_is_in_class(space, w_name) + w_value = w_obj.getdictvalue_attr_is_in_class(space, name) if w_value is None: # fast method path: a function object in the class, # nothing in the instance @@ -85,15 +85,15 @@ """An optimized version of space.call_method() based on the same principle as above. """ - w_name = space.wrap(methname) w_getattribute = space.lookup(w_obj, '__getattribute__') if w_getattribute is object_getattribute(space): w_descr = space.lookup(w_obj, methname) if type(w_descr) is function.Function: - w_value = w_obj.getdictvalue_attr_is_in_class(space, w_name) + w_value = w_obj.getdictvalue_attr_is_in_class(space, methname) if w_value is None: # fast method path: a function object in the class, # nothing in the instance return space.call_function(w_descr, w_obj, *arg_w) + w_name = space.wrap(methname) w_meth = space.getattr(w_obj, w_name) return space.call_function(w_meth, *arg_w) Modified: pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py Tue Oct 27 10:01:24 2009 @@ -80,15 +80,19 @@ space = self.space w_lookup_type = space.type(w_lookup) if space.is_w(w_lookup_type, space.w_str): - res = self.getcell(space.str_w(w_lookup), False) - if res is None: - return None - return res.w_value + return self.impl_getitem_str(space.str_w(w_lookup)) + elif _is_sane_hash(space, w_lookup_type): return None else: return self._as_rdict().getitem(w_lookup) + def impl_getitem_str(self, lookup): + res = self.getcell(lookup, False) + if res is None: + return None + return res.w_value + def impl_iter(self): return ModuleDictIteratorImplementation(self.space, self) Modified: pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py Tue Oct 27 10:01:24 2009 @@ -109,6 +109,10 @@ #return w_value or None raise NotImplementedError("abstract base class") + def impl_getitem_str(self, w_key): + #return w_value or None + raise NotImplementedError("abstract base class") + def impl_setitem_str(self, w_key, w_value, shadows_type=True): raise NotImplementedError("abstract base class") @@ -159,8 +163,8 @@ # CALL_LIKELY_BUILTIN opcode is set. Otherwise it won't even be seen # by the annotator def impl_get_builtin_indexed(self, i): - w_key = self.space.wrap(OPTIMIZED_BUILTINS[i]) - return self.impl_getitem(w_key) + key = OPTIMIZED_BUILTINS[i] + return self.impl_getitem_str(key) # this method will only be seen whan a certain config option is used def impl_shadows_anything(self): @@ -187,6 +191,9 @@ def impl_fallback_getitem(self, w_key): return self.r_dict_content.get(w_key, None) + def impl_fallback_getitem_str(self, key): + return self.r_dict_content.get(self.space.wrap(key), None) + def impl_fallback_iter(self): return RDictIteratorImplementation(self.space, self) @@ -202,8 +209,8 @@ self.r_dict_content.clear() def impl_fallback_get_builtin_indexed(self, i): - w_key = self.space.wrap(OPTIMIZED_BUILTINS[i]) - return self.impl_fallback_getitem(w_key) + key = OPTIMIZED_BUILTINS[i] + return self.impl_fallback_getitem_str(key) def impl_fallback_shadows_anything(self): return True @@ -214,6 +221,7 @@ implementation_methods = [ ("getitem", 1), + ("getitem_str", 1), ("length", 0), ("setitem_str", 3), ("setitem", 2), @@ -322,15 +330,18 @@ def impl_length(self): return len(self.content) + def impl_getitem_str(self, key): + return self.content.get(key, None) + def impl_getitem(self, w_key): space = self.space # -- This is called extremely often. Hack for performance -- if type(w_key) is space.StringObjectCls: - return self.content.get(w_key.unwrap(space), None) + return self.impl_getitem_str(w_key.unwrap(space)) # -- End of performance hack -- w_lookup_type = space.type(w_key) if space.is_w(w_lookup_type, space.w_str): - return self.content.get(space.str_w(w_key), None) + return self.impl_getitem_str(space.str_w(w_key)) elif _is_sane_hash(space, w_lookup_type): return None else: @@ -562,6 +573,8 @@ def impl_length(self): self.info.lengths += 1 return len(self.content) + def impl_getitem_str(self, key): + return self.impl_getitem(self.space.wrap(key)) def impl_getitem(self, w_key): self.info.gets += 1 self._read(w_key) Modified: pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py Tue Oct 27 10:01:24 2009 @@ -84,285 +84,6 @@ return getattr(self, attrname) is not None def getdictvalue_w(self, space, attr): - return self.getdictvalue(space, space.wrap(attr)) - - def getdictvalue(self, space, w_attr): - if self._inlined_dict_valid(): - return self.impl_getitem(w_attr) - w_dict = self.getdict() - return w_dict.getitem(w_attr) - - def getdictvalue_attr_is_in_class(self, space, w_attr): - return self.getdictvalue(space, w_attr) - - def setdictvalue(self, space, w_attr, w_value, shadows_type=True): - if self._inlined_dict_valid(): - # XXX don't ignore shadows_type - self.impl_setitem(w_attr, w_value) - return True - w_dict = self.getdict() - w_dict.setitem(w_attr, w_value) - return True - - def deldictvalue(self, space, w_attr): - if self._inlined_dict_valid(): - try: - self.impl_delitem(w_attr) - except KeyError: - return False - return True - w_dict = self.getdict() - try: - w_dict.delitem(w_attr) - except KeyError: - return False - return True - - def setdict(self, space, w_dict): - make_rdict(self) # invalidate attributes on self - self.w__dict__ = check_new_dictionary(space, w_dict) - - def _as_rdict(self): - make_rdict(self) - return self.getdict() - - def initialize_as_rdict(self): - return self.getdict().initialize_as_rdict() - - for methname, _ in implementation_methods: - implname = "impl_" + methname - meth = func_with_new_name(getattr(dictimplclass, implname).im_func, - implname) - if not hasattr(InlineDictMixin, implname): - setattr(InlineDictMixin, implname, meth) - return InlineDictMixin -import py -from pypy.interpreter.typedef import check_new_dictionary -from pypy.objspace.std.dictmultiobject import W_DictMultiObject -from pypy.objspace.std.dictmultiobject import StrDictImplementation -from pypy.objspace.std.dictmultiobject import IteratorImplementation -from pypy.objspace.std.dictmultiobject import implementation_methods -from pypy.tool.sourcetools import func_with_new_name - -def make_mixin(config): - if config.objspace.std.withsharingdict: - from pypy.objspace.std.sharingdict import SharedDictImplementation - return make_inlinedict_mixin(SharedDictImplementation, "structure") - else: - return make_inlinedict_mixin(StrDictImplementation, "content") - -def make_indirection_method(methname, numargs): - # *args don't work, the call normalization gets confused - args = ", ".join(["a" + str(i) for i in range(numargs)]) - code = """def f(self, %s): - return self.w_obj.%s(%s) -""" % (args, methname, args) - d = {} - exec py.code.Source(code).compile() in d - func = d["f"] - func.func_name = methname + "_indirect" - func.func_defaults = getattr(W_DictMultiObject, methname).func_defaults - return func - -def make_inlinedict_mixin(dictimplclass, attrname): - assert dictimplclass.__base__ is W_DictMultiObject - class IndirectionIterImplementation(IteratorImplementation): - def __init__(self, space, dictimpl, itemlist): - IteratorImplementation.__init__(self, space, dictimpl) - self.itemlist = itemlist - - def next_entry(self): - return self.itemlist[self.pos] - - class IndirectionDictImplementation(W_DictMultiObject): - def __init__(self, space, w_obj): - self.space = space - self.w_obj = w_obj - - def impl_iter(self): - # XXX sucky - items = [] - for w_item in self.impl_items(): - w_key, w_value = self.space.viewiterable(w_item) - items.append((w_key, w_value)) - return IndirectionIterImplementation(self.space, self, items) - - IndirectionDictImplementation.__name__ = "IndirectionDictImplementation" + dictimplclass.__name__ - - for methname, numargs in implementation_methods: - implname = "impl_" + methname - if implname != "impl_iter": - setattr(IndirectionDictImplementation, implname, - make_indirection_method(implname, numargs)) - - init_dictattributes = func_with_new_name(dictimplclass.__init__.im_func, - "init_dictattributes") - make_rdict = func_with_new_name(dictimplclass._as_rdict.im_func, - "make_rdict") - - class InlineDictMixin(object): - - def user_setup(self, space, w_subtype): - self.space = space - self.w__class__ = w_subtype - self.w__dict__ = None - init_dictattributes(self, space) - assert getattr(self, attrname) is not None - self.user_setup_slots(w_subtype.nslots) - - def getdict(self): - w__dict__ = self.w__dict__ - if w__dict__ is None: - w__dict__ = IndirectionDictImplementation(self.space, self) - self.w__dict__ = w__dict__ - assert isinstance(w__dict__, W_DictMultiObject) - return w__dict__ - - def _inlined_dict_valid(self): - return getattr(self, attrname) is not None - - def getdictvalue_w(self, space, attr): - if self._inlined_dict_valid(): - return self.impl_getitem_str(attr) - w_dict = self.getdict() - return w_dict.getitem_str(attr) - - def getdictvalue(self, space, w_attr): - if self._inlined_dict_valid(): - return self.impl_getitem(w_attr) - w_dict = self.getdict() - return w_dict.getitem(w_attr) - - def getdictvalue_attr_is_in_class(self, space, attr): - return self.getdictvalue_w(space, attr) - - def setdictvalue(self, space, w_attr, w_value, shadows_type=True): - if self._inlined_dict_valid(): - # XXX don't ignore shadows_type - self.impl_setitem(w_attr, w_value) - return True - w_dict = self.getdict() - w_dict.setitem(w_attr, w_value) - return True - - def deldictvalue(self, space, w_attr): - if self._inlined_dict_valid(): - try: - self.impl_delitem(w_attr) - except KeyError: - return False - return True - w_dict = self.getdict() - try: - w_dict.delitem(w_attr) - except KeyError: - return False - return True - - def setdict(self, space, w_dict): - make_rdict(self) # invalidate attributes on self - self.w__dict__ = check_new_dictionary(space, w_dict) - - def _as_rdict(self): - make_rdict(self) - return self.getdict() - - def initialize_as_rdict(self): - return self.getdict().initialize_as_rdict() - - for methname, _ in implementation_methods: - implname = "impl_" + methname - meth = func_with_new_name(getattr(dictimplclass, implname).im_func, - implname) - if not hasattr(InlineDictMixin, implname): - setattr(InlineDictMixin, implname, meth) - return InlineDictMixin -import py -from pypy.interpreter.typedef import check_new_dictionary -from pypy.objspace.std.dictmultiobject import W_DictMultiObject -from pypy.objspace.std.dictmultiobject import StrDictImplementation -from pypy.objspace.std.dictmultiobject import IteratorImplementation -from pypy.objspace.std.dictmultiobject import implementation_methods -from pypy.tool.sourcetools import func_with_new_name - -def make_mixin(config): - if config.objspace.std.withsharingdict: - from pypy.objspace.std.sharingdict import SharedDictImplementation - return make_inlinedict_mixin(SharedDictImplementation, "structure") - else: - return make_inlinedict_mixin(StrDictImplementation, "content") - -def make_indirection_method(methname, numargs): - # *args don't work, the call normalization gets confused - args = ", ".join(["a" + str(i) for i in range(numargs)]) - code = """def f(self, %s): - return self.w_obj.%s(%s) -""" % (args, methname, args) - d = {} - exec py.code.Source(code).compile() in d - func = d["f"] - func.func_name = methname + "_indirect" - func.func_defaults = getattr(W_DictMultiObject, methname).func_defaults - return func - -def make_inlinedict_mixin(dictimplclass, attrname): - assert dictimplclass.__base__ is W_DictMultiObject - class IndirectionIterImplementation(IteratorImplementation): - def __init__(self, space, dictimpl, itemlist): - IteratorImplementation.__init__(self, space, dictimpl) - self.itemlist = itemlist - - def next_entry(self): - return self.itemlist[self.pos] - - class IndirectionDictImplementation(W_DictMultiObject): - def __init__(self, space, w_obj): - self.space = space - self.w_obj = w_obj - - def impl_iter(self): - # XXX sucky - items = [] - for w_item in self.impl_items(): - w_key, w_value = self.space.viewiterable(w_item) - items.append((w_key, w_value)) - return IndirectionIterImplementation(self.space, self, items) - - IndirectionDictImplementation.__name__ = "IndirectionDictImplementation" + dictimplclass.__name__ - - for methname, numargs in implementation_methods: - implname = "impl_" + methname - if implname != "impl_iter": - setattr(IndirectionDictImplementation, implname, - make_indirection_method(implname, numargs)) - - init_dictattributes = func_with_new_name(dictimplclass.__init__.im_func, - "init_dictattributes") - make_rdict = func_with_new_name(dictimplclass._as_rdict.im_func, - "make_rdict") - - class InlineDictMixin(object): - - def user_setup(self, space, w_subtype): - self.space = space - self.w__class__ = w_subtype - self.w__dict__ = None - init_dictattributes(self, space) - assert getattr(self, attrname) is not None - self.user_setup_slots(w_subtype.nslots) - - def getdict(self): - w__dict__ = self.w__dict__ - if w__dict__ is None: - w__dict__ = IndirectionDictImplementation(self.space, self) - self.w__dict__ = w__dict__ - assert isinstance(w__dict__, W_DictMultiObject) - return w__dict__ - - def _inlined_dict_valid(self): - return getattr(self, attrname) is not None - - def getdictvalue_w(self, space, attr): if self._inlined_dict_valid(): return self.impl_getitem_str(attr) w_dict = self.getdict() Modified: pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py Tue Oct 27 10:01:24 2009 @@ -708,7 +708,7 @@ e = None if w_descr is not None: if not self.is_data_descr(w_descr): - w_value = w_obj.getdictvalue_attr_is_in_class(self, w_name) + w_value = w_obj.getdictvalue_attr_is_in_class(self, name) if w_value is not None: return w_value try: @@ -717,7 +717,7 @@ if not e.match(self, self.w_AttributeError): raise else: - w_value = w_obj.getdictvalue(self, w_name) + w_value = w_obj.getdictvalue_w(self, name) if w_value is not None: return w_value @@ -729,6 +729,13 @@ else: raiseattrerror(self, w_obj, name) + def finditem_str(self, w_obj, key): + # performance shortcut to avoid creating the OperationError(KeyError) + if (isinstance(w_obj, self.DictObjectCls) and + not w_obj.user_overridden_class): + return w_obj.getitem_str(key) + return ObjSpace.finditem_str(self, w_obj, key) + def finditem(self, w_obj, w_key): # performance shortcut to avoid creating the OperationError(KeyError) if (isinstance(w_obj, self.DictObjectCls) and Modified: pypy/branch/shrink-multidict/pypy/objspace/std/proxyobject.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/proxyobject.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/proxyobject.py Tue Oct 27 10:01:24 2009 @@ -34,6 +34,9 @@ raise OperationError(space.w_TypeError, space.wrap("You cannot override __class__ for transparent proxies")) + def getdictvalue_w(self, space, attr): + return self.getdictvalue(space, space.wrap(attr)) + def getdictvalue(self, space, w_attr): try: return space.call_function(self.w_controller, space.wrap('__getattribute__'), Modified: pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py Tue Oct 27 10:01:24 2009 @@ -85,15 +85,17 @@ space = self.space w_lookup_type = space.type(w_lookup) if space.is_w(w_lookup_type, space.w_str): - lookup = space.str_w(w_lookup) - i = self.structure.lookup_position(lookup) - if i == -1: - return None - return self.entries[i] + return self.impl_getitem_str(space.str_w(w_lookup)) elif _is_sane_hash(space, w_lookup_type): return None else: - return self._as_rdict().get(w_lookup) + return self._as_rdict().getitem(w_lookup) + + def impl_getitem_str(self, lookup): + i = self.structure.lookup_position(lookup) + if i == -1: + return None + return self.entries[i] def impl_setitem(self, w_key, w_value): space = self.space @@ -155,9 +157,9 @@ return [space.newtuple([space.wrap(key), self.entries[item]]) for (key, item) in self.structure.keys.iteritems()] def impl_clear(self): - SharedDictImplementation.__init__(self, self.space) - - + space = self.space + self.structure = space.fromcache(State).empty_structure + self.entries = space.fromcache(State).emptylist def _as_rdict(self): r_dict_content = self.initialize_as_rdict() for k, i in self.structure.keys.items(): Modified: pypy/branch/shrink-multidict/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/test/test_dictmultiobject.py Tue Oct 27 10:01:24 2009 @@ -550,7 +550,11 @@ raises(KeyError, "d['def']") -class C: pass + +class FakeString(str): + def unwrap(self, space): + self.unwrapped = True + return str(self) # the minimal 'space' needed to use a W_DictMultiObject class FakeSpace: @@ -600,7 +604,7 @@ w_StopIteration = StopIteration w_None = None - StringObjectCls = None # xxx untested: shortcut in StrDictImpl.getitem + StringObjectCls = FakeString w_dict = None iter = iter viewiterable = list @@ -658,12 +662,14 @@ self.impl.setitem(self.string, 1000) assert self.impl.length() == 1 assert self.impl.getitem(self.string) == 1000 + assert self.impl.getitem_str(self.string) == 1000 self.check_not_devolved() def test_setitem_str(self): self.impl.setitem_str(self.fakespace.str_w(self.string), 1000) assert self.impl.length() == 1 assert self.impl.getitem(self.string) == 1000 + assert self.impl.getitem_str(self.string) == 1000 self.check_not_devolved() def test_delitem(self): @@ -719,6 +725,12 @@ class TestStrDictImplementation(BaseTestRDictImplementation): ImplementionClass = StrDictImplementation + def test_str_shortcut(self): + self.fill_impl() + s = FakeString(self.string) + assert self.impl.getitem(s) == 1000 + assert s.unwrapped + ## class TestMeasuringDictImplementation(BaseTestRDictImplementation): ## ImplementionClass = MeasuringDictImplementation ## DevolvedClass = MeasuringDictImplementation @@ -736,7 +748,6 @@ ImplementionClass = SharedDictImplementation - class BaseTestDevolvedDictImplementation(BaseTestRDictImplementation): def fill_impl(self): BaseTestRDictImplementation.fill_impl(self) Modified: pypy/branch/shrink-multidict/pypy/objspace/std/test/test_inlinedict.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/test/test_inlinedict.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/test/test_inlinedict.py Tue Oct 27 10:01:24 2009 @@ -125,257 +125,3 @@ assert self.space.int_w(w_a.content['x']) == 12 assert self.space.int_w(w_a.content['y']) == 13 -from pypy.conftest import gettestobjspace -from pypy.objspace.std.inlinedict import make_inlinedict_mixin -from pypy.objspace.std.dictmultiobject import StrDictImplementation -from pypy.objspace.std.test.test_dictmultiobject import FakeSpace -from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation -from pypy.objspace.std.sharingdict import SharedDictImplementation - -class FakeSubtype: - nslots = 0 - -class TestMixin(object): - Mixin = make_inlinedict_mixin(StrDictImplementation, "content") - class FakeObject(Mixin): - def user_setup_slots(self, nslots): - pass - - fakespace = FakeSpace() - - def make_obj(self): - obj = self.FakeObject() - obj.user_setup(self.fakespace, FakeSubtype) - obj.setdictvalue(self.fakespace, "hello", 1) - obj.setdictvalue(self.fakespace, "world", 2) - assert obj._inlined_dict_valid() - assert obj.w__dict__ is None - return obj - - def test_setgetdel_dictvalue(self): - obj = self.make_obj() - assert obj.getdictvalue(self.fakespace, "hello") == 1 - assert obj.getdictvalue(self.fakespace, "world") == 2 - assert obj.getdictvalue(self.fakespace, "bla") is None - assert not obj.deldictvalue(self.fakespace, "bla") - obj.deldictvalue(self.fakespace, "world") - assert obj.getdictvalue(self.fakespace, "world") is None - obj.deldictvalue(self.fakespace, "hello") - assert obj.getdictvalue(self.fakespace, "hello") is None - - - def test_getdict(self): - obj = self.make_obj() - w_dict = obj.getdict() - assert obj.getdict() or w_dict # always get the same dict - assert obj.w__dict__ is w_dict - - assert w_dict.getitem("hello") == 1 - assert w_dict.getitem("world") == 2 - w_dict.setitem("hello", 4) - w_dict.setitem("world", 5) - assert obj.getdictvalue(self.fakespace, "hello") == 4 - assert obj.getdictvalue(self.fakespace, "world") == 5 - - def test_dict_devolves_via_object(self): - obj = self.make_obj() - obj.setdictvalue(self.fakespace, 4, 1) - obj.setdictvalue(self.fakespace, 5, 2) - w_dict = obj.w__dict__ - assert w_dict is not None - assert dict(w_dict.r_dict_content) == {4: 1, 5: 2, "hello": 1, "world": 2} - assert obj.getdictvalue(self.fakespace, "hello") == 1 - assert obj.getdictvalue(self.fakespace, "world") == 2 - assert obj.getdictvalue(self.fakespace, 4) == 1 - assert obj.getdictvalue(self.fakespace, 5) == 2 - obj.deldictvalue(self.fakespace, "world") - assert obj.getdictvalue(self.fakespace, "world") is None - obj.deldictvalue(self.fakespace, "hello") - assert obj.getdictvalue(self.fakespace, "hello") is None - - def test_dict_devolves_via_dict(self): - obj = self.make_obj() - w_dict = obj.getdict() - w_dict.setitem(4, 1) - w_dict.setitem(5, 2) - assert dict(w_dict.r_dict_content) == {4: 1, 5: 2, "hello": 1, "world": 2} - assert obj.getdictvalue(self.fakespace, "hello") == 1 - assert obj.getdictvalue(self.fakespace, "world") == 2 - assert obj.getdictvalue(self.fakespace, 4) == 1 - assert obj.getdictvalue(self.fakespace, 5) == 2 - obj.deldictvalue(self.fakespace, "world") - assert obj.getdictvalue(self.fakespace, "world") is None - obj.deldictvalue(self.fakespace, "hello") - assert obj.getdictvalue(self.fakespace, "hello") is None - -class TestMixinShared(TestMixin): - Mixin = make_inlinedict_mixin(SharedDictImplementation, "structure") - class FakeObject(Mixin): - def user_setup_slots(self, nslots): - pass - -class TestIndirectDict(BaseTestRDictImplementation): - Mixin = make_inlinedict_mixin(StrDictImplementation, "content") - class FakeObject(Mixin): - def user_setup_slots(self, nslots): - pass - - def get_impl(self): - obj = self.FakeObject() - obj.user_setup(self.fakespace, FakeSubtype) - return obj.getdict() - - -class TestIndirectDictShared(TestIndirectDict): - Mixin = make_inlinedict_mixin(SharedDictImplementation, "structure") - class FakeObject(Mixin): - def user_setup_slots(self, nslots): - pass - - - - -class TestInlineDict(object): - def setup_class(cls): - cls.space = gettestobjspace(**{"objspace.std.withinlineddict": True}) - - def test_simple(self): - w_a = self.space.appexec([], """(): - class A(object): - pass - a = A() - a.x = 12 - a.y = 13 - return a - """) - assert w_a.w__dict__ is None - assert self.space.int_w(w_a.content['x']) == 12 - assert self.space.int_w(w_a.content['y']) == 13 - -from pypy.conftest import gettestobjspace -from pypy.objspace.std.inlinedict import make_inlinedict_mixin -from pypy.objspace.std.dictmultiobject import StrDictImplementation -from pypy.objspace.std.test.test_dictmultiobject import FakeSpace -from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation -from pypy.objspace.std.sharingdict import SharedDictImplementation - -class FakeSubtype: - nslots = 0 - -class TestMixin(object): - Mixin = make_inlinedict_mixin(StrDictImplementation, "content") - class FakeObject(Mixin): - def user_setup_slots(self, nslots): - pass - - fakespace = FakeSpace() - - def make_obj(self): - obj = self.FakeObject() - obj.user_setup(self.fakespace, FakeSubtype) - obj.setdictvalue(self.fakespace, "hello", 1) - obj.setdictvalue(self.fakespace, "world", 2) - assert obj._inlined_dict_valid() - assert obj.w__dict__ is None - return obj - - def test_setgetdel_dictvalue(self): - obj = self.make_obj() - assert obj.getdictvalue(self.fakespace, "hello") == 1 - assert obj.getdictvalue(self.fakespace, "world") == 2 - assert obj.getdictvalue(self.fakespace, "bla") is None - assert not obj.deldictvalue(self.fakespace, "bla") - obj.deldictvalue(self.fakespace, "world") - assert obj.getdictvalue(self.fakespace, "world") is None - obj.deldictvalue(self.fakespace, "hello") - assert obj.getdictvalue(self.fakespace, "hello") is None - - - def test_getdict(self): - obj = self.make_obj() - w_dict = obj.getdict() - assert obj.getdict() or w_dict # always get the same dict - assert obj.w__dict__ is w_dict - - assert w_dict.getitem("hello") == 1 - assert w_dict.getitem("world") == 2 - w_dict.setitem("hello", 4) - w_dict.setitem("world", 5) - assert obj.getdictvalue(self.fakespace, "hello") == 4 - assert obj.getdictvalue(self.fakespace, "world") == 5 - - def test_dict_devolves_via_object(self): - obj = self.make_obj() - obj.setdictvalue(self.fakespace, 4, 1) - obj.setdictvalue(self.fakespace, 5, 2) - w_dict = obj.w__dict__ - assert w_dict is not None - assert dict(w_dict.r_dict_content) == {4: 1, 5: 2, "hello": 1, "world": 2} - assert obj.getdictvalue(self.fakespace, "hello") == 1 - assert obj.getdictvalue(self.fakespace, "world") == 2 - assert obj.getdictvalue(self.fakespace, 4) == 1 - assert obj.getdictvalue(self.fakespace, 5) == 2 - obj.deldictvalue(self.fakespace, "world") - assert obj.getdictvalue(self.fakespace, "world") is None - obj.deldictvalue(self.fakespace, "hello") - assert obj.getdictvalue(self.fakespace, "hello") is None - - def test_dict_devolves_via_dict(self): - obj = self.make_obj() - w_dict = obj.getdict() - w_dict.setitem(4, 1) - w_dict.setitem(5, 2) - assert dict(w_dict.r_dict_content) == {4: 1, 5: 2, "hello": 1, "world": 2} - assert obj.getdictvalue(self.fakespace, "hello") == 1 - assert obj.getdictvalue(self.fakespace, "world") == 2 - assert obj.getdictvalue(self.fakespace, 4) == 1 - assert obj.getdictvalue(self.fakespace, 5) == 2 - obj.deldictvalue(self.fakespace, "world") - assert obj.getdictvalue(self.fakespace, "world") is None - obj.deldictvalue(self.fakespace, "hello") - assert obj.getdictvalue(self.fakespace, "hello") is None - -class TestMixinShared(TestMixin): - Mixin = make_inlinedict_mixin(SharedDictImplementation, "structure") - class FakeObject(Mixin): - def user_setup_slots(self, nslots): - pass - -class TestIndirectDict(BaseTestRDictImplementation): - Mixin = make_inlinedict_mixin(StrDictImplementation, "content") - class FakeObject(Mixin): - def user_setup_slots(self, nslots): - pass - - def get_impl(self): - obj = self.FakeObject() - obj.user_setup(self.fakespace, FakeSubtype) - return obj.getdict() - - -class TestIndirectDictShared(TestIndirectDict): - Mixin = make_inlinedict_mixin(SharedDictImplementation, "structure") - class FakeObject(Mixin): - def user_setup_slots(self, nslots): - pass - - - - -class TestInlineDict(object): - def setup_class(cls): - cls.space = gettestobjspace(**{"objspace.std.withinlineddict": True}) - - def test_simple(self): - w_a = self.space.appexec([], """(): - class A(object): - pass - a = A() - a.x = 12 - a.y = 13 - return a - """) - assert w_a.w__dict__ is None - assert self.space.int_w(w_a.content['x']) == 12 - assert self.space.int_w(w_a.content['y']) == 13 - From arigo at codespeak.net Tue Oct 27 10:45:43 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Oct 2009 10:45:43 +0100 (CET) Subject: [pypy-svn] r68778 - in pypy/branch/logging/pypy: config doc/config rlib rlib/test Message-ID: <20091027094543.6D4DD1683DA@codespeak.net> Author: arigo Date: Tue Oct 27 10:45:43 2009 New Revision: 68778 Added: pypy/branch/logging/pypy/doc/config/translation.rlog.txt (contents, props changed) Modified: pypy/branch/logging/pypy/config/translationoption.py pypy/branch/logging/pypy/rlib/rlog.py pypy/branch/logging/pypy/rlib/test/test_rlog.py Log: Add a general translation option, --rlog or --no-rlog, to enable or disable rlog in translated programs. Modified: pypy/branch/logging/pypy/config/translationoption.py ============================================================================== --- pypy/branch/logging/pypy/config/translationoption.py (original) +++ pypy/branch/logging/pypy/config/translationoption.py Tue Oct 27 10:45:43 2009 @@ -41,6 +41,11 @@ }, cmdline="-b --backend"), + BoolOption("rlog", + "Includes logging code (at runtime, set PYPYLOG=file)", + default=True, + cmdline="--rlog"), + # gc ChoiceOption("gc", "Garbage Collection Strategy", ["boehm", "ref", "marksweep", "semispace", "statistics", Added: pypy/branch/logging/pypy/doc/config/translation.rlog.txt ============================================================================== --- (empty file) +++ pypy/branch/logging/pypy/doc/config/translation.rlog.txt Tue Oct 27 10:45:43 2009 @@ -0,0 +1,3 @@ +Enable general logging. When true, the translated program will look +for the PYPYLOG env var, and if set to a file name, it will dump +various information into that file. Modified: pypy/branch/logging/pypy/rlib/rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog.py (original) +++ pypy/branch/logging/pypy/rlib/rlog.py Tue Oct 27 10:45:43 2009 @@ -26,18 +26,26 @@ _about_ = has_log def compute_result_annotation(self): - from pypy.annotation import model as annmodel - return annmodel.s_Bool + translator = self.bookkeeper.annotator.translator + if translator.config.translation.rlog: + from pypy.annotation import model as annmodel + return annmodel.s_Bool + else: + return self.bookkeeper.immutablevalue(False) def specialize_call(self, hop): - from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem import lltype - logwriter = get_logwriter(hop.rtyper) - annhelper = hop.rtyper.getannmixlevel() - c_func = annhelper.constfunc(logwriter.has_log, [], - annmodel.s_Bool) - hop.exception_cannot_occur() - return hop.genop('direct_call', [c_func], resulttype=lltype.Bool) + translator = hop.rtyper.annotator.translator + if translator.config.translation.rlog: + from pypy.annotation import model as annmodel + logwriter = get_logwriter(hop.rtyper) + annhelper = hop.rtyper.getannmixlevel() + c_func = annhelper.constfunc(logwriter.has_log, [], + annmodel.s_Bool) + hop.exception_cannot_occur() + return hop.genop('direct_call', [c_func], resulttype=lltype.Bool) + else: + return hop.inputconst(lltype.Bool, False) class DebugLogEntry(ExtRegistryEntry): @@ -45,56 +53,60 @@ def compute_result_annotation(self, s_category, s_message, **kwds_s): from pypy.annotation import model as annmodel - assert s_category.is_constant() - assert s_message.is_constant() translator = self.bookkeeper.annotator.translator - try: - logcategories = translator._logcategories - except AttributeError: - logcategories = translator._logcategories = {} - try: - cat = logcategories[s_category.const] - except KeyError: - num = len(logcategories) + 1 - logcategories[s_category.const] = LogCategory(s_category.const, - s_message.const, - num) - else: - assert cat.message == s_message.const, ( - "log category %r is used with different messages:\n\t%s\n\t%s" - % (s_category.const, cat.message, s_message.const)) + if translator.config.translation.rlog: + assert s_category.is_constant() + assert s_message.is_constant() + translator = self.bookkeeper.annotator.translator + try: + logcategories = translator._logcategories + except AttributeError: + logcategories = translator._logcategories = {} + try: + cat = logcategories[s_category.const] + except KeyError: + num = len(logcategories) + 1 + logcategories[s_category.const] = LogCategory(s_category.const, + s_message.const, + num) + else: + assert cat.message == s_message.const, ( + "log category %r is used with different messages:\n\t%s\n" + "\t%s" % (s_category.const, cat.message, s_message.const)) return annmodel.s_None def specialize_call(self, hop, **kwds_i): - from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem import lltype - logwriter = get_logwriter(hop.rtyper) translator = hop.rtyper.annotator.translator - cat = translator._logcategories[hop.args_s[0].const] - ann = { - 'd': annmodel.SomeInteger(), - 'f': annmodel.SomeFloat(), - 's': annmodel.SomeString(can_be_None=True), - } - annhelper = hop.rtyper.getannmixlevel() - args_s = [ann[t] for t in cat.types] - c_func = annhelper.constfunc(cat.gen_call(logwriter), args_s, - annmodel.s_None) - args_v = [c_func] - for name, typechar in cat.entries: - arg = kwds_i['i_'+name] - if typechar == 'd': - v = hop.inputarg(lltype.Signed, arg=arg) - elif typechar == 'f': - v = hop.inputarg(lltype.Float, arg=arg) - elif typechar == 's': - v = hop.inputarg(hop.rtyper.type_system.rstr.string_repr, - arg=arg) - else: - assert 0, typechar - args_v.append(v) - hop.exception_cannot_occur() - hop.genop('direct_call', args_v) + if translator.config.translation.rlog: + from pypy.annotation import model as annmodel + logwriter = get_logwriter(hop.rtyper) + translator = hop.rtyper.annotator.translator + cat = translator._logcategories[hop.args_s[0].const] + ann = { + 'd': annmodel.SomeInteger(), + 'f': annmodel.SomeFloat(), + 's': annmodel.SomeString(can_be_None=True), + } + annhelper = hop.rtyper.getannmixlevel() + args_s = [ann[t] for t in cat.types] + c_func = annhelper.constfunc(cat.gen_call(logwriter), args_s, + annmodel.s_None) + args_v = [c_func] + for name, typechar in cat.entries: + arg = kwds_i['i_'+name] + if typechar == 'd': + v = hop.inputarg(lltype.Signed, arg=arg) + elif typechar == 'f': + v = hop.inputarg(lltype.Float, arg=arg) + elif typechar == 's': + v = hop.inputarg(hop.rtyper.type_system.rstr.string_repr, + arg=arg) + else: + assert 0, typechar + args_v.append(v) + hop.exception_cannot_occur() + hop.genop('direct_call', args_v) return hop.inputconst(lltype.Void, None) def get_logwriter(rtyper): Modified: pypy/branch/logging/pypy/rlib/test/test_rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/test/test_rlog.py (original) +++ pypy/branch/logging/pypy/rlib/test/test_rlog.py Tue Oct 27 10:45:43 2009 @@ -250,6 +250,19 @@ assert entries[1][2] == [133, 132.5] assert entries[2][2] == ['hi there'] - def test_interpret(self): + def test_interpret_f(self): interpret(self.f.im_func, [132], malloc_check=False) self.check_result() + + def test_interpret_g(self): + def never_called(): + xyz + def g(): + rlog.debug_log("Aa", "hello %(foo)d", foo=5) + if rlog.has_log(): + never_called() + from pypy.config import translationoption + config = translationoption.get_combined_translation_config( + overrides={"translation.rlog": False}) + interpret(g, [], config=config) + assert not os.path.exists(self.pypylog) From cfbolz at codespeak.net Tue Oct 27 11:11:33 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 27 Oct 2009 11:11:33 +0100 (CET) Subject: [pypy-svn] r68779 - in pypy/trunk/pypy: config doc/config Message-ID: <20091027101133.B637B1683D8@codespeak.net> Author: cfbolz Date: Tue Oct 27 11:11:32 2009 New Revision: 68779 Removed: pypy/trunk/pypy/doc/config/translation.backendopt.heap2stack.txt Modified: pypy/trunk/pypy/config/translationoption.py Log: also remove translation option of heap2stack optimization Modified: pypy/trunk/pypy/config/translationoption.py ============================================================================== --- pypy/trunk/pypy/config/translationoption.py (original) +++ pypy/trunk/pypy/config/translationoption.py Tue Oct 27 11:11:32 2009 @@ -27,7 +27,6 @@ requires={ "ootype": [ ("translation.backendopt.constfold", False), - ("translation.backendopt.heap2stack", False), ("translation.backendopt.clever_malloc_removal", False), ("translation.gc", "boehm"), # it's not really used, but some jit code expects a value here ] @@ -202,9 +201,6 @@ BoolOption("mallocs", "Remove mallocs", default=True), BoolOption("constfold", "Constant propagation", default=True), - BoolOption("heap2stack", "Escape analysis and stack allocation", - default=False, - requires=[("translation.stackless", False)]), # control profile based inlining StrOption("profile_based_inline", "Use call count profiling to drive inlining" From cfbolz at codespeak.net Tue Oct 27 11:15:23 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 27 Oct 2009 11:15:23 +0100 (CET) Subject: [pypy-svn] r68780 - pypy/trunk/pypy/translator/backendopt/test Message-ID: <20091027101523.55FC81683D8@codespeak.net> Author: cfbolz Date: Tue Oct 27 11:15:22 2009 New Revision: 68780 Modified: pypy/trunk/pypy/translator/backendopt/test/test_escape.py Log: Make the test not depend on dictionary order. Modified: pypy/trunk/pypy/translator/backendopt/test/test_escape.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/test/test_escape.py (original) +++ pypy/trunk/pypy/translator/backendopt/test/test_escape.py Tue Oct 27 11:15:22 2009 @@ -366,5 +366,5 @@ t, adi, graph = build_adi(main, [int]) graphs = malloc_like_graphs(adi) - assert [g.name for g in graphs] == ["f", "h"] + assert set([g.name for g in graphs]) == set(["f", "h"]) From arigo at codespeak.net Tue Oct 27 11:46:51 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Oct 2009 11:46:51 +0100 (CET) Subject: [pypy-svn] r68781 - in pypy/branch/logging/pypy: config rlib rlib/test rpython/memory rpython/memory/gc rpython/memory/gctransform translator/c Message-ID: <20091027104651.6CBDE1683D8@codespeak.net> Author: arigo Date: Tue Oct 27 11:46:49 2009 New Revision: 68781 Modified: pypy/branch/logging/pypy/config/translationoption.py pypy/branch/logging/pypy/rlib/rlog.py pypy/branch/logging/pypy/rlib/rlog_parsing.py pypy/branch/logging/pypy/rlib/test/test_rlog.py pypy/branch/logging/pypy/rpython/memory/gc/semispace.py pypy/branch/logging/pypy/rpython/memory/gctransform/framework.py pypy/branch/logging/pypy/rpython/memory/gctypelayout.py pypy/branch/logging/pypy/translator/c/gc.py Log: For the GC, put both the debug_log() calls at the end of collecting (because it can allocate more memory). Works by overriding the time at which the first debug_log() is called. Modified: pypy/branch/logging/pypy/config/translationoption.py ============================================================================== --- pypy/branch/logging/pypy/config/translationoption.py (original) +++ pypy/branch/logging/pypy/config/translationoption.py Tue Oct 27 11:46:49 2009 @@ -72,12 +72,8 @@ "ref": [("translation.gcrootfinder", "n/a")], "none": [("translation.gcrootfinder", "n/a")], }), - OptionDescription("gcconfig", "Configure garbage collectors", [ - BoolOption("debugprint", "Turn on debug printing for the GC", - default=False), - BoolOption("removetypeptr", "Remove the typeptr from every object", - default=False, cmdline="--gcremovetypeptr"), - ]), + BoolOption("gcremovetypeptr", "Remove the typeptr from every object", + default=False, cmdline="--gcremovetypeptr"), ChoiceOption("gcrootfinder", "Strategy for finding GC Roots (framework GCs only)", ["n/a", "shadowstack", "asmgcc"], Modified: pypy/branch/logging/pypy/rlib/rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog.py (original) +++ pypy/branch/logging/pypy/rlib/rlog.py Tue Oct 27 11:46:49 2009 @@ -16,7 +16,7 @@ def has_log(): return True -def debug_log(_category, _message, **_kwds): +def debug_log(_category, _message, _time=None, **_kwds): getattr(_log, _category)(_message % _kwds) # ____________________________________________________________ @@ -66,13 +66,19 @@ cat = logcategories[s_category.const] except KeyError: num = len(logcategories) + 1 - logcategories[s_category.const] = LogCategory(s_category.const, - s_message.const, - num) + cat = LogCategory(s_category.const, s_message.const, num) + logcategories[s_category.const] = cat else: assert cat.message == s_message.const, ( "log category %r is used with different messages:\n\t%s\n" "\t%s" % (s_category.const, cat.message, s_message.const)) + for entry, _ in cat.entries: + name = 's_' + entry + assert name in kwds_s, "missing log entry %r" % (entry,) + del kwds_s[name] + if 's__time' in kwds_s: + del kwds_s['s__time'] + assert not kwds_s, "unexpected log entries %r" % (kwds_s.keys(),) return annmodel.s_None def specialize_call(self, hop, **kwds_i): @@ -83,17 +89,22 @@ logwriter = get_logwriter(hop.rtyper) translator = hop.rtyper.annotator.translator cat = translator._logcategories[hop.args_s[0].const] + types = cat.types + entries = cat.entries + if 'i__time' in kwds_i: + types = types + ['f'] + entries = entries + [('_time', 'f')] ann = { 'd': annmodel.SomeInteger(), 'f': annmodel.SomeFloat(), 's': annmodel.SomeString(can_be_None=True), } annhelper = hop.rtyper.getannmixlevel() - args_s = [ann[t] for t in cat.types] + args_s = [ann[t] for t in types] c_func = annhelper.constfunc(cat.gen_call(logwriter), args_s, annmodel.s_None) args_v = [c_func] - for name, typechar in cat.entries: + for name, typechar in entries: arg = kwds_i['i_'+name] if typechar == 'd': v = hop.inputarg(lltype.Signed, arg=arg) @@ -149,7 +160,11 @@ def call(*args): if not logwriter.enabled: return - if not logwriter.add_entry(self): + if len(args) > len(self.types): + now = args[len(self.types)] + else: + now = 0.0 + if not logwriter.add_entry(self, now): return i = 0 for typechar in types: @@ -205,6 +220,7 @@ self.write_int(-1) self.write_float(1.0) self.initialized_file = True + open_file._dont_inline_ = True def define_new_category(self, cat): if not self.initialized_file: @@ -216,13 +232,15 @@ self.write_str(cat.category) self.write_str(cat.message) self.initialized_index[cat.index] = None + define_new_category._dont_inline_ = True - def add_entry(self, cat): + def add_entry(self, cat, now=0.0): if cat.index not in self.initialized_index: self.define_new_category(cat) if not self.enabled: return False - now = self.get_time() + if now == 0.0: + now = self.get_time() timestamp_delta = now - self.curtime self.curtime = now self.write_int(cat.index) Modified: pypy/branch/logging/pypy/rlib/rlog_parsing.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog_parsing.py (original) +++ pypy/branch/logging/pypy/rlib/rlog_parsing.py Tue Oct 27 11:46:49 2009 @@ -1,3 +1,4 @@ +import autopath import struct from pypy.rlib.rarithmetic import intmask from pypy.rlib import rlog @@ -71,3 +72,17 @@ def parse_log(filename): logparser = LogParser(open(filename, 'rb')) return logparser.enum_entries() + + +if __name__ == '__main__': + import sys, re + r_replace = re.compile(r"%\(\w+\)") + for curtime, cat, entries in parse_log(sys.argv[1]): + try: + printcode = cat.printcode + except AttributeError: + code = cat.category + ' ' + message = cat.message.replace('\n', '\n' + ' '*len(code)) + message = r_replace.sub("%", message) + printcode = cat.printcode = code + message + print printcode % tuple(entries) Modified: pypy/branch/logging/pypy/rlib/test/test_rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/test/test_rlog.py (original) +++ pypy/branch/logging/pypy/rlib/test/test_rlog.py Tue Oct 27 11:46:49 2009 @@ -88,6 +88,24 @@ 17, 123.0, 515, "hellooo", 17, 0.0, 2873, "woooooorld"] +def test_logwriter_force_time(): + class FakeCategory: + def __init__(self, index, category, message): + self.index = index + self.category = category + self.message = message + # + logwriter = MyLogWriter() + cat5 = FakeCategory(5, "F5", "foobar") + logwriter.add_entry(cat5, now=100.0) + logwriter.add_entry(cat5) + # + assert logwriter.content == [ + ord('R'), ord('L'), ord('o'), ord('g'), ord('\n'), -1, 1.0, + 0, 5, "F5", "foobar", + 5, 100.0, + 5, 23.0] + SIZEOF_FLOAT = rlog.LLLogWriter.SIZEOF_FLOAT @@ -210,7 +228,8 @@ def f(x): assert rlog.has_log() - rlog.debug_log("Aa", "hello %(foo)d %(bar)f", foo=x, bar=-7.3) + rlog.debug_log("Aa", "hello %(foo)d %(bar)f", foo=x, bar=-7.3, + _time=0.5) rlog.debug_log("Aa", "hello %(foo)d %(bar)f", foo=x+1, bar=x+0.5) rlog.debug_log("Ab", "<<%(baz)s>>", baz="hi there") assert rlog.has_log() @@ -232,7 +251,7 @@ entries = list(rlog_parsing.parse_log(self.pypylog)) assert len(entries) == 3 # - assert isinstance(entries[0][0], float) + assert entries[0][0] == 0.5 assert isinstance(entries[1][0], float) assert isinstance(entries[2][0], float) # Modified: pypy/branch/logging/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/logging/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/logging/pypy/rpython/memory/gc/semispace.py Tue Oct 27 11:46:49 2009 @@ -9,6 +9,7 @@ from pypy.rlib.debug import ll_assert from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib import rlog from pypy.rpython.memory.gc.base import MovingGCBase import sys, os, time @@ -61,8 +62,7 @@ self.max_space_size = self.param_max_space_size self.red_zone = 0 - if self.config.gcconfig.debugprint: - self.program_start_time = time.time() + self.program_start_time = time.time() self.tospace = llarena.arena_malloc(self.space_size, True) ll_assert(bool(self.tospace), "couldn't allocate tospace") self.top_of_space = self.tospace + self.space_size @@ -213,18 +213,8 @@ # (this is also a hook for the HybridGC) def semispace_collect(self, size_changing=False): - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void) - llop.debug_print(lltype.Void, - ".----------- Full collection ------------------") - start_usage = self.free - self.tospace - llop.debug_print(lltype.Void, - "| used before collection: ", - start_usage, "bytes") - start_time = time.time() - else: - start_time = 0 # Help the flow space - start_usage = 0 # Help the flow space + start_time = time.time() + start_usage = self.free - self.tospace #llop.debug_print(lltype.Void, 'semispace_collect', int(size_changing)) # Switch the spaces. We copy everything over to the empty space @@ -254,41 +244,39 @@ self.record_red_zone() self.execute_finalizers() #llop.debug_print(lltype.Void, 'collected', self.space_size, size_changing, self.top_of_space - self.free) - if self.config.gcconfig.debugprint: + if rlog.has_log(): + rlog.debug_log("gc-full-{", + ".----------- Full collection ------------------\n" + "| used before collection: %(start_usage)d bytes", + start_usage=start_usage, + _time=start_time) end_time = time.time() elapsed_time = end_time - start_time self.total_collection_time += elapsed_time self.total_collection_count += 1 total_program_time = end_time - self.program_start_time end_usage = self.free - self.tospace - llop.debug_print(lltype.Void, - "| used after collection: ", - end_usage, "bytes") - llop.debug_print(lltype.Void, - "| freed: ", - start_usage - end_usage, "bytes") - llop.debug_print(lltype.Void, - "| size of each semispace: ", - self.space_size, "bytes") - llop.debug_print(lltype.Void, - "| fraction of semispace now used: ", - end_usage * 100.0 / self.space_size, "%") ct = self.total_collection_time cc = self.total_collection_count - llop.debug_print(lltype.Void, - "| number of semispace_collects: ", - cc) - llop.debug_print(lltype.Void, - "| i.e.: ", - cc / total_program_time, "per second") - llop.debug_print(lltype.Void, - "| total time in semispace_collect: ", - ct, "seconds") - llop.debug_print(lltype.Void, - "| i.e.: ", - ct * 100.0 / total_program_time, "%") - llop.debug_print(lltype.Void, - "`----------------------------------------------") + rlog.debug_log( + "gc-full-}", + "| used after collection: %(end_usage)d bytes\n" + "| freed: %(freed)d bytes\n" + "| size of each semispace: %(semispace)d bytes\n" + "| fraction of semispace now used: %(now_used)f %%\n" + "| number of semispace_collects: %(cc)d\n" + "| i.e.: %(cc_sec)f per second\n" + "| total time in semispace_collect: %(ct)f seconds\n" + "| i.e.: %(ct_frac)f %%\n" + "`----------------------------------------------", + end_usage = end_usage, + freed = start_usage - end_usage, + semispace = self.space_size, + now_used = end_usage * 100.0 / self.space_size, + cc = cc, + cc_sec = cc / total_program_time, + ct = ct, + ct_frac = ct * 100.0 / total_program_time) def starting_full_collect(self): pass # hook for the HybridGC Modified: pypy/branch/logging/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/logging/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/logging/pypy/rpython/memory/gctransform/framework.py Tue Oct 27 11:46:49 2009 @@ -130,7 +130,7 @@ if hasattr(translator, '_jit2gc'): self.layoutbuilder = translator._jit2gc['layoutbuilder'] else: - if translator.config.translation.gcconfig.removetypeptr: + if translator.config.translation.gcremovetypeptr: lltype2vtable = translator.rtyper.lltype2vtable else: lltype2vtable = {} @@ -888,7 +888,7 @@ def gct_getfield(self, hop): if (hop.spaceop.args[1].value == 'typeptr' and hop.spaceop.args[0].concretetype.TO._hints.get('typeptr') and - self.translator.config.translation.gcconfig.removetypeptr): + self.translator.config.translation.gcremovetypeptr): self.transform_getfield_typeptr(hop) else: GCTransformer.gct_getfield(self, hop) @@ -896,7 +896,7 @@ def gct_setfield(self, hop): if (hop.spaceop.args[1].value == 'typeptr' and hop.spaceop.args[0].concretetype.TO._hints.get('typeptr') and - self.translator.config.translation.gcconfig.removetypeptr): + self.translator.config.translation.gcremovetypeptr): self.transform_setfield_typeptr(hop) else: GCTransformer.gct_setfield(self, hop) @@ -991,11 +991,11 @@ class JITTransformerLayoutBuilder(TransformerLayoutBuilder): - # for the JIT: currently does not support removetypeptr + # for the JIT: currently does not support gcremovetypeptr def __init__(self, config): from pypy.rpython.memory.gc.base import choose_gc_from_config try: - assert not config.translation.gcconfig.removetypeptr + assert not config.translation.gcremovetypeptr except AttributeError: # for some tests pass GCClass, _ = choose_gc_from_config(config) Modified: pypy/branch/logging/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/logging/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/logging/pypy/rpython/memory/gctypelayout.py Tue Oct 27 11:46:49 2009 @@ -219,8 +219,7 @@ type_id = self.type_info_group.add_member(fullinfo) self.id_of_type[TYPE] = type_id # store the vtable of the type (if any) immediately thereafter - # (note that if gcconfig.removetypeptr is False, lltype2vtable - # is empty) + # (note that if gcremovetypeptr is False, lltype2vtable is empty) vtable = self.lltype2vtable.get(TYPE, None) if vtable is not None: # check that if we have a vtable, we are not varsize Modified: pypy/branch/logging/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/logging/pypy/translator/c/gc.py (original) +++ pypy/branch/logging/pypy/translator/c/gc.py Tue Oct 27 11:46:49 2009 @@ -352,7 +352,7 @@ def need_no_typeptr(self): config = self.db.translator.config - return config.translation.gcconfig.removetypeptr + return config.translation.gcremovetypeptr def OP_GC_GETTYPEPTR_GROUP(self, funcgen, op): # expands to a number of steps, as per rpython/lltypesystem/opimpl.py, From fijal at codespeak.net Tue Oct 27 11:59:53 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Oct 2009 11:59:53 +0100 (CET) Subject: [pypy-svn] r68782 - in pypy/branch/gc-dump-heap/pypy/rpython/memory: gc test Message-ID: <20091027105953.5C8C71683D6@codespeak.net> Author: fijal Date: Tue Oct 27 11:59:51 2009 New Revision: 68782 Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py Log: Compute total amount of objects, because (of course) varsized structs can have different sizes. Should fix the apparent difference between number of observed objects and memory consumed Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py Tue Oct 27 11:59:51 2009 @@ -651,7 +651,8 @@ self._tracked_dict.add(adr) idx = llop.get_member_index(lltype.Signed, self.get_type_id(adr)) self._ll_typeid_map[idx].count += 1 - self._ll_typeid_map[idx].size = self.get_size(adr) + totsize = self.get_size(adr) + self.size_gc_header() + self._ll_typeid_map[idx].size += llmemory.raw_malloc_usage(totsize) self.trace(adr, self.track_heap_parent, adr) def _track_heap_root(self, root): Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py Tue Oct 27 11:59:51 2009 @@ -4,7 +4,7 @@ from pypy.translator.c import gc from pypy.annotation import model as annmodel from pypy.annotation import policy as annpolicy -from pypy.rpython.lltypesystem import lltype, llmemory, llarena +from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi from pypy.rpython.memory.gctransform import framework from pypy.rpython.lltypesystem.lloperation import llop, void from pypy.rpython.memory.gc.marksweep import X_CLONE, X_POOL, X_POOL_PTR @@ -809,8 +809,9 @@ s = lltype.malloc(S) l1.append(s) l2.append(s) - l3.append(s) - l4.append(s) + if i < 3: + l3.append(s) + l4.append(s) # We cheat here and only read the table which we later on # process ourselves, otherwise this test takes ages llop.gc__collect(lltype.Void) @@ -820,6 +821,7 @@ b = 0 c = 0 d = 0 + e = 0 for i in range(len(tb)): if tb[i].count == 10: a += 1 @@ -830,13 +832,19 @@ if tb[i].count == 4: b += 1 c += tb[i].links[nr] + e += tb[i].size return d * 1000 + c * 100 + b * 10 + a return f def test_gc_dump_heap(self): run = self.runner("gc_dump_heap") res = run([]) - assert res == 4011 + assert res % 10000 == 2611 + totsize = (res / 10000) + size_of_int = rffi.sizeof(lltype.Signed) + assert (totsize - 26 * size_of_int) % 4 == 0 + # ^^^ a crude assumption that totsize - varsize would be dividable by 4 + # (and give fixedsize) # ________________________________________________________________ From fijal at codespeak.net Tue Oct 27 12:02:32 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 27 Oct 2009 12:02:32 +0100 (CET) Subject: [pypy-svn] r68783 - pypy/branch/gc-dump-heap/pypy/tool Message-ID: <20091027110232.D90601683D6@codespeak.net> Author: fijal Date: Tue Oct 27 12:02:32 2009 New Revision: 68783 Modified: pypy/branch/gc-dump-heap/pypy/tool/gcdump.py Log: * Adapt to new format (totalsize instead of per-instance) * Add number of objects as a field Modified: pypy/branch/gc-dump-heap/pypy/tool/gcdump.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/tool/gcdump.py (original) +++ pypy/branch/gc-dump-heap/pypy/tool/gcdump.py Tue Oct 27 12:02:32 2009 @@ -38,17 +38,18 @@ return '(%d) %s' % (no, name) def process(f, gcdump, typeids): - f.write("events: B\n\n") + f.write("events: number B\n\n") for tid, name in enumerate(typeids): if not tid % 100: sys.stderr.write("%d%%.." % (tid / len(typeids) * 100)) f.write("fn=%s\n" % getname(name)) - f.write("0 %d\n" % (gcdump[tid].count * gcdump[tid].size)) + f.write("0 %d %d\n" % (gcdump[tid].count, gcdump[tid].size)) for subtid, no in enumerate(gcdump[tid].links): if no != 0: f.write("cfn=%s\n" % getname(typeids[subtid])) f.write("calls=0 %d\n" % no) - f.write("0 %d\n" % (gcdump[subtid].count * gcdump[subtid].size)) + f.write("0 %d %d\n" % (gcdump[subtid].count, + gcdump[subtid].size)) f.write("\n") sys.stderr.write("100%\n") From arigo at codespeak.net Tue Oct 27 13:38:13 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Oct 2009 13:38:13 +0100 (CET) Subject: [pypy-svn] r68784 - in pypy/branch/logging/pypy/rlib: . test Message-ID: <20091027123813.0DDA21683D7@codespeak.net> Author: arigo Date: Tue Oct 27 13:38:07 2009 New Revision: 68784 Added: pypy/branch/logging/pypy/rlib/autopath.py - copied unchanged from r68778, pypy/branch/logging/pypy/tool/autopath.py pypy/branch/logging/pypy/rlib/rlog_ll.py (contents, props changed) Modified: pypy/branch/logging/pypy/rlib/rlog.py pypy/branch/logging/pypy/rlib/rlog_parsing.py pypy/branch/logging/pypy/rlib/test/test_rlog.py Log: Split rlog_ll.py from rlog.py. Try to remove all dependencies of rlog_ll.py on the GC. Modified: pypy/branch/logging/pypy/rlib/rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog.py (original) +++ pypy/branch/logging/pypy/rlib/rlog.py Tue Oct 27 13:38:07 2009 @@ -1,8 +1,6 @@ -import py, os, time, struct +import py, time, struct from pypy.tool.ansi_print import ansi_log from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib.rarithmetic import r_uint, r_singlefloat -from pypy.rlib.objectmodel import we_are_translated from pypy.tool.sourcetools import func_with_new_name from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rpython.annlowlevel import hlstr @@ -16,7 +14,7 @@ def has_log(): return True -def debug_log(_category, _message, _time=None, **_kwds): +def debug_log(_category, _message, **_kwds): getattr(_log, _category)(_message % _kwds) # ____________________________________________________________ @@ -76,8 +74,6 @@ name = 's_' + entry assert name in kwds_s, "missing log entry %r" % (entry,) del kwds_s[name] - if 's__time' in kwds_s: - del kwds_s['s__time'] assert not kwds_s, "unexpected log entries %r" % (kwds_s.keys(),) return annmodel.s_None @@ -89,22 +85,17 @@ logwriter = get_logwriter(hop.rtyper) translator = hop.rtyper.annotator.translator cat = translator._logcategories[hop.args_s[0].const] - types = cat.types - entries = cat.entries - if 'i__time' in kwds_i: - types = types + ['f'] - entries = entries + [('_time', 'f')] ann = { 'd': annmodel.SomeInteger(), 'f': annmodel.SomeFloat(), 's': annmodel.SomeString(can_be_None=True), } annhelper = hop.rtyper.getannmixlevel() - args_s = [ann[t] for t in types] + args_s = [ann[t] for t in cat.types] c_func = annhelper.constfunc(cat.gen_call(logwriter), args_s, annmodel.s_None) args_v = [c_func] - for name, typechar in entries: + for name, typechar in cat.entries: arg = kwds_i['i_'+name] if typechar == 'd': v = hop.inputarg(lltype.Signed, arg=arg) @@ -124,6 +115,8 @@ try: return rtyper.annotator.translator._logwriter except AttributeError: + # XXX detect lltype vs. ootype + from pypy.rlib.rlog_ll import LLLogWriter logwriter = LLLogWriter() logwriter._register(rtyper) rtyper.annotator.translator._logwriter = logwriter @@ -135,8 +128,11 @@ r_entry = re.compile(r"%\((\w+)\)([sdf])") +SIZEOF_FLOAT = struct.calcsize("f") + class LogCategory(object): + seen_by = None def __init__(self, category, message, index): self.category = category @@ -160,11 +156,7 @@ def call(*args): if not logwriter.enabled: return - if len(args) > len(self.types): - now = args[len(self.types)] - else: - now = 0.0 - if not logwriter.add_entry(self, now): + if not logwriter.add_entry(self): return i = 0 for typechar in types: @@ -178,9 +170,6 @@ assert self.logwriter is logwriter return self.call - def _freeze_(self): - return True - class AbstractLogWriter(object): get_time = time.time @@ -188,8 +177,6 @@ def __init__(self): self.enabled = True self.initialized_file = False - self.initialized_index = {} - self.fd = -1 self.curtime = 0.0 # def has_log(): @@ -198,17 +185,8 @@ return self.enabled self.has_log = has_log - def get_filename(self): - return os.environ.get('PYPYLOG') - def open_file(self): - from pypy.rpython.lltypesystem import lltype, rffi - filename = self.get_filename() - if filename: - self.fd = os.open(filename, - os.O_WRONLY | os.O_CREAT | os.O_TRUNC, - 0666) - self.enabled = self.fd >= 0 + self.do_open_file() # write the header if self.enabled: self.create_buffer() @@ -231,16 +209,15 @@ self.write_int(cat.index) self.write_str(cat.category) self.write_str(cat.message) - self.initialized_index[cat.index] = None + cat.seen_by = self define_new_category._dont_inline_ = True - def add_entry(self, cat, now=0.0): - if cat.index not in self.initialized_index: + def add_entry(self, cat): + if cat.seen_by is not self: self.define_new_category(cat) if not self.enabled: return False - if now == 0.0: - now = self.get_time() + now = self.get_time() timestamp_delta = now - self.curtime self.curtime = now self.write_int(cat.index) @@ -261,88 +238,3 @@ def add_subentry_f(self, float): self.write_float(float) - -# ____________________________________________________________ - - -class LLLogWriter(AbstractLogWriter): - BUFSIZE = 8192 - SIZEOF_FLOAT = struct.calcsize("f") - - def do_write(self, fd, buf, size): - if we_are_translated(): - from pypy.rpython.lltypesystem import rffi - self._os_write(rffi.cast(rffi.INT, fd), - buf, - rffi.cast(rffi.SIZE_T, size)) - else: - l = [buf[i] for i in range(size)] - s = ''.join(l) - os.write(fd, s) - self.writecount += 1 - - def _register(self, rtyper): - from pypy.rpython.lltypesystem import rffi - from pypy.rpython.module.ll_os import underscore_on_windows - self._os_write = rffi.llexternal(underscore_on_windows+'write', - [rffi.INT, rffi.CCHARP, rffi.SIZE_T], - rffi.SIZE_T) - # register flush() to be called at program exit - def flush_log_cache(): - if self.initialized_file: - self._flush() - annhelper = rtyper.getannmixlevel() - annhelper.register_atexit(flush_log_cache) - - def create_buffer(self): - from pypy.rpython.lltypesystem import lltype, rffi - self.buffer = lltype.malloc(rffi.CCHARP.TO, self.BUFSIZE, flavor='raw') - self.buffer_position = 0 - self.writecount = 0 - - def write_int(self, n): - self._write_int_noflush(n) - if self.buffer_position > self.BUFSIZE-48: - self._flush() - - def _write_int_noflush(self, n): - p = self.buffer_position - buf = self.buffer - n = r_uint(n) - while n > 0x7F: - buf[p] = chr((n & 0x7F) | 0x80) - p += 1 - n >>= 7 - buf[p] = chr(n) - self.buffer_position = p + 1 - - def write_str(self, s): - self._write_int_noflush(len(s)) - p = self.buffer_position - if p + len(s) > self.BUFSIZE-24: - self._flush() - os.write(self.fd, s) - self.writecount += 1 - else: - buf = self.buffer - for i in range(len(s)): - buf[p + i] = s[i] - self.buffer_position = p + len(s) - - def write_float(self, f): - from pypy.rpython.lltypesystem import rffi - p = self.buffer_position - ptr = rffi.cast(rffi.FLOATP, rffi.ptradd(self.buffer, p)) - ptr[0] = r_singlefloat(f) - self.buffer_position = p + self.SIZEOF_FLOAT - if self.buffer_position > self.BUFSIZE-48: - self._flush() - - def _flush(self): - if self.buffer_position > 0: - self.do_write(self.fd, self.buffer, self.buffer_position) - self.buffer_position = 0 - - def _close(self): - self._flush() - os.close(self.fd) Added: pypy/branch/logging/pypy/rlib/rlog_ll.py ============================================================================== --- (empty file) +++ pypy/branch/logging/pypy/rlib/rlog_ll.py Tue Oct 27 13:38:07 2009 @@ -0,0 +1,117 @@ +import os +from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.rarithmetic import r_uint, r_singlefloat +from pypy.rlib.rlog import AbstractLogWriter, SIZEOF_FLOAT +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.module.ll_os_environ import os_getenv +from pypy.rpython.module.ll_os import underscore_on_windows + +os_write = rffi.llexternal(underscore_on_windows+'write', + [rffi.INT, rffi.CCHARP, rffi.SIZE_T], + rffi.SIZE_T) +os_open = rffi.llexternal(underscore_on_windows+'open', + [rffi.CCHARP, rffi.INT, rffi.MODE_T], + rffi.INT) + +l_pypylog = rffi.str2charp('PYPYLOG') + +# ____________________________________________________________ +# +# Important: logging on lltype must not use the GC at all +# + +class LLLogWriter(AbstractLogWriter): + BUFSIZE = 8192 + + fd = -1 + + def ll_get_filename(self): + return os_getenv(l_pypylog) + + def do_open_file(self): + l_result = self.ll_get_filename() + if l_result and l_result[0] != '\x00': + flags = rffi.cast(rffi.INT, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + mode = rffi.cast(rffi.MODE_T, 0666) + self.fd = rffi.cast(lltype.Signed, os_open(l_result, flags, mode)) + self.enabled = self.fd >= 0 + + def do_write(self, fd, buf, size): + if we_are_translated(): + os_write(rffi.cast(rffi.INT, fd), + buf, + rffi.cast(rffi.SIZE_T, size)) + else: + l = [buf[i] for i in range(size)] + s = ''.join(l) + os.write(fd, s) + self.writecount += 1 + + def _register(self, rtyper): + # register flush() to be called at program exit + def flush_log_cache(): + if self.initialized_file: + self._flush() + annhelper = rtyper.getannmixlevel() + annhelper.register_atexit(flush_log_cache) + + def create_buffer(self): + self.buffer = lltype.malloc(rffi.CCHARP.TO, self.BUFSIZE, flavor='raw') + self.buffer_position = 0 + self.writecount = 0 + + def write_int(self, n): + self._write_int_noflush(n) + if self.buffer_position > self.BUFSIZE-48: + self._flush() + + def _write_int_noflush(self, n): + p = self.buffer_position + buf = self.buffer + n = r_uint(n) + while n > 0x7F: + buf[p] = chr((n & 0x7F) | 0x80) + p += 1 + n >>= 7 + buf[p] = chr(n) + self.buffer_position = p + 1 + + def write_str(self, s): + start = 0 + length = len(s) + self._write_int_noflush(length) + while self.buffer_position + length > self.BUFSIZE - 24: + count = self.BUFSIZE - self.buffer_position + if count > length: + count = length + self._write_raw_data(s, start, count) + start += count + length -= count + self._flush() + self._write_raw_data(s, start, length) + + def _write_raw_data(self, s, start, length): + p = self.buffer_position + buf = self.buffer + i = 0 + while i < length: + buf[p + i] = s[start + i] + i += 1 + self.buffer_position = p + length + + def write_float(self, f): + p = self.buffer_position + ptr = rffi.cast(rffi.FLOATP, rffi.ptradd(self.buffer, p)) + ptr[0] = r_singlefloat(f) + self.buffer_position = p + SIZEOF_FLOAT + if self.buffer_position > self.BUFSIZE-48: + self._flush() + + def _flush(self): + if self.buffer_position > 0: + self.do_write(self.fd, self.buffer, self.buffer_position) + self.buffer_position = 0 + + def _close(self): + self._flush() + os.close(self.fd) Modified: pypy/branch/logging/pypy/rlib/rlog_parsing.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog_parsing.py (original) +++ pypy/branch/logging/pypy/rlib/rlog_parsing.py Tue Oct 27 13:38:07 2009 @@ -3,7 +3,7 @@ from pypy.rlib.rarithmetic import intmask from pypy.rlib import rlog -SIZEOF_FLOAT = rlog.LLLogWriter.SIZEOF_FLOAT +SIZEOF_FLOAT = rlog.SIZEOF_FLOAT class LogParser(object): Modified: pypy/branch/logging/pypy/rlib/test/test_rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/test/test_rlog.py (original) +++ pypy/branch/logging/pypy/rlib/test/test_rlog.py Tue Oct 27 13:38:07 2009 @@ -1,5 +1,5 @@ import struct, os -from pypy.rlib import rlog, rlog_parsing +from pypy.rlib import rlog, rlog_ll, rlog_parsing from pypy.rlib.rarithmetic import intmask from pypy.tool.udir import udir from pypy.rpython.test.test_llinterp import interpret @@ -35,8 +35,8 @@ def get_time(self): return 123.0 - def get_filename(self): - return str(self._path) + def do_open_file(self): + self.enabled = True def create_buffer(self): self.content = [] def write_int(self, n): @@ -51,6 +51,7 @@ def test_logwriter(): class FakeCategory: + seen_by = None def __init__(self, index, category, message): self.index = index self.category = category @@ -88,26 +89,8 @@ 17, 123.0, 515, "hellooo", 17, 0.0, 2873, "woooooorld"] -def test_logwriter_force_time(): - class FakeCategory: - def __init__(self, index, category, message): - self.index = index - self.category = category - self.message = message - # - logwriter = MyLogWriter() - cat5 = FakeCategory(5, "F5", "foobar") - logwriter.add_entry(cat5, now=100.0) - logwriter.add_entry(cat5) - # - assert logwriter.content == [ - ord('R'), ord('L'), ord('o'), ord('g'), ord('\n'), -1, 1.0, - 0, 5, "F5", "foobar", - 5, 100.0, - 5, 23.0] - -SIZEOF_FLOAT = rlog.LLLogWriter.SIZEOF_FLOAT +SIZEOF_FLOAT = rlog.SIZEOF_FLOAT class TestLLLogWriter: COUNTER = 0 @@ -117,9 +100,10 @@ self.path = path TestLLLogWriter.COUNTER += 1 # - class MyLLLogWriter(rlog.LLLogWriter): - def get_filename(self): - return str(path) + class MyLLLogWriter(rlog_ll.LLLogWriter): + def ll_get_filename(self): + from pypy.rpython.lltypesystem import rffi + return rffi.str2charp(str(path)) # logwriter = MyLLLogWriter() logwriter.open_file() @@ -203,7 +187,7 @@ logwriter.write_str(s) logwriter._close() self.check(slist) - assert logwriter.writecount <= 9 + assert logwriter.writecount <= 10 def test_write_float(self): import math @@ -228,8 +212,7 @@ def f(x): assert rlog.has_log() - rlog.debug_log("Aa", "hello %(foo)d %(bar)f", foo=x, bar=-7.3, - _time=0.5) + rlog.debug_log("Aa", "hello %(foo)d %(bar)f", foo=x, bar=-7.3) rlog.debug_log("Aa", "hello %(foo)d %(bar)f", foo=x+1, bar=x+0.5) rlog.debug_log("Ab", "<<%(baz)s>>", baz="hi there") assert rlog.has_log() @@ -251,7 +234,7 @@ entries = list(rlog_parsing.parse_log(self.pypylog)) assert len(entries) == 3 # - assert entries[0][0] == 0.5 + assert isinstance(entries[0][0], float) assert isinstance(entries[1][0], float) assert isinstance(entries[2][0], float) # From arigo at codespeak.net Tue Oct 27 13:44:31 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Oct 2009 13:44:31 +0100 (CET) Subject: [pypy-svn] r68785 - pypy/branch/logging/pypy/rpython/memory/gc Message-ID: <20091027124431.9FF4C1683D7@codespeak.net> Author: arigo Date: Tue Oct 27 13:44:31 2009 New Revision: 68785 Modified: pypy/branch/logging/pypy/rpython/memory/gc/semispace.py Log: We no longer need the '_time' hack. Modified: pypy/branch/logging/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/logging/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/logging/pypy/rpython/memory/gc/semispace.py Tue Oct 27 13:44:31 2009 @@ -213,8 +213,18 @@ # (this is also a hook for the HybridGC) def semispace_collect(self, size_changing=False): - start_time = time.time() - start_usage = self.free - self.tospace + if rlog.has_log(): + start_usage = self.free - self.tospace + start_time = time.time() + rlog.debug_log("gc-full-{", + ".----------- Full collection ------------------\n" + "| used before collection: %(start_usage)d bytes", + start_usage = start_usage) + if size_changing: + rlog.debug_log("gc-full-s", "| size changing") + else: + start_time = 0 # Help the flow space + start_usage = 0 # Help the flow space #llop.debug_print(lltype.Void, 'semispace_collect', int(size_changing)) # Switch the spaces. We copy everything over to the empty space @@ -245,11 +255,6 @@ self.execute_finalizers() #llop.debug_print(lltype.Void, 'collected', self.space_size, size_changing, self.top_of_space - self.free) if rlog.has_log(): - rlog.debug_log("gc-full-{", - ".----------- Full collection ------------------\n" - "| used before collection: %(start_usage)d bytes", - start_usage=start_usage, - _time=start_time) end_time = time.time() elapsed_time = end_time - start_time self.total_collection_time += elapsed_time @@ -258,8 +263,7 @@ end_usage = self.free - self.tospace ct = self.total_collection_time cc = self.total_collection_count - rlog.debug_log( - "gc-full-}", + rlog.debug_log("gc-full-}", "| used after collection: %(end_usage)d bytes\n" "| freed: %(freed)d bytes\n" "| size of each semispace: %(semispace)d bytes\n" From arigo at codespeak.net Tue Oct 27 13:44:40 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Oct 2009 13:44:40 +0100 (CET) Subject: [pypy-svn] r68786 - pypy/branch/logging/pypy/rpython/memory/gc Message-ID: <20091027124440.9F3661683D9@codespeak.net> Author: arigo Date: Tue Oct 27 13:44:40 2009 New Revision: 68786 Modified: pypy/branch/logging/pypy/rpython/memory/gc/generation.py Log: Adapt the generation GC. Modified: pypy/branch/logging/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/logging/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/logging/pypy/rpython/memory/gc/generation.py Tue Oct 27 13:44:40 2009 @@ -7,6 +7,7 @@ from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE from pypy.rlib.objectmodel import free_non_gc_object from pypy.rlib.debug import ll_assert +from pypy.rlib import rlog from pypy.rpython.lltypesystem.lloperation import llop # The following flag is never set on young objects, i.e. the ones living @@ -85,8 +86,7 @@ if self.auto_nursery_size: newsize = nursery_size_from_env() if newsize <= 0: - newsize = estimate_best_nursery_size( - self.config.gcconfig.debugprint) + newsize = estimate_best_nursery_size() if newsize > 0: self.set_nursery_size(newsize) @@ -116,13 +116,6 @@ while (self.min_nursery_size << (scale+1)) <= newsize: scale += 1 self.nursery_scale = scale - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, "SSS nursery_size =", newsize) - llop.debug_print(lltype.Void, "SSS largest_young_fixedsize =", - self.largest_young_fixedsize) - llop.debug_print(lltype.Void, "SSS largest_young_var_basesize =", - self.largest_young_var_basesize) - llop.debug_print(lltype.Void, "SSS nursery_scale =", scale) # we get the following invariant: assert self.nursery_size >= (self.min_nursery_size << scale) @@ -132,6 +125,19 @@ # a new nursery (e.g. if it invokes finalizers). self.semispace_collect() + rlog.debug_log( + "gc-init-scale", + ".--- set_nursery_size() ---\n" + "| nursery_size = %(nursery_size)d\n" + "| largest_young_fixedsize = %(largest_young_fixedsize)d\n" + "| largest_young_var_basesize = %(largest_young_var_basesize)d\n" + "| nursery_scale = %(scale)d\n" + "`--------------------------", + nursery_size = newsize, + largest_young_fixedsize = self.largest_young_fixedsize, + largest_young_var_basesize = self.largest_young_var_basesize, + scale = scale) + @staticmethod def get_young_fixedsize(nursery_size): return nursery_size // 2 - 1 @@ -249,11 +255,7 @@ self.weakrefs_grow_older() self.ids_grow_older() self.reset_nursery() - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, "major collect, size changing", size_changing) SemiSpaceGC.semispace_collect(self, size_changing) - if self.config.gcconfig.debugprint and not size_changing: - llop.debug_print(lltype.Void, "percent survived", float(self.free - self.tospace) / self.space_size) def make_a_copy(self, obj, objsize): tid = self.header(obj).tid @@ -330,13 +332,10 @@ ll_assert(self.nursery_size <= self.top_of_space - self.free, "obtain_free_space failed to do its job") if self.nursery: - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, "--- minor collect ---") - llop.debug_print(lltype.Void, "nursery:", - self.nursery, "to", self.nursery_top) + rlog.debug_log("gc-minor-{", ".--- minor collect ---") # a nursery-only collection scan = beginning = self.free - self.collect_oldrefs_to_nursery() + oldobj_count = self.collect_oldrefs_to_nursery() self.collect_roots_in_nursery() scan = self.scan_objects_just_copied_out_of_nursery(scan) # at this point, all static and old objects have got their @@ -347,10 +346,12 @@ self.update_young_objects_with_id() # mark the nursery as free and fill it with zeroes again llarena.arena_reset(self.nursery, self.nursery_size, 2) - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, - "survived (fraction of the size):", - float(scan - beginning) / self.nursery_size) + self.nursery_free = self.nursery + rlog.debug_log("gc-minor-}", + "| tracked older objects: %(oldobj)d\n" + "`------ survived (fraction of the size): %(survived)f", + oldobj = oldobj_count, + survived = float(scan - beginning) / self.nursery_size) #self.debug_check_consistency() # -- quite expensive else: # no nursery - this occurs after a full collect, triggered either @@ -359,7 +360,7 @@ self.nursery = self.free self.nursery_top = self.nursery + self.nursery_size self.free = self.nursery_top - self.nursery_free = self.nursery + self.nursery_free = self.nursery return self.nursery_free # NB. we can use self.copy() to move objects out of the nursery, @@ -376,8 +377,7 @@ hdr = self.header(obj) hdr.tid |= GCFLAG_NO_YOUNG_PTRS self.trace_and_drag_out_of_nursery(obj) - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, "collect_oldrefs_to_nursery", count) + return count def collect_roots_in_nursery(self): # we don't need to trace prebuilt GcStructs during a minor collect: @@ -588,16 +588,15 @@ pass return -1 -def best_nursery_size_for_L2cache(L2cache, debugprint=False): - if debugprint: - llop.debug_print(lltype.Void, "CCC L2cache =", L2cache) +def best_nursery_size_for_L2cache(L2cache): + rlog.debug_log("gc-init-L2", "\tL2cache = %(L2cache)d", L2cache=L2cache) # Heuristically, the best nursery size to choose is about half # of the L2 cache. XXX benchmark some more. return L2cache // 2 if sys.platform == 'linux2': - def estimate_best_nursery_size(debugprint=False): + def estimate_best_nursery_size(): """Try to estimate the best nursery size at run-time, depending on the machine we are running on. """ @@ -649,7 +648,7 @@ L2cache = number if L2cache < sys.maxint: - return best_nursery_size_for_L2cache(L2cache, debugprint) + return best_nursery_size_for_L2cache(L2cache) else: # Print a warning even in non-debug builds llop.debug_print(lltype.Void, @@ -676,7 +675,7 @@ rffi.INT, sandboxsafe=True) - def estimate_best_nursery_size(debugprint=False): + def estimate_best_nursery_size(): """Try to estimate the best nursery size at run-time, depending on the machine we are running on. """ @@ -704,7 +703,7 @@ finally: lltype.free(l2cache_p, flavor='raw') if L2cache > 0: - return best_nursery_size_for_L2cache(L2cache, debugprint) + return best_nursery_size_for_L2cache(L2cache) else: # Print a warning even in non-debug builds llop.debug_print(lltype.Void, @@ -712,5 +711,5 @@ return -1 else: - def estimate_best_nursery_size(debugprint=False): + def estimate_best_nursery_size(): return -1 # XXX implement me for other platforms From arigo at codespeak.net Tue Oct 27 14:02:46 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Oct 2009 14:02:46 +0100 (CET) Subject: [pypy-svn] r68787 - pypy/branch/logging/pypy/rpython/memory/gc Message-ID: <20091027130246.E52461683D7@codespeak.net> Author: arigo Date: Tue Oct 27 14:02:46 2009 New Revision: 68787 Modified: pypy/branch/logging/pypy/rpython/memory/gc/generation.py Log: Revert a minor change. Modified: pypy/branch/logging/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/logging/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/logging/pypy/rpython/memory/gc/generation.py Tue Oct 27 14:02:46 2009 @@ -346,7 +346,6 @@ self.update_young_objects_with_id() # mark the nursery as free and fill it with zeroes again llarena.arena_reset(self.nursery, self.nursery_size, 2) - self.nursery_free = self.nursery rlog.debug_log("gc-minor-}", "| tracked older objects: %(oldobj)d\n" "`------ survived (fraction of the size): %(survived)f", @@ -360,7 +359,7 @@ self.nursery = self.free self.nursery_top = self.nursery + self.nursery_size self.free = self.nursery_top - self.nursery_free = self.nursery + self.nursery_free = self.nursery return self.nursery_free # NB. we can use self.copy() to move objects out of the nursery, From arigo at codespeak.net Tue Oct 27 14:02:54 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Oct 2009 14:02:54 +0100 (CET) Subject: [pypy-svn] r68788 - pypy/branch/logging/pypy/rpython/memory/gc Message-ID: <20091027130254.6BD691683D9@codespeak.net> Author: arigo Date: Tue Oct 27 14:02:53 2009 New Revision: 68788 Modified: pypy/branch/logging/pypy/rpython/memory/gc/hybrid.py Log: Adapt the hybrid GC too. Modified: pypy/branch/logging/pypy/rpython/memory/gc/hybrid.py ============================================================================== --- pypy/branch/logging/pypy/rpython/memory/gc/hybrid.py (original) +++ pypy/branch/logging/pypy/rpython/memory/gc/hybrid.py Tue Oct 27 14:02:53 2009 @@ -10,6 +10,7 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.debug import ll_assert from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib import rlog from pypy.rpython.lltypesystem import rffi # _______in the semispaces_________ ______external (non-moving)_____ @@ -117,7 +118,7 @@ def setup(self): self.large_objects_collect_trigger = self.param_space_size - if self.config.gcconfig.debugprint: + if self.config.rlog: self._initial_trigger = self.large_objects_collect_trigger self.rawmalloced_objects_to_trace = self.AddressStack() self.count_semispaceonly_collects = 0 @@ -271,11 +272,10 @@ def _check_rawsize_alloced(self, size_estimate, can_collect=True): self.large_objects_collect_trigger -= size_estimate if can_collect and self.large_objects_collect_trigger < 0: - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, "allocated", - self._initial_trigger - - self.large_objects_collect_trigger, - "bytes, triggering full collection") + rlog.debug_log("gc-full-trigger", + "allocated %(bytes)d bytes, triggering full collection", + bytes = self._initial_trigger - + self.large_objects_collect_trigger) self.semispace_collect() def malloc_varsize_marknsweep(self, totalsize, resizable=False): @@ -341,7 +341,7 @@ None) ll_assert(not self.rawmalloced_objects_to_trace.non_empty(), "rawmalloced_objects_to_trace should be empty at start") - if self.config.gcconfig.debugprint: + if self.config.rlog: self._nonmoving_copy_count = 0 self._nonmoving_copy_size = 0 @@ -419,7 +419,7 @@ newaddr = self.allocate_external_object(totalsize_incl_hash) if not newaddr: return llmemory.NULL # can't raise MemoryError during a collect() - if self.config.gcconfig.debugprint: + if self.config.rlog: self._nonmoving_copy_count += 1 self._nonmoving_copy_size += raw_malloc_usage(totalsize) @@ -464,11 +464,12 @@ def finished_full_collect(self): ll_assert(not self.rawmalloced_objects_to_trace.non_empty(), "rawmalloced_objects_to_trace should be empty at end") - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, - "| [hybrid] made nonmoving: ", - self._nonmoving_copy_size, "bytes in", - self._nonmoving_copy_count, "objs") + if self.config.rlog: + rlog.debug_log("gc-hybr-1", + "| made nonmoving: " + "%(nonmoving_size)d bytes in %(nonmoving_objs)d objs", + nonmoving_size = self._nonmoving_copy_size, + nonmoving_objs = self._nonmoving_copy_count) # sweep the nonmarked rawmalloced objects if self.is_collecting_gen3(): self.sweep_rawmalloced_objects(generation=3) @@ -479,7 +480,7 @@ self.large_objects_collect_trigger = self.space_size if self.is_collecting_gen3(): self.count_semispaceonly_collects = 0 - if self.config.gcconfig.debugprint: + if self.config.rlog: self._initial_trigger = self.large_objects_collect_trigger def sweep_rawmalloced_objects(self, generation): @@ -513,17 +514,18 @@ surviving_objects = self.AddressStack() # Help the flow space alive_count = alive_size = dead_count = dead_size = 0 + counting_sizes = rlog.has_log() while objects.non_empty(): obj = objects.pop() tid = self.header(obj).tid if tid & GCFLAG_UNVISITED: - if self.config.gcconfig.debugprint: + if counting_sizes: dead_count+=1 dead_size+=raw_malloc_usage(self.get_size_incl_hash(obj)) addr = obj - self.gcheaderbuilder.size_gc_header llmemory.raw_free(addr) else: - if self.config.gcconfig.debugprint: + if counting_sizes: alive_count+=1 alive_size+=raw_malloc_usage(self.get_size_incl_hash(obj)) if generation == 3: @@ -554,17 +556,16 @@ self.gen3_rawmalloced_objects = surviving_objects elif generation == -2: self.gen2_resizable_objects = surviving_objects - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, - "| [hyb] gen", generation, - "nonmoving now alive: ", - alive_size, "bytes in", - alive_count, "objs") - llop.debug_print(lltype.Void, - "| [hyb] gen", generation, - "nonmoving freed: ", - dead_size, "bytes in", - dead_count, "objs") + rlog.debug_log("gc-hybr-2", + "| gen %(sweep_gen)d nonmoving now alive: " + "%(alive_size)d bytes in %(alive_count)d objs\n" + "| freed: " + "%(dead_size)d bytes in %(dead_count)d objs", + sweep_gen = generation, + alive_size = alive_size, + alive_count = alive_count, + dead_size = dead_size, + dead_count = dead_count) def id(self, ptr): obj = llmemory.cast_ptr_to_adr(ptr) From cfbolz at codespeak.net Tue Oct 27 14:16:35 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 27 Oct 2009 14:16:35 +0100 (CET) Subject: [pypy-svn] r68789 - pypy/branch/shrink-multidict/pypy/doc/config Message-ID: <20091027131635.CE44C1683D7@codespeak.net> Author: cfbolz Date: Tue Oct 27 14:16:35 2009 New Revision: 68789 Removed: pypy/branch/shrink-multidict/pypy/doc/config/objspace.std.withbucketdict.txt Log: this option is gone From arigo at codespeak.net Tue Oct 27 14:20:29 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Oct 2009 14:20:29 +0100 (CET) Subject: [pypy-svn] r68790 - pypy/branch/logging/pypy/rpython/memory/gc Message-ID: <20091027132029.49FA01683D7@codespeak.net> Author: arigo Date: Tue Oct 27 14:20:28 2009 New Revision: 68790 Modified: pypy/branch/logging/pypy/rpython/memory/gc/hybrid.py Log: Translation fix. Modified: pypy/branch/logging/pypy/rpython/memory/gc/hybrid.py ============================================================================== --- pypy/branch/logging/pypy/rpython/memory/gc/hybrid.py (original) +++ pypy/branch/logging/pypy/rpython/memory/gc/hybrid.py Tue Oct 27 14:20:28 2009 @@ -118,8 +118,7 @@ def setup(self): self.large_objects_collect_trigger = self.param_space_size - if self.config.rlog: - self._initial_trigger = self.large_objects_collect_trigger + self._initial_trigger = self.large_objects_collect_trigger self.rawmalloced_objects_to_trace = self.AddressStack() self.count_semispaceonly_collects = 0 @@ -480,8 +479,7 @@ self.large_objects_collect_trigger = self.space_size if self.is_collecting_gen3(): self.count_semispaceonly_collects = 0 - if self.config.rlog: - self._initial_trigger = self.large_objects_collect_trigger + self._initial_trigger = self.large_objects_collect_trigger def sweep_rawmalloced_objects(self, generation): # free all the rawmalloced objects of the specified generation From arigo at codespeak.net Tue Oct 27 14:20:54 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Oct 2009 14:20:54 +0100 (CET) Subject: [pypy-svn] r68791 - pypy/branch/logging/pypy/rpython/memory/gc Message-ID: <20091027132054.2DD291683D7@codespeak.net> Author: arigo Date: Tue Oct 27 14:20:53 2009 New Revision: 68791 Modified: pypy/branch/logging/pypy/rpython/memory/gc/marksweep.py Log: Adapt the mark'n'sweep GC. Modified: pypy/branch/logging/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/branch/logging/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/branch/logging/pypy/rpython/memory/gc/marksweep.py Tue Oct 27 14:20:53 2009 @@ -8,6 +8,7 @@ from pypy.rlib.objectmodel import free_non_gc_object from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib import rlog from pypy.rpython.memory.gc.base import GCBase @@ -241,9 +242,7 @@ # 3. walk the list of objects-with-del and for the ones not marked: # call __del__, move the object to the list of object-without-del import time - from pypy.rpython.lltypesystem.lloperation import llop - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, 'collecting...') + rlog.debug_log("gc-{", "collecting...") start_time = time.time() self.collect_in_progress = True size_gc_header = self.gcheaderbuilder.size_gc_header @@ -406,31 +405,22 @@ 256 * 1024 * 1024) self.total_collection_time += collect_time self.prev_collect_end_time = end_time - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, - " malloced since previous collection:", - old_malloced, "bytes") - llop.debug_print(lltype.Void, - " heap usage at start of collection: ", - self.heap_usage + old_malloced, "bytes") - llop.debug_print(lltype.Void, - " freed: ", - freed_size, "bytes") - llop.debug_print(lltype.Void, - " new heap usage: ", - curr_heap_size, "bytes") - llop.debug_print(lltype.Void, - " total time spent collecting: ", - self.total_collection_time, "seconds") - llop.debug_print(lltype.Void, - " collecting time: ", - collect_time) - llop.debug_print(lltype.Void, - " computing time: ", - collect_time) - llop.debug_print(lltype.Void, - " new threshold: ", - self.bytes_malloced_threshold) + rlog.debug_log("gc-}", + " malloced since previous collect.: %(old_malloced)d bytes\n" + " heap usage at start of collection: %(old_heap)d bytes\n" + " freed: %(freed_size)d bytes\n" + " new heap usage: %(new_heap)d bytes\n" + " total time spent collecting: %(total_col_time)f seconds\n" + " collecting time: %(collect_time)f\n" + " new threshold: %(threshold)d\n", + old_malloced = old_malloced, + old_heap = self.heap_usage + old_malloced, + freed_size = freed_size, + new_heap = curr_heap_size, + total_col_time = self.total_collection_time, + collect_time = collect_time, + threshold = self.bytes_malloced_threshold) + ## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, ## size_gc_header) assert self.heap_usage + old_malloced == curr_heap_size + freed_size From cfbolz at codespeak.net Tue Oct 27 14:27:30 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 27 Oct 2009 14:27:30 +0100 (CET) Subject: [pypy-svn] r68792 - pypy/branch/shrink-multidict/pypy/doc/config Message-ID: <20091027132730.DA8061683D9@codespeak.net> Author: cfbolz Date: Tue Oct 27 14:27:30 2009 New Revision: 68792 Added: pypy/branch/shrink-multidict/pypy/doc/config/objspace.std.withinlineddict.txt (contents, props changed) Log: document the new option Added: pypy/branch/shrink-multidict/pypy/doc/config/objspace.std.withinlineddict.txt ============================================================================== --- (empty file) +++ pypy/branch/shrink-multidict/pypy/doc/config/objspace.std.withinlineddict.txt Tue Oct 27 14:27:30 2009 @@ -0,0 +1,2 @@ +Make instances smaller by creating the __dict__ only when somebody actually +accesses it. Also makes attribute accesses a tiny bit faster. From arigo at codespeak.net Tue Oct 27 16:04:29 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Oct 2009 16:04:29 +0100 (CET) Subject: [pypy-svn] r68794 - pypy/branch/logging/pypy/rpython/memory/test Message-ID: <20091027150429.B19BF1683D6@codespeak.net> Author: arigo Date: Tue Oct 27 16:04:28 2009 New Revision: 68794 Modified: pypy/branch/logging/pypy/rpython/memory/test/test_transformed_gc.py Log: Fix test. Modified: pypy/branch/logging/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/logging/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/logging/pypy/rpython/memory/test/test_transformed_gc.py Tue Oct 27 16:04:28 2009 @@ -23,7 +23,7 @@ t = TranslationContext() # XXX XXX XXX mess t.config.translation.gc = gcname - t.config.translation.gcconfig.removetypeptr = True + t.config.translation.gcremovetypeptr = True if stacklessgc: t.config.translation.gcrootfinder = "stackless" t.config.set(**extraconfigopts) From arigo at codespeak.net Tue Oct 27 16:16:38 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Oct 2009 16:16:38 +0100 (CET) Subject: [pypy-svn] r68795 - in pypy/branch/logging/pypy: config doc/config translator/c/test Message-ID: <20091027151638.61C7B1683D6@codespeak.net> Author: arigo Date: Tue Oct 27 16:16:37 2009 New Revision: 68795 Added: pypy/branch/logging/pypy/doc/config/translation.gcremovetypeptr.txt - copied unchanged from r68778, pypy/branch/logging/pypy/doc/config/translation.gcconfig.removetypeptr.txt Removed: pypy/branch/logging/pypy/doc/config/translation.gcconfig.debugprint.txt pypy/branch/logging/pypy/doc/config/translation.gcconfig.removetypeptr.txt pypy/branch/logging/pypy/doc/config/translation.gcconfig.txt Modified: pypy/branch/logging/pypy/config/translationoption.py pypy/branch/logging/pypy/translator/c/test/test_newgc.py Log: More small fixes. Modified: pypy/branch/logging/pypy/config/translationoption.py ============================================================================== --- pypy/branch/logging/pypy/config/translationoption.py (original) +++ pypy/branch/logging/pypy/config/translationoption.py Tue Oct 27 16:16:37 2009 @@ -99,7 +99,7 @@ BoolOption("jit", "generate a JIT", default=False, requires=[("translation.thread", False), - ("translation.gcconfig.removetypeptr", False)], + ("translation.gcremovetypeptr", False)], suggests=[("translation.gc", "hybrid"), # or "boehm" ("translation.gcrootfinder", "asmgcc"), ("translation.list_comprehension_operations", True)]), @@ -364,7 +364,7 @@ elif word == 'jit': config.translation.suggest(jit=True) elif word == 'removetypeptr': - config.translation.gcconfig.suggest(removetypeptr=True) + config.translation.suggest(gcremovetypeptr=True) else: raise ValueError(word) Modified: pypy/branch/logging/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/logging/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/logging/pypy/translator/c/test/test_newgc.py Tue Oct 27 16:16:37 2009 @@ -39,8 +39,8 @@ t = Translation(main, standalone=True, gc=cls.gcpolicy, policy=annpolicy.StrictAnnotatorPolicy(), taggedpointers=cls.taggedpointers, - removetypeptr=cls.removetypeptr, - debugprint=True) + gcremovetypeptr=cls.removetypeptr, + rlog=True) # to see, run the tests with PYPYLOG=log t.disable(['backendopt']) t.set_backend_extra_options(c_debug_defines=True) t.rtype() From arigo at codespeak.net Tue Oct 27 18:19:53 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Oct 2009 18:19:53 +0100 (CET) Subject: [pypy-svn] r68800 - pypy/extradoc/sprintinfo/ddorf2009 Message-ID: <20091027171953.EC43A1683D6@codespeak.net> Author: arigo Date: Tue Oct 27 18:19:53 2009 New Revision: 68800 Modified: pypy/extradoc/sprintinfo/ddorf2009/people.txt Log: My dates, and the accomodation I share with Antonio. Modified: pypy/extradoc/sprintinfo/ddorf2009/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/ddorf2009/people.txt (original) +++ pypy/extradoc/sprintinfo/ddorf2009/people.txt Tue Oct 27 18:19:53 2009 @@ -9,7 +9,8 @@ Name Arrive/Depart Accomodation ==================== ============== ===================== Carl Friedrich Bolz always there private -Antonio Cuni 6/11 - 13/11 ??? +Antonio Cuni 6/11 - 13/11 Hotel Haus Hillesheim +Armin Rigo 6/11 - 13/11 Hotel Haus Hillesheim ==================== ============== ===================== ==================== ============== ===================== From arigo at codespeak.net Tue Oct 27 18:40:53 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 27 Oct 2009 18:40:53 +0100 (CET) Subject: [pypy-svn] r68801 - in pypy/trunk/pypy: config jit/backend jit/backend/x86 Message-ID: <20091027174053.B5D791683D6@codespeak.net> Author: arigo Date: Tue Oct 27 18:40:53 2009 New Revision: 68801 Added: pypy/trunk/pypy/jit/backend/x86/detect_sse2.py Modified: pypy/trunk/pypy/config/translationoption.py pypy/trunk/pypy/jit/backend/detect_cpu.py pypy/trunk/pypy/jit/backend/x86/runner.py Log: Detect if the CPU is old enough to not support SSE2, and just disable float support in that case. Modified: pypy/trunk/pypy/config/translationoption.py ============================================================================== --- pypy/trunk/pypy/config/translationoption.py (original) +++ pypy/trunk/pypy/config/translationoption.py Tue Oct 27 18:40:53 2009 @@ -102,7 +102,7 @@ ("translation.gcrootfinder", "asmgcc"), ("translation.list_comprehension_operations", True)]), ChoiceOption("jit_backend", "choose the backend for the JIT", - ["auto", "x86", "llvm"], + ["auto", "x86", "x86-without-sse2", "llvm"], default="auto", cmdline="--jit-backend"), ChoiceOption("jit_debug", "the amount of debugging dumps made by the JIT", ["off", "profile", "steps", "detailed"], Modified: pypy/trunk/pypy/jit/backend/detect_cpu.py ============================================================================== --- pypy/trunk/pypy/jit/backend/detect_cpu.py (original) +++ pypy/trunk/pypy/jit/backend/detect_cpu.py Tue Oct 27 18:40:53 2009 @@ -1,13 +1,14 @@ """ Processor auto-detection """ +import autopath import sys, os class ProcessorAutodetectError(Exception): pass -def autodetect(): +def autodetect_main_model(): mach = None try: import platform @@ -40,11 +41,21 @@ except KeyError: raise ProcessorAutodetectError, "unsupported processor '%s'" % mach +def autodetect(): + model = autodetect_main_model() + if model in ('i386', 'x86'): + from pypy.jit.backend.x86.detect_sse2 import detect_sse2 + if not detect_sse2(): + model = 'x86-without-sse2' + return model + def getcpuclass(backend_name="auto"): if backend_name == "auto": backend_name = autodetect() if backend_name in ('i386', 'x86'): from pypy.jit.backend.x86.runner import CPU + elif backend_name == 'x86-without-sse2': + from pypy.jit.backend.x86.runner import CPU386_NO_SSE2 as CPU elif backend_name == 'cli': from pypy.jit.backend.cli.runner import CliCPU as CPU elif backend_name == 'llvm': Added: pypy/trunk/pypy/jit/backend/x86/detect_sse2.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/jit/backend/x86/detect_sse2.py Tue Oct 27 18:40:53 2009 @@ -0,0 +1,27 @@ +import autopath +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rlib.rmmap import alloc, free + + +def detect_sse2(): + data = alloc(4096) + pos = 0 + for c in ("\xB8\x01\x00\x00\x00" # MOV EAX, 1 + "\x53" # PUSH EBX + "\x0F\xA2" # CPUID + "\x5B" # POP EBX + "\x92" # XCHG EAX, EDX + "\xC3"): # RET + data[pos] = c + pos += 1 + fnptr = rffi.cast(lltype.Ptr(lltype.FuncType([], lltype.Signed)), data) + code = fnptr() + free(data, 4096) + return bool(code & (1<<25)) and bool(code & (1<<26)) + + +if __name__ == '__main__': + if detect_sse2(): + print 'Processor supports sse2.' + else: + print 'Missing processor support for sse2.' Modified: pypy/trunk/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/runner.py Tue Oct 27 18:40:53 2009 @@ -88,6 +88,10 @@ return CPU386.cast_adr_to_int(adr) +class CPU386_NO_SSE2(CPU386): + supports_floats = False + + CPU = CPU386 import pypy.jit.metainterp.executor From fijal at codespeak.net Wed Oct 28 09:20:23 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 28 Oct 2009 09:20:23 +0100 (CET) Subject: [pypy-svn] r68802 - pypy/branch/gc-dump-heap/pypy/doc/jit Message-ID: <20091028082023.60D4E1683DE@codespeak.net> Author: fijal Date: Wed Oct 28 09:20:21 2009 New Revision: 68802 Modified: pypy/branch/gc-dump-heap/pypy/doc/jit/pyjitpl5.txt Log: Leave a couple of XXXs. optimization phase is not complete Modified: pypy/branch/gc-dump-heap/pypy/doc/jit/pyjitpl5.txt ============================================================================== --- pypy/branch/gc-dump-heap/pypy/doc/jit/pyjitpl5.txt (original) +++ pypy/branch/gc-dump-heap/pypy/doc/jit/pyjitpl5.txt Wed Oct 28 09:20:21 2009 @@ -32,6 +32,7 @@ longer suitable. can_enter_jit goes at the end of a application level loop. In the Python interpreter, this is the JUMP_ABSOLUTE bytecode. The Python interpreter defines its hints in pypy/module/pypyjit/interp_jit.py. +XXX by overloading default frame behavior from pyframe The interpreter wishing to use the PyPy's JIT must define a list of *green* variables and a list of *red* variables. The *green* variables are loop @@ -98,6 +99,8 @@ contain values that may change during the running of a loop. There are three kinds of normal boxes: BoxInt, BoxPtr, and BoxFloat, and four kinds of constant boxes: ConstInt, ConstPtr, ConstFloat, and ConstAddr. +XXX ConstAddr is only a hack for translation not to translate pointers, +XXX nothing more The meta-interpreter starts interpreting the JIT bytecode. Each operation is executed and then recorded in a list of operations, called the trace. From arigo at codespeak.net Wed Oct 28 09:31:02 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 09:31:02 +0100 (CET) Subject: [pypy-svn] r68803 - in pypy/branch/logging/pypy/jit/metainterp: . test Message-ID: <20091028083102.D9B921683DE@codespeak.net> Author: arigo Date: Wed Oct 28 09:31:02 2009 New Revision: 68803 Modified: pypy/branch/logging/pypy/jit/metainterp/logger.py pypy/branch/logging/pypy/jit/metainterp/test/oparser.py pypy/branch/logging/pypy/jit/metainterp/test/test_oparser.py Log: Rename ConstClass into ConstAddr. It can be much more than a class -- e.g. it can be a function pointer. Modified: pypy/branch/logging/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/branch/logging/pypy/jit/metainterp/logger.py (original) +++ pypy/branch/logging/pypy/jit/metainterp/logger.py Wed Oct 28 09:31:02 2009 @@ -66,7 +66,7 @@ elif isinstance(arg, BoxFloat): return 'f' + str(mv) elif isinstance(arg, self.ts.ConstAddr): - return 'ConstClass(cls' + str(mv) + ')' + return 'ConstAddr(adr' + str(mv) + ')' else: return '?' Modified: pypy/branch/logging/pypy/jit/metainterp/test/oparser.py ============================================================================== --- pypy/branch/logging/pypy/jit/metainterp/test/oparser.py (original) +++ pypy/branch/logging/pypy/jit/metainterp/test/oparser.py Wed Oct 28 09:31:02 2009 @@ -147,8 +147,8 @@ info = arg.strip("'\"") return ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, llstr(info))) - if arg.startswith('ConstClass('): - name = arg[len('ConstClass('):-1] + if arg.startswith('ConstAddr('): + name = arg[len('ConstAddr('):-1] return self.get_const(name, 'class') elif arg == 'None': return None @@ -160,6 +160,9 @@ elif arg.startswith('ConstPtr('): name = arg[len('ConstPtr('):-1] return self.get_const(name, 'ptr') + elif arg.startswith('ConstClass('): # compatibility + name = arg[len('ConstClass('):-1] + return self.get_const(name, 'class') return self.vars[arg] def parse_op(self, line): Modified: pypy/branch/logging/pypy/jit/metainterp/test/test_oparser.py ============================================================================== --- pypy/branch/logging/pypy/jit/metainterp/test/test_oparser.py (original) +++ pypy/branch/logging/pypy/jit/metainterp/test/test_oparser.py Wed Oct 28 09:31:02 2009 @@ -165,10 +165,10 @@ # Loop0 (loop), 12 ops [i0, i1] debug_merge_point('(no jitdriver.get_printable_location!)') -i3 = call(ConstClass(cls2), i0, descr=) +i3 = call(ConstAddr(adr2), i0, descr=) guard_no_exception(, descr=) [i0, i1, i3] i5 = int_add(i1, 2) -i7 = call(ConstClass(cls6), i0, descr=) +i7 = call(ConstAddr(adr6), i0, descr=) p9 = guard_exception(4, descr=) [i5, i0, i7] i11 = int_sub(i5, 1) i12 = int_sub(i0, 1) @@ -179,10 +179,10 @@ # Loop1 (entry bridge), 12 ops [i0, i1] debug_merge_point('(no jitdriver.get_printable_location!)') -i3 = call(ConstClass(cls2), i0, descr=) +i3 = call(ConstAddr(adr2), i0, descr=) p5 = guard_exception(4, descr=) [i0, i1, i3] i7 = int_add(i1, 1) -i9 = call(ConstClass(cls8), i0, descr=) +i9 = call(ConstAddr(adr8), i0, descr=) p11 = guard_exception(4, descr=) [i7, i0, i9] i12 = int_sub(i7, 1) i13 = int_sub(i0, 1) @@ -194,7 +194,7 @@ [i0, i1, i2] p4 = guard_exception(4, descr=) [i0, i1, i2] i6 = int_add(i1, 1) -i8 = call(ConstClass(cls7), i0, descr=) +i8 = call(ConstAddr(adr7), i0, descr=) p10 = guard_exception(4, descr=) [i6, i0, i8] i11 = int_sub(i6, 1) i12 = int_sub(i0, 1) From arigo at codespeak.net Wed Oct 28 09:41:27 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 09:41:27 +0100 (CET) Subject: [pypy-svn] r68804 - pypy/trunk/pypy/doc Message-ID: <20091028084127.A0FE11683DE@codespeak.net> Author: arigo Date: Wed Oct 28 09:41:26 2009 New Revision: 68804 Modified: pypy/trunk/pypy/doc/translation.txt Log: Mostly kill the section talking about heap2stack. Modified: pypy/trunk/pypy/doc/translation.txt ============================================================================== --- pypy/trunk/pypy/doc/translation.txt (original) +++ pypy/trunk/pypy/doc/translation.txt Wed Oct 28 09:41:26 2009 @@ -548,42 +548,8 @@ Another technique to reduce the memory allocation penalty is to use stack allocation for objects that can be proved not to life longer than the stack -frame they have been allocated in. If this is the case it is possible to -allocate the object on the stack. This makes allocation faster, since stack -allocation is just the increase of a pointer, and makes deallocation basically -free since deallocation happens automatically when the function returns. -Therefore we wrote an analysis, which analyses which malloc positions lead to -mallocs which "escape" the current function, e.g. have references to them -stored into a place where they can be accessed by something outside of the -stack of frames starting with the frame where the malloc occured. - -For this we choose a naive, pessimistic approach. The analysis -assumes that an object escapes if one of the following situation occurs: - - * the object is returned - - * the object is raised as an exception - - * the object is stored into a field of any another object - -The algorithm uses abstract interpretation together with a fix point search to -find a solution. - -After using the escape analysis to find malloc sites that don't escape, we -replace the mallocs by stack allocations. This cannot be done in all cases, -namely if the allocated object is variable-sized or if the allocation occurs in -a loop. Both cases should be avoided because they make stack overflows more -likely. Also objects that have a finalizer cannot be allocated on the stack, -since the finalizer might resurrect the object. - -The resulting performance improvements by this optimization were not as big -we hoped. We think that this is due to the fact that the Boehm garbage -collector becomes slower when the stack is bigger, thus compensating -any speed improvement achieved by having faster allocation. We did not -implement stack allocation with any of the other GCs that PyPy can -use. - -Enable this optimization with :config:`translation.backendopt.heap2stack`. +frame they have been allocated in. This proved not to really gain us any +speed, so over time it was removed again. The Stackless Transform From arigo at codespeak.net Wed Oct 28 10:33:14 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 10:33:14 +0100 (CET) Subject: [pypy-svn] r68805 - in pypy/branch/logging/pypy/rlib: . test Message-ID: <20091028093314.29DDC1683DB@codespeak.net> Author: arigo Date: Wed Oct 28 10:33:13 2009 New Revision: 68805 Modified: pypy/branch/logging/pypy/rlib/rlog.py pypy/branch/logging/pypy/rlib/rlog_parsing.py pypy/branch/logging/pypy/rlib/test/test_rlog.py Log: Add (big) docstrings on the two public functions of pypy.rlib.rlog. Small tweaks. Modified: pypy/branch/logging/pypy/rlib/rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog.py (original) +++ pypy/branch/logging/pypy/rlib/rlog.py Wed Oct 28 10:33:13 2009 @@ -12,9 +12,39 @@ def has_log(): + """Check if logging is enabled. If translated with --no-rjit, this + returns the constant False. Otherwise, it returns True or False + depending on the presence of the PYPYLOG env var. (Note that the + debug_log() function also checks has_log() by itself, so it's only + useful to explicitly call has_log() to protect large pieces of code.) + """ return True def debug_log(_category, _message, **_kwds): + """Logging main function. If translated with --no-rjit, all calls to + this function are ignored. Otherwise, if the PYPYLOG env var is set, + it records an entry in the log. Arguments: + + * category: a short string, more or less of the form + 'mainarea-subarea-exactlocation'. To make log parsing easier, + whenever multiple log entries belong together, start with + an entry 'mainarea-subarea-{' and end with 'mainarea-subarea-}'. + Note that the category should be unique (use different + 'exactlocation' if needed). + + * message: freeform string displayed to the user, with embedded + %-formatting commands like '%(foo)d' and '%(bar)s'. The message + must be a constant string; it is only recorded once per log file. + + * keywords arguments: values to record in the entry, like foo=5 + and bar='hello'. + + Supported argument types: + 'd': signed integer + 'f': float + 's': printable string or none + 'r': random binary string or none + """ getattr(_log, _category)(_message % _kwds) # ____________________________________________________________ @@ -89,6 +119,7 @@ 'd': annmodel.SomeInteger(), 'f': annmodel.SomeFloat(), 's': annmodel.SomeString(can_be_None=True), + 'r': annmodel.SomeString(can_be_None=True), } annhelper = hop.rtyper.getannmixlevel() args_s = [ann[t] for t in cat.types] @@ -101,7 +132,7 @@ v = hop.inputarg(lltype.Signed, arg=arg) elif typechar == 'f': v = hop.inputarg(lltype.Float, arg=arg) - elif typechar == 's': + elif typechar == 's' or typechar == 'r': v = hop.inputarg(hop.rtyper.type_system.rstr.string_repr, arg=arg) else: @@ -126,7 +157,7 @@ import re -r_entry = re.compile(r"%\((\w+)\)([sdf])") +r_entry = re.compile(r"%\((\w+)\)([srdf])") SIZEOF_FLOAT = struct.calcsize("f") @@ -236,5 +267,7 @@ s = '(null)' self.write_str(s) + add_subentry_r = add_subentry_s + def add_subentry_f(self, float): self.write_float(float) Modified: pypy/branch/logging/pypy/rlib/rlog_parsing.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog_parsing.py (original) +++ pypy/branch/logging/pypy/rlib/rlog_parsing.py Wed Oct 28 10:33:13 2009 @@ -23,9 +23,11 @@ nextc = self.f.read(1) if not nextc: raise EOFError + lastbyte = ord(nextc) + if lastbyte < 0x80: + return lastbyte shift = 0 result = 0 - lastbyte = ord(nextc) while lastbyte & 0x80: result |= ((lastbyte & 0x7F) << shift) shift += 7 @@ -47,6 +49,7 @@ readers = { 'd': self.read_int, 's': self.read_str, + 'r': self.read_str, 'f': self.read_float, } curtime = 0.0 Modified: pypy/branch/logging/pypy/rlib/test/test_rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/test/test_rlog.py (original) +++ pypy/branch/logging/pypy/rlib/test/test_rlog.py Wed Oct 28 10:33:13 2009 @@ -215,6 +215,7 @@ rlog.debug_log("Aa", "hello %(foo)d %(bar)f", foo=x, bar=-7.3) rlog.debug_log("Aa", "hello %(foo)d %(bar)f", foo=x+1, bar=x+0.5) rlog.debug_log("Ab", "<<%(baz)s>>", baz="hi there") + rlog.debug_log("Ac", "[%(foo)r]", foo="\x00") assert rlog.has_log() def setup_method(self, _): @@ -232,14 +233,16 @@ def check_result(self): entries = list(rlog_parsing.parse_log(self.pypylog)) - assert len(entries) == 3 + assert len(entries) == 4 # assert isinstance(entries[0][0], float) assert isinstance(entries[1][0], float) assert isinstance(entries[2][0], float) + assert isinstance(entries[3][0], float) # Aa = entries[0][1] Ab = entries[2][1] + Ac = entries[3][1] assert entries[1][1] is Aa assert Aa.category == 'Aa' assert Aa.message == 'hello %(foo)d %(bar)f' @@ -247,10 +250,14 @@ assert Ab.category == 'Ab' assert Ab.message == '<<%(baz)s>>' assert Ab.entries == [('baz', 's')] + assert Ac.category == 'Ac' + assert Ac.message == '[%(foo)r]' + assert Ac.entries == [('foo', 'r')] # assert entries[0][2] == [132, roughly(-7.3)] assert entries[1][2] == [133, 132.5] assert entries[2][2] == ['hi there'] + assert entries[3][2] == ['\x00'] def test_interpret_f(self): interpret(self.f.im_func, [132], malloc_check=False) From arigo at codespeak.net Wed Oct 28 11:25:05 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 11:25:05 +0100 (CET) Subject: [pypy-svn] r68807 - in pypy/branch/logging/pypy/jit/metainterp: . test Message-ID: <20091028102505.094311683D3@codespeak.net> Author: arigo Date: Wed Oct 28 11:25:05 2009 New Revision: 68807 Modified: pypy/branch/logging/pypy/jit/metainterp/logger.py pypy/branch/logging/pypy/jit/metainterp/optimizeopt.py pypy/branch/logging/pypy/jit/metainterp/pyjitpl.py pypy/branch/logging/pypy/jit/metainterp/resume.py pypy/branch/logging/pypy/jit/metainterp/simple_optimize.py pypy/branch/logging/pypy/jit/metainterp/test/test_basic.py pypy/branch/logging/pypy/jit/metainterp/test/test_logger.py pypy/branch/logging/pypy/jit/metainterp/test/test_loop.py pypy/branch/logging/pypy/jit/metainterp/test/test_optimizeopt.py Log: Use rlog instead of PYPYJITLOG and PYPYJITRESUMELOG. In-progress. Modified: pypy/branch/logging/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/branch/logging/pypy/jit/metainterp/logger.py (original) +++ pypy/branch/logging/pypy/jit/metainterp/logger.py Wed Oct 28 11:25:05 2009 @@ -1,48 +1,37 @@ import os -from pypy.rlib.objectmodel import compute_unique_id +from pypy.rlib import rlog from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.history import Const, ConstInt, Box, \ BoxInt, ConstAddr, ConstFloat, BoxFloat, AbstractFailDescr -from pypy.rlib.streamio import open_file_as_stream class Logger(object): def __init__(self, ts, guard_number=False): - self.log_stream = None self.ts = ts self.guard_number=guard_number - def create_log(self, extension='.ops'): - if self.log_stream is not None: - return self.log_stream - s = os.environ.get('PYPYJITLOG') - if not s: - return None - s += extension - try: - self.log_stream = open_file_as_stream(s, 'w') - except OSError: - os.write(2, "could not create log file\n") - return None - return self.log_stream - - def log_loop(self, inputargs, operations, number=0, type=None): - if self.log_stream is None: + def log_loop(self, inputargs, operations, number=-1, type="unoptimized"): + if not rlog.has_log(): return - if type is not None: - self.log_stream.write("# Loop%d (%s), %d ops\n" % (number, - type, - len(operations))) + rlog.debug_log("jit-log-loop-{", + "# Loop%(number)d (%(type)s), %(length)d ops", + number = number, + type = type, + length = len(operations)) self._log_operations(inputargs, operations, {}) + rlog.debug_log("jit-log-loop-}", + "# End") def log_bridge(self, inputargs, operations, number=-1): - if self.log_stream is None: + if not rlog.has_log(): return - if number != -1: - self.log_stream.write("# bridge out of Guard%d, %d ops\n" % (number, - len(operations))) + rlog.debug_log("jit-log-bridge-{", + "# Bridge out of Guard%(guard)d, %(length)d ops", + guard = number, + length = len(operations)) self._log_operations(inputargs, operations, {}) - + rlog.debug_log("jit-log-bridge-}", + "# End") def repr_of_descr(self, descr): return descr.repr_of_descr() @@ -73,12 +62,16 @@ def _log_operations(self, inputargs, operations, memo): if inputargs is not None: args = ", ".join([self.repr_of_arg(memo, arg) for arg in inputargs]) - self.log_stream.write('[' + args + ']\n') + rlog.debug_log("jit-log-head", + "[%(inputargs)s]", + inputargs = args) for i in range(len(operations)): op = operations[i] if op.opnum == rop.DEBUG_MERGE_POINT: loc = op.args[0]._get_str() - self.log_stream.write("debug_merge_point('%s')\n" % (loc,)) + rlog.debug_log("jit-log-mgpt", + "debug_merge_point(%(loc)r)", + loc = loc) continue args = ", ".join([self.repr_of_arg(memo, arg) for arg in op.args]) if op.result is not None: @@ -99,6 +92,9 @@ for arg in op.fail_args]) + ']' else: fail_args = '' - self.log_stream.write(res + op.getopname() + - '(' + args + ')' + fail_args + '\n') - self.log_stream.flush() + rlog.debug_log("jit-log-insn", + "%(res)s%(opname)s(%(args)s)%(fail_args)s", + res = res, + opname = op.getopname(), + args = args, + fail_args = fail_args) Modified: pypy/branch/logging/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/logging/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/logging/pypy/jit/metainterp/optimizeopt.py Wed Oct 28 11:25:05 2009 @@ -518,8 +518,7 @@ def store_final_boxes_in_guard(self, op): descr = op.descr assert isinstance(descr, compile.ResumeGuardDescr) - modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo, - self.metainterp_sd.globaldata.storedebug) + modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo) newboxes = modifier.finish(self.values) if len(newboxes) > self.metainterp_sd.options.failargs_limit: raise compile.GiveUp Modified: pypy/branch/logging/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/logging/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/logging/pypy/jit/metainterp/pyjitpl.py Wed Oct 28 11:25:05 2009 @@ -1027,9 +1027,6 @@ self.profiler.start() self.profiler.initialized = True self.globaldata.initialized = True - self.logger_noopt.create_log('.noopt') - self.logger_ops.create_log('.ops') - self.globaldata.storedebug = os.environ.get('PYPYJITRESUMELOG') def _setup_class_sizes(self): class_sizes = {} Modified: pypy/branch/logging/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/logging/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/logging/pypy/jit/metainterp/resume.py Wed Oct 28 11:25:05 2009 @@ -3,6 +3,7 @@ from pypy.jit.metainterp.resoperation import rop from pypy.rpython.lltypesystem import rffi from pypy.rlib import rarithmetic +from pypy.rlib import rlog from pypy.rlib.objectmodel import we_are_translated # Logic to encode the chain of frames and the state of the boxes at a @@ -179,10 +180,9 @@ class ResumeDataVirtualAdder(object): - def __init__(self, storage, memo, debug_storage=None): + def __init__(self, storage, memo): self.storage = storage self.memo = memo - self.debug_storage = debug_storage #self.virtuals = [] #self.vfieldboxes = [] @@ -258,8 +258,8 @@ self._number_virtuals(liveboxes) storage.rd_consts = self.memo.consts - if self.debug_storage: - dump_storage(self.debug_storage, storage, liveboxes) + if rlog.has_log(): + dump_storage(storage, liveboxes) return liveboxes[:] def _number_virtuals(self, liveboxes): @@ -425,38 +425,48 @@ # ____________________________________________________________ -def dump_storage(logname, storage, liveboxes): +def dump_storage(storage, liveboxes): "For profiling only." - import os from pypy.rlib import objectmodel - assert logname is not None # annotator hack - fd = os.open(logname, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0666) - os.write(fd, 'Log(%d, [\n' % objectmodel.compute_unique_id(storage)) - frameinfo = storage.rd_frame_info_list - while True: - os.write(fd, '\t("%s", %d, %d) at %xd,\n' % ( - frameinfo.jitcode, frameinfo.pc, frameinfo.exception_target, - objectmodel.compute_unique_id(frameinfo))) - frameinfo = frameinfo.prev - if frameinfo is None: - break - os.write(fd, '\t],\n\t[\n') + rlog.debug_log("jit-resume-{", + "Storage entry %(id)d", + id = objectmodel.compute_unique_id(storage)) + try: + frameinfo = storage.rd_frame_info_list + except AttributeError: + pass # for tests + else: + while True: + rlog.debug_log("jit-resume-frameinfo", + "(%(jitcode)r, %(pc)d, %(exception_target)d) at %(id)d", + jitcode = frameinfo.jitcode, + pc = frameinfo.pc, + exception_target = frameinfo.exception_target, + id = objectmodel.compute_unique_id(frameinfo)) + frameinfo = frameinfo.prev + if frameinfo is None: + break numb = storage.rd_numb while True: - os.write(fd, '\t\t%s at %xd,\n' % ([untag(i) for i in numb.nums], - objectmodel.compute_unique_id(numb))) + rlog.debug_log("jit-resume-numb", + "%(nums)s at %(id)d", + nums = str([untag(i) for i in numb.nums]), + id = objectmodel.compute_unique_id(numb)) numb = numb.prev if numb is None: break - os.write(fd, '\t], [\n') for const in storage.rd_consts: - os.write(fd, '\t"%s",\n' % (const.repr_rpython(),)) - os.write(fd, '\t], [\n') + rlog.debug_log("jit-resume-const", + "%(const)s", + const = const.repr_rpython()) for box in liveboxes: - os.write(fd, '\t"%s",\n' % (box.repr_rpython(),)) - os.write(fd, '\t], [\n') + rlog.debug_log("jit-resume-box", + "%(box)s", + box = box.repr_rpython()) if storage.rd_virtuals is not None: for virtual in storage.rd_virtuals: - os.write(fd, '\t%s,\n' % (virtual.repr_rpython(),)) - os.write(fd, '\t])\n') - os.close(fd) + rlog.debug_log("jit-resume-virtual", + "%(virtual)s", + virtual = virtual.repr_rpython()) + rlog.debug_log("jit-resume-}", + "End") Modified: pypy/branch/logging/pypy/jit/metainterp/simple_optimize.py ============================================================================== --- pypy/branch/logging/pypy/jit/metainterp/simple_optimize.py (original) +++ pypy/branch/logging/pypy/jit/metainterp/simple_optimize.py Wed Oct 28 11:25:05 2009 @@ -22,8 +22,7 @@ if op.is_guard(): descr = op.descr assert isinstance(descr, compile.ResumeGuardDescr) - modifier = resume.ResumeDataVirtualAdder(descr, memo, - metainterp_sd.globaldata.storedebug) + modifier = resume.ResumeDataVirtualAdder(descr, memo) newboxes = modifier.finish(EMPTY_VALUES) descr.store_final_boxes(op, newboxes) newoperations.append(op) Modified: pypy/branch/logging/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/logging/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/logging/pypy/jit/metainterp/test/test_basic.py Wed Oct 28 11:25:05 2009 @@ -4,7 +4,7 @@ from pypy.rlib.jit import OPTIMIZER_FULL, OPTIMIZER_SIMPLE from pypy.jit.metainterp.warmspot import ll_meta_interp, get_stats from pypy.jit.backend.llgraph import runner -from pypy.jit.metainterp import support, codewriter, pyjitpl, history +from pypy.jit.metainterp import support, codewriter, pyjitpl, history, resume from pypy.jit.metainterp.policy import JitPolicy, StopAtXPolicy from pypy import conftest from pypy.rlib.rarithmetic import ovfcheck @@ -830,7 +830,12 @@ return r() is None # assert f(30) == 1 - res = self.meta_interp(f, [30], no_stats=True) + old_dump_storage = resume.dump_storage + resume.dump_storage = lambda *args: None + try: + res = self.meta_interp(f, [30], no_stats=True) + finally: + resume.dump_storage = old_dump_storage assert res == 1 def test_pass_around(self): Modified: pypy/branch/logging/pypy/jit/metainterp/test/test_logger.py ============================================================================== --- pypy/branch/logging/pypy/jit/metainterp/test/test_logger.py (original) +++ pypy/branch/logging/pypy/jit/metainterp/test/test_logger.py Wed Oct 28 11:25:05 2009 @@ -3,18 +3,30 @@ from pypy.jit.metainterp import logger from pypy.jit.metainterp.typesystem import llhelper from StringIO import StringIO +from pypy.rlib import rlog from pypy.jit.metainterp.test.test_optimizeopt import equaloplists from pypy.jit.metainterp.history import AbstractDescr, LoopToken, BasicFailDescr class Descr(AbstractDescr): pass +def capturing(func, *args, **kwds): + log_stream = StringIO() + def write_to_log_stream(_cat, _msg, **_kwds): + print >> log_stream, _msg % _kwds + old_debug_log = rlog.debug_log + try: + rlog.debug_log = write_to_log_stream + func(*args, **kwds) + finally: + rlog.debug_log = old_debug_log + return log_stream.getvalue() + class Logger(logger.Logger): def log_loop(self, loop, namespace={}): - self.log_stream = StringIO() self.namespace = namespace - logger.Logger.log_loop(self, loop.inputargs, loop.operations) - return self.log_stream.getvalue() + return capturing(logger.Logger.log_loop, self, + loop.inputargs, loop.operations) def repr_of_descr(self, descr): for k, v in self.namespace.items(): @@ -92,7 +104,7 @@ loop = pure_parse(inp, namespace=namespace) logger = Logger(self.ts) output = logger.log_loop(loop) - assert output.splitlines()[-1] == "jump(i0, descr=)" + assert output.splitlines()[-2] == "jump(i0, descr=)" pure_parse(output) def test_guard(self): @@ -104,7 +116,7 @@ loop = pure_parse(inp, namespace=namespace) logger = Logger(self.ts, guard_number=True) output = logger.log_loop(loop) - assert output.splitlines()[-1] == "guard_true(i0, descr=) [i0]" + assert output.splitlines()[-2] == "guard_true(i0, descr=) [i0]" pure_parse(output) def boom(): @@ -112,20 +124,16 @@ namespace['fdescr'].get_index = boom logger = Logger(self.ts, guard_number=False) output = logger.log_loop(loop) - assert output.splitlines()[-1].startswith("guard_true(i0, descr=<") + assert output.splitlines()[-2].startswith("guard_true(i0, descr=<") def test_intro_loop(self): bare_logger = logger.Logger(self.ts) - bare_logger.log_stream = StringIO() - bare_logger.log_loop([], [], 1, "foo") - output = bare_logger.log_stream.getvalue() + output = capturing(bare_logger.log_loop, [], [], 1, "foo") assert output.splitlines()[0] == "# Loop1 (foo), 0 ops" pure_parse(output) def test_intro_bridge(self): bare_logger = logger.Logger(self.ts) - bare_logger.log_stream = StringIO() - bare_logger.log_bridge([], [], 3) - output = bare_logger.log_stream.getvalue() - assert output.splitlines()[0] == "# bridge out of Guard3, 0 ops" + output = capturing(bare_logger.log_bridge, [], [], 3) + assert output.splitlines()[0] == "# Bridge out of Guard3, 0 ops" pure_parse(output) Modified: pypy/branch/logging/pypy/jit/metainterp/test/test_loop.py ============================================================================== --- pypy/branch/logging/pypy/jit/metainterp/test/test_loop.py (original) +++ pypy/branch/logging/pypy/jit/metainterp/test/test_loop.py Wed Oct 28 11:25:05 2009 @@ -594,25 +594,6 @@ res = self.meta_interp(f, [200]) - def test_dump_storage(self): - import os - from pypy.tool.udir import udir - logfile = udir.join('resume.log') - os.environ['PYPYJITRESUMELOG'] = str(logfile) - try: - jitdriver = JitDriver(greens = [], reds = ['i', 'n']) - - def f(n): - i = 0 - while i < n: - jitdriver.can_enter_jit(n=n, i=i) - jitdriver.jit_merge_point(n=n, i=i) - i += 1 - return i - res = self.meta_interp(f, [10]) - data = logfile.read() # assert did not crash - finally: - del os.environ['PYPYJITRESUMELOG'] class TestOOtype(LoopTest, OOJitMixin): pass Modified: pypy/branch/logging/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/logging/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/logging/pypy/jit/metainterp/test/test_optimizeopt.py Wed Oct 28 11:25:05 2009 @@ -25,7 +25,6 @@ class Fake(object): failargs_limit = 1000 - storedebug = None class FakeMetaInterpStaticData(object): From arigo at codespeak.net Wed Oct 28 11:35:19 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 11:35:19 +0100 (CET) Subject: [pypy-svn] r68808 - pypy/branch/logging/pypy/jit/metainterp Message-ID: <20091028103519.F1E881683D3@codespeak.net> Author: arigo Date: Wed Oct 28 11:35:19 2009 New Revision: 68808 Modified: pypy/branch/logging/pypy/jit/metainterp/history.py pypy/branch/logging/pypy/jit/metainterp/resume.py Log: Translation fixes. Modified: pypy/branch/logging/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/logging/pypy/jit/metainterp/history.py (original) +++ pypy/branch/logging/pypy/jit/metainterp/history.py Wed Oct 28 11:35:19 2009 @@ -75,6 +75,9 @@ def repr_rpython(self, box, typechars): n = self.seen.setdefault(box, len(self.seen)) return '%s/%s%d' % (box._get_hash_(), typechars, n) + def _freeze_(self): + self.seen.clear() + return False repr_rpython = ReprRPython().repr_rpython Modified: pypy/branch/logging/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/logging/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/logging/pypy/jit/metainterp/resume.py Wed Oct 28 11:35:19 2009 @@ -437,9 +437,14 @@ pass # for tests else: while True: + try: + jitcodename = frameinfo.jitcode.name + except AttributeError: + jitcodename = str(objectmodel.compute_unique_id( + frameinfo.jitcode)) rlog.debug_log("jit-resume-frameinfo", - "(%(jitcode)r, %(pc)d, %(exception_target)d) at %(id)d", - jitcode = frameinfo.jitcode, + "(%(jitcode)s, %(pc)d, %(exception_target)d) at %(id)d", + jitcode = jitcodename, pc = frameinfo.pc, exception_target = frameinfo.exception_target, id = objectmodel.compute_unique_id(frameinfo)) From arigo at codespeak.net Wed Oct 28 12:01:36 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 12:01:36 +0100 (CET) Subject: [pypy-svn] r68809 - in pypy/branch/logging/pypy/rlib: . test Message-ID: <20091028110136.35C421683D4@codespeak.net> Author: arigo Date: Wed Oct 28 12:01:35 2009 New Revision: 68809 Modified: pypy/branch/logging/pypy/rlib/rlog.py pypy/branch/logging/pypy/rlib/rlog_parsing.py pypy/branch/logging/pypy/rlib/test/test_rlog.py Log: Avoid printing '(null)' even for empty strings. Accept both high-level and low-level strings. Modified: pypy/branch/logging/pypy/rlib/rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog.py (original) +++ pypy/branch/logging/pypy/rlib/rlog.py Wed Oct 28 12:01:35 2009 @@ -260,12 +260,19 @@ def add_subentry_d(self, num): self.write_int(num) - def add_subentry_s(self, llstr): - if llstr: - s = hlstr(llstr) + def add_subentry_s(self, s): + if s is None: + s = '(null)' # 's' is a high-level string -- but None right now + elif isinstance(s, str): + pass # 's' is already a high-level string else: - s = '(null)' + # in this case, assume that 's' is a low-level string + if s: + s = hlstr(s) + else: + s = '(null)' self.write_str(s) + add_subentry_s._annspecialcase_ = 'specialize:argtype(1)' add_subentry_r = add_subentry_s Modified: pypy/branch/logging/pypy/rlib/rlog_parsing.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog_parsing.py (original) +++ pypy/branch/logging/pypy/rlib/rlog_parsing.py Wed Oct 28 12:01:35 2009 @@ -84,7 +84,7 @@ try: printcode = cat.printcode except AttributeError: - code = cat.category + ' ' + code = '[%s] ' % cat.category message = cat.message.replace('\n', '\n' + ' '*len(code)) message = r_replace.sub("%", message) printcode = cat.printcode = code + message Modified: pypy/branch/logging/pypy/rlib/test/test_rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/test/test_rlog.py (original) +++ pypy/branch/logging/pypy/rlib/test/test_rlog.py Wed Oct 28 12:01:35 2009 @@ -81,7 +81,7 @@ logwriter = MyLogWriter() call = cat.gen_call(logwriter) call(515, llstr("hellooo")) - call(2873, llstr("woooooorld")) + call(2873, "woooooorld") # assert logwriter.content == [ ord('R'), ord('L'), ord('o'), ord('g'), ord('\n'), -1, 1.0, @@ -216,6 +216,8 @@ rlog.debug_log("Aa", "hello %(foo)d %(bar)f", foo=x+1, bar=x+0.5) rlog.debug_log("Ab", "<<%(baz)s>>", baz="hi there") rlog.debug_log("Ac", "[%(foo)r]", foo="\x00") + rlog.debug_log("Ac", "[%(foo)r]", foo="") + rlog.debug_log("Ac", "[%(foo)r]", foo=None) assert rlog.has_log() def setup_method(self, _): @@ -233,17 +235,17 @@ def check_result(self): entries = list(rlog_parsing.parse_log(self.pypylog)) - assert len(entries) == 4 + assert len(entries) == 6 # - assert isinstance(entries[0][0], float) - assert isinstance(entries[1][0], float) - assert isinstance(entries[2][0], float) - assert isinstance(entries[3][0], float) + for entry in entries: + assert isinstance(entry[0], float) # Aa = entries[0][1] Ab = entries[2][1] Ac = entries[3][1] assert entries[1][1] is Aa + assert entries[4][1] is Ac + assert entries[5][1] is Ac assert Aa.category == 'Aa' assert Aa.message == 'hello %(foo)d %(bar)f' assert Aa.entries == [('foo', 'd'), ('bar', 'f')] @@ -258,6 +260,8 @@ assert entries[1][2] == [133, 132.5] assert entries[2][2] == ['hi there'] assert entries[3][2] == ['\x00'] + assert entries[4][2] == [''] + assert entries[5][2] == ['(null)'] def test_interpret_f(self): interpret(self.f.im_func, [132], malloc_check=False) From arigo at codespeak.net Wed Oct 28 12:20:20 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 12:20:20 +0100 (CET) Subject: [pypy-svn] r68811 - pypy/branch/logging/pypy/rlib Message-ID: <20091028112020.4EC811683D4@codespeak.net> Author: arigo Date: Wed Oct 28 12:20:19 2009 New Revision: 68811 Modified: pypy/branch/logging/pypy/rlib/rlog_parsing.py Log: Allow an optional second argument to filter out matches based on the category. It can contain "?" or "*". Modified: pypy/branch/logging/pypy/rlib/rlog_parsing.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog_parsing.py (original) +++ pypy/branch/logging/pypy/rlib/rlog_parsing.py Wed Oct 28 12:20:19 2009 @@ -78,9 +78,16 @@ if __name__ == '__main__': - import sys, re + import sys, re, fnmatch + filename = sys.argv[1] + if len(sys.argv) > 2: + limit = sys.argv[2] + '*' + else: + limit = '*' r_replace = re.compile(r"%\(\w+\)") - for curtime, cat, entries in parse_log(sys.argv[1]): + for curtime, cat, entries in parse_log(filename): + if not fnmatch.fnmatch(cat.category, limit): + continue try: printcode = cat.printcode except AttributeError: From arigo at codespeak.net Wed Oct 28 12:22:25 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 12:22:25 +0100 (CET) Subject: [pypy-svn] r68812 - pypy/branch/logging/pypy/jit/metainterp Message-ID: <20091028112225.BB8591683D4@codespeak.net> Author: arigo Date: Wed Oct 28 12:22:25 2009 New Revision: 68812 Modified: pypy/branch/logging/pypy/jit/metainterp/logger.py Log: Separate the log entries the for opt and noopt cases. Modified: pypy/branch/logging/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/branch/logging/pypy/jit/metainterp/logger.py (original) +++ pypy/branch/logging/pypy/jit/metainterp/logger.py Wed Oct 28 12:22:25 2009 @@ -10,28 +10,44 @@ self.ts = ts self.guard_number=guard_number - def log_loop(self, inputargs, operations, number=-1, type="unoptimized"): + def log_loop(self, inputargs, operations, number=0, type=None): if not rlog.has_log(): return - rlog.debug_log("jit-log-loop-{", - "# Loop%(number)d (%(type)s), %(length)d ops", - number = number, - type = type, - length = len(operations)) - self._log_operations(inputargs, operations, {}) - rlog.debug_log("jit-log-loop-}", - "# End") + if type is None: + rlog.debug_log("jit-log-noopt-loop-{", + "# Unoptimized loop, %(length)d ops", + length = len(operations)) + self._log_operations(inputargs, operations, {}) + rlog.debug_log("jit-log-noopt-loop-}", + "# End") + else: + rlog.debug_log("jit-log-opt-loop-{", + "# Loop%(number)d (%(type)s), %(length)d ops", + number = number, + type = type, + length = len(operations)) + self._log_operations(inputargs, operations, {}) + rlog.debug_log("jit-log-opt-loop-}", + "# End") def log_bridge(self, inputargs, operations, number=-1): if not rlog.has_log(): return - rlog.debug_log("jit-log-bridge-{", - "# Bridge out of Guard%(guard)d, %(length)d ops", - guard = number, - length = len(operations)) - self._log_operations(inputargs, operations, {}) - rlog.debug_log("jit-log-bridge-}", - "# End") + if number == -1: + rlog.debug_log("jit-log-noopt-bridge-{", + "# Unoptimized bridge, %(length)d ops", + length = len(operations)) + self._log_operations(inputargs, operations, {}) + rlog.debug_log("jit-log-noopt-bridge-}", + "# End") + else: + rlog.debug_log("jit-log-opt-bridge-{", + "# Bridge out of Guard%(guard)d, %(length)d ops", + guard = number, + length = len(operations)) + self._log_operations(inputargs, operations, {}) + rlog.debug_log("jit-log-opt-bridge-}", + "# End") def repr_of_descr(self, descr): return descr.repr_of_descr() From arigo at codespeak.net Wed Oct 28 12:30:44 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 12:30:44 +0100 (CET) Subject: [pypy-svn] r68813 - pypy/branch/logging/pypy/rlib Message-ID: <20091028113044.763381683D4@codespeak.net> Author: arigo Date: Wed Oct 28 12:30:43 2009 New Revision: 68813 Modified: pypy/branch/logging/pypy/rlib/rlog_parsing.py Log: Move the code to its own function and support limited syntax highlighting. Modified: pypy/branch/logging/pypy/rlib/rlog_parsing.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog_parsing.py (original) +++ pypy/branch/logging/pypy/rlib/rlog_parsing.py Wed Oct 28 12:30:43 2009 @@ -76,14 +76,8 @@ logparser = LogParser(open(filename, 'rb')) return logparser.enum_entries() - -if __name__ == '__main__': - import sys, re, fnmatch - filename = sys.argv[1] - if len(sys.argv) > 2: - limit = sys.argv[2] + '*' - else: - limit = '*' +def dump_log(filename, limit='*', highlight=False): + import re r_replace = re.compile(r"%\(\w+\)") for curtime, cat, entries in parse_log(filename): if not fnmatch.fnmatch(cat.category, limit): @@ -94,5 +88,22 @@ code = '[%s] ' % cat.category message = cat.message.replace('\n', '\n' + ' '*len(code)) message = r_replace.sub("%", message) - printcode = cat.printcode = code + message + printcode = code + message + if highlight: + if cat.category.endswith('{'): + printcode = '\x1B[1m%s\x1B[0m' % (printcode,) + elif cat.category.endswith('}'): + printcode = '\x1B[31m%s\x1B[0m' % (printcode,) + cat.printcode = printcode print printcode % tuple(entries) + + +if __name__ == '__main__': + import sys, fnmatch + filename = sys.argv[1] + if len(sys.argv) > 2: + limit = sys.argv[2] + '*' + else: + limit = '*' + highlight = sys.stdout.isatty() + dump_log(filename, limit, highlight) From arigo at codespeak.net Wed Oct 28 13:18:35 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 13:18:35 +0100 (CET) Subject: [pypy-svn] r68814 - in pypy/branch/logging/pypy/rlib: . test Message-ID: <20091028121835.786D11683D4@codespeak.net> Author: arigo Date: Wed Oct 28 13:18:34 2009 New Revision: 68814 Modified: pypy/branch/logging/pypy/rlib/rlog.py pypy/branch/logging/pypy/rlib/rlog_parsing.py pypy/branch/logging/pypy/rlib/test/test_rlog.py Log: Small fixes and a new function, untested so far... Modified: pypy/branch/logging/pypy/rlib/rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog.py (original) +++ pypy/branch/logging/pypy/rlib/rlog.py Wed Oct 28 13:18:34 2009 @@ -1,6 +1,7 @@ import py, time, struct from pypy.tool.ansi_print import ansi_log from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.nonconst import NonConstant from pypy.tool.sourcetools import func_with_new_name from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rpython.annlowlevel import hlstr @@ -185,6 +186,12 @@ types = unrolling_iterable(self.types) # def call(*args): + if NonConstant(False): + logwriter.add_subentry_d(123) + logwriter.add_subentry_s('abc') + logwriter.add_subentry_r('abc') + logwriter.add_subentry_f(123.4) + # ^^^ annotation hacks if not logwriter.enabled: return if not logwriter.add_entry(self): Modified: pypy/branch/logging/pypy/rlib/rlog_parsing.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog_parsing.py (original) +++ pypy/branch/logging/pypy/rlib/rlog_parsing.py Wed Oct 28 13:18:34 2009 @@ -1,11 +1,21 @@ import autopath -import struct +import struct, re, fnmatch from pypy.rlib.rarithmetic import intmask from pypy.rlib import rlog SIZEOF_FLOAT = rlog.SIZEOF_FLOAT +class LogCategory(object): + + def __init__(self, category, message, index): + self.category = category + self.message = message + self.index = index + self.types = re.findall(r"%\(\w+\)(\w)", message) + self.messagestr = re.sub(r"%\(\w+\)", "%", message) + + class LogParser(object): def __init__(self, file, has_signature=True): @@ -64,7 +74,7 @@ category = self.read_str() message = self.read_str() assert index not in categories - categories[index] = rlog.LogCategory(category, message, index) + categories[index] = LogCategory(category, message, index) else: curtime += self.read_float() cat = categories[c] @@ -77,8 +87,6 @@ return logparser.enum_entries() def dump_log(filename, limit='*', highlight=False): - import re - r_replace = re.compile(r"%\(\w+\)") for curtime, cat, entries in parse_log(filename): if not fnmatch.fnmatch(cat.category, limit): continue @@ -86,8 +94,7 @@ printcode = cat.printcode except AttributeError: code = '[%s] ' % cat.category - message = cat.message.replace('\n', '\n' + ' '*len(code)) - message = r_replace.sub("%", message) + message = cat.messagestr.replace('\n', '\n' + ' '*len(code)) printcode = code + message if highlight: if cat.category.endswith('{'): @@ -97,9 +104,24 @@ cat.printcode = printcode print printcode % tuple(entries) +def extract_sections(filename, limit): + """Extract sections between 'limit-{' and 'limit-}'. + Yields multiline strings, each a complete section. + Accept * and ? in 'limit'. + """ + pieces = None + for curtime, cat, entries in parse_log(filename): + if fnmatch.fnmatch(cat.category, limit + '-{'): + pieces = [] + if pieces is not None: + pieces.append(cat.messagestr % tuple(entries)) + if fnmatch.fnmatch(cat.category, limit + '-}'): + yield '\n'.join(pieces) + pieces = None + if __name__ == '__main__': - import sys, fnmatch + import sys filename = sys.argv[1] if len(sys.argv) > 2: limit = sys.argv[2] + '*' Modified: pypy/branch/logging/pypy/rlib/test/test_rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/test/test_rlog.py (original) +++ pypy/branch/logging/pypy/rlib/test/test_rlog.py Wed Oct 28 13:18:34 2009 @@ -248,13 +248,16 @@ assert entries[5][1] is Ac assert Aa.category == 'Aa' assert Aa.message == 'hello %(foo)d %(bar)f' - assert Aa.entries == [('foo', 'd'), ('bar', 'f')] + assert Aa.messagestr == 'hello %d %f' + assert Aa.types == ['d', 'f'] assert Ab.category == 'Ab' assert Ab.message == '<<%(baz)s>>' - assert Ab.entries == [('baz', 's')] + assert Ab.messagestr == '<<%s>>' + assert Ab.types == ['s'] assert Ac.category == 'Ac' assert Ac.message == '[%(foo)r]' - assert Ac.entries == [('foo', 'r')] + assert Ac.messagestr == '[%r]' + assert Ac.types == ['r'] # assert entries[0][2] == [132, roughly(-7.3)] assert entries[1][2] == [133, 132.5] From arigo at codespeak.net Wed Oct 28 13:19:17 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 13:19:17 +0100 (CET) Subject: [pypy-svn] r68815 - pypy/branch/logging/pypy/jit/backend/x86/test Message-ID: <20091028121917.580CE1683D4@codespeak.net> Author: arigo Date: Wed Oct 28 13:19:16 2009 New Revision: 68815 Modified: pypy/branch/logging/pypy/jit/backend/x86/test/test_zrpy_gc.py Log: Fix test. Modified: pypy/branch/logging/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/logging/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/logging/pypy/jit/backend/x86/test/test_zrpy_gc.py Wed Oct 28 13:19:16 2009 @@ -77,7 +77,6 @@ # t = TranslationContext() t.config.translation.gc = gc - t.config.translation.gcconfig.debugprint = True for name, value in kwds.items(): setattr(t.config.translation, name, value) ann = t.buildannotator(policy=annpolicy.StrictAnnotatorPolicy()) From arigo at codespeak.net Wed Oct 28 13:21:44 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 13:21:44 +0100 (CET) Subject: [pypy-svn] r68816 - in pypy/branch/logging/pypy/jit: backend metainterp/test Message-ID: <20091028122144.68F381683D4@codespeak.net> Author: arigo Date: Wed Oct 28 13:21:44 2009 New Revision: 68816 Modified: pypy/branch/logging/pypy/jit/backend/loopviewer.py pypy/branch/logging/pypy/jit/metainterp/test/oparser.py Log: Fix loopviewer.py. Also give it the option "-n" to view the unoptimized loops instead of the optimized ones. Modified: pypy/branch/logging/pypy/jit/backend/loopviewer.py ============================================================================== --- pypy/branch/logging/pypy/jit/backend/loopviewer.py (original) +++ pypy/branch/logging/pypy/jit/backend/loopviewer.py Wed Oct 28 13:21:44 2009 @@ -1,29 +1,35 @@ #!/usr/bin/env python -""" Usage: loopviewer.py [loopnum] loopfile +""" Usage: loopviewer.py [-i# -n] loopfile + -i: specifies the loop number, default to the last one + -n: show the unoptimized loops instead of the optimized ones """ import autopath import py import sys -from pypy.jit.metainterp.test.oparser import parse, split_logs_into_loops +from getopt import gnu_getopt +from pypy.rlib.rlog_parsing import extract_sections +from pypy.jit.metainterp.test.oparser import parse from pypy.jit.metainterp.history import ConstInt from pypy.rpython.lltypesystem import llmemory, lltype -def main(loopnum, loopfile): - data = py.path.local(loopfile).read() - loops = split_logs_into_loops(data) +def main(loopfile, loopnum=-1, noopt=False): + if noopt: + word = 'noopt' + else: + word = 'opt' + loops = list(extract_sections(loopfile, 'jit-log-%s-*' % word)) inp = loops[loopnum] - loop = parse(inp, no_namespace=True) + loop = parse(inp, no_namespace=True, enforce_fail_args=False) loop.show() if __name__ == '__main__': - if len(sys.argv) == 2: - loopnum = -1 - loopfile = sys.argv[1] - elif len(sys.argv) == 3: - loopnum = int(sys.argv[1]) - loopfile = sys.argv[2] - else: + options, args = gnu_getopt(sys.argv[1:], 'i:n') + if len(args) != 1: print __doc__ sys.exit(1) - main(loopnum, loopfile) + [loopfile] = args + options = dict(options) + loopnum = int(options.get('-i', -1)) + noopt = '-n' in options + main(loopfile, loopnum=loopnum, noopt=noopt) Modified: pypy/branch/logging/pypy/jit/metainterp/test/oparser.py ============================================================================== --- pypy/branch/logging/pypy/jit/metainterp/test/oparser.py (original) +++ pypy/branch/logging/pypy/jit/metainterp/test/oparser.py Wed Oct 28 13:21:44 2009 @@ -51,7 +51,7 @@ class OpParser(object): def __init__(self, input, cpu, namespace, type_system, boxkinds, - invent_fail_descr=default_fail_descr): + invent_fail_descr=default_fail_descr, enforce_fail_args=True): self.input = input self.vars = {} self.cpu = cpu @@ -63,6 +63,7 @@ else: self._cache = {} self.invent_fail_descr = invent_fail_descr + self.enforce_fail_args = enforce_fail_args self.looptoken = LoopToken() def get_const(self, name, typ): @@ -203,8 +204,12 @@ i = line.find('[', endnum) + 1 j = line.find(']', i) if i <= 0 or j <= 0: - raise ParseError("missing fail_args for guard operation") - fail_args = [] + if self.enforce_fail_args: + raise ParseError("missing fail_args for guard operation") + i = j = -1 + fail_args = None + else: + fail_args = [] if i < j: for arg in line[i:j].split(','): arg = arg.strip() @@ -306,11 +311,11 @@ def parse(input, cpu=None, namespace=None, type_system='lltype', boxkinds=None, invent_fail_descr=default_fail_descr, - no_namespace=False): + no_namespace=False, enforce_fail_args=True): if namespace is None and not no_namespace: namespace = {} return OpParser(input, cpu, namespace, type_system, boxkinds, - invent_fail_descr).parse() + invent_fail_descr, enforce_fail_args).parse() def pure_parse(*args, **kwds): kwds['invent_fail_descr'] = None From arigo at codespeak.net Wed Oct 28 13:31:28 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 13:31:28 +0100 (CET) Subject: [pypy-svn] r68817 - pypy/branch/logging/pypy/rlib Message-ID: <20091028123128.486441683D4@codespeak.net> Author: arigo Date: Wed Oct 28 13:31:27 2009 New Revision: 68817 Modified: pypy/branch/logging/pypy/rlib/rlog.py Log: Really fix annotation, with more hacks, argh. The next issue is a segfault in test_zrpy_gc. Unhappy. Modified: pypy/branch/logging/pypy/rlib/rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog.py (original) +++ pypy/branch/logging/pypy/rlib/rlog.py Wed Oct 28 13:31:27 2009 @@ -4,7 +4,7 @@ from pypy.rlib.nonconst import NonConstant from pypy.tool.sourcetools import func_with_new_name from pypy.rpython.extregistry import ExtRegistryEntry -from pypy.rpython.annlowlevel import hlstr +from pypy.rpython.annlowlevel import llstr, hlstr _log = py.log.Producer("rlog") py.log.setconsumer("rlog", ansi_log) @@ -187,10 +187,14 @@ # def call(*args): if NonConstant(False): - logwriter.add_subentry_d(123) - logwriter.add_subentry_s('abc') - logwriter.add_subentry_r('abc') - logwriter.add_subentry_f(123.4) + logwriter.add_subentry_d(NonConstant(-123)) + logwriter.add_subentry_s(NonConstant('abc')) + logwriter.add_subentry_s(None) + logwriter.add_subentry_s(llstr('abc')) + logwriter.add_subentry_r(NonConstant('abc')) + logwriter.add_subentry_r(None) + logwriter.add_subentry_r(llstr('abc')) + logwriter.add_subentry_f(NonConstant(123.4)) # ^^^ annotation hacks if not logwriter.enabled: return From arigo at codespeak.net Wed Oct 28 13:31:52 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 13:31:52 +0100 (CET) Subject: [pypy-svn] r68818 - in pypy/branch/logging/pypy/jit: backend tool Message-ID: <20091028123152.1B7C81683D4@codespeak.net> Author: arigo Date: Wed Oct 28 13:31:51 2009 New Revision: 68818 Added: pypy/branch/logging/pypy/jit/tool/loopviewer.py - copied unchanged from r68816, pypy/branch/logging/pypy/jit/backend/loopviewer.py pypy/branch/logging/pypy/jit/tool/showstats.py - copied unchanged from r68809, pypy/branch/logging/pypy/jit/backend/showstats.py Removed: pypy/branch/logging/pypy/jit/backend/loopviewer.py pypy/branch/logging/pypy/jit/backend/showstats.py Log: Move these two untested helpers to pypy/jit/tool/. From cfbolz at codespeak.net Wed Oct 28 15:02:42 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 28 Oct 2009 15:02:42 +0100 (CET) Subject: [pypy-svn] r68819 - in pypy/branch/shrink-multidict/pypy/module/sys: . test Message-ID: <20091028140242.166D01683DC@codespeak.net> Author: cfbolz Date: Wed Oct 28 15:02:40 2009 New Revision: 68819 Modified: pypy/branch/shrink-multidict/pypy/module/sys/__init__.py pypy/branch/shrink-multidict/pypy/module/sys/test/test_sysmodule.py Log: bah, there were no tests for sys.exc_value, .exc_type and .exc_traceback Modified: pypy/branch/shrink-multidict/pypy/module/sys/__init__.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/module/sys/__init__.py (original) +++ pypy/branch/shrink-multidict/pypy/module/sys/__init__.py Wed Oct 28 15:02:40 2009 @@ -100,12 +100,11 @@ w_modules = self.get('modules') self.space.setitem(w_modules, w_name, w_module) - def getdictvalue(self, space, w_attr): + def getdictvalue_w(self, space, attr): """ specialize access to dynamic exc_* attributes. """ - value = MixedModule.getdictvalue(self, space, w_attr) + value = MixedModule.getdictvalue_w(self, space, attr) if value is not None: return value - attr = space.str_w(w_attr) if attr == 'exc_type': operror = space.getexecutioncontext().sys_exc_info() if operror is None: @@ -126,6 +125,9 @@ return space.wrap(operror.application_traceback) return None + def getdictvalue(self, space, w_attr): + return self.getdictvalue_w(space, space.str_w(w_attr)) + def get_w_default_encoder(self): if self.w_default_encoder is not None: # XXX is this level of caching ok? CPython has some shortcuts Modified: pypy/branch/shrink-multidict/pypy/module/sys/test/test_sysmodule.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/module/sys/test/test_sysmodule.py (original) +++ pypy/branch/shrink-multidict/pypy/module/sys/test/test_sysmodule.py Wed Oct 28 15:02:40 2009 @@ -65,6 +65,26 @@ assert exc_val2 ==e2 assert tb2.tb_lineno - tb.tb_lineno == 5 + def test_dynamic_attributes(self): + try: + raise Exception + except Exception,e: + import sys + exc_type = sys.exc_type + exc_val = sys.exc_value + tb = sys.exc_traceback + try: + raise Exception # 7 lines below the previous one + except Exception,e2: + exc_type2 = sys.exc_type + exc_val2 = sys.exc_value + tb2 = sys.exc_traceback + assert exc_type ==Exception + assert exc_val ==e + assert exc_type2 ==Exception + assert exc_val2 ==e2 + assert tb2.tb_lineno - tb.tb_lineno == 7 + def test_exc_info_normalization(self): import sys try: From cfbolz at codespeak.net Wed Oct 28 15:26:32 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 28 Oct 2009 15:26:32 +0100 (CET) Subject: [pypy-svn] r68820 - in pypy/branch/shrink-multidict/pypy: interpreter module/operator module/sys objspace objspace/std Message-ID: <20091028142632.D3CA11683D4@codespeak.net> Author: cfbolz Date: Wed Oct 28 15:26:31 2009 New Revision: 68820 Modified: pypy/branch/shrink-multidict/pypy/interpreter/baseobjspace.py pypy/branch/shrink-multidict/pypy/interpreter/main.py pypy/branch/shrink-multidict/pypy/interpreter/mixedmodule.py pypy/branch/shrink-multidict/pypy/interpreter/pyopcode.py pypy/branch/shrink-multidict/pypy/module/operator/__init__.py pypy/branch/shrink-multidict/pypy/module/sys/__init__.py pypy/branch/shrink-multidict/pypy/objspace/descroperation.py pypy/branch/shrink-multidict/pypy/objspace/std/callmethod.py pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py pypy/branch/shrink-multidict/pypy/objspace/std/proxyobject.py pypy/branch/shrink-multidict/pypy/objspace/std/typeobject.py pypy/branch/shrink-multidict/pypy/objspace/std/typetype.py pypy/branch/shrink-multidict/pypy/objspace/taint.py Log: make getdictvalue take an unwrapped string and kill getdictvalue_w Modified: pypy/branch/shrink-multidict/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/shrink-multidict/pypy/interpreter/baseobjspace.py Wed Oct 28 15:26:31 2009 @@ -25,20 +25,14 @@ def getdict(self): return None - def getdictvalue_w(self, space, attr): + def getdictvalue(self, space, attr): w_dict = self.getdict() if w_dict is not None: return space.finditem_str(w_dict, attr) return None - def getdictvalue(self, space, w_attr): - w_dict = self.getdict() - if w_dict is not None: - return space.finditem(w_dict, w_attr) - return None - def getdictvalue_attr_is_in_class(self, space, attr): - return self.getdictvalue_w(space, attr) + return self.getdictvalue(space, attr) def setdictvalue(self, space, w_attr, w_value, shadows_type=True): w_dict = self.getdict() @@ -285,7 +279,7 @@ self.timer.stop("startup " + modname) def finish(self): - w_exitfunc = self.sys.getdictvalue_w(self, 'exitfunc') + w_exitfunc = self.sys.getdictvalue(self, 'exitfunc') if w_exitfunc is not None: self.call_function(w_exitfunc) from pypy.interpreter.module import Module @@ -778,7 +772,7 @@ w_type = self.type(w_obj) w_mro = self.getattr(w_type, self.wrap("__mro__")) for w_supertype in self.unpackiterable(w_mro): - w_value = w_supertype.getdictvalue_w(self, name) + w_value = w_supertype.getdictvalue(self, name) if w_value is not None: return w_value return None Modified: pypy/branch/shrink-multidict/pypy/interpreter/main.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/interpreter/main.py (original) +++ pypy/branch/shrink-multidict/pypy/interpreter/main.py Wed Oct 28 15:26:31 2009 @@ -149,11 +149,11 @@ w_traceback) # call sys.excepthook if present - w_hook = space.sys.getdictvalue_w(space, 'excepthook') + w_hook = space.sys.getdictvalue(space, 'excepthook') if w_hook is not None: # hack: skip it if it wasn't modified by the user, # to do instead the faster verbose/nonverbose thing below - w_original = space.sys.getdictvalue_w(space, '__excepthook__') + w_original = space.sys.getdictvalue(space, '__excepthook__') if w_original is None or not space.is_w(w_hook, w_original): space.call_function(w_hook, w_type, w_value, w_traceback) return False # done Modified: pypy/branch/shrink-multidict/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/interpreter/mixedmodule.py (original) +++ pypy/branch/shrink-multidict/pypy/interpreter/mixedmodule.py Wed Oct 28 15:26:31 2009 @@ -32,7 +32,7 @@ def get(self, name): space = self.space - w_value = self.getdictvalue_w(space, name) + w_value = self.getdictvalue(space, name) if w_value is None: raise OperationError(space.w_AttributeError, space.wrap(name)) return w_value @@ -41,18 +41,12 @@ w_builtin = self.get(name) return self.space.call_function(w_builtin, *args_w) - def getdictvalue_w(self, space, name): + def getdictvalue(self, space, name): w_value = space.finditem_str(self.w_dict, name) if self.lazy and w_value is None: return self._load_lazily(space, name) return w_value - def getdictvalue(self, space, w_name): - w_value = space.finditem(self.w_dict, w_name) - if self.lazy and w_value is None: - return self._load_lazily(space, space.str_w(w_name)) - return w_value - def _load_lazily(self, space, name): w_name = space.new_interned_str(name) try: Modified: pypy/branch/shrink-multidict/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/shrink-multidict/pypy/interpreter/pyopcode.py Wed Oct 28 15:26:31 2009 @@ -673,25 +673,24 @@ return f.LOAD_GLOBAL(nameindex) # fall-back - def _load_global(f, w_varname): - w_value = f.space.finditem(f.w_globals, w_varname) + def _load_global(f, varname): + w_value = f.space.finditem_str(f.w_globals, varname) if w_value is None: # not in the globals, now look in the built-ins - w_value = f.get_builtin().getdictvalue(f.space, w_varname) + w_value = f.get_builtin().getdictvalue(f.space, varname) if w_value is None: - f._load_global_failed(w_varname) + f._load_global_failed(varname) return w_value _load_global._always_inline_ = True - def _load_global_failed(f, w_varname): - varname = f.space.str_w(w_varname) + def _load_global_failed(f, varname): message = "global name '%s' is not defined" % varname raise OperationError(f.space.w_NameError, f.space.wrap(message)) _load_global_failed._dont_inline_ = True def LOAD_GLOBAL(f, nameindex, *ignored): - f.pushvalue(f._load_global(f.getname_w(nameindex))) + f.pushvalue(f._load_global(f.getname_u(nameindex))) LOAD_GLOBAL._always_inline_ = True def DELETE_FAST(f, varindex, *ignored): @@ -782,7 +781,7 @@ else: w_flag = None - w_import = f.get_builtin().getdictvalue_w(f.space, '__import__') + w_import = f.get_builtin().getdictvalue(f.space, '__import__') if w_import is None: raise OperationError(space.w_ImportError, space.wrap("__import__ not found")) @@ -989,8 +988,8 @@ def CALL_LIKELY_BUILTIN(f, oparg, *ignored): # overridden by faster version in the standard object space. from pypy.module.__builtin__ import OPTIMIZED_BUILTINS - w_varname = f.space.wrap(OPTIMIZED_BUILTINS[oparg >> 8]) - w_function = f._load_global(w_varname) + varname = OPTIMIZED_BUILTINS[oparg >> 8] + w_function = f._load_global(varname) nargs = oparg&0xFF try: w_result = f.space.call_valuestack(w_function, nargs, f) Modified: pypy/branch/shrink-multidict/pypy/module/operator/__init__.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/module/operator/__init__.py (original) +++ pypy/branch/shrink-multidict/pypy/module/operator/__init__.py Wed Oct 28 15:26:31 2009 @@ -9,7 +9,7 @@ def __init__(self, space, w_name): def create_lambda(name, alsoname): - return lambda space : self.getdictvalue(space, space.wrap(alsoname)) + return lambda space : self.getdictvalue(space, alsoname) MixedModule.__init__(self, space, w_name) for name, alsoname in self.mapping.iteritems(): Modified: pypy/branch/shrink-multidict/pypy/module/sys/__init__.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/module/sys/__init__.py (original) +++ pypy/branch/shrink-multidict/pypy/module/sys/__init__.py Wed Oct 28 15:26:31 2009 @@ -100,9 +100,9 @@ w_modules = self.get('modules') self.space.setitem(w_modules, w_name, w_module) - def getdictvalue_w(self, space, attr): + def getdictvalue(self, space, attr): """ specialize access to dynamic exc_* attributes. """ - value = MixedModule.getdictvalue_w(self, space, attr) + value = MixedModule.getdictvalue(self, space, attr) if value is not None: return value if attr == 'exc_type': @@ -125,9 +125,6 @@ return space.wrap(operror.application_traceback) return None - def getdictvalue(self, space, w_attr): - return self.getdictvalue_w(space, space.str_w(w_attr)) - def get_w_default_encoder(self): if self.w_default_encoder is not None: # XXX is this level of caching ok? CPython has some shortcuts Modified: pypy/branch/shrink-multidict/pypy/objspace/descroperation.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/descroperation.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/descroperation.py Wed Oct 28 15:26:31 2009 @@ -38,7 +38,7 @@ return space.get(w_descr, w_obj) w_value = w_obj.getdictvalue_attr_is_in_class(space, name) else: - w_value = w_obj.getdictvalue_w(space, name) + w_value = w_obj.getdictvalue(space, name) if w_value is not None: return w_value if w_descr is not None: Modified: pypy/branch/shrink-multidict/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/callmethod.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/callmethod.py Wed Oct 28 15:26:31 2009 @@ -54,7 +54,7 @@ if w_descr is None: # this handles directly the common case # module.function(args..) - w_value = w_obj.getdictvalue_w(space, name) + w_value = w_obj.getdictvalue(space, name) elif type(w_descr) is function.Function: w_value = w_obj.getdictvalue_attr_is_in_class(space, name) if w_value is None: Modified: pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py Wed Oct 28 15:26:31 2009 @@ -83,20 +83,14 @@ def _inlined_dict_valid(self): return getattr(self, attrname) is not None - def getdictvalue_w(self, space, attr): + def getdictvalue(self, space, attr): if self._inlined_dict_valid(): return self.impl_getitem_str(attr) w_dict = self.getdict() return w_dict.getitem_str(attr) - def getdictvalue(self, space, w_attr): - if self._inlined_dict_valid(): - return self.impl_getitem(w_attr) - w_dict = self.getdict() - return w_dict.getitem(w_attr) - def getdictvalue_attr_is_in_class(self, space, attr): - return self.getdictvalue_w(space, attr) + return self.getdictvalue(space, attr) def setdictvalue(self, space, w_attr, w_value, shadows_type=True): if self._inlined_dict_valid(): Modified: pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py Wed Oct 28 15:26:31 2009 @@ -717,7 +717,7 @@ if not e.match(self, self.w_AttributeError): raise else: - w_value = w_obj.getdictvalue_w(self, name) + w_value = w_obj.getdictvalue(self, name) if w_value is not None: return w_value Modified: pypy/branch/shrink-multidict/pypy/objspace/std/proxyobject.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/proxyobject.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/proxyobject.py Wed Oct 28 15:26:31 2009 @@ -34,13 +34,10 @@ raise OperationError(space.w_TypeError, space.wrap("You cannot override __class__ for transparent proxies")) - def getdictvalue_w(self, space, attr): - return self.getdictvalue(space, space.wrap(attr)) - - def getdictvalue(self, space, w_attr): + def getdictvalue(self, space, attr): try: return space.call_function(self.w_controller, space.wrap('__getattribute__'), - w_attr) + space.wrap(attr)) except OperationError, e: if not e.match(space, space.w_AttributeError): raise @@ -67,19 +64,12 @@ return False def getdict(self): - return self.getdictvalue(self.space, self.space.wrap('__dict__')) + return self.getdictvalue(self.space, '__dict__') def setdict(self, space, w_dict): if not self.setdictvalue(space, space.wrap('__dict__'), w_dict): baseobjspace.W_Root.setdict(self, space, w_dict) -## def __getattr__(self, attr): -## # NOT_RPYTHON -## try: -## return self.getdictvalue(self.space, self.space.wrap(attr)) -## except OperationError, e: -## raise AttributeError(attr) - W_Transparent.__name__ = name return W_Transparent Modified: pypy/branch/shrink-multidict/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/typeobject.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/typeobject.py Wed Oct 28 15:26:31 2009 @@ -131,10 +131,7 @@ def compute_default_mro(w_self): return compute_C3_mro(w_self.space, w_self) - def getdictvalue(w_self, space, w_attr): - return w_self.getdictvalue_w(space, space.str_w(w_attr)) - - def getdictvalue_w(w_self, space, attr): + def getdictvalue(w_self, space, attr): w_value = w_self.dict_w.get(attr, None) if w_self.lazyloaders and w_value is None: if attr in w_self.lazyloaders: @@ -172,7 +169,7 @@ if w_class is w_starttype: look = True elif look: - w_value = w_class.getdictvalue_w(space, name) + w_value = w_class.getdictvalue(space, name) if w_value is not None: return w_value return None @@ -182,7 +179,7 @@ def _lookup(w_self, key): space = w_self.space for w_class in w_self.mro_w: - w_value = w_class.getdictvalue_w(space, key) + w_value = w_class.getdictvalue(space, key) if w_value is not None: return w_value return None @@ -193,7 +190,7 @@ # attribute was found space = w_self.space for w_class in w_self.mro_w: - w_value = w_class.getdictvalue_w(space, key) + w_value = w_class.getdictvalue(space, key) if w_value is not None: return w_class, w_value return None, None @@ -273,7 +270,7 @@ "NOT_RPYTHON. Forces the lazy attributes to be computed." if 'lazyloaders' in w_self.__dict__: for attr in w_self.lazyloaders.keys(): - w_self.getdictvalue_w(w_self.space, attr) + w_self.getdictvalue(w_self.space, attr) del w_self.lazyloaders return False Modified: pypy/branch/shrink-multidict/pypy/objspace/std/typetype.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/typetype.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/typetype.py Wed Oct 28 15:26:31 2009 @@ -174,7 +174,7 @@ return space.wrap("""type(object) -> the object's type type(name, bases, dict) -> a new type""") w_type = _check(space, w_type) - w_result = w_type.getdictvalue_w(space, '__doc__') + w_result = w_type.getdictvalue(space, '__doc__') if w_result is None: return space.w_None else: Modified: pypy/branch/shrink-multidict/pypy/objspace/taint.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/taint.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/taint.py Wed Oct 28 15:26:31 2009 @@ -20,11 +20,8 @@ ## def getdict(self): ## return taint(self.w_obj.getdict()) -## def getdictvalue_w(self, space, attr): -## return taint(self.w_obj.getdictvalue_w(space, attr)) - -## def getdictvalue(self, space, w_attr): -## return taint(self.w_obj.getdictvalue(space, w_attr)) +## def getdictvalue(self, space, attr): +## return taint(self.w_obj.getdictvalue(space, attr)) ## def setdictvalue(self, space, w_attr, w_value): ## return self.w_obj.setdictvalue(space, w_attr, w_value) From cfbolz at codespeak.net Wed Oct 28 16:21:36 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 28 Oct 2009 16:21:36 +0100 (CET) Subject: [pypy-svn] r68821 - in pypy/branch/shrink-multidict/pypy: interpreter jit/tl module/__builtin__ objspace objspace/std objspace/std/test translator/goal Message-ID: <20091028152136.79DA01683D2@codespeak.net> Author: cfbolz Date: Wed Oct 28 16:21:35 2009 New Revision: 68821 Modified: pypy/branch/shrink-multidict/pypy/interpreter/baseobjspace.py pypy/branch/shrink-multidict/pypy/interpreter/pyopcode.py pypy/branch/shrink-multidict/pypy/jit/tl/pypyjit.py pypy/branch/shrink-multidict/pypy/jit/tl/pypyjit_demo.py pypy/branch/shrink-multidict/pypy/module/__builtin__/interp_classobj.py pypy/branch/shrink-multidict/pypy/objspace/descroperation.py pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py pypy/branch/shrink-multidict/pypy/objspace/std/proxyobject.py pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py pypy/branch/shrink-multidict/pypy/objspace/std/test/test_inlinedict.py pypy/branch/shrink-multidict/pypy/objspace/taint.py pypy/branch/shrink-multidict/pypy/translator/goal/gcbench.py Log: Change setdictvalue to also take a string. Now it's asymmetric to deldictvalue, I might not care though. Modified: pypy/branch/shrink-multidict/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/shrink-multidict/pypy/interpreter/baseobjspace.py Wed Oct 28 16:21:35 2009 @@ -34,10 +34,10 @@ def getdictvalue_attr_is_in_class(self, space, attr): return self.getdictvalue(space, attr) - def setdictvalue(self, space, w_attr, w_value, shadows_type=True): + def setdictvalue(self, space, attr, w_value, shadows_type=True): w_dict = self.getdict() if w_dict is not None: - space.set_str_keyed_item(w_dict, w_attr, w_value, shadows_type) + space.set_str_keyed_item(w_dict, attr, w_value, shadows_type) return True return False @@ -563,8 +563,8 @@ """shortcut for space.int_w(space.hash(w_obj))""" return self.int_w(self.hash(w_obj)) - def set_str_keyed_item(self, w_obj, w_key, w_value, shadows_type=True): - return self.setitem(w_obj, w_key, w_value) + def set_str_keyed_item(self, w_obj, key, w_value, shadows_type=True): + return self.setitem(w_obj, self.wrap(key), w_value) def finditem_str(self, w_obj, key): return self.finditem(w_obj, self.wrap(key)) Modified: pypy/branch/shrink-multidict/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/shrink-multidict/pypy/interpreter/pyopcode.py Wed Oct 28 16:21:35 2009 @@ -619,9 +619,9 @@ f.pushvalue(w_newclass) def STORE_NAME(f, varindex, *ignored): - w_varname = f.getname_w(varindex) + varname = f.getname_u(varindex) w_newvalue = f.popvalue() - f.space.set_str_keyed_item(f.w_locals, w_varname, w_newvalue) + f.space.set_str_keyed_item(f.w_locals, varname, w_newvalue) def DELETE_NAME(f, varindex, *ignored): w_varname = f.getname_w(varindex) @@ -656,9 +656,9 @@ f.space.delattr(w_obj, w_attributename) def STORE_GLOBAL(f, nameindex, *ignored): - w_varname = f.getname_w(nameindex) + varname = f.getname_u(nameindex) w_newvalue = f.popvalue() - f.space.set_str_keyed_item(f.w_globals, w_varname, w_newvalue) + f.space.set_str_keyed_item(f.w_globals, varname, w_newvalue) def DELETE_GLOBAL(f, nameindex, *ignored): w_varname = f.getname_w(nameindex) Modified: pypy/branch/shrink-multidict/pypy/jit/tl/pypyjit.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/jit/tl/pypyjit.py (original) +++ pypy/branch/shrink-multidict/pypy/jit/tl/pypyjit.py Wed Oct 28 16:21:35 2009 @@ -41,6 +41,7 @@ config.objspace.usemodules._weakref = False config.objspace.usemodules._sre = False set_pypy_opt_level(config, level='jit') +config.objspace.std.withinlineddict = True if BACKEND == 'c': config.objspace.std.multimethods = 'mrd' Modified: pypy/branch/shrink-multidict/pypy/jit/tl/pypyjit_demo.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/jit/tl/pypyjit_demo.py (original) +++ pypy/branch/shrink-multidict/pypy/jit/tl/pypyjit_demo.py Wed Oct 28 16:21:35 2009 @@ -1,7 +1,6 @@ base = object class Number(base): - __slots__ = ('val', ) def __init__(self, val=0): self.val = val @@ -28,10 +27,10 @@ return x import time -#t1 = time.time() -#f(10000000, Number(), 1) -#t2 = time.time() -#print t2 - t1 +t1 = time.time() +f(10000000, Number(), 1) +t2 = time.time() +print t2 - t1 t1 = time.time() f(10000000, 0, 1) t2 = time.time() Modified: pypy/branch/shrink-multidict/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/shrink-multidict/pypy/module/__builtin__/interp_classobj.py Wed Oct 28 16:21:35 2009 @@ -383,7 +383,7 @@ if w_meth is not None: space.call_function(w_meth, w_name, w_value) else: - self.setdictvalue(space, w_name, w_value) + self.setdictvalue(space, name, w_value) def descr_delattr(self, space, w_name): name = unwrap_attr(space, w_name) Modified: pypy/branch/shrink-multidict/pypy/objspace/descroperation.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/descroperation.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/descroperation.py Wed Oct 28 16:21:35 2009 @@ -54,7 +54,7 @@ space.set(w_descr, w_obj, w_value) return shadows_type = True - if w_obj.setdictvalue(space, w_name, w_value, shadows_type): + if w_obj.setdictvalue(space, name, w_value, shadows_type): return raiseattrerror(space, w_obj, name, w_descr) Modified: pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py Wed Oct 28 16:21:35 2009 @@ -46,12 +46,11 @@ def impl_setitem(self, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_str): - self.impl_setitem_str(w_key, w_value) + self.impl_setitem_str(self.space.str_w(w_key), w_value) else: self._as_rdict().setitem(w_key, w_value) - def impl_setitem_str(self, w_key, w_value, shadows_type=True): - name = self.space.str_w(w_key) + def impl_setitem_str(self, name, w_value, shadows_type=True): self.getcell(name).w_value = w_value if name in self.unshadowed_builtins: @@ -221,5 +220,5 @@ if cell is not None: f.cache_for_globals[nameindex] = cell return cell.w_value - return f._load_global(f.getname_w(nameindex)) + return f._load_global(f.getname_u(nameindex)) load_global_fill_cache._dont_inline_ = True Modified: pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py Wed Oct 28 16:21:35 2009 @@ -100,8 +100,8 @@ else: return None - def set_str_keyed_item(w_dict, w_key, w_value, shadows_type=True): - w_dict.setitem_str(w_key, w_value, shadows_type) + def set_str_keyed_item(w_dict, key, w_value, shadows_type=True): + w_dict.setitem_str(key, w_value, shadows_type) # _________________________________________________________________ # implementation methods @@ -113,7 +113,7 @@ #return w_value or None raise NotImplementedError("abstract base class") - def impl_setitem_str(self, w_key, w_value, shadows_type=True): + def impl_setitem_str(self, key, w_value, shadows_type=True): raise NotImplementedError("abstract base class") def impl_setitem(self, w_key, w_value): @@ -179,8 +179,8 @@ def impl_fallback_setitem(self, w_key, w_value): self.r_dict_content[w_key] = w_value - def impl_fallback_setitem_str(self, w_key, w_value, shadows_type=True): - return self.impl_fallback_setitem(w_key, w_value) + def impl_fallback_setitem_str(self, key, w_value, shadows_type=True): + return self.impl_fallback_setitem(self.space.wrap(key), w_value) def impl_fallback_delitem(self, w_key): del self.r_dict_content[w_key] @@ -309,12 +309,12 @@ def impl_setitem(self, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_str): - self.impl_setitem_str(w_key, w_value) + self.impl_setitem_str(self.space.str_w(w_key), w_value) else: self._as_rdict().setitem(w_key, w_value) - def impl_setitem_str(self, w_key, w_value, shadows_type=True): - self.content[self.space.str_w(w_key)] = w_value + def impl_setitem_str(self, key, w_value, shadows_type=True): + self.content[key] = w_value def impl_delitem(self, w_key): space = self.space @@ -396,11 +396,11 @@ else: self._shadows_anything = False - def impl_setitem_str(self, w_key, w_value, shadows_type=True): + def impl_setitem_str(self, key, w_value, shadows_type=True): if shadows_type: self._shadows_anything = True StrDictImplementation.impl_setitem_str( - self, w_key, w_value, shadows_type) + self, key, w_value, shadows_type) def impl_setitem(self, w_key, w_value): space = self.space @@ -410,7 +410,7 @@ if w_obj is not None: self._shadows_anything = True StrDictImplementation.impl_setitem_str( - self, w_key, w_value, False) + self, self.space.str_w(w_key), w_value, False) else: self._as_rdict().setitem(w_key, w_value) @@ -426,8 +426,7 @@ StrDictImplementation.__init__(self, space) self.shadowed = [None] * len(BUILTIN_TO_INDEX) - def impl_setitem_str(self, w_key, w_value, shadows_type=True): - key = self.space.str_w(w_key) + def impl_setitem_str(self, key, w_value, shadows_type=True): i = BUILTIN_TO_INDEX.get(key, -1) if i != -1: self.shadowed[i] = w_value @@ -557,9 +556,9 @@ self.info.writes += 1 self.content[w_key] = w_value self.info.maxcontents = max(self.info.maxcontents, len(self.content)) - def impl_setitem_str(self, w_key, w_value, shadows_type=True): + def impl_setitem_str(self, key, w_value, shadows_type=True): self.info.setitem_strs += 1 - self.impl_setitem(w_key, w_value) + self.impl_setitem(self.space.wrap(key), w_value) def impl_delitem(self, w_key): if not self.info.seen_non_string_in_write \ and not self.info.seen_non_string_in_read_first \ Modified: pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py Wed Oct 28 16:21:35 2009 @@ -92,10 +92,10 @@ def getdictvalue_attr_is_in_class(self, space, attr): return self.getdictvalue(space, attr) - def setdictvalue(self, space, w_attr, w_value, shadows_type=True): + def setdictvalue(self, space, attr, w_value, shadows_type=True): if self._inlined_dict_valid(): # XXX don't ignore shadows_type - self.impl_setitem(w_attr, w_value) + self.impl_setitem_str(attr, w_value) return True w_dict = self.getdict() w_dict.setitem(w_attr, w_value) Modified: pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py Wed Oct 28 16:21:35 2009 @@ -743,13 +743,13 @@ return w_obj.getitem(w_key) return ObjSpace.finditem(self, w_obj, w_key) - def set_str_keyed_item(self, w_obj, w_key, w_value, shadows_type=True): + def set_str_keyed_item(self, w_obj, key, w_value, shadows_type=True): # performance shortcut to avoid creating the OperationError(KeyError) if (isinstance(w_obj, self.DictObjectCls) and not w_obj.user_overridden_class): - w_obj.set_str_keyed_item(w_key, w_value, shadows_type) + w_obj.set_str_keyed_item(key, w_value, shadows_type) else: - self.setitem(w_obj, w_key, w_value) + self.setitem(w_obj, self.wrap(key), w_value) def getindex_w(self, w_obj, w_exception, objdescr=None): # Performance shortcut for the common case of w_obj being an int. Modified: pypy/branch/shrink-multidict/pypy/objspace/std/proxyobject.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/proxyobject.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/proxyobject.py Wed Oct 28 16:21:35 2009 @@ -43,10 +43,10 @@ raise return None - def setdictvalue(self, space, w_attr, w_value, shadows_type=True): + def setdictvalue(self, space, attr, w_value, shadows_type=True): try: space.call_function(self.w_controller, space.wrap('__setattr__'), - w_attr, w_value) + space.wrap(attr), w_value) return True except OperationError, e: if not e.match(space, space.w_AttributeError): @@ -67,7 +67,7 @@ return self.getdictvalue(self.space, '__dict__') def setdict(self, space, w_dict): - if not self.setdictvalue(space, space.wrap('__dict__'), w_dict): + if not self.setdictvalue(space, '__dict__', w_dict): baseobjspace.W_Root.setdict(self, space, w_dict) W_Transparent.__name__ = name Modified: pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py Wed Oct 28 16:21:35 2009 @@ -100,13 +100,12 @@ def impl_setitem(self, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_str): - self.impl_setitem_str(w_key, w_value) + self.impl_setitem_str(self.space.str_w(w_key), w_value) else: self._as_rdict().setitem(w_key, w_value) @unroll_safe - def impl_setitem_str(self, w_key, w_value, shadows_type=True): - key = self.space.str_w(w_key) + def impl_setitem_str(self, key, w_value, shadows_type=True): i = self.structure.lookup_position(key) if i != -1: self.entries[i] = w_value Modified: pypy/branch/shrink-multidict/pypy/objspace/std/test/test_inlinedict.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/test/test_inlinedict.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/test/test_inlinedict.py Wed Oct 28 16:21:35 2009 @@ -50,22 +50,6 @@ assert obj.getdictvalue(self.fakespace, "hello") == 4 assert obj.getdictvalue(self.fakespace, "world") == 5 - def test_dict_devolves_via_object(self): - obj = self.make_obj() - obj.setdictvalue(self.fakespace, 4, 1) - obj.setdictvalue(self.fakespace, 5, 2) - w_dict = obj.w__dict__ - assert w_dict is not None - assert dict(w_dict.r_dict_content) == {4: 1, 5: 2, "hello": 1, "world": 2} - assert obj.getdictvalue(self.fakespace, "hello") == 1 - assert obj.getdictvalue(self.fakespace, "world") == 2 - assert obj.getdictvalue(self.fakespace, 4) == 1 - assert obj.getdictvalue(self.fakespace, 5) == 2 - obj.deldictvalue(self.fakespace, "world") - assert obj.getdictvalue(self.fakespace, "world") is None - obj.deldictvalue(self.fakespace, "hello") - assert obj.getdictvalue(self.fakespace, "hello") is None - def test_dict_devolves_via_dict(self): obj = self.make_obj() w_dict = obj.getdict() Modified: pypy/branch/shrink-multidict/pypy/objspace/taint.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/taint.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/taint.py Wed Oct 28 16:21:35 2009 @@ -23,8 +23,8 @@ ## def getdictvalue(self, space, attr): ## return taint(self.w_obj.getdictvalue(space, attr)) -## def setdictvalue(self, space, w_attr, w_value): -## return self.w_obj.setdictvalue(space, w_attr, w_value) +## def setdictvalue(self, space, attr, w_value): +## return self.w_obj.setdictvalue(space, attr, w_value) ## ... Modified: pypy/branch/shrink-multidict/pypy/translator/goal/gcbench.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/translator/goal/gcbench.py (original) +++ pypy/branch/shrink-multidict/pypy/translator/goal/gcbench.py Wed Oct 28 16:21:35 2009 @@ -87,9 +87,21 @@ else: return Node(make_tree(depth-1), make_tree(depth-1)) +def get_memory_usage(): + import os + pid = os.getpid() + f = file("/proc/%s/status" % (pid, )) + lines = f.readlines() + f.close() + for l in lines: + if l.startswith("VmData:\t"): + return int(l[len("VmData:\t"):-len(" kB")]) + print "\n".join(lines) + assert 0 + def print_diagnostics(): "ought to print free/total memory" - pass + print "total memory:", get_memory_usage() def time_construction(depth): niters = num_iters(depth) From arigo at codespeak.net Wed Oct 28 16:33:26 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 16:33:26 +0100 (CET) Subject: [pypy-svn] r68822 - pypy/branch/logging/pypy/jit/backend/x86/test Message-ID: <20091028153326.CF0061683D7@codespeak.net> Author: arigo Date: Wed Oct 28 16:33:26 2009 New Revision: 68822 Modified: pypy/branch/logging/pypy/jit/backend/x86/test/test_zrpy_gc.py Log: Crash less obscurely than "Fatal RPython error: KeyError". Modified: pypy/branch/logging/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/logging/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/logging/pypy/jit/backend/x86/test/test_zrpy_gc.py Wed Oct 28 16:33:26 2009 @@ -5,7 +5,7 @@ soon as possible (at least in a simple case). """ -import weakref, random +import weakref, random, os import py from pypy.annotation import policy as annpolicy from pypy.rlib import rgc @@ -144,7 +144,11 @@ x.foo = 5 return weakref.ref(x) def main_allfuncs(name, n, x): - num = name_to_func[name] + try: + num = name_to_func[name] + except KeyError: + os.write(2, "unknown name: '" + name + "'\n") + raise n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s = funcs[num][0](n, x) while n > 0: myjitdriver.can_enter_jit(num=num, n=n, x=x, x0=x0, x1=x1, From arigo at codespeak.net Wed Oct 28 16:35:43 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 16:35:43 +0100 (CET) Subject: [pypy-svn] r68823 - in pypy/branch/logging/pypy: annotation translator/c translator/c/src Message-ID: <20091028153543.2C7541683D9@codespeak.net> Author: arigo Date: Wed Oct 28 16:35:42 2009 New Revision: 68823 Modified: pypy/branch/logging/pypy/annotation/annrpython.py pypy/branch/logging/pypy/translator/c/extfunc.py pypy/branch/logging/pypy/translator/c/genc.py pypy/branch/logging/pypy/translator/c/src/main.h pypy/branch/logging/pypy/translator/c/src/rtyper.h Log: Kill old code from the .h files and from extfunc.py. Building the list of strings of arguments cannot really be done in C; instead, write code that does it in RPython using rffi.charp2str(). Modified: pypy/branch/logging/pypy/annotation/annrpython.py ============================================================================== --- pypy/branch/logging/pypy/annotation/annrpython.py (original) +++ pypy/branch/logging/pypy/annotation/annrpython.py Wed Oct 28 16:35:42 2009 @@ -224,7 +224,12 @@ # graph -- it's already low-level operations! for a, s_newarg in zip(graph.getargs(), cells): s_oldarg = self.binding(a) - assert s_oldarg.contains(s_newarg) + assert (s_oldarg == s_newarg or + pair(s_oldarg, s_newarg).union() == s_oldarg) + # Note that we don't use s_oldarg.contains(s_newarg) here, + # to let merging occur. E.g. if the old function has an + # argument which is a list of strings, the new call site + # argument will be unified with it. else: assert not self.frozen for a in cells: Modified: pypy/branch/logging/pypy/translator/c/extfunc.py ============================================================================== --- pypy/branch/logging/pypy/translator/c/extfunc.py (original) +++ pypy/branch/logging/pypy/translator/c/extfunc.py Wed Oct 28 16:35:42 2009 @@ -30,52 +30,6 @@ if LIST_OF_STR is not None: yield ('RPyListOfString', LIST_OF_STR) -def predeclare_utility_functions(db, rtyper): - # Common utility functions - def RPyString_New(length=lltype.Signed): - return mallocstr(length) - - # !!! - # be extremely careful passing a gc tracked object - # from such an helper result to another one - # as argument, this could result in leaks - # Such result should be only from C code - # returned directly as results - - LIST_OF_STR = find_list_of_str(rtyper) - if LIST_OF_STR is not None: - p = lltype.Ptr(LIST_OF_STR) - - def _RPyListOfString_New(length=lltype.Signed): - return LIST_OF_STR.ll_newlist(length) - - def _RPyListOfString_New(length=lltype.Signed): - return LIST_OF_STR.ll_newlist(length) - - def _RPyListOfString_SetItem(l=p, - index=lltype.Signed, - newstring=lltype.Ptr(STR)): - rlist.ll_setitem_nonneg(rlist.dum_nocheck, l, index, newstring) - - def _RPyListOfString_GetItem(l=p, - index=lltype.Signed): - return rlist.ll_getitem_fast(l, index) - - def _RPyListOfString_Length(l=p): - return rlist.ll_length(l) - - for fname, f in locals().items(): - if isinstance(f, types.FunctionType): - # XXX this is painful :( - if (LIST_OF_STR, fname) in db.helper2ptr: - yield (fname, db.helper2ptr[LIST_OF_STR, fname]) - else: - # hack: the defaults give the type of the arguments - graph = rtyper.annotate_helper(f, f.func_defaults) - db.helper2ptr[LIST_OF_STR, fname] = graph - yield (fname, graph) - - def predeclare_extfuncs(db, rtyper): modules = {} def module_name(c_name): @@ -128,7 +82,6 @@ def predeclare_all(db, rtyper): for fn in [predeclare_common_types, - predeclare_utility_functions, predeclare_exception_data, predeclare_extfuncs, ]: @@ -138,7 +91,6 @@ def get_all(db, rtyper): for fn in [predeclare_common_types, - predeclare_utility_functions, predeclare_exception_data, predeclare_extfuncs, ]: Modified: pypy/branch/logging/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/logging/pypy/translator/c/genc.py (original) +++ pypy/branch/logging/pypy/translator/c/genc.py Wed Oct 28 16:35:42 2009 @@ -156,7 +156,7 @@ list(db.gcpolicy.gc_startup_code()) # build entrypoint and eventually other things to expose - pf = self.getentrypointptr() + pf = self.getentrypointptr(db) if isinstance(pf, list): for one_pf in pf: db.get(one_pf) @@ -214,7 +214,7 @@ if db is None: db = self.build_database() - pf = self.getentrypointptr() + pf = self.getentrypointptr(db) if self.modulename is None: self.modulename = uniquemodulename('testing') modulename = self.modulename @@ -306,7 +306,7 @@ _module = None _wrapper = None - def getentrypointptr(self): # xxx + def getentrypointptr(self, db): # xxx if self._wrapper is None: self._wrapper = new_wrapper(self.entrypoint, self.translator) return self._wrapper @@ -314,7 +314,7 @@ def compile(self): assert self.c_source_filename assert not self._compiled - export_symbols = [self.db.get(self.getentrypointptr()), + export_symbols = [self.db.get(self.getentrypointptr(self.db)), 'RPython_StartupCode', ] if self.config.translation.countmallocs: @@ -338,7 +338,7 @@ fname = 'wrap_' + self.c_source_filename.purebasename modfile = self.c_source_filename.new(purebasename=fname, ext=".py") - entrypoint_ptr = self.getentrypointptr() + entrypoint_ptr = self.getentrypointptr(self.db) wrapped_entrypoint_c_name = self.db.get(entrypoint_ptr) CODE = """ @@ -405,6 +405,7 @@ class CStandaloneBuilder(CBuilder): standalone = True executable_name = None + _ll_entrypoint_ptr = None def getprofbased(self): profbased = None @@ -424,11 +425,30 @@ return (profbased and isinstance(profbased, tuple) and profbased[0] is ProfOpt) - def getentrypointptr(self): - # XXX check that the entrypoint has the correct - # signature: list-of-strings -> int - bk = self.translator.annotator.bookkeeper - return getfunctionptr(bk.getdesc(self.entrypoint).getuniquegraph()) + def getentrypointptr(self, db): + if self._ll_entrypoint_ptr is None: + # + # Annotate and rtype a '_ll_entrypoint(int argc, char *argv[])' + # function, using rffi to decode the arguments into a real + # list of strings to call self.entrypoint(). + from pypy.annotation import model as annmodel + from pypy.rpython.lltypesystem import rffi + from pypy.rpython.annlowlevel import MixLevelHelperAnnotator + real_entrypoint = self.entrypoint + # + def _ll_entrypoint(argc, argv): + list = [rffi.charp2str(argv[i]) for i in range(argc)] + return real_entrypoint(list) + # + annhelper = MixLevelHelperAnnotator(db.translator.rtyper) + self._ll_entrypoint_ptr = annhelper.delayedfunction( + _ll_entrypoint, + [annmodel.SomeInteger(), + annmodel.lltype_to_annotation(rffi.CCHARPP)], + annmodel.SomeInteger()) + annhelper.finish() + # + return self._ll_entrypoint_ptr def cmdexec(self, args='', env=None): assert self._compiled Modified: pypy/branch/logging/pypy/translator/c/src/main.h ============================================================================== --- pypy/branch/logging/pypy/translator/c/src/main.h (original) +++ pypy/branch/logging/pypy/translator/c/src/main.h Wed Oct 28 16:35:42 2009 @@ -26,15 +26,7 @@ errmsg = RPython_StartupCode(); if (errmsg) goto error; - list = _RPyListOfString_New(argc); - if (RPyExceptionOccurred()) goto memory_out; - for (i=0; irs_chars.items, buf, length); - return rps; -} - #endif /* PYPY_NOT_MAIN_FILE */ From arigo at codespeak.net Wed Oct 28 16:42:39 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 16:42:39 +0100 (CET) Subject: [pypy-svn] r68824 - pypy/branch/logging/pypy/translator/c Message-ID: <20091028154239.0AA831683D9@codespeak.net> Author: arigo Date: Wed Oct 28 16:42:39 2009 New Revision: 68824 Modified: pypy/branch/logging/pypy/translator/c/genc.py Log: Rewrite this without using range() and not making a possibly variable-sized list. Modified: pypy/branch/logging/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/logging/pypy/translator/c/genc.py (original) +++ pypy/branch/logging/pypy/translator/c/genc.py Wed Oct 28 16:42:39 2009 @@ -437,7 +437,11 @@ real_entrypoint = self.entrypoint # def _ll_entrypoint(argc, argv): - list = [rffi.charp2str(argv[i]) for i in range(argc)] + list = [''] * argc + i = 0 + while i < argc: + list[i] = rffi.charp2str(argv[i]) + i += 1 return real_entrypoint(list) # annhelper = MixLevelHelperAnnotator(db.translator.rtyper) From cfbolz at codespeak.net Wed Oct 28 17:44:38 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 28 Oct 2009 17:44:38 +0100 (CET) Subject: [pypy-svn] r68825 - pypy/branch/shrink-multidict/lib-python/modified-2.5.2/test Message-ID: <20091028164438.4C01C168476@codespeak.net> Author: cfbolz Date: Wed Oct 28 17:44:37 2009 New Revision: 68825 Modified: pypy/branch/shrink-multidict/lib-python/modified-2.5.2/test/test_repr.py Log: fix test to match 68763 Modified: pypy/branch/shrink-multidict/lib-python/modified-2.5.2/test/test_repr.py ============================================================================== --- pypy/branch/shrink-multidict/lib-python/modified-2.5.2/test/test_repr.py (original) +++ pypy/branch/shrink-multidict/lib-python/modified-2.5.2/test/test_repr.py Wed Oct 28 17:44:37 2009 @@ -148,8 +148,7 @@ # Functions eq(repr(hash), '') # Methods - self.failUnless(repr(''.split).find( - "bound method str.split of '' at 0x") > -1) + self.assertEquals(repr(''.split), "") def test_xrange(self): import warnings From cfbolz at codespeak.net Wed Oct 28 17:51:49 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 28 Oct 2009 17:51:49 +0100 (CET) Subject: [pypy-svn] r68826 - pypy/trunk/pypy/interpreter/test Message-ID: <20091028165149.CDBF2168476@codespeak.net> Author: cfbolz Date: Wed Oct 28 17:51:49 2009 New Revision: 68826 Modified: pypy/trunk/pypy/interpreter/test/test_function.py Log: fix XXX from rev 4872 Modified: pypy/trunk/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_function.py (original) +++ pypy/trunk/pypy/interpreter/test/test_function.py Wed Oct 28 17:51:49 2009 @@ -16,7 +16,7 @@ assert f.func_defaults == None assert f.func_dict == {} assert type(f.func_globals) == dict - #self.assertEquals(f.func_closure, None) XXX + assert f.func_closure is None assert f.func_doc == None assert f.func_name == 'f' assert f.__module__ == 'mymodulename' From arigo at codespeak.net Wed Oct 28 17:53:47 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 17:53:47 +0100 (CET) Subject: [pypy-svn] r68827 - pypy/branch/logging/pypy/rlib Message-ID: <20091028165347.DCBEE168476@codespeak.net> Author: arigo Date: Wed Oct 28 17:53:47 2009 New Revision: 68827 Modified: pypy/branch/logging/pypy/rlib/rlog.py pypy/branch/logging/pypy/rlib/rlog_ll.py Log: Add syntax: 'PYPYLOG=+filename' means 'always force a flush'. Obscure for now, will document it is proves useful. Modified: pypy/branch/logging/pypy/rlib/rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog.py (original) +++ pypy/branch/logging/pypy/rlib/rlog.py Wed Oct 28 17:53:47 2009 @@ -185,7 +185,7 @@ self.logwriter = logwriter types = unrolling_iterable(self.types) # - def call(*args): + def really_call(*args): if NonConstant(False): logwriter.add_subentry_d(NonConstant(-123)) logwriter.add_subentry_s(NonConstant('abc')) @@ -196,8 +196,6 @@ logwriter.add_subentry_r(llstr('abc')) logwriter.add_subentry_f(NonConstant(123.4)) # ^^^ annotation hacks - if not logwriter.enabled: - return if not logwriter.add_entry(self): return i = 0 @@ -205,8 +203,16 @@ methname = 'add_subentry_' + typechar getattr(logwriter, methname)(args[i]) i = i + 1 - call = func_with_new_name(call, 'debug_log_' + self.category) + if logwriter.always_flush: + logwriter._flush() + really_call = func_with_new_name(really_call, + 'debug_log_' + self.category) + # + def call(*args): + if logwriter.enabled: + really_call(*args) call._always_inline_ = True + # self.call = call else: assert self.logwriter is logwriter @@ -215,6 +221,7 @@ class AbstractLogWriter(object): get_time = time.time + always_flush = False def __init__(self): self.enabled = True Modified: pypy/branch/logging/pypy/rlib/rlog_ll.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog_ll.py (original) +++ pypy/branch/logging/pypy/rlib/rlog_ll.py Wed Oct 28 17:53:47 2009 @@ -31,6 +31,9 @@ def do_open_file(self): l_result = self.ll_get_filename() if l_result and l_result[0] != '\x00': + if l_result[0] == '+': + self.always_flush = True + l_result = rffi.ptradd(l_result, 1) flags = rffi.cast(rffi.INT, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) mode = rffi.cast(rffi.MODE_T, 0666) self.fd = rffi.cast(lltype.Signed, os_open(l_result, flags, mode)) From cfbolz at codespeak.net Wed Oct 28 18:19:54 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 28 Oct 2009 18:19:54 +0100 (CET) Subject: [pypy-svn] r68828 - pypy/trunk/pypy/interpreter/test Message-ID: <20091028171954.7EAAC16849D@codespeak.net> Author: cfbolz Date: Wed Oct 28 18:19:54 2009 New Revision: 68828 Modified: pypy/trunk/pypy/interpreter/test/test_function.py Log: a test about writing the func_code attribute of functions Modified: pypy/trunk/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_function.py (original) +++ pypy/trunk/pypy/interpreter/test/test_function.py Wed Oct 28 18:19:54 2009 @@ -70,6 +70,22 @@ f = g(42) raises(TypeError, FuncType, f.func_code, f.func_globals, 'f2', None, None) + def test_write_code(self): + def f(): + return 42 + def g(): + return 41 + assert f() == 42 + assert g() == 41 + raises(TypeError, "f.func_code = 1") + f.func_code = g.func_code + assert f() == 41 + def h(): + return f() # a closure + raises(ValueError, "f.func_code = h.func_code") + + + class AppTestFunction: def test_simple_call(self): def func(arg1, arg2): From arigo at codespeak.net Wed Oct 28 19:38:37 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 19:38:37 +0100 (CET) Subject: [pypy-svn] r68832 - pypy/branch/logging/pypy/rpython/module Message-ID: <20091028183837.1A7FE1684A9@codespeak.net> Author: arigo Date: Wed Oct 28 19:38:36 2009 New Revision: 68832 Modified: pypy/branch/logging/pypy/rpython/module/ll_time.py Log: Partly revert r47057, which was as far as I can tell an attempt to speed up things by not releasing the GIL. Makes no sense for _nowrapper external calls. Modified: pypy/branch/logging/pypy/rpython/module/ll_time.py ============================================================================== --- pypy/branch/logging/pypy/rpython/module/ll_time.py (original) +++ pypy/branch/logging/pypy/rpython/module/ll_time.py Wed Oct 28 19:38:36 2009 @@ -76,11 +76,11 @@ if self.GETTIMEOFDAY_NO_TZ: c_gettimeofday = self.llexternal('gettimeofday', [self.TIMEVALP], rffi.INT, - _nowrapper=True, threadsafe=False) + _nowrapper=True) else: c_gettimeofday = self.llexternal('gettimeofday', [self.TIMEVALP, rffi.VOIDP], rffi.INT, - _nowrapper=True, threadsafe=False) + _nowrapper=True) else: c_gettimeofday = None @@ -88,12 +88,12 @@ self.configure(CConfigForFTime) c_ftime = self.llexternal(FTIME, [lltype.Ptr(self.TIMEB)], lltype.Void, - _nowrapper=True, threadsafe=False) + _nowrapper=True) else: c_ftime = None # to not confuse the flow space c_time = self.llexternal('time', [rffi.VOIDP], rffi.TIME_T, - _nowrapper=True, threadsafe=False) + _nowrapper=True) def time_time_llimpl(): void = lltype.nullptr(rffi.VOIDP.TO) From arigo at codespeak.net Wed Oct 28 19:50:44 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 19:50:44 +0100 (CET) Subject: [pypy-svn] r68833 - pypy/branch/logging/pypy/rlib Message-ID: <20091028185044.571071684A9@codespeak.net> Author: arigo Date: Wed Oct 28 19:50:43 2009 New Revision: 68833 Modified: pypy/branch/logging/pypy/rlib/rlog_parsing.py Log: Enhance the command-line interface. Modified: pypy/branch/logging/pypy/rlib/rlog_parsing.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog_parsing.py (original) +++ pypy/branch/logging/pypy/rlib/rlog_parsing.py Wed Oct 28 19:50:43 2009 @@ -1,5 +1,15 @@ +""" +Utilities to parse a log file. When used from the command-line, +the syntax is: + + python rlog_parsing.py [-f|--follow] [-l|--limit=..] logfile + + -f, --follow wait for file growth and display it too + -l, --limit=CAT only shows log entries of category 'CAT*' + (supports * and ? special characters) +""" import autopath -import struct, re, fnmatch +import struct, re, fnmatch, time from pypy.rlib.rarithmetic import intmask from pypy.rlib import rlog @@ -86,8 +96,28 @@ logparser = LogParser(open(filename, 'rb')) return logparser.enum_entries() -def dump_log(filename, limit='*', highlight=False): - for curtime, cat, entries in parse_log(filename): + +class FollowFile(object): + def __init__(self, f): + self.tell = f.tell + self.seek = f.seek + self._read = f.read + def read(self, size): + buf = self._read(size) + try: + while len(buf) < size: + time.sleep(1) + buf += self._read(size - len(buf)) + except KeyboardInterrupt: + sys.exit(0) + return buf + +def dump_log(filename, limit='*', highlight=False, follow=False): + f = open(filename, 'rb') + if follow: + f = FollowFile(f) + logparser = LogParser(f) + for curtime, cat, entries in logparser.enum_entries(): if not fnmatch.fnmatch(cat.category, limit): continue try: @@ -121,11 +151,15 @@ if __name__ == '__main__': - import sys - filename = sys.argv[1] - if len(sys.argv) > 2: - limit = sys.argv[2] + '*' - else: - limit = '*' + import sys, getopt + options, args = getopt.gnu_getopt(sys.argv[1:], 'l:f', + ['limit=', 'follow']) + if len(args) != 1: + print __doc__ + sys.exit(2) + [filename] = args + options = dict(options) + limit = options.get('-l', options.get('--limit', '')) + '*' + follow = '-f' in options or '--follow' in options highlight = sys.stdout.isatty() - dump_log(filename, limit, highlight) + dump_log(filename, limit=limit, highlight=highlight, follow=follow) From arigo at codespeak.net Wed Oct 28 19:51:01 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 19:51:01 +0100 (CET) Subject: [pypy-svn] r68834 - pypy/branch/logging/pypy/rlib Message-ID: <20091028185101.07F3A1684A9@codespeak.net> Author: arigo Date: Wed Oct 28 19:51:00 2009 New Revision: 68834 Modified: pypy/branch/logging/pypy/rlib/rlog.py Log: argh argh, can't use '*args' in RPython if we want to be sure that we don't use the GC on lltype. Modified: pypy/branch/logging/pypy/rlib/rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog.py (original) +++ pypy/branch/logging/pypy/rlib/rlog.py Wed Oct 28 19:51:00 2009 @@ -1,6 +1,5 @@ import py, time, struct from pypy.tool.ansi_print import ansi_log -from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.nonconst import NonConstant from pypy.tool.sourcetools import func_with_new_name from pypy.rpython.extregistry import ExtRegistryEntry @@ -183,37 +182,39 @@ def gen_call(self, logwriter): if self.call is None: self.logwriter = logwriter - types = unrolling_iterable(self.types) + # argh argh, can't use '*args' in RPython if we want to be + # sure that we don't use the GC on lltype + argnames = ', '.join([name for name, typechar in self.entries]) # - def really_call(*args): - if NonConstant(False): - logwriter.add_subentry_d(NonConstant(-123)) - logwriter.add_subentry_s(NonConstant('abc')) - logwriter.add_subentry_s(None) - logwriter.add_subentry_s(llstr('abc')) - logwriter.add_subentry_r(NonConstant('abc')) - logwriter.add_subentry_r(None) - logwriter.add_subentry_r(llstr('abc')) - logwriter.add_subentry_f(NonConstant(123.4)) - # ^^^ annotation hacks - if not logwriter.add_entry(self): - return - i = 0 - for typechar in types: - methname = 'add_subentry_' + typechar - getattr(logwriter, methname)(args[i]) - i = i + 1 - if logwriter.always_flush: - logwriter._flush() - really_call = func_with_new_name(really_call, - 'debug_log_' + self.category) + source = """ + def really_call(%s): + if not logwriter.add_entry(cat): + return + """ % argnames # - def call(*args): - if logwriter.enabled: - really_call(*args) - call._always_inline_ = True + for name, typechar in self.entries: + source += """ + logwriter.add_subentry_%s(%s) + """ % (typechar, name) # - self.call = call + source += """ + if logwriter.always_flush: + logwriter._flush() + + really_call = func_with_new_name(really_call, + 'debug_log_' + cat.category) + + def call(%s): + if logwriter.enabled: + really_call(%s) + call._always_inline_ = True + """ % (argnames, argnames) + # + miniglobals = {'logwriter': logwriter, + 'cat': self, + 'func_with_new_name': func_with_new_name} + exec py.code.Source(source).compile() in miniglobals + self.call = miniglobals['call'] else: assert self.logwriter is logwriter return self.call @@ -262,6 +263,16 @@ define_new_category._dont_inline_ = True def add_entry(self, cat): + if NonConstant(False): + self.add_subentry_d(NonConstant(-123)) + self.add_subentry_s(NonConstant('abc')) + self.add_subentry_s(None) + self.add_subentry_s(llstr('abc')) + self.add_subentry_r(NonConstant('abc')) + self.add_subentry_r(None) + self.add_subentry_r(llstr('abc')) + self.add_subentry_f(NonConstant(123.4)) + # ^^^ annotation hacks if cat.seen_by is not self: self.define_new_category(cat) if not self.enabled: From arigo at codespeak.net Wed Oct 28 20:43:08 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 28 Oct 2009 20:43:08 +0100 (CET) Subject: [pypy-svn] r68835 - in pypy/branch/logging/pypy: rlib rlib/test rpython Message-ID: <20091028194308.D82EA1684A9@codespeak.net> Author: arigo Date: Wed Oct 28 20:43:08 2009 New Revision: 68835 Modified: pypy/branch/logging/pypy/rlib/rlog.py pypy/branch/logging/pypy/rlib/rlog_ll.py pypy/branch/logging/pypy/rlib/test/test_rlog.py pypy/branch/logging/pypy/rpython/llinterp.py Log: Managed to write a test that checks that rlog does not use the GC. Fixed a few remaining places thus found. Phew. Modified: pypy/branch/logging/pypy/rlib/rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog.py (original) +++ pypy/branch/logging/pypy/rlib/rlog.py Wed Oct 28 20:43:08 2009 @@ -163,6 +163,7 @@ class LogCategory(object): + _alloc_flavor_ = 'raw' seen_by = None def __init__(self, category, message, index): @@ -221,6 +222,7 @@ class AbstractLogWriter(object): + _alloc_flavor_ = "raw" get_time = time.time always_flush = False @@ -240,8 +242,11 @@ # write the header if self.enabled: self.create_buffer() - for c in 'RLog\n': - self.write_int(ord(c)) + self.write_int(ord('R')) + self.write_int(ord('L')) + self.write_int(ord('o')) + self.write_int(ord('g')) + self.write_int(ord('\n')) # Write two numbers at the start, to ensure that the log is # considered invalid on machines with different endianness # or word size. They also play the role of version numbers. Modified: pypy/branch/logging/pypy/rlib/rlog_ll.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog_ll.py (original) +++ pypy/branch/logging/pypy/rlib/rlog_ll.py Wed Oct 28 20:43:08 2009 @@ -8,10 +8,10 @@ os_write = rffi.llexternal(underscore_on_windows+'write', [rffi.INT, rffi.CCHARP, rffi.SIZE_T], - rffi.SIZE_T) + rffi.SIZE_T, _nowrapper=True) os_open = rffi.llexternal(underscore_on_windows+'open', [rffi.CCHARP, rffi.INT, rffi.MODE_T], - rffi.INT) + rffi.INT, _nowrapper=True) l_pypylog = rffi.str2charp('PYPYLOG') @@ -21,6 +21,7 @@ # class LLLogWriter(AbstractLogWriter): + _alloc_flavor_ = "raw" BUFSIZE = 8192 fd = -1 Modified: pypy/branch/logging/pypy/rlib/test/test_rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/test/test_rlog.py (original) +++ pypy/branch/logging/pypy/rlib/test/test_rlog.py Wed Oct 28 20:43:08 2009 @@ -2,7 +2,7 @@ from pypy.rlib import rlog, rlog_ll, rlog_parsing from pypy.rlib.rarithmetic import intmask from pypy.tool.udir import udir -from pypy.rpython.test.test_llinterp import interpret +from pypy.rpython.test.test_llinterp import interpret, get_interpreter def test_log_direct(): @@ -267,7 +267,9 @@ assert entries[5][2] == ['(null)'] def test_interpret_f(self): - interpret(self.f.im_func, [132], malloc_check=False) + interp, graph = get_interpreter(self.f.im_func, [132]) + interp.heap = None # make sure this does not use any GC operation + interp.eval_graph(graph, [132]) self.check_result() def test_interpret_g(self): Modified: pypy/branch/logging/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/logging/pypy/rpython/llinterp.py (original) +++ pypy/branch/logging/pypy/rpython/llinterp.py Wed Oct 28 20:43:08 2009 @@ -604,9 +604,13 @@ def op_setfield(self, obj, fieldname, fieldvalue): # obj should be pointer - FIELDTYPE = getattr(lltype.typeOf(obj).TO, fieldname) - if FIELDTYPE is not lltype.Void: + STRUCT = lltype.typeOf(obj).TO + FIELDTYPE = getattr(STRUCT, fieldname) + if (STRUCT._gckind == 'gc' and isinstance(FIELDTYPE, lltype.Ptr) + and FIELDTYPE.TO._gckind == 'gc'): self.heap.setfield(obj, fieldname, fieldvalue) + elif FIELDTYPE is not lltype.Void: + setattr(obj, fieldname, fieldvalue) def op_bare_setfield(self, obj, fieldname, fieldvalue): # obj should be pointer @@ -654,9 +658,13 @@ def op_setarrayitem(self, array, index, item): # array should be a pointer - ITEMTYPE = lltype.typeOf(array).TO.OF - if ITEMTYPE is not lltype.Void: + ARRAY = lltype.typeOf(array).TO + ITEMTYPE = ARRAY.OF + if (ARRAY._gckind == 'gc' and isinstance(ITEMTYPE, lltype.Ptr) + and ITEMTYPE.TO._gckind == 'gc'): self.heap.setarrayitem(array, index, item) + elif ITEMTYPE is not lltype.Void: + array[index] = item def op_bare_setarrayitem(self, array, index, item): # array should be a pointer @@ -721,9 +729,12 @@ result = self.heap.malloc(obj, zero=zero, flavor='raw') self.alloca_objects.append(result) return result - ptr = self.heap.malloc(obj, zero=zero, flavor=flavor) - if flavor == 'raw' and self.llinterpreter.malloc_check: - self.llinterpreter.remember_malloc(ptr, self) + if flavor == 'raw': + ptr = lltype.malloc(obj, zero=zero, flavor=flavor) + if self.llinterpreter.malloc_check: + self.llinterpreter.remember_malloc(ptr, self) + else: + ptr = self.heap.malloc(obj, zero=zero, flavor=flavor) return ptr def op_malloc_varsize(self, obj, flags, size): @@ -731,9 +742,12 @@ zero = flags.get('zero', False) assert flavor in ('gc', 'raw') try: - ptr = self.heap.malloc(obj, size, zero=zero, flavor=flavor) - if flavor == 'raw' and self.llinterpreter.malloc_check: - self.llinterpreter.remember_malloc(ptr, self) + if flavor == 'raw': + ptr = lltype.malloc(obj, size, zero=zero, flavor=flavor) + if self.llinterpreter.malloc_check: + self.llinterpreter.remember_malloc(ptr, self) + else: + ptr = self.heap.malloc(obj, size, zero=zero, flavor=flavor) return ptr except MemoryError: self.make_llexception() @@ -761,9 +775,12 @@ def op_free(self, obj, flavor): assert isinstance(flavor, str) - if flavor == 'raw' and self.llinterpreter.malloc_check: - self.llinterpreter.remember_free(obj) - self.heap.free(obj, flavor=flavor) + if flavor == 'raw': + if self.llinterpreter.malloc_check: + self.llinterpreter.remember_free(obj) + lltype.free(obj, flavor=flavor) + else: + self.heap.free(obj, flavor=flavor) def op_zero_gc_pointers_inside(self, obj): raise NotImplementedError("zero_gc_pointers_inside") From arigo at codespeak.net Thu Oct 29 11:21:25 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 29 Oct 2009 11:21:25 +0100 (CET) Subject: [pypy-svn] r68837 - pypy/branch/logging/pypy/jit/metainterp/test Message-ID: <20091029102125.74C03168471@codespeak.net> Author: arigo Date: Thu Oct 29 11:21:23 2009 New Revision: 68837 Modified: pypy/branch/logging/pypy/jit/metainterp/test/oparser.py Log: Remove this outdated function. Modified: pypy/branch/logging/pypy/jit/metainterp/test/oparser.py ============================================================================== --- pypy/branch/logging/pypy/jit/metainterp/test/oparser.py (original) +++ pypy/branch/logging/pypy/jit/metainterp/test/oparser.py Thu Oct 29 11:21:23 2009 @@ -322,23 +322,6 @@ return parse(*args, **kwds) -def split_logs_into_loops(text): - lines = text.splitlines() - parts = [] - last_with_hash = False - lines_of_part = [] - for i, line in enumerate(lines): - if (line.startswith("[") and last_with_hash - and len(lines_of_part) > 1): - parts.append("\n".join(lines_of_part[:-1])) - lines_of_part = [lines_of_part[-1], line] - else: - lines_of_part.append(line) - last_with_hash = line.startswith("#") - parts.append("\n".join(lines_of_part)) - return parts - - def _box_counter_more_than(s): if s.isdigit(): Box._counter = max(Box._counter, int(s)+1) From arigo at codespeak.net Thu Oct 29 11:22:19 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 29 Oct 2009 11:22:19 +0100 (CET) Subject: [pypy-svn] r68838 - in pypy/branch/logging/pypy: rpython/memory/test translator/c Message-ID: <20091029102219.C3E63168471@codespeak.net> Author: arigo Date: Thu Oct 29 11:22:19 2009 New Revision: 68838 Modified: pypy/branch/logging/pypy/rpython/memory/test/test_transformed_gc.py pypy/branch/logging/pypy/translator/c/genc.py Log: Urgh. test_transformed_gc used to make a CStandaloneBuilder, but the entry point was not taking a list of strings at all. Fixed by adding a minimal CTestBuilder. Modified: pypy/branch/logging/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/logging/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/logging/pypy/rpython/memory/test/test_transformed_gc.py Thu Oct 29 11:22:19 2009 @@ -98,7 +98,7 @@ cleanup() return res - from pypy.translator.c.genc import CStandaloneBuilder + from pypy.translator.c.genc import CTestBuilder s_args = annmodel.SomePtr(lltype.Ptr(ARGS)) t = rtype(entrypoint, [s_args], gcname=cls.gcname, @@ -109,11 +109,10 @@ if fixup: fixup(t) - cbuild = CStandaloneBuilder(t, entrypoint, config=t.config, - gcpolicy=cls.gcpolicy) + cbuild = CTestBuilder(t, entrypoint, config=t.config, + gcpolicy=cls.gcpolicy) db = cbuild.generate_graphs_for_llinterp() - entrypointptr = cbuild.getentrypointptr() - entrygraph = entrypointptr._obj.graph + entrygraph = t._graphof(cbuild.entrypoint) if conftest.option.view: t.viewcg() Modified: pypy/branch/logging/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/logging/pypy/translator/c/genc.py (original) +++ pypy/branch/logging/pypy/translator/c/genc.py Thu Oct 29 11:22:19 2009 @@ -301,6 +301,14 @@ pass +class CTestBuilder(CBuilder): + # only for tests + standalone = True + def getentrypointptr(self, db): + bk = self.translator.annotator.bookkeeper + return getfunctionptr(bk.getdesc(self.entrypoint).getuniquegraph()) + + class CExtModuleBuilder(CBuilder): standalone = False _module = None From arigo at codespeak.net Thu Oct 29 11:33:04 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 29 Oct 2009 11:33:04 +0100 (CET) Subject: [pypy-svn] r68839 - pypy/branch/logging/pypy/rpython/memory/gc Message-ID: <20091029103304.DED77168471@codespeak.net> Author: arigo Date: Thu Oct 29 11:33:04 2009 New Revision: 68839 Modified: pypy/branch/logging/pypy/rpython/memory/gc/generation.py pypy/branch/logging/pypy/rpython/memory/gc/hybrid.py pypy/branch/logging/pypy/rpython/memory/gc/semispace.py Log: Check against using a helper (like rlog.debug_log()) in the GC that would end up mallocating more objects using the GC. Done with a flag to detect recursion. Modified: pypy/branch/logging/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/logging/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/logging/pypy/rpython/memory/gc/generation.py Thu Oct 29 11:33:04 2009 @@ -328,10 +328,16 @@ def collect_nursery(self): if self.nursery_size > self.top_of_space - self.free: # the semispace is running out, do a full collect + # (also handles the case of a recursive collect done by mistake) self.obtain_free_space(self.nursery_size) ll_assert(self.nursery_size <= self.top_of_space - self.free, "obtain_free_space failed to do its job") if self.nursery: + ll_assert(not self.collection_in_progress, + "should not reach this point") + self.collection_in_progress = True + old_top_of_space = self.top_of_space + self.top_of_space = self.free # temp., to detect recursion rlog.debug_log("gc-minor-{", ".--- minor collect ---") # a nursery-only collection scan = beginning = self.free @@ -351,6 +357,8 @@ "`------ survived (fraction of the size): %(survived)f", oldobj = oldobj_count, survived = float(scan - beginning) / self.nursery_size) + self.top_of_space = old_top_of_space + self.collection_in_progress = False #self.debug_check_consistency() # -- quite expensive else: # no nursery - this occurs after a full collect, triggered either Modified: pypy/branch/logging/pypy/rpython/memory/gc/hybrid.py ============================================================================== --- pypy/branch/logging/pypy/rpython/memory/gc/hybrid.py (original) +++ pypy/branch/logging/pypy/rpython/memory/gc/hybrid.py Thu Oct 29 11:33:04 2009 @@ -182,6 +182,8 @@ # In order to keep malloc_varsize_clear() as compact as possible, # we recompute what we need in this slow path instead of passing # it all as function arguments. + ll_assert(not self.collection_in_progress, + "malloc_varsize_slowpath() while a collection is in progress") size_gc_header = self.gcheaderbuilder.size_gc_header nonvarsize = size_gc_header + self.fixed_size(typeid) itemsize = self.varsize_item_sizes(typeid) @@ -222,6 +224,8 @@ return llmemory.cast_ptr_to_adr(gcref) def realloc(self, ptr, newlength, fixedsize, itemsize, lengthofs, grow): + ll_assert(not self.collection_in_progress, + "realloc() while a collection is in progress") size_gc_header = self.size_gc_header() addr = llmemory.cast_ptr_to_adr(ptr) ll_assert(self.header(addr).tid & GCFLAG_EXTERNAL, Modified: pypy/branch/logging/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/logging/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/logging/pypy/rpython/memory/gc/semispace.py Thu Oct 29 11:33:04 2009 @@ -61,6 +61,7 @@ self.space_size = self.param_space_size self.max_space_size = self.param_max_space_size self.red_zone = 0 + self.collection_in_progress = False self.program_start_time = time.time() self.tospace = llarena.arena_malloc(self.space_size, True) @@ -213,6 +214,18 @@ # (this is also a hook for the HybridGC) def semispace_collect(self, size_changing=False): + ll_assert(not self.collection_in_progress, + "allocating memory while the GC is collecting!") + self.collection_in_progress = True + # temporarily set 'top_of_space' to a value that will prevent + # all mallocs from succeeding and cause semispace_collect() + # to be called again -- and hitting the ll_assert() above. + tospace = self.fromspace + fromspace = self.tospace + self.fromspace = fromspace + self.tospace = tospace + self.top_of_space = tospace + if rlog.has_log(): start_usage = self.free - self.tospace start_time = time.time() @@ -225,17 +238,11 @@ else: start_time = 0 # Help the flow space start_usage = 0 # Help the flow space - #llop.debug_print(lltype.Void, 'semispace_collect', int(size_changing)) # Switch the spaces. We copy everything over to the empty space # (self.fromspace at the beginning of the collection), and clear the old # one (self.tospace at the beginning). Their purposes will be reversed # for the next collection. - tospace = self.fromspace - fromspace = self.tospace - self.fromspace = fromspace - self.tospace = tospace - self.top_of_space = tospace + self.space_size scan = self.free = tospace self.starting_full_collect() self.collect_roots() @@ -248,6 +255,9 @@ self.invalidate_weakrefs() self.update_objects_with_id() self.finished_full_collect() + self.top_of_space = self.tospace + self.space_size + ll_assert(self.free <= self.top_of_space, "space overflowed") + self.collection_in_progress = False self.debug_check_consistency() if not size_changing: llarena.arena_reset(fromspace, self.space_size, True) @@ -601,6 +611,8 @@ def identityhash(self, gcobj): # The following code should run at most twice. while 1: + ll_assert(not self.collection_in_progress, + "identityhash() while the GC is collecting!") obj = llmemory.cast_ptr_to_adr(gcobj) hdr = self.header(obj) # From arigo at codespeak.net Thu Oct 29 11:40:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 29 Oct 2009 11:40:42 +0100 (CET) Subject: [pypy-svn] r68840 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091029104042.EC260168495@codespeak.net> Author: arigo Date: Thu Oct 29 11:40:42 2009 New Revision: 68840 Modified: pypy/trunk/pypy/jit/metainterp/resume.py Log: Remove this old flag, not used any more. Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Thu Oct 29 11:40:42 2009 @@ -90,8 +90,6 @@ UNASSIGNED = tag(-1, TAGBOX) NULLREF = tag(-1, TAGCONST) -VIRTUAL_FLAG = int((sys.maxint+1) // 2) -assert not (VIRTUAL_FLAG & (VIRTUAL_FLAG-1)) # a power of two class ResumeDataLoopMemo(object): From arigo at codespeak.net Thu Oct 29 11:51:22 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 29 Oct 2009 11:51:22 +0100 (CET) Subject: [pypy-svn] r68841 - pypy/branch/logging/pypy/rlib Message-ID: <20091029105122.66983168495@codespeak.net> Author: arigo Date: Thu Oct 29 11:51:20 2009 New Revision: 68841 Modified: pypy/branch/logging/pypy/rlib/rlog_ll.py Log: Mumble mumble. Must also read this env var with _nowrapper=True. Modified: pypy/branch/logging/pypy/rlib/rlog_ll.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog_ll.py (original) +++ pypy/branch/logging/pypy/rlib/rlog_ll.py Thu Oct 29 11:51:20 2009 @@ -3,7 +3,6 @@ from pypy.rlib.rarithmetic import r_uint, r_singlefloat from pypy.rlib.rlog import AbstractLogWriter, SIZEOF_FLOAT from pypy.rpython.lltypesystem import lltype, rffi -from pypy.rpython.module.ll_os_environ import os_getenv from pypy.rpython.module.ll_os import underscore_on_windows os_write = rffi.llexternal(underscore_on_windows+'write', @@ -12,6 +11,8 @@ os_open = rffi.llexternal(underscore_on_windows+'open', [rffi.CCHARP, rffi.INT, rffi.MODE_T], rffi.INT, _nowrapper=True) +os_getenv = rffi.llexternal('getenv', [rffi.CCHARP], rffi.CCHARP, + _nowrapper=True) l_pypylog = rffi.str2charp('PYPYLOG') From fijal at codespeak.net Thu Oct 29 11:55:52 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 29 Oct 2009 11:55:52 +0100 (CET) Subject: [pypy-svn] r68842 - pypy/branch/logging/pypy/rlib Message-ID: <20091029105552.452E216849C@codespeak.net> Author: fijal Date: Thu Oct 29 11:55:51 2009 New Revision: 68842 Modified: pypy/branch/logging/pypy/rlib/rlog.py Log: typo, I think I can see what is armin thinking about :) Modified: pypy/branch/logging/pypy/rlib/rlog.py ============================================================================== --- pypy/branch/logging/pypy/rlib/rlog.py (original) +++ pypy/branch/logging/pypy/rlib/rlog.py Thu Oct 29 11:55:51 2009 @@ -12,7 +12,7 @@ def has_log(): - """Check if logging is enabled. If translated with --no-rjit, this + """Check if logging is enabled. If translated with --no-rlog, this returns the constant False. Otherwise, it returns True or False depending on the presence of the PYPYLOG env var. (Note that the debug_log() function also checks has_log() by itself, so it's only @@ -21,7 +21,7 @@ return True def debug_log(_category, _message, **_kwds): - """Logging main function. If translated with --no-rjit, all calls to + """Logging main function. If translated with --no-rlog, all calls to this function are ignored. Otherwise, if the PYPYLOG env var is set, it records an entry in the log. Arguments: From arigo at codespeak.net Thu Oct 29 12:31:33 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 29 Oct 2009 12:31:33 +0100 (CET) Subject: [pypy-svn] r68843 - pypy/branch/logging/pypy/jit/metainterp Message-ID: <20091029113133.6D28B16849D@codespeak.net> Author: arigo Date: Thu Oct 29 12:31:32 2009 New Revision: 68843 Modified: pypy/branch/logging/pypy/jit/metainterp/history.py Log: Use compute_unique_id() instead of keeping our own dict of boxes. The idea is to avoid keeping boxes alive forever. Modified: pypy/branch/logging/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/logging/pypy/jit/metainterp/history.py (original) +++ pypy/branch/logging/pypy/jit/metainterp/history.py Thu Oct 29 12:31:32 2009 @@ -3,7 +3,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.ootypesystem import ootype from pypy.rlib.objectmodel import we_are_translated, r_dict, Symbolic -from pypy.rlib.objectmodel import compute_hash +from pypy.rlib.objectmodel import compute_hash, compute_unique_id from pypy.rlib.rarithmetic import intmask from pypy.tool.uid import uid from pypy.conftest import option @@ -69,17 +69,9 @@ except AttributeError: return box.value -class ReprRPython: - def __init__(self): - self.seen = {} - def repr_rpython(self, box, typechars): - n = self.seen.setdefault(box, len(self.seen)) - return '%s/%s%d' % (box._get_hash_(), typechars, n) - def _freeze_(self): - self.seen.clear() - return False - -repr_rpython = ReprRPython().repr_rpython +def repr_rpython(box, typechars): + return '%s/%s%d' % (box._get_hash_(), typechars, + compute_unique_id(box)) class AbstractValue(object): From arigo at codespeak.net Thu Oct 29 12:31:44 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 29 Oct 2009 12:31:44 +0100 (CET) Subject: [pypy-svn] r68844 - pypy/branch/logging/pypy/jit/metainterp/test Message-ID: <20091029113144.AD94C1684A9@codespeak.net> Author: arigo Date: Thu Oct 29 12:31:44 2009 New Revision: 68844 Modified: pypy/branch/logging/pypy/jit/metainterp/test/test_oparser.py Log: Kill these tests for now... Modified: pypy/branch/logging/pypy/jit/metainterp/test/test_oparser.py ============================================================================== --- pypy/branch/logging/pypy/jit/metainterp/test/test_oparser.py (original) +++ pypy/branch/logging/pypy/jit/metainterp/test/test_oparser.py Thu Oct 29 12:31:44 2009 @@ -1,7 +1,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.jit.metainterp.test.oparser import parse, split_logs_into_loops +from pypy.jit.metainterp.test.oparser import parse from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.history import AbstractDescr, BoxInt, LoopToken,\ BoxFloat @@ -160,78 +160,3 @@ ''' loop = parse(x) # assert did not explode - -examplelog = '''\ -# Loop0 (loop), 12 ops -[i0, i1] -debug_merge_point('(no jitdriver.get_printable_location!)') -i3 = call(ConstAddr(adr2), i0, descr=) -guard_no_exception(, descr=) [i0, i1, i3] -i5 = int_add(i1, 2) -i7 = call(ConstAddr(adr6), i0, descr=) -p9 = guard_exception(4, descr=) [i5, i0, i7] -i11 = int_sub(i5, 1) -i12 = int_sub(i0, 1) -i14 = int_gt(i12, 3) -guard_true(i14, descr=) [i11, i12] -debug_merge_point('(no jitdriver.get_printable_location!)') -jump(i12, i11, descr=) -# Loop1 (entry bridge), 12 ops -[i0, i1] -debug_merge_point('(no jitdriver.get_printable_location!)') -i3 = call(ConstAddr(adr2), i0, descr=) -p5 = guard_exception(4, descr=) [i0, i1, i3] -i7 = int_add(i1, 1) -i9 = call(ConstAddr(adr8), i0, descr=) -p11 = guard_exception(4, descr=) [i7, i0, i9] -i12 = int_sub(i7, 1) -i13 = int_sub(i0, 1) -i15 = int_gt(i13, 3) -guard_true(i15, descr=) [i12, i13] -debug_merge_point('(no jitdriver.get_printable_location!)') -jump(i13, i12, descr=) -# bridge out of Guard5, 10 ops -[i0, i1, i2] -p4 = guard_exception(4, descr=) [i0, i1, i2] -i6 = int_add(i1, 1) -i8 = call(ConstAddr(adr7), i0, descr=) -p10 = guard_exception(4, descr=) [i6, i0, i8] -i11 = int_sub(i6, 1) -i12 = int_sub(i0, 1) -i14 = int_gt(i12, 3) -guard_true(i14, descr=) [i11, i12] -debug_merge_point('(no jitdriver.get_printable_location!)') -jump(i12, i11, descr=) -# bridge out of Guard9, 6 ops -[i0, i1, i2] -i4 = int_add(i0, 2) -i6 = int_sub(i1, 1) -i8 = int_gt(i6, 3) -guard_true(i8, descr=) [i4, i6] -debug_merge_point('(no jitdriver.get_printable_location!)') -jump(i6, i4, descr=) -# bridge out of Guard12, 6 ops -[i0, i1, i2] -i4 = int_add(i0, 2) -i6 = int_sub(i1, 1) -i8 = int_gt(i6, 3) -guard_true(i8, descr=) [i4, i6] -debug_merge_point('(no jitdriver.get_printable_location!)') -jump(i6, i4, descr=) -''' - -def test_split_logs_into_loops(): - parts = split_logs_into_loops(examplelog) - assert len(parts) == 5 - assert "\n".join(parts) == examplelog.strip() - for part, typ in zip(parts, - ["Loop0", "Loop1", - "bridge out of Guard5", - "bridge out of Guard9", - "bridge out of Guard12"]): - assert part.startswith("# %s" % typ) - -def test_parse_no_namespace(): - parts = split_logs_into_loops(examplelog) - for part in parts: - loop = parse(part, no_namespace=True) From arigo at codespeak.net Thu Oct 29 12:45:40 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 29 Oct 2009 12:45:40 +0100 (CET) Subject: [pypy-svn] r68845 - pypy/branch/logging/pypy/rpython/memory/gc Message-ID: <20091029114540.C16E016849C@codespeak.net> Author: arigo Date: Thu Oct 29 12:45:40 2009 New Revision: 68845 Modified: pypy/branch/logging/pypy/rpython/memory/gc/semispace.py Log: Fix. Modified: pypy/branch/logging/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/logging/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/logging/pypy/rpython/memory/gc/semispace.py Thu Oct 29 12:45:40 2009 @@ -227,7 +227,7 @@ self.top_of_space = tospace if rlog.has_log(): - start_usage = self.free - self.tospace + start_usage = self.free - fromspace start_time = time.time() rlog.debug_log("gc-full-{", ".----------- Full collection ------------------\n" From cfbolz at codespeak.net Thu Oct 29 15:58:17 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 29 Oct 2009 15:58:17 +0100 (CET) Subject: [pypy-svn] r68846 - in pypy/trunk/pypy: interpreter interpreter/test objspace Message-ID: <20091029145817.A86E416849C@codespeak.net> Author: cfbolz Date: Thu Oct 29 15:58:17 2009 New Revision: 68846 Modified: pypy/trunk/pypy/interpreter/function.py pypy/trunk/pypy/interpreter/gateway.py pypy/trunk/pypy/interpreter/test/test_function.py pypy/trunk/pypy/objspace/descroperation.py Log: Make it impossible to change the code object of builtin functions. This gets rid of a getfield and a guard for all builtin function calls. Modified: pypy/trunk/pypy/interpreter/function.py ============================================================================== --- pypy/trunk/pypy/interpreter/function.py (original) +++ pypy/trunk/pypy/interpreter/function.py Thu Oct 29 15:58:17 2009 @@ -16,12 +16,20 @@ funccallunrolling = unrolling_iterable(range(4)) + at jit.purefunction +def _get_immutable_code(func): + assert not func.can_change_code + return func.code + class Function(Wrappable): """A function is a code object captured with some environment: an object space, a dictionary of globals, default arguments, and an arbitrary 'closure' passed to the code object.""" - def __init__(self, space, code, w_globals=None, defs_w=[], closure=None, forcename=None): + can_change_code = True + + def __init__(self, space, code, w_globals=None, defs_w=[], closure=None, + forcename=None): self.space = space self.name = forcename or code.co_name self.w_doc = None # lazily read from code.getdocstring() @@ -48,7 +56,12 @@ return self.getcode().funcrun_obj(self, w_obj, args) def getcode(self): - return jit.hint(self.code, promote=True) + if jit.we_are_jitted(): + if not self.can_change_code: + self = jit.hint(self, promote=True) + return _get_immutable_code(self) + return jit.hint(self.code, promote=True) + return self.code def funccall(self, *args_w): # speed hack from pypy.interpreter import gateway @@ -368,6 +381,9 @@ def fset_func_code(space, self, w_code): from pypy.interpreter.pycode import PyCode + if not self.can_change_code: + raise OperationError(space.w_TypeError, + space.wrap("Cannot change code attribute of builtin functions")) code = space.interp_w(Code, w_code) closure_len = 0 if self.closure: @@ -568,7 +584,11 @@ "'%s' object is not callable" % typename)) return space.wrap(ClassMethod(w_function)) +class FunctionWithFixedCode(Function): + can_change_code = False + class BuiltinFunction(Function): + can_change_code = False def __init__(self, func): assert isinstance(func, Function) Modified: pypy/trunk/pypy/interpreter/gateway.py ============================================================================== --- pypy/trunk/pypy/interpreter/gateway.py (original) +++ pypy/trunk/pypy/interpreter/gateway.py Thu Oct 29 15:58:17 2009 @@ -16,6 +16,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter import eval from pypy.interpreter.function import Function, Method, ClassMethod +from pypy.interpreter.function import FunctionWithFixedCode from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable from pypy.interpreter.baseobjspace import Wrappable, SpaceCache, DescrMismatch from pypy.interpreter.argument import Arguments, Signature @@ -788,7 +789,7 @@ space = cache.space defs = gateway._getdefaults(space) # needs to be implemented by subclass code = gateway._code - fn = Function(space, code, None, defs, forcename = gateway.name) + fn = FunctionWithFixedCode(space, code, None, defs, forcename = gateway.name) if not space.config.translating: # for tests and py.py fn._freeze_() if gateway.as_classmethod: Modified: pypy/trunk/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_function.py (original) +++ pypy/trunk/pypy/interpreter/test/test_function.py Thu Oct 29 15:58:17 2009 @@ -84,6 +84,11 @@ return f() # a closure raises(ValueError, "f.func_code = h.func_code") + def test_write_code_builtin_forbidden(self): + def f(*args): + return 42 + raises(TypeError, "dir.func_code = f.func_code") + raises(TypeError, "list.append.im_func.func_code = f.func_code") class AppTestFunction: Modified: pypy/trunk/pypy/objspace/descroperation.py ============================================================================== --- pypy/trunk/pypy/objspace/descroperation.py (original) +++ pypy/trunk/pypy/objspace/descroperation.py Thu Oct 29 15:58:17 2009 @@ -90,9 +90,7 @@ def get_and_call_function(space, w_descr, w_obj, *args_w): descr = space.interpclass_w(w_descr) # a special case for performance and to avoid infinite recursion - if type(descr) is Function: - # the fastcall paths are purely for performance, but the resulting - # increase of speed is huge + if isinstance(descr, Function): return descr.funccall(w_obj, *args_w) else: args = Arguments(space, list(args_w)) From cfbolz at codespeak.net Thu Oct 29 16:01:28 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 29 Oct 2009 16:01:28 +0100 (CET) Subject: [pypy-svn] r68847 - pypy/trunk/pypy/objspace Message-ID: <20091029150128.1ECEA16849C@codespeak.net> Author: cfbolz Date: Thu Oct 29 16:01:27 2009 New Revision: 68847 Modified: pypy/trunk/pypy/objspace/descroperation.py Log: Restore accidentally killed comment. Thanks Armin Modified: pypy/trunk/pypy/objspace/descroperation.py ============================================================================== --- pypy/trunk/pypy/objspace/descroperation.py (original) +++ pypy/trunk/pypy/objspace/descroperation.py Thu Oct 29 16:01:27 2009 @@ -91,6 +91,8 @@ descr = space.interpclass_w(w_descr) # a special case for performance and to avoid infinite recursion if isinstance(descr, Function): + # the fastcall paths are purely for performance, but the resulting + # increase of speed is huge return descr.funccall(w_obj, *args_w) else: args = Arguments(space, list(args_w)) From cfbolz at codespeak.net Thu Oct 29 16:13:36 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 29 Oct 2009 16:13:36 +0100 (CET) Subject: [pypy-svn] r68848 - in pypy/trunk/pypy: interpreter interpreter/test objspace Message-ID: <20091029151336.B40C416849C@codespeak.net> Author: cfbolz Date: Thu Oct 29 16:13:36 2009 New Revision: 68848 Modified: pypy/trunk/pypy/interpreter/function.py pypy/trunk/pypy/interpreter/gateway.py pypy/trunk/pypy/interpreter/test/test_function.py pypy/trunk/pypy/objspace/descroperation.py Log: revert 68846 and 68847 for now, they broke untested stuff Modified: pypy/trunk/pypy/interpreter/function.py ============================================================================== --- pypy/trunk/pypy/interpreter/function.py (original) +++ pypy/trunk/pypy/interpreter/function.py Thu Oct 29 16:13:36 2009 @@ -16,20 +16,12 @@ funccallunrolling = unrolling_iterable(range(4)) - at jit.purefunction -def _get_immutable_code(func): - assert not func.can_change_code - return func.code - class Function(Wrappable): """A function is a code object captured with some environment: an object space, a dictionary of globals, default arguments, and an arbitrary 'closure' passed to the code object.""" - can_change_code = True - - def __init__(self, space, code, w_globals=None, defs_w=[], closure=None, - forcename=None): + def __init__(self, space, code, w_globals=None, defs_w=[], closure=None, forcename=None): self.space = space self.name = forcename or code.co_name self.w_doc = None # lazily read from code.getdocstring() @@ -56,12 +48,7 @@ return self.getcode().funcrun_obj(self, w_obj, args) def getcode(self): - if jit.we_are_jitted(): - if not self.can_change_code: - self = jit.hint(self, promote=True) - return _get_immutable_code(self) - return jit.hint(self.code, promote=True) - return self.code + return jit.hint(self.code, promote=True) def funccall(self, *args_w): # speed hack from pypy.interpreter import gateway @@ -381,9 +368,6 @@ def fset_func_code(space, self, w_code): from pypy.interpreter.pycode import PyCode - if not self.can_change_code: - raise OperationError(space.w_TypeError, - space.wrap("Cannot change code attribute of builtin functions")) code = space.interp_w(Code, w_code) closure_len = 0 if self.closure: @@ -584,11 +568,7 @@ "'%s' object is not callable" % typename)) return space.wrap(ClassMethod(w_function)) -class FunctionWithFixedCode(Function): - can_change_code = False - class BuiltinFunction(Function): - can_change_code = False def __init__(self, func): assert isinstance(func, Function) Modified: pypy/trunk/pypy/interpreter/gateway.py ============================================================================== --- pypy/trunk/pypy/interpreter/gateway.py (original) +++ pypy/trunk/pypy/interpreter/gateway.py Thu Oct 29 16:13:36 2009 @@ -16,7 +16,6 @@ from pypy.interpreter.error import OperationError from pypy.interpreter import eval from pypy.interpreter.function import Function, Method, ClassMethod -from pypy.interpreter.function import FunctionWithFixedCode from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable from pypy.interpreter.baseobjspace import Wrappable, SpaceCache, DescrMismatch from pypy.interpreter.argument import Arguments, Signature @@ -789,7 +788,7 @@ space = cache.space defs = gateway._getdefaults(space) # needs to be implemented by subclass code = gateway._code - fn = FunctionWithFixedCode(space, code, None, defs, forcename = gateway.name) + fn = Function(space, code, None, defs, forcename = gateway.name) if not space.config.translating: # for tests and py.py fn._freeze_() if gateway.as_classmethod: Modified: pypy/trunk/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_function.py (original) +++ pypy/trunk/pypy/interpreter/test/test_function.py Thu Oct 29 16:13:36 2009 @@ -84,11 +84,6 @@ return f() # a closure raises(ValueError, "f.func_code = h.func_code") - def test_write_code_builtin_forbidden(self): - def f(*args): - return 42 - raises(TypeError, "dir.func_code = f.func_code") - raises(TypeError, "list.append.im_func.func_code = f.func_code") class AppTestFunction: Modified: pypy/trunk/pypy/objspace/descroperation.py ============================================================================== --- pypy/trunk/pypy/objspace/descroperation.py (original) +++ pypy/trunk/pypy/objspace/descroperation.py Thu Oct 29 16:13:36 2009 @@ -90,7 +90,7 @@ def get_and_call_function(space, w_descr, w_obj, *args_w): descr = space.interpclass_w(w_descr) # a special case for performance and to avoid infinite recursion - if isinstance(descr, Function): + if type(descr) is Function: # the fastcall paths are purely for performance, but the resulting # increase of speed is huge return descr.funccall(w_obj, *args_w) From cfbolz at codespeak.net Thu Oct 29 16:26:37 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 29 Oct 2009 16:26:37 +0100 (CET) Subject: [pypy-svn] r68849 - pypy/trunk/pypy/interpreter/test Message-ID: <20091029152637.793FC1684A8@codespeak.net> Author: cfbolz Date: Thu Oct 29 16:26:37 2009 New Revision: 68849 Modified: pypy/trunk/pypy/interpreter/test/test_function.py Log: a test that checks that builtin functions have no __get__ Modified: pypy/trunk/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_function.py (original) +++ pypy/trunk/pypy/interpreter/test/test_function.py Thu Oct 29 16:26:37 2009 @@ -221,6 +221,13 @@ meth = func.__get__(obj, object) assert meth() == obj + def test_no_get_builtin(self): + assert not hasattr(dir, '__get__') + class A(object): + ord = ord + a = A() + assert a.ord('a') == 97 + def test_call_builtin(self): s = 'hello' raises(TypeError, len) From cfbolz at codespeak.net Thu Oct 29 16:51:01 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 29 Oct 2009 16:51:01 +0100 (CET) Subject: [pypy-svn] r68850 - pypy/trunk/pypy/interpreter/test Message-ID: <20091029155101.10F9E16849C@codespeak.net> Author: cfbolz Date: Thu Oct 29 16:51:01 2009 New Revision: 68850 Modified: pypy/trunk/pypy/interpreter/test/test_function.py Log: A test that checks that using a builtin function as a special method works as expected (e.g. the builtin function is not bound to the object before being called). Modified: pypy/trunk/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_function.py (original) +++ pypy/trunk/pypy/interpreter/test/test_function.py Thu Oct 29 16:51:01 2009 @@ -228,6 +228,14 @@ a = A() assert a.ord('a') == 97 + def test_builtin_as_special_method_is_not_bound(self): + class A(object): + __getattr__ = len + a = A() + assert a.a == 1 + assert a.ab == 2 + assert a.abcdefghij == 10 + def test_call_builtin(self): s = 'hello' raises(TypeError, len) From hpk at codespeak.net Thu Oct 29 17:28:48 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 29 Oct 2009 17:28:48 +0100 (CET) Subject: [pypy-svn] r68851 - pypy/extradoc/sprintinfo/ddorf2009 Message-ID: <20091029162848.9C718168487@codespeak.net> Author: hpk Date: Thu Oct 29 17:28:47 2009 New Revision: 68851 Modified: pypy/extradoc/sprintinfo/ddorf2009/people.txt Log: tentative travel plan Modified: pypy/extradoc/sprintinfo/ddorf2009/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/ddorf2009/people.txt (original) +++ pypy/extradoc/sprintinfo/ddorf2009/people.txt Thu Oct 29 17:28:47 2009 @@ -11,6 +11,7 @@ Carl Friedrich Bolz always there private Antonio Cuni 6/11 - 13/11 Hotel Haus Hillesheim Armin Rigo 6/11 - 13/11 Hotel Haus Hillesheim +Holger Krekel 9/11 - 13/11 ? ==================== ============== ===================== ==================== ============== ===================== From fijal at codespeak.net Fri Oct 30 11:51:40 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 30 Oct 2009 11:51:40 +0100 (CET) Subject: [pypy-svn] r68853 - pypy/extradoc/planning Message-ID: <20091030105140.7CDBC1684A9@codespeak.net> Author: fijal Date: Fri Oct 30 11:51:39 2009 New Revision: 68853 Modified: pypy/extradoc/planning/jit.txt Log: Mention recent observation Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Fri Oct 30 11:51:39 2009 @@ -38,6 +38,8 @@ - improve test running, compile only once +- module/__builtin__/app_inspect.py forces frames for globals() and locals(). + It's a horrible idea, should be fixed (and rest reviewed). META ----- From arigo at codespeak.net Fri Oct 30 13:10:01 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Oct 2009 13:10:01 +0100 (CET) Subject: [pypy-svn] r68854 - pypy/branch/logging2 Message-ID: <20091030121001.C3AA0168483@codespeak.net> Author: arigo Date: Fri Oct 30 13:10:00 2009 New Revision: 68854 Added: pypy/branch/logging2/ - copied from r68853, pypy/trunk/ Log: A different branch in which to try a less over-engineered version of logging. From arigo at codespeak.net Fri Oct 30 13:29:21 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Oct 2009 13:29:21 +0100 (CET) Subject: [pypy-svn] r68855 - in pypy/branch/logging2/pypy: rlib rlib/test rpython rpython/lltypesystem Message-ID: <20091030122921.36DCF168483@codespeak.net> Author: arigo Date: Fri Oct 30 13:29:20 2009 New Revision: 68855 Modified: pypy/branch/logging2/pypy/rlib/debug.py pypy/branch/logging2/pypy/rlib/test/test_debug.py pypy/branch/logging2/pypy/rpython/llinterp.py pypy/branch/logging2/pypy/rpython/lltypesystem/lloperation.py pypy/branch/logging2/pypy/rpython/lltypesystem/opimpl.py Log: Add the debug_start and debug_stop operations that mark the start and the stop of some operation that we might want to profile and display. Also rewrite a bit debug_print. Modified: pypy/branch/logging2/pypy/rlib/debug.py ============================================================================== --- pypy/branch/logging2/pypy/rlib/debug.py (original) +++ pypy/branch/logging2/pypy/rlib/debug.py Fri Oct 30 13:29:20 2009 @@ -1,4 +1,4 @@ - +import sys, time from pypy.rpython.extregistry import ExtRegistryEntry def ll_assert(x, msg): @@ -21,7 +21,6 @@ def debug_print(*args): - import sys for arg in args: print >> sys.stderr, arg, print >> sys.stderr @@ -38,6 +37,26 @@ hop.genop('debug_print', vlist) +def debug_start(category): + print >> sys.stderr, '[%s] debug_start %s' % (time.clock(), category) + +def debug_stop(category): + print >> sys.stderr, '[%s] debug_stop %s' % (time.clock(), category) + +class Entry(ExtRegistryEntry): + _about_ = debug_start, debug_stop + + def compute_result_annotation(self, s_category): + return None + + def specialize_call(self, hop): + fn = self.instance + string_repr = hop.rtyper.type_system.rstr.string_repr + vlist = hop.inputargs(string_repr) + hop.exception_cannot_occur() + hop.genop(fn.__name__, vlist) + + def llinterpcall(RESTYPE, pythonfunction, *args): """When running on the llinterp, this causes the llinterp to call to the provided Python function with the run-time value of the given args. Modified: pypy/branch/logging2/pypy/rlib/test/test_debug.py ============================================================================== --- pypy/branch/logging2/pypy/rlib/test/test_debug.py (original) +++ pypy/branch/logging2/pypy/rlib/test/test_debug.py Fri Oct 30 13:29:20 2009 @@ -1,6 +1,7 @@ import py from pypy.rlib.debug import check_annotation, make_sure_not_resized +from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rpython.test.test_llinterp import interpret def test_check_annotation(): @@ -37,3 +38,41 @@ py.test.raises(TooLateForChange, interpret, f, [], list_comprehension_operations=True) + + +class DebugTests: + + def test_debug_print_start_stop(self): + import sys + from cStringIO import StringIO + + def f(x): + debug_start("mycat") + debug_print("foo", 2, "bar", x) + debug_stop("mycat") + + olderr = sys.stderr + try: + sys.stderr = c = StringIO() + f(3) + finally: + sys.stderr = olderr + assert 'mycat' in c.getvalue() + assert 'foo 2 bar 3' in c.getvalue() + + try: + sys.stderr = c = StringIO() + self.interpret(f, [3]) + finally: + sys.stderr = olderr + assert 'mycat' in c.getvalue() + assert 'foo 2 bar 3' in c.getvalue() + + +class TestLLType(DebugTests): + def interpret(self, f, args): + interpret(f, args, type_system='lltype') + +class TestOOType(DebugTests): + def interpret(self, f, args): + interpret(f, args, type_system='ootype') Modified: pypy/branch/logging2/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/logging2/pypy/rpython/llinterp.py (original) +++ pypy/branch/logging2/pypy/rpython/llinterp.py Fri Oct 30 13:29:20 2009 @@ -522,20 +522,6 @@ from pypy.translator.tool.lltracker import track track(*ll_objects) - def op_debug_print(self, *ll_args): - from pypy.rpython.lltypesystem.rstr import STR - line = [] - for arg in ll_args: - T = lltype.typeOf(arg) - if T == lltype.Ptr(STR): - arg = ''.join(arg.chars) - line.append(str(arg)) - line = ' '.join(line) - print line - tracer = self.llinterpreter.tracer - if tracer: - tracer.dump('\n[debug] %s\n' % (line,)) - def op_debug_pdb(self, *ll_args): if self.llinterpreter.tracer: self.llinterpreter.tracer.flush() Modified: pypy/branch/logging2/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/logging2/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/logging2/pypy/rpython/lltypesystem/lloperation.py Fri Oct 30 13:29:20 2009 @@ -523,6 +523,8 @@ # __________ debugging __________ 'debug_view': LLOp(), 'debug_print': LLOp(canrun=True), + 'debug_start': LLOp(canrun=True), + 'debug_stop': LLOp(canrun=True), 'debug_pdb': LLOp(), 'debug_assert': LLOp(tryfold=True), 'debug_fatalerror': LLOp(), Modified: pypy/branch/logging2/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/logging2/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/logging2/pypy/rpython/lltypesystem/opimpl.py Fri Oct 30 13:29:20 2009 @@ -3,6 +3,7 @@ from pypy.tool.sourcetools import func_with_new_name from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem.lloperation import opimpls +from pypy.rlib import debug # ____________________________________________________________ # Implementation of the 'canfold' operations @@ -411,10 +412,22 @@ raise TypeError("cannot fold getfield on mutable array") return p[index] +def _normalize(x): + TYPE = lltype.typeOf(x) + if (isinstance(TYPE, lltype.Ptr) and TYPE.TO._name == 'rpy_string' + or getattr(TYPE, '_name', '') == 'String'): # ootype + from pypy.rpython.annlowlevel import hlstr + return hlstr(x) + return x + def op_debug_print(*args): - for arg in args: - print arg, - print + debug.debug_print(*map(_normalize, args)) + +def op_debug_start(category): + debug.debug_start(_normalize(category)) + +def op_debug_stop(category): + debug.debug_stop(_normalize(category)) def op_gc_stack_bottom(): pass # marker for trackgcroot.py From arigo at codespeak.net Fri Oct 30 13:31:04 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Oct 2009 13:31:04 +0100 (CET) Subject: [pypy-svn] r68856 - in pypy/branch/logging2/pypy/translator/c: . src test Message-ID: <20091030123104.29E86168483@codespeak.net> Author: arigo Date: Fri Oct 30 13:31:03 2009 New Revision: 68856 Added: pypy/branch/logging2/pypy/translator/c/src/debug.h (contents, props changed) Modified: pypy/branch/logging2/pypy/translator/c/funcgen.py pypy/branch/logging2/pypy/translator/c/genc.py pypy/branch/logging2/pypy/translator/c/src/g_include.h pypy/branch/logging2/pypy/translator/c/test/test_standalone.py Log: The C implementation of debug_start and debug_stop, as well as small rewrites in debug_print. For now only for x86 gcc; need to generalize READ_TIMESTAMP... Modified: pypy/branch/logging2/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/logging2/pypy/translator/c/funcgen.py (original) +++ pypy/branch/logging2/pypy/translator/c/funcgen.py Fri Oct 30 13:31:03 2009 @@ -714,6 +714,7 @@ from pypy.rpython.lltypesystem.rstr import STR format = [] argv = [] + free_line = "" for arg in op.args: T = arg.concretetype if T == Ptr(STR): @@ -722,6 +723,7 @@ else: format.append('%s') argv.append('RPyString_AsCharP(%s)' % self.expr(arg)) + free_line = "RPyString_FreeCache();" continue elif T == Signed: format.append('%d') @@ -741,9 +743,21 @@ else: raise Exception("don't know how to debug_print %r" % (T,)) argv.append(self.expr(arg)) - return "fprintf(stderr, %s%s); RPyString_FreeCache();" % ( - c_string_constant(' '.join(format) + '\n\000'), - ''.join([', ' + s for s in argv])) + argv.insert(0, c_string_constant(' '.join(format) + '\n')) + return ("if (PYPY_DEBUG_ENABLED) { fprintf(PYPY_DEBUG_FILE, %s); %s}" + % (', '.join(argv), free_line)) + + def OP_DEBUG_START(self, op): + arg = op.args[0] + assert isinstance(arg, Constant) + return "PYPY_DEBUG_START(%s);" % ( + c_string_constant(''.join(arg.value.chars)),) + + def OP_DEBUG_STOP(self, op): + arg = op.args[0] + assert isinstance(arg, Constant) + return "PYPY_DEBUG_STOP(%s);" % ( + c_string_constant(''.join(arg.value.chars)),) def OP_DEBUG_ASSERT(self, op): return 'RPyAssert(%s, %s);' % (self.expr(op.args[0]), Modified: pypy/branch/logging2/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/logging2/pypy/translator/c/genc.py (original) +++ pypy/branch/logging2/pypy/translator/c/genc.py Fri Oct 30 13:31:03 2009 @@ -430,12 +430,14 @@ bk = self.translator.annotator.bookkeeper return getfunctionptr(bk.getdesc(self.entrypoint).getuniquegraph()) - def cmdexec(self, args='', env=None): + def cmdexec(self, args='', env=None, err=False): assert self._compiled res = self.translator.platform.execute(self.executable_name, args, env=env) if res.returncode != 0: raise Exception("Returned %d" % (res.returncode,)) + if err: + return res.out, res.err return res.out def compile(self): Added: pypy/branch/logging2/pypy/translator/c/src/debug.h ============================================================================== --- (empty file) +++ pypy/branch/logging2/pypy/translator/c/src/debug.h Fri Oct 30 13:31:03 2009 @@ -0,0 +1,82 @@ +/************************************************************/ + /*** C header subsection: debug_print & related tools ***/ + + +/* macros used by the generated code */ +#define PYPY_DEBUG_ENABLED (pypy_debug_enabled && pypy_debug_is_ready()) +#define PYPY_DEBUG_FILE pypy_debug_file +#define PYPY_DEBUG_START(cat) if (pypy_debug_enabled) pypy_debug_start(cat) +#define PYPY_DEBUG_STOP(cat) if (pypy_debug_enabled) pypy_debug_stop(cat) + + +/************************************************************/ + +/* prototypes (internal use only) */ +bool_t pypy_debug_is_ready(void); +void pypy_debug_start(const char *category); +void pypy_debug_stop(const char *category); + +extern bool_t pypy_debug_enabled; +extern FILE *pypy_debug_file; + + +/* implementations */ + +#ifndef PYPY_NOT_MAIN_FILE +#include + +static bool_t pypy_debug_initialized = 0; +bool_t pypy_debug_enabled = 1; /* set to 0 if PYPYLOG is not defined */ +FILE *pypy_debug_file; + +static void pypy_debug_open(void) +{ + char *filename = getenv("PYPYLOG"); + if (filename && filename[0]) + { + if (filename[0] == '-' && filename[1] == 0) + pypy_debug_file = stderr; + else + pypy_debug_file = fopen(filename, "w"); + } + else + { + pypy_debug_file = NULL; + } + pypy_debug_enabled = (pypy_debug_file != NULL); + pypy_debug_initialized = 1; +} + +bool_t pypy_debug_is_ready(void) +{ + if (!pypy_debug_initialized) + pypy_debug_open(); + return pypy_debug_enabled; +} + + +/* XXXXXXXXXX x86 Pentium only! */ +#define READ_TIMESTAMP(val) \ + __asm__ __volatile__("rdtsc" : "=A" (val)) + + +static void pypy_debug_category(const char *start, const char *category) +{ + long long timestamp; + if (!pypy_debug_is_ready()) + return; + READ_TIMESTAMP(timestamp); + fprintf(pypy_debug_file, "{%llx} -%s- %s\n", timestamp, start, category); +} + +void pypy_debug_start(const char *category) +{ + pypy_debug_category("start", category); +} + +void pypy_debug_stop(const char *category) +{ + pypy_debug_category("stop", category); +} + +#endif /* PYPY_NOT_MAIN_FILE */ Modified: pypy/branch/logging2/pypy/translator/c/src/g_include.h ============================================================================== --- pypy/branch/logging2/pypy/translator/c/src/g_include.h (original) +++ pypy/branch/logging2/pypy/translator/c/src/g_include.h Fri Oct 30 13:31:03 2009 @@ -51,6 +51,7 @@ /*** modules ***/ #ifdef HAVE_RTYPER /* only if we have an RTyper */ # include "src/rtyper.h" +# include "src/debug.h" #ifndef AVR # include "src/ll_os.h" # include "src/ll_strtod.h" Modified: pypy/branch/logging2/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/branch/logging2/pypy/translator/c/test/test_standalone.py (original) +++ pypy/branch/logging2/pypy/translator/c/test/test_standalone.py Fri Oct 30 13:31:03 2009 @@ -2,7 +2,7 @@ import sys, os, re from pypy.rlib.rarithmetic import r_longlong -from pypy.rlib.debug import ll_assert, debug_print +from pypy.rlib.debug import ll_assert, debug_print, debug_start, debug_stop from pypy.translator.translator import TranslationContext from pypy.translator.backendopt import all from pypy.translator.c.genc import CStandaloneBuilder, ExternalCompilationInfo @@ -11,9 +11,22 @@ from pypy.tool.autopath import pypydir -class TestStandalone(object): +class StandaloneTests(object): config = None - + + def compile(self, entry_point): + t = TranslationContext(self.config) + t.buildannotator().build_types(entry_point, [s_list_of_strings]) + t.buildrtyper().specialize() + + cbuilder = CStandaloneBuilder(t, entry_point, t.config) + cbuilder.generate_source() + cbuilder.compile() + return t, cbuilder + + +class TestStandalone(StandaloneTests): + def test_hello_world(self): def entry_point(argv): os.write(1, "hello world\n") @@ -23,13 +36,7 @@ os.write(1, " '" + str(s) + "'\n") return 0 - t = TranslationContext(self.config) - t.buildannotator().build_types(entry_point, [s_list_of_strings]) - t.buildrtyper().specialize() - - cbuilder = CStandaloneBuilder(t, entry_point, t.config) - cbuilder.generate_source() - cbuilder.compile() + t, cbuilder = self.compile(entry_point) data = cbuilder.cmdexec('hi there') assert data.startswith('''hello world\nargument count: 2\n 'hi'\n 'there'\n''') @@ -43,13 +50,7 @@ print [len(s) for s in argv] return 0 - t = TranslationContext(self.config) - t.buildannotator().build_types(entry_point, [s_list_of_strings]) - t.buildrtyper().specialize() - - cbuilder = CStandaloneBuilder(t, entry_point, t.config) - cbuilder.generate_source() - cbuilder.compile() + t, cbuilder = self.compile(entry_point) data = cbuilder.cmdexec('hi there') assert data.startswith('''hello simpler world\n''' '''argument count: 2\n''' @@ -130,13 +131,7 @@ print m, x return 0 - t = TranslationContext(self.config) - t.buildannotator().build_types(entry_point, [s_list_of_strings]) - t.buildrtyper().specialize() - - cbuilder = CStandaloneBuilder(t, entry_point, t.config) - cbuilder.generate_source() - cbuilder.compile() + t, cbuilder = self.compile(entry_point) data = cbuilder.cmdexec('hi there') assert map(float, data.split()) == [0.0, 0.0] @@ -173,13 +168,8 @@ os.setpgrp() return 0 - t = TranslationContext(self.config) - t.buildannotator().build_types(entry_point, [s_list_of_strings]) - t.buildrtyper().specialize() - - cbuilder = CStandaloneBuilder(t, entry_point, t.config) - cbuilder.generate_source() - cbuilder.compile() + t, cbuilder = self.compile(entry_point) + cbuilder.cmdexec("") def test_profopt_mac_osx_bug(self): @@ -223,12 +213,7 @@ print "BAD POS" os.close(fd) return 0 - t = TranslationContext(self.config) - t.buildannotator().build_types(entry_point, [s_list_of_strings]) - t.buildrtyper().specialize() - cbuilder = CStandaloneBuilder(t, entry_point, t.config) - cbuilder.generate_source() - cbuilder.compile() + t, cbuilder = self.compile(entry_point) data = cbuilder.cmdexec('hi there') assert data.strip() == "OK" @@ -270,6 +255,36 @@ assert " ll_strtod.h" in makefile assert " ll_strtod.o" in makefile + def test_debug_print_start_stop(self): + def entry_point(argv): + debug_start("mycat") + debug_print("foo", 2, "bar", 3) + debug_stop("mycat") + return 0 + t, cbuilder = self.compile(entry_point) + # check with PYPYLOG undefined + out, err = cbuilder.cmdexec("", err=True, env={}) + assert not out + assert not err + # check with PYPYLOG defined to an empty string + out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': ''}) + assert not out + assert not err + # check with PYPYLOG=- + out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': '-'}) + assert not out + assert 'mycat' in err + assert 'foo 2 bar 3' in err + # check with PYPYLOG=somefilename + path = udir.join('test_debug_xxx.log') + out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': str(path)}) + assert not out + assert not err + assert path.check(file=1) + assert 'mycat' in path.read() + assert 'foo 2 bar 3' in path.read() + + class TestMaemo(TestStandalone): def setup_class(cls): from pypy.translator.platform.maemo import check_scratchbox From arigo at codespeak.net Fri Oct 30 14:44:40 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Oct 2009 14:44:40 +0100 (CET) Subject: [pypy-svn] r68860 - in pypy/branch/logging2/pypy/translator/c: src test Message-ID: <20091030134440.7AE5A168495@codespeak.net> Author: arigo Date: Fri Oct 30 14:44:39 2009 New Revision: 68860 Modified: pypy/branch/logging2/pypy/translator/c/src/debug.h pypy/branch/logging2/pypy/translator/c/test/test_standalone.py Log: Add the option to set PYPYLOG to 'prof:filename'. Document and test it. Modified: pypy/branch/logging2/pypy/translator/c/src/debug.h ============================================================================== --- pypy/branch/logging2/pypy/translator/c/src/debug.h (original) +++ pypy/branch/logging2/pypy/translator/c/src/debug.h Fri Oct 30 14:44:39 2009 @@ -1,12 +1,20 @@ /************************************************************/ /*** C header subsection: debug_print & related tools ***/ +/* values of the PYPYLOG environment variable: + + (empty) logging is turned off + - logging goes to stderr + filename logging goes to the given file; overwritten at process start + prof:filename logs only debug_start and debug_stop, not debug_print +*/ + /* macros used by the generated code */ -#define PYPY_DEBUG_ENABLED (pypy_debug_enabled && pypy_debug_is_ready()) +#define PYPY_DEBUG_ENABLED (pypy_debug_full && pypy_debug_is_ready()) #define PYPY_DEBUG_FILE pypy_debug_file -#define PYPY_DEBUG_START(cat) if (pypy_debug_enabled) pypy_debug_start(cat) -#define PYPY_DEBUG_STOP(cat) if (pypy_debug_enabled) pypy_debug_stop(cat) +#define PYPY_DEBUG_START(cat) if (pypy_debug_profiling) pypy_debug_start(cat) +#define PYPY_DEBUG_STOP(cat) if (pypy_debug_profiling) pypy_debug_stop(cat) /************************************************************/ @@ -16,7 +24,8 @@ void pypy_debug_start(const char *category); void pypy_debug_stop(const char *category); -extern bool_t pypy_debug_enabled; +extern bool_t pypy_debug_full; +extern bool_t pypy_debug_profiling; extern FILE *pypy_debug_file; @@ -26,7 +35,9 @@ #include static bool_t pypy_debug_initialized = 0; -bool_t pypy_debug_enabled = 1; /* set to 0 if PYPYLOG is not defined */ +bool_t pypy_debug_full = 1; /* set to 0 if PYPYLOG is not defined + or starts with 'prof:' */ +bool_t pypy_debug_profiling = 1; /* set to 0 if PYPYLOG is not defined */ FILE *pypy_debug_file; static void pypy_debug_open(void) @@ -34,6 +45,15 @@ char *filename = getenv("PYPYLOG"); if (filename && filename[0]) { + if (filename[0] == 'p' && + filename[1] == 'r' && + filename[2] == 'o' && + filename[3] == 'f' && + filename[4] == ':') + { + pypy_debug_full = 0; + filename += 5; + } if (filename[0] == '-' && filename[1] == 0) pypy_debug_file = stderr; else @@ -43,7 +63,8 @@ { pypy_debug_file = NULL; } - pypy_debug_enabled = (pypy_debug_file != NULL); + pypy_debug_full &= (pypy_debug_file != NULL); + pypy_debug_profiling &= (pypy_debug_file != NULL); pypy_debug_initialized = 1; } @@ -51,7 +72,7 @@ { if (!pypy_debug_initialized) pypy_debug_open(); - return pypy_debug_enabled; + return pypy_debug_full; } @@ -63,7 +84,9 @@ static void pypy_debug_category(const char *start, const char *category) { long long timestamp; - if (!pypy_debug_is_ready()) + if (!pypy_debug_initialized) + pypy_debug_open(); + if (!pypy_debug_profiling) return; READ_TIMESTAMP(timestamp); fprintf(pypy_debug_file, "{%llx} -%s- %s\n", timestamp, start, category); Modified: pypy/branch/logging2/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/branch/logging2/pypy/translator/c/test/test_standalone.py (original) +++ pypy/branch/logging2/pypy/translator/c/test/test_standalone.py Fri Oct 30 14:44:39 2009 @@ -266,11 +266,11 @@ out, err = cbuilder.cmdexec("", err=True, env={}) assert not out assert not err - # check with PYPYLOG defined to an empty string + # check with PYPYLOG defined to an empty string (same as undefined) out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': ''}) assert not out assert not err - # check with PYPYLOG=- + # check with PYPYLOG=- (means print to stderr) out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': '-'}) assert not out assert 'mycat' in err @@ -283,6 +283,15 @@ assert path.check(file=1) assert 'mycat' in path.read() assert 'foo 2 bar 3' in path.read() + # check with PYPYLOG=prof:somefilename (only start/stop events) + path = udir.join('test_debug_xxx_prof.log') + out, err = cbuilder.cmdexec("", err=True, + env={'PYPYLOG': 'prof:%s' % path}) + assert not out + assert not err + assert path.check(file=1) + assert 'mycat' in path.read() + assert 'foo 2 bar 3' not in path.read() class TestMaemo(TestStandalone): From arigo at codespeak.net Fri Oct 30 15:22:00 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Oct 2009 15:22:00 +0100 (CET) Subject: [pypy-svn] r68866 - in pypy/branch/logging2/pypy: config rlib rlib/test rpython/lltypesystem translator/c/src translator/c/test Message-ID: <20091030142200.1CD4B168489@codespeak.net> Author: arigo Date: Fri Oct 30 15:21:57 2009 New Revision: 68866 Modified: pypy/branch/logging2/pypy/config/translationoption.py pypy/branch/logging2/pypy/rlib/debug.py pypy/branch/logging2/pypy/rlib/test/test_debug.py pypy/branch/logging2/pypy/rpython/lltypesystem/lloperation.py pypy/branch/logging2/pypy/rpython/lltypesystem/opimpl.py pypy/branch/logging2/pypy/translator/c/src/debug.h pypy/branch/logging2/pypy/translator/c/test/test_standalone.py Log: Add debug_level(), a function/operation to read the debug level from a program. Modified: pypy/branch/logging2/pypy/config/translationoption.py ============================================================================== --- pypy/branch/logging2/pypy/config/translationoption.py (original) +++ pypy/branch/logging2/pypy/config/translationoption.py Fri Oct 30 15:21:57 2009 @@ -40,6 +40,9 @@ }, cmdline="-b --backend"), + BoolOption("log", "Include debug prints in the translation (PYPYLOG=...)", + default=True, cmdline="--log"), + # gc ChoiceOption("gc", "Garbage Collection Strategy", ["boehm", "ref", "marksweep", "semispace", "statistics", Modified: pypy/branch/logging2/pypy/rlib/debug.py ============================================================================== --- pypy/branch/logging2/pypy/rlib/debug.py (original) +++ pypy/branch/logging2/pypy/rlib/debug.py Fri Oct 30 15:21:57 2009 @@ -57,6 +57,37 @@ hop.genop(fn.__name__, vlist) +def debug_level(): + """Returns the current debug level. It is: + 0: all calls to debug_print/debug_start/debug_stop are ignored + 1: profiling: debug_start/debug_stop handled, but debug_print ignored + 2: all handled + The value returned depend on the runtime presence of the PYPYLOG env var, + unless debuglevel == 0 in the config. + """ + return 2 + +class Entry(ExtRegistryEntry): + _about_ = debug_level + + def compute_result_annotation(self): + from pypy.annotation import model as annmodel + t = self.bookkeeper.annotator.translator + if t.config.translation.log: + return annmodel.SomeInteger() + else: + return self.bookkeeper.immutablevalue(0) + + def specialize_call(self, hop): + from pypy.rpython.lltypesystem import lltype + t = hop.rtyper.annotator.translator + hop.exception_cannot_occur() + if t.config.translation.log: + return hop.genop('debug_level', [], resulttype=lltype.Signed) + else: + return hop.inputconst(lltype.Signed, 0) + + def llinterpcall(RESTYPE, pythonfunction, *args): """When running on the llinterp, this causes the llinterp to call to the provided Python function with the run-time value of the given args. Modified: pypy/branch/logging2/pypy/rlib/test/test_debug.py ============================================================================== --- pypy/branch/logging2/pypy/rlib/test/test_debug.py (original) +++ pypy/branch/logging2/pypy/rlib/test/test_debug.py Fri Oct 30 15:21:57 2009 @@ -1,7 +1,7 @@ import py from pypy.rlib.debug import check_annotation, make_sure_not_resized -from pypy.rlib.debug import debug_print, debug_start, debug_stop +from pypy.rlib.debug import debug_print, debug_start, debug_stop, debug_level from pypy.rpython.test.test_llinterp import interpret def test_check_annotation(): @@ -50,11 +50,13 @@ debug_start("mycat") debug_print("foo", 2, "bar", x) debug_stop("mycat") + return debug_level() olderr = sys.stderr try: sys.stderr = c = StringIO() - f(3) + res = f(3) + assert res == 2 finally: sys.stderr = olderr assert 'mycat' in c.getvalue() @@ -62,7 +64,8 @@ try: sys.stderr = c = StringIO() - self.interpret(f, [3]) + res = self.interpret(f, [3]) + assert res == 2 finally: sys.stderr = olderr assert 'mycat' in c.getvalue() @@ -71,8 +74,8 @@ class TestLLType(DebugTests): def interpret(self, f, args): - interpret(f, args, type_system='lltype') + return interpret(f, args, type_system='lltype') class TestOOType(DebugTests): def interpret(self, f, args): - interpret(f, args, type_system='ootype') + return interpret(f, args, type_system='ootype') Modified: pypy/branch/logging2/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/logging2/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/logging2/pypy/rpython/lltypesystem/lloperation.py Fri Oct 30 15:21:57 2009 @@ -525,6 +525,7 @@ 'debug_print': LLOp(canrun=True), 'debug_start': LLOp(canrun=True), 'debug_stop': LLOp(canrun=True), + 'debug_level': LLOp(canrun=True), 'debug_pdb': LLOp(), 'debug_assert': LLOp(tryfold=True), 'debug_fatalerror': LLOp(), Modified: pypy/branch/logging2/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/logging2/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/logging2/pypy/rpython/lltypesystem/opimpl.py Fri Oct 30 15:21:57 2009 @@ -429,6 +429,9 @@ def op_debug_stop(category): debug.debug_stop(_normalize(category)) +def op_debug_level(): + return debug.debug_level() + def op_gc_stack_bottom(): pass # marker for trackgcroot.py Modified: pypy/branch/logging2/pypy/translator/c/src/debug.h ============================================================================== --- pypy/branch/logging2/pypy/translator/c/src/debug.h (original) +++ pypy/branch/logging2/pypy/translator/c/src/debug.h Fri Oct 30 15:21:57 2009 @@ -11,21 +11,33 @@ /* macros used by the generated code */ -#define PYPY_DEBUG_ENABLED (pypy_debug_full && pypy_debug_is_ready()) -#define PYPY_DEBUG_FILE pypy_debug_file -#define PYPY_DEBUG_START(cat) if (pypy_debug_profiling) pypy_debug_start(cat) -#define PYPY_DEBUG_STOP(cat) if (pypy_debug_profiling) pypy_debug_stop(cat) +#define PYPY_DEBUG_ENABLED \ + (pypy_debug_level >= PYDEBUG_FULL && pypy_debug_is_ready_full()) +#define PYPY_DEBUG_FILE \ + pypy_debug_file +#define PYPY_DEBUG_START(cat) \ + if (pypy_debug_level >= PYDEBUG_PROFILE) pypy_debug_start(cat) +#define PYPY_DEBUG_STOP(cat) \ + if (pypy_debug_level >= PYDEBUG_PROFILE) pypy_debug_stop(cat) +#define OP_DEBUG_LEVEL(r) \ + if (pypy_debug_level == PYDEBUG_UNINITIALIZED) pypy_debug_open(); \ + r = pypy_debug_level /************************************************************/ +#define PYDEBUG_OFF 0 +#define PYDEBUG_PROFILE 1 +#define PYDEBUG_FULL 2 +#define PYDEBUG_UNINITIALIZED 3 + /* prototypes (internal use only) */ -bool_t pypy_debug_is_ready(void); +void pypy_debug_open(void); +bool_t pypy_debug_is_ready_full(void); void pypy_debug_start(const char *category); void pypy_debug_stop(const char *category); -extern bool_t pypy_debug_full; -extern bool_t pypy_debug_profiling; +extern int pypy_debug_level; extern FILE *pypy_debug_file; @@ -34,15 +46,13 @@ #ifndef PYPY_NOT_MAIN_FILE #include -static bool_t pypy_debug_initialized = 0; -bool_t pypy_debug_full = 1; /* set to 0 if PYPYLOG is not defined - or starts with 'prof:' */ -bool_t pypy_debug_profiling = 1; /* set to 0 if PYPYLOG is not defined */ +int pypy_debug_level = PYDEBUG_UNINITIALIZED; FILE *pypy_debug_file; -static void pypy_debug_open(void) +void pypy_debug_open(void) { char *filename = getenv("PYPYLOG"); + pypy_debug_level = PYDEBUG_FULL; if (filename && filename[0]) { if (filename[0] == 'p' && @@ -51,7 +61,7 @@ filename[3] == 'f' && filename[4] == ':') { - pypy_debug_full = 0; + pypy_debug_level = PYDEBUG_PROFILE; filename += 5; } if (filename[0] == '-' && filename[1] == 0) @@ -63,16 +73,15 @@ { pypy_debug_file = NULL; } - pypy_debug_full &= (pypy_debug_file != NULL); - pypy_debug_profiling &= (pypy_debug_file != NULL); - pypy_debug_initialized = 1; + if (pypy_debug_file == NULL) + pypy_debug_level = PYDEBUG_OFF; } -bool_t pypy_debug_is_ready(void) +bool_t pypy_debug_is_ready_full(void) { - if (!pypy_debug_initialized) + if (pypy_debug_level == PYDEBUG_UNINITIALIZED) pypy_debug_open(); - return pypy_debug_full; + return pypy_debug_level == PYDEBUG_FULL; } @@ -84,9 +93,9 @@ static void pypy_debug_category(const char *start, const char *category) { long long timestamp; - if (!pypy_debug_initialized) + if (!pypy_debug_level == PYDEBUG_UNINITIALIZED) pypy_debug_open(); - if (!pypy_debug_profiling) + if (pypy_debug_level < PYDEBUG_PROFILE) return; READ_TIMESTAMP(timestamp); fprintf(pypy_debug_file, "{%llx} -%s- %s\n", timestamp, start, category); Modified: pypy/branch/logging2/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/branch/logging2/pypy/translator/c/test/test_standalone.py (original) +++ pypy/branch/logging2/pypy/translator/c/test/test_standalone.py Fri Oct 30 15:21:57 2009 @@ -2,7 +2,8 @@ import sys, os, re from pypy.rlib.rarithmetic import r_longlong -from pypy.rlib.debug import ll_assert, debug_print, debug_start, debug_stop +from pypy.rlib.debug import ll_assert +from pypy.rlib.debug import debug_print, debug_start, debug_stop, debug_level from pypy.translator.translator import TranslationContext from pypy.translator.backendopt import all from pypy.translator.c.genc import CStandaloneBuilder, ExternalCompilationInfo @@ -257,6 +258,7 @@ def test_debug_print_start_stop(self): def entry_point(argv): + os.write(1, str(debug_level()) + '\n') debug_start("mycat") debug_print("foo", 2, "bar", 3) debug_stop("mycat") @@ -264,21 +266,21 @@ t, cbuilder = self.compile(entry_point) # check with PYPYLOG undefined out, err = cbuilder.cmdexec("", err=True, env={}) - assert not out + assert out.strip() == '0' assert not err # check with PYPYLOG defined to an empty string (same as undefined) out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': ''}) - assert not out + assert out.strip() == '0' assert not err # check with PYPYLOG=- (means print to stderr) out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': '-'}) - assert not out + assert out.strip() == '2' assert 'mycat' in err assert 'foo 2 bar 3' in err # check with PYPYLOG=somefilename path = udir.join('test_debug_xxx.log') out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': str(path)}) - assert not out + assert out.strip() == '2' assert not err assert path.check(file=1) assert 'mycat' in path.read() @@ -287,7 +289,7 @@ path = udir.join('test_debug_xxx_prof.log') out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': 'prof:%s' % path}) - assert not out + assert out.strip() == '1' assert not err assert path.check(file=1) assert 'mycat' in path.read() From arigo at codespeak.net Fri Oct 30 15:30:18 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Oct 2009 15:30:18 +0100 (CET) Subject: [pypy-svn] r68867 - in pypy/branch/logging2/pypy: rlib translator/c/src translator/c/test Message-ID: <20091030143018.484851684B0@codespeak.net> Author: arigo Date: Fri Oct 30 15:30:17 2009 New Revision: 68867 Modified: pypy/branch/logging2/pypy/rlib/debug.py pypy/branch/logging2/pypy/translator/c/src/debug.h pypy/branch/logging2/pypy/translator/c/test/test_standalone.py Log: Test and fixes to ensure that if --no-log is given, all the debug_xxx calls are removed during translation. Modified: pypy/branch/logging2/pypy/rlib/debug.py ============================================================================== --- pypy/branch/logging2/pypy/rlib/debug.py (original) +++ pypy/branch/logging2/pypy/rlib/debug.py Fri Oct 30 15:30:17 2009 @@ -34,7 +34,9 @@ def specialize_call(self, hop): vlist = hop.inputargs(*hop.args_r) hop.exception_cannot_occur() - hop.genop('debug_print', vlist) + t = hop.rtyper.annotator.translator + if t.config.translation.log: + hop.genop('debug_print', vlist) def debug_start(category): @@ -54,7 +56,9 @@ string_repr = hop.rtyper.type_system.rstr.string_repr vlist = hop.inputargs(string_repr) hop.exception_cannot_occur() - hop.genop(fn.__name__, vlist) + t = hop.rtyper.annotator.translator + if t.config.translation.log: + hop.genop(fn.__name__, vlist) def debug_level(): Modified: pypy/branch/logging2/pypy/translator/c/src/debug.h ============================================================================== --- pypy/branch/logging2/pypy/translator/c/src/debug.h (original) +++ pypy/branch/logging2/pypy/translator/c/src/debug.h Fri Oct 30 15:30:17 2009 @@ -53,6 +53,7 @@ { char *filename = getenv("PYPYLOG"); pypy_debug_level = PYDEBUG_FULL; + pypy_debug_file = NULL; if (filename && filename[0]) { if (filename[0] == 'p' && @@ -69,10 +70,6 @@ else pypy_debug_file = fopen(filename, "w"); } - else - { - pypy_debug_file = NULL; - } if (pypy_debug_file == NULL) pypy_debug_level = PYDEBUG_OFF; } @@ -93,7 +90,7 @@ static void pypy_debug_category(const char *start, const char *category) { long long timestamp; - if (!pypy_debug_level == PYDEBUG_UNINITIALIZED) + if (pypy_debug_level == PYDEBUG_UNINITIALIZED) pypy_debug_open(); if (pypy_debug_level < PYDEBUG_PROFILE) return; Modified: pypy/branch/logging2/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/branch/logging2/pypy/translator/c/test/test_standalone.py (original) +++ pypy/branch/logging2/pypy/translator/c/test/test_standalone.py Fri Oct 30 15:30:17 2009 @@ -294,6 +294,17 @@ assert path.check(file=1) assert 'mycat' in path.read() assert 'foo 2 bar 3' not in path.read() + # + # finally, check compiling with logging disabled + from pypy.config.pypyoption import get_pypy_config + config = get_pypy_config(translating=True) + config.translation.log = False + self.config = config + t, cbuilder = self.compile(entry_point) + path = udir.join('test_debug_does_not_show_up.log') + out = cbuilder.cmdexec("", env={'PYPYLOG': str(path)}) + assert out.strip() == '0' + assert path.check(file=0) class TestMaemo(TestStandalone): From arigo at codespeak.net Fri Oct 30 17:45:25 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Oct 2009 17:45:25 +0100 (CET) Subject: [pypy-svn] r68869 - in pypy/branch/logging2/pypy: rlib rlib/test rpython/lltypesystem translator/c translator/c/src translator/c/test Message-ID: <20091030164525.B4BA116847C@codespeak.net> Author: arigo Date: Fri Oct 30 17:45:24 2009 New Revision: 68869 Modified: pypy/branch/logging2/pypy/rlib/debug.py pypy/branch/logging2/pypy/rlib/test/test_debug.py pypy/branch/logging2/pypy/rpython/lltypesystem/lloperation.py pypy/branch/logging2/pypy/rpython/lltypesystem/opimpl.py pypy/branch/logging2/pypy/translator/c/funcgen.py pypy/branch/logging2/pypy/translator/c/src/debug.h pypy/branch/logging2/pypy/translator/c/test/test_standalone.py Log: Move all the code to support variants of logging to debug.h. Implement a number of variants, documented at the start of debug.h. Modified: pypy/branch/logging2/pypy/rlib/debug.py ============================================================================== --- pypy/branch/logging2/pypy/rlib/debug.py (original) +++ pypy/branch/logging2/pypy/rlib/debug.py Fri Oct 30 17:45:24 2009 @@ -61,35 +61,30 @@ hop.genop(fn.__name__, vlist) -def debug_level(): - """Returns the current debug level. It is: - 0: all calls to debug_print/debug_start/debug_stop are ignored - 1: profiling: debug_start/debug_stop handled, but debug_print ignored - 2: all handled - The value returned depend on the runtime presence of the PYPYLOG env var, - unless debuglevel == 0 in the config. - """ - return 2 +def have_debug_prints(): + # returns True if the next calls to debug_print show up, + # and False if they would not have any effect. + return True class Entry(ExtRegistryEntry): - _about_ = debug_level + _about_ = have_debug_prints def compute_result_annotation(self): from pypy.annotation import model as annmodel t = self.bookkeeper.annotator.translator if t.config.translation.log: - return annmodel.SomeInteger() + return annmodel.s_Bool else: - return self.bookkeeper.immutablevalue(0) + return self.bookkeeper.immutablevalue(False) def specialize_call(self, hop): from pypy.rpython.lltypesystem import lltype t = hop.rtyper.annotator.translator hop.exception_cannot_occur() if t.config.translation.log: - return hop.genop('debug_level', [], resulttype=lltype.Signed) + return hop.genop('have_debug_prints', [], resulttype=lltype.Bool) else: - return hop.inputconst(lltype.Signed, 0) + return hop.inputconst(lltype.Bool, False) def llinterpcall(RESTYPE, pythonfunction, *args): Modified: pypy/branch/logging2/pypy/rlib/test/test_debug.py ============================================================================== --- pypy/branch/logging2/pypy/rlib/test/test_debug.py (original) +++ pypy/branch/logging2/pypy/rlib/test/test_debug.py Fri Oct 30 17:45:24 2009 @@ -1,7 +1,8 @@ import py from pypy.rlib.debug import check_annotation, make_sure_not_resized -from pypy.rlib.debug import debug_print, debug_start, debug_stop, debug_level +from pypy.rlib.debug import debug_print, debug_start, debug_stop +from pypy.rlib.debug import have_debug_prints from pypy.rpython.test.test_llinterp import interpret def test_check_annotation(): @@ -50,13 +51,13 @@ debug_start("mycat") debug_print("foo", 2, "bar", x) debug_stop("mycat") - return debug_level() + return have_debug_prints() olderr = sys.stderr try: sys.stderr = c = StringIO() res = f(3) - assert res == 2 + assert res == True finally: sys.stderr = olderr assert 'mycat' in c.getvalue() @@ -65,7 +66,7 @@ try: sys.stderr = c = StringIO() res = self.interpret(f, [3]) - assert res == 2 + assert res == True finally: sys.stderr = olderr assert 'mycat' in c.getvalue() Modified: pypy/branch/logging2/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/logging2/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/logging2/pypy/rpython/lltypesystem/lloperation.py Fri Oct 30 17:45:24 2009 @@ -525,7 +525,7 @@ 'debug_print': LLOp(canrun=True), 'debug_start': LLOp(canrun=True), 'debug_stop': LLOp(canrun=True), - 'debug_level': LLOp(canrun=True), + 'have_debug_prints': LLOp(canrun=True), 'debug_pdb': LLOp(), 'debug_assert': LLOp(tryfold=True), 'debug_fatalerror': LLOp(), Modified: pypy/branch/logging2/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/logging2/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/logging2/pypy/rpython/lltypesystem/opimpl.py Fri Oct 30 17:45:24 2009 @@ -429,8 +429,8 @@ def op_debug_stop(category): debug.debug_stop(_normalize(category)) -def op_debug_level(): - return debug.debug_level() +def op_have_debug_prints(): + return debug.have_debug_prints() def op_gc_stack_bottom(): pass # marker for trackgcroot.py Modified: pypy/branch/logging2/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/logging2/pypy/translator/c/funcgen.py (original) +++ pypy/branch/logging2/pypy/translator/c/funcgen.py Fri Oct 30 17:45:24 2009 @@ -744,8 +744,9 @@ raise Exception("don't know how to debug_print %r" % (T,)) argv.append(self.expr(arg)) argv.insert(0, c_string_constant(' '.join(format) + '\n')) - return ("if (PYPY_DEBUG_ENABLED) { fprintf(PYPY_DEBUG_FILE, %s); %s}" - % (', '.join(argv), free_line)) + return ( + "if (PYPY_HAVE_DEBUG_PRINTS) { fprintf(PYPY_DEBUG_FILE, %s); %s}" + % (', '.join(argv), free_line)) def OP_DEBUG_START(self, op): arg = op.args[0] Modified: pypy/branch/logging2/pypy/translator/c/src/debug.h ============================================================================== --- pypy/branch/logging2/pypy/translator/c/src/debug.h (original) +++ pypy/branch/logging2/pypy/translator/c/src/debug.h Fri Oct 30 17:45:24 2009 @@ -2,42 +2,41 @@ /*** C header subsection: debug_print & related tools ***/ /* values of the PYPYLOG environment variable: + ("top-level" debug_prints means not between debug_start and debug_stop) - (empty) logging is turned off - - logging goes to stderr - filename logging goes to the given file; overwritten at process start - prof:filename logs only debug_start and debug_stop, not debug_print + (empty) logging is turned off, apart from top-level debug_prints + that go to stderr + fname logging for profiling: includes all debug_start/debug_stop + but not any nested debug_print + :fname full logging + prefix:fname conditional logging + + Conditional logging means that it only includes the debug_start/debug_stop + sections whose name match 'prefix'. Other sections are ignored, including + all debug_prints that occur while this section is running and all nested + subsections. + + Note that 'fname' can be '-' to send the logging data to stderr. */ /* macros used by the generated code */ -#define PYPY_DEBUG_ENABLED \ - (pypy_debug_level >= PYDEBUG_FULL && pypy_debug_is_ready_full()) -#define PYPY_DEBUG_FILE \ - pypy_debug_file -#define PYPY_DEBUG_START(cat) \ - if (pypy_debug_level >= PYDEBUG_PROFILE) pypy_debug_start(cat) -#define PYPY_DEBUG_STOP(cat) \ - if (pypy_debug_level >= PYDEBUG_PROFILE) pypy_debug_stop(cat) -#define OP_DEBUG_LEVEL(r) \ - if (pypy_debug_level == PYDEBUG_UNINITIALIZED) pypy_debug_open(); \ - r = pypy_debug_level +#define PYPY_HAVE_DEBUG_PRINTS (pypy_ignoring_nested_prints ? 0 : \ + (pypy_debug_ensure_opened(), 1)) +#define PYPY_DEBUG_FILE pypy_debug_file +#define PYPY_DEBUG_START(cat) pypy_debug_start(cat) +#define PYPY_DEBUG_STOP(cat) pypy_debug_stop(cat) +#define OP_HAVE_DEBUG_PRINTS(r) r = !pypy_ignoring_nested_prints /************************************************************/ -#define PYDEBUG_OFF 0 -#define PYDEBUG_PROFILE 1 -#define PYDEBUG_FULL 2 -#define PYDEBUG_UNINITIALIZED 3 - /* prototypes (internal use only) */ -void pypy_debug_open(void); -bool_t pypy_debug_is_ready_full(void); +void pypy_debug_ensure_opened(void); void pypy_debug_start(const char *category); void pypy_debug_stop(const char *category); -extern int pypy_debug_level; +extern int pypy_ignoring_nested_prints; extern FILE *pypy_debug_file; @@ -45,40 +44,46 @@ #ifndef PYPY_NOT_MAIN_FILE #include +#include -int pypy_debug_level = PYDEBUG_UNINITIALIZED; -FILE *pypy_debug_file; +int pypy_ignoring_nested_prints = 0; +FILE *pypy_debug_file = NULL; +static bool_t debug_ready = 0; +static bool_t debug_profile = 0; +static char *debug_prefix = NULL; -void pypy_debug_open(void) +static void pypy_debug_open(void) { char *filename = getenv("PYPYLOG"); - pypy_debug_level = PYDEBUG_FULL; - pypy_debug_file = NULL; if (filename && filename[0]) { - if (filename[0] == 'p' && - filename[1] == 'r' && - filename[2] == 'o' && - filename[3] == 'f' && - filename[4] == ':') + char *colon = strchr(filename, ':'); + if (!colon) { - pypy_debug_level = PYDEBUG_PROFILE; - filename += 5; + /* PYPYLOG=filename --- profiling version */ + debug_profile = 1; } - if (filename[0] == '-' && filename[1] == 0) - pypy_debug_file = stderr; else + { + /* PYPYLOG=prefix:filename --- conditional logging */ + int n = colon - filename; + debug_prefix = malloc(n + 1); + memcpy(debug_prefix, filename, n); + debug_prefix[n] = '\0'; + filename = colon + 1; + } + if (strcmp(filename, "-") != 0) pypy_debug_file = fopen(filename, "w"); } - if (pypy_debug_file == NULL) - pypy_debug_level = PYDEBUG_OFF; + if (!pypy_debug_file) + pypy_debug_file = stderr; + debug_ready = 1; } -bool_t pypy_debug_is_ready_full(void) +void pypy_debug_ensure_opened(void) { - if (pypy_debug_level == PYDEBUG_UNINITIALIZED) + if (!debug_ready) pypy_debug_open(); - return pypy_debug_level == PYDEBUG_FULL; } @@ -87,25 +92,58 @@ __asm__ __volatile__("rdtsc" : "=A" (val)) -static void pypy_debug_category(const char *start, const char *category) +static bool_t startswith(const char *str, const char *substr) +{ + while (*substr) + if (*str++ != *substr++) + return 0; + return 1; +} + +static void display_startstop(const char *start, const char *category) { long long timestamp; - if (pypy_debug_level == PYDEBUG_UNINITIALIZED) - pypy_debug_open(); - if (pypy_debug_level < PYDEBUG_PROFILE) - return; READ_TIMESTAMP(timestamp); fprintf(pypy_debug_file, "{%llx} -%s- %s\n", timestamp, start, category); } void pypy_debug_start(const char *category) { - pypy_debug_category("start", category); + if (debug_profile) + { + /* profiling version */ + pypy_debug_ensure_opened(); + pypy_ignoring_nested_prints++; /* disable nested debug_print */ + } + else + { + /* non-profiling version */ + if (pypy_ignoring_nested_prints > 0) + { + /* already ignoring the parent section */ + pypy_ignoring_nested_prints++; + return; + } + pypy_debug_ensure_opened(); + if (!debug_prefix || !startswith(category, debug_prefix)) + { + /* wrong section name, or no PYPYLOG at all, skip it */ + pypy_ignoring_nested_prints = 1; + return; + } + } + display_startstop("start", category); } void pypy_debug_stop(const char *category) { - pypy_debug_category("stop", category); + if (pypy_ignoring_nested_prints > 0) + { + pypy_ignoring_nested_prints--; + if (!debug_profile) + return; + } + display_startstop("stop", category); } #endif /* PYPY_NOT_MAIN_FILE */ Modified: pypy/branch/logging2/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/branch/logging2/pypy/translator/c/test/test_standalone.py (original) +++ pypy/branch/logging2/pypy/translator/c/test/test_standalone.py Fri Oct 30 17:45:24 2009 @@ -2,8 +2,8 @@ import sys, os, re from pypy.rlib.rarithmetic import r_longlong -from pypy.rlib.debug import ll_assert -from pypy.rlib.debug import debug_print, debug_start, debug_stop, debug_level +from pypy.rlib.debug import ll_assert, have_debug_prints +from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.translator.translator import TranslationContext from pypy.translator.backendopt import all from pypy.translator.c.genc import CStandaloneBuilder, ExternalCompilationInfo @@ -258,42 +258,104 @@ def test_debug_print_start_stop(self): def entry_point(argv): - os.write(1, str(debug_level()) + '\n') - debug_start("mycat") - debug_print("foo", 2, "bar", 3) - debug_stop("mycat") + x = "got:" + if have_debug_prints(): x += "a" + debug_print("toplevel") + debug_start ("mycat") + if have_debug_prints(): x += "b" + debug_print ("foo", 2, "bar", 3) + debug_start ("cat2") + if have_debug_prints(): x += "c" + debug_print ("baz") + debug_stop ("cat2") + if have_debug_prints(): x += "d" + debug_print ("bok") + debug_stop ("mycat") + os.write(1, x + '.\n') return 0 t, cbuilder = self.compile(entry_point) # check with PYPYLOG undefined out, err = cbuilder.cmdexec("", err=True, env={}) - assert out.strip() == '0' - assert not err + assert out.strip() == 'got:a.' + assert 'toplevel' in err + assert 'mycat' not in err + assert 'foo 2 bar 3' not in err + assert 'cat2' not in err + assert 'baz' not in err + assert 'bok' not in err # check with PYPYLOG defined to an empty string (same as undefined) out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': ''}) - assert out.strip() == '0' - assert not err - # check with PYPYLOG=- (means print to stderr) - out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': '-'}) - assert out.strip() == '2' + assert out.strip() == 'got:a.' + assert 'toplevel' in err + assert 'mycat' not in err + assert 'foo 2 bar 3' not in err + assert 'cat2' not in err + assert 'baz' not in err + assert 'bok' not in err + # check with PYPYLOG=:- (means print to stderr) + out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': ':-'}) + assert out.strip() == 'got:abcd.' + assert 'toplevel' in err assert 'mycat' in err assert 'foo 2 bar 3' in err - # check with PYPYLOG=somefilename + assert 'cat2' in err + assert 'baz' in err + assert 'bok' in err + # check with PYPYLOG=:somefilename path = udir.join('test_debug_xxx.log') + out, err = cbuilder.cmdexec("", err=True, + env={'PYPYLOG': ':%s' % path}) + assert out.strip() == 'got:abcd.' + assert not err + assert path.check(file=1) + data = path.read() + assert 'toplevel' in data + assert 'mycat' in data + assert 'foo 2 bar 3' in data + assert 'cat2' in data + assert 'baz' in data + assert 'bok' in data + # check with PYPYLOG=somefilename + path = udir.join('test_debug_xxx_prof.log') out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': str(path)}) - assert out.strip() == '2' + assert out.strip() == 'got:a.' assert not err assert path.check(file=1) + data = path.read() + assert 'toplevel' in data + assert 'mycat' in data + assert 'foo 2 bar 3' not in data + assert 'cat2' in data + assert 'baz' not in data + assert 'bok' not in data + # check with PYPYLOG=myc:somefilename (includes mycat but not cat2) + path = udir.join('test_debug_xxx_myc.log') + out, err = cbuilder.cmdexec("", err=True, + env={'PYPYLOG': 'myc:%s' % path}) + assert out.strip() == 'got:abd.' + assert not err + assert path.check(file=1) + data = path.read() + assert 'toplevel' in path.read() assert 'mycat' in path.read() assert 'foo 2 bar 3' in path.read() - # check with PYPYLOG=prof:somefilename (only start/stop events) - path = udir.join('test_debug_xxx_prof.log') + assert 'cat2' not in data + assert 'baz' not in data + assert 'bok' in data + # check with PYPYLOG=cat:somefilename (includes cat2 but not mycat) + path = udir.join('test_debug_xxx_cat.log') out, err = cbuilder.cmdexec("", err=True, - env={'PYPYLOG': 'prof:%s' % path}) - assert out.strip() == '1' + env={'PYPYLOG': 'cat:%s' % path}) + assert out.strip() == 'got:a.' assert not err assert path.check(file=1) - assert 'mycat' in path.read() + data = path.read() + assert 'toplevel' in path.read() + assert 'mycat' not in path.read() assert 'foo 2 bar 3' not in path.read() + assert 'cat2' not in data # because it is nested + assert 'baz' not in data + assert 'bok' not in data # # finally, check compiling with logging disabled from pypy.config.pypyoption import get_pypy_config @@ -302,8 +364,10 @@ self.config = config t, cbuilder = self.compile(entry_point) path = udir.join('test_debug_does_not_show_up.log') - out = cbuilder.cmdexec("", env={'PYPYLOG': str(path)}) - assert out.strip() == '0' + out, err = cbuilder.cmdexec("", err=True, + env={'PYPYLOG': ':%s' % path}) + assert out.strip() == 'got:.' + assert not err assert path.check(file=0) From arigo at codespeak.net Fri Oct 30 18:28:18 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Oct 2009 18:28:18 +0100 (CET) Subject: [pypy-svn] r68870 - in pypy/branch/logging2/pypy: config jit/backend/x86/test rpython/lltypesystem rpython/memory rpython/memory/gc rpython/memory/gctransform rpython/memory/test translator/c Message-ID: <20091030172818.BA1B316849C@codespeak.net> Author: arigo Date: Fri Oct 30 18:28:17 2009 New Revision: 68870 Modified: pypy/branch/logging2/pypy/config/translationoption.py pypy/branch/logging2/pypy/jit/backend/x86/test/test_zrpy_gc.py pypy/branch/logging2/pypy/rpython/lltypesystem/opimpl.py pypy/branch/logging2/pypy/rpython/memory/gc/generation.py pypy/branch/logging2/pypy/rpython/memory/gc/hybrid.py pypy/branch/logging2/pypy/rpython/memory/gc/marksweep.py pypy/branch/logging2/pypy/rpython/memory/gc/semispace.py pypy/branch/logging2/pypy/rpython/memory/gctransform/framework.py pypy/branch/logging2/pypy/rpython/memory/gctypelayout.py pypy/branch/logging2/pypy/rpython/memory/test/test_transformed_gc.py pypy/branch/logging2/pypy/translator/c/gc.py Log: Use the new debug_start/debug_stop sectionning functions in the GC. Change calls to debug_print to go via the rlib.debug module instead of directly to llop.debug_print(). This removes the gcconfig.debugprint option. While I'm at it, remove the whole gcconfig group in which there was only removetypeptr left. Modified: pypy/branch/logging2/pypy/config/translationoption.py ============================================================================== --- pypy/branch/logging2/pypy/config/translationoption.py (original) +++ pypy/branch/logging2/pypy/config/translationoption.py Fri Oct 30 18:28:17 2009 @@ -69,12 +69,8 @@ "ref": [("translation.gcrootfinder", "n/a")], "none": [("translation.gcrootfinder", "n/a")], }), - OptionDescription("gcconfig", "Configure garbage collectors", [ - BoolOption("debugprint", "Turn on debug printing for the GC", - default=False), - BoolOption("removetypeptr", "Remove the typeptr from every object", - default=False, cmdline="--gcremovetypeptr"), - ]), + BoolOption("gcremovetypeptr", "Remove the typeptr from every object", + default=False, cmdline="--gcremovetypeptr"), ChoiceOption("gcrootfinder", "Strategy for finding GC Roots (framework GCs only)", ["n/a", "shadowstack", "asmgcc"], @@ -100,7 +96,7 @@ BoolOption("jit", "generate a JIT", default=False, requires=[("translation.thread", False), - ("translation.gcconfig.removetypeptr", False)], + ("translation.gcremovetypeptr", False)], suggests=[("translation.gc", "hybrid"), # or "boehm" ("translation.gcrootfinder", "asmgcc"), ("translation.list_comprehension_operations", True)]), @@ -362,7 +358,7 @@ elif word == 'jit': config.translation.suggest(jit=True) elif word == 'removetypeptr': - config.translation.gcconfig.suggest(removetypeptr=True) + config.translation.suggest(gcremovetypeptr=True) else: raise ValueError(word) Modified: pypy/branch/logging2/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/logging2/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/logging2/pypy/jit/backend/x86/test/test_zrpy_gc.py Fri Oct 30 18:28:17 2009 @@ -15,6 +15,7 @@ from pypy.jit.backend.x86.runner import CPU386 from pypy.jit.backend.llsupport.gc import GcRefList, GcRootMap_asmgcc from pypy.jit.backend.x86.regalloc import X86StackManager +from pypy.tool.udir import udir stack_pos = X86StackManager.stack_pos @@ -77,7 +78,6 @@ # t = TranslationContext() t.config.translation.gc = gc - t.config.translation.gcconfig.debugprint = True for name, value in kwds.items(): setattr(t.config.translation, name, value) ann = t.buildannotator(policy=annpolicy.StrictAnnotatorPolicy()) @@ -92,7 +92,8 @@ def run(cbuilder, args=''): # - data = cbuilder.cmdexec(args) + pypylog = udir.join('test_zrpy_gc.log') + data = cbuilder.cmdexec(args, env={'PYPYLOG': str(pypylog)}) return data.strip() def compile_and_run(f, gc, **kwds): @@ -164,7 +165,9 @@ cls.cbuilder = compile(get_entry(allfuncs), "hybrid", gcrootfinder="asmgcc", jit=True) def run(self, name, n=2000): - res = self.cbuilder.cmdexec("%s %d" %(name, n)) + pypylog = udir.join('TestCompileHybrid.log') + res = self.cbuilder.cmdexec("%s %d" %(name, n), + env={'PYPYLOG': str(pypylog)}) assert int(res) == 20 def run_orig(self, name, n, x): Modified: pypy/branch/logging2/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/logging2/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/logging2/pypy/rpython/lltypesystem/opimpl.py Fri Oct 30 18:28:17 2009 @@ -413,11 +413,12 @@ return p[index] def _normalize(x): - TYPE = lltype.typeOf(x) - if (isinstance(TYPE, lltype.Ptr) and TYPE.TO._name == 'rpy_string' - or getattr(TYPE, '_name', '') == 'String'): # ootype - from pypy.rpython.annlowlevel import hlstr - return hlstr(x) + if not isinstance(x, str): + TYPE = lltype.typeOf(x) + if (isinstance(TYPE, lltype.Ptr) and TYPE.TO._name == 'rpy_string' + or getattr(TYPE, '_name', '') == 'String'): # ootype + from pypy.rpython.annlowlevel import hlstr + return hlstr(x) return x def op_debug_print(*args): Modified: pypy/branch/logging2/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/logging2/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/logging2/pypy/rpython/memory/gc/generation.py Fri Oct 30 18:28:17 2009 @@ -7,6 +7,7 @@ from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE from pypy.rlib.objectmodel import free_non_gc_object from pypy.rlib.debug import ll_assert +from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rpython.lltypesystem.lloperation import llop # The following flag is never set on young objects, i.e. the ones living @@ -85,8 +86,7 @@ if self.auto_nursery_size: newsize = nursery_size_from_env() if newsize <= 0: - newsize = estimate_best_nursery_size( - self.config.gcconfig.debugprint) + newsize = estimate_best_nursery_size() if newsize > 0: self.set_nursery_size(newsize) @@ -102,6 +102,7 @@ self.nursery_free = NULL def set_nursery_size(self, newsize): + debug_start("gc-set-nursery-size") if newsize < self.min_nursery_size: newsize = self.min_nursery_size if newsize > self.space_size // 2: @@ -116,13 +117,12 @@ while (self.min_nursery_size << (scale+1)) <= newsize: scale += 1 self.nursery_scale = scale - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, "SSS nursery_size =", newsize) - llop.debug_print(lltype.Void, "SSS largest_young_fixedsize =", - self.largest_young_fixedsize) - llop.debug_print(lltype.Void, "SSS largest_young_var_basesize =", - self.largest_young_var_basesize) - llop.debug_print(lltype.Void, "SSS nursery_scale =", scale) + debug_print("nursery_size =", newsize) + debug_print("largest_young_fixedsize =", + self.largest_young_fixedsize) + debug_print("largest_young_var_basesize =", + self.largest_young_var_basesize) + debug_print("nursery_scale =", scale) # we get the following invariant: assert self.nursery_size >= (self.min_nursery_size << scale) @@ -131,6 +131,7 @@ # be done after changing the bounds, because it might re-create # a new nursery (e.g. if it invokes finalizers). self.semispace_collect() + debug_stop("gc-set-nursery-size") @staticmethod def get_young_fixedsize(nursery_size): @@ -249,11 +250,7 @@ self.weakrefs_grow_older() self.ids_grow_older() self.reset_nursery() - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, "major collect, size changing", size_changing) SemiSpaceGC.semispace_collect(self, size_changing) - if self.config.gcconfig.debugprint and not size_changing: - llop.debug_print(lltype.Void, "percent survived", float(self.free - self.tospace) / self.space_size) def make_a_copy(self, obj, objsize): tid = self.header(obj).tid @@ -330,10 +327,9 @@ ll_assert(self.nursery_size <= self.top_of_space - self.free, "obtain_free_space failed to do its job") if self.nursery: - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, "--- minor collect ---") - llop.debug_print(lltype.Void, "nursery:", - self.nursery, "to", self.nursery_top) + debug_start("gc-minor") + debug_print("--- minor collect ---") + debug_print("nursery:", self.nursery, "to", self.nursery_top) # a nursery-only collection scan = beginning = self.free self.collect_oldrefs_to_nursery() @@ -347,10 +343,9 @@ self.update_young_objects_with_id() # mark the nursery as free and fill it with zeroes again llarena.arena_reset(self.nursery, self.nursery_size, 2) - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, - "survived (fraction of the size):", - float(scan - beginning) / self.nursery_size) + debug_print("survived (fraction of the size):", + float(scan - beginning) / self.nursery_size) + debug_stop("gc-minor") #self.debug_check_consistency() # -- quite expensive else: # no nursery - this occurs after a full collect, triggered either @@ -376,8 +371,7 @@ hdr = self.header(obj) hdr.tid |= GCFLAG_NO_YOUNG_PTRS self.trace_and_drag_out_of_nursery(obj) - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, "collect_oldrefs_to_nursery", count) + debug_print("collect_oldrefs_to_nursery", count) def collect_roots_in_nursery(self): # we don't need to trace prebuilt GcStructs during a minor collect: @@ -588,19 +582,18 @@ pass return -1 -def best_nursery_size_for_L2cache(L2cache, debugprint=False): - if debugprint: - llop.debug_print(lltype.Void, "CCC L2cache =", L2cache) +def best_nursery_size_for_L2cache(L2cache): # Heuristically, the best nursery size to choose is about half # of the L2 cache. XXX benchmark some more. return L2cache // 2 if sys.platform == 'linux2': - def estimate_best_nursery_size(debugprint=False): + def estimate_best_nursery_size(): """Try to estimate the best nursery size at run-time, depending on the machine we are running on. """ + debug_start("gc-L2cache") L2cache = sys.maxint try: fd = os.open('/proc/cpuinfo', os.O_RDONLY, 0644) @@ -648,10 +641,13 @@ if number < L2cache: L2cache = number + debug_print("L2cache =", L2cache) + debug_stop("gc-L2cache") + if L2cache < sys.maxint: - return best_nursery_size_for_L2cache(L2cache, debugprint) + return best_nursery_size_for_L2cache(L2cache) else: - # Print a warning even in non-debug builds + # Print a top-level warning even in non-debug builds llop.debug_print(lltype.Void, "Warning: cannot find your CPU L2 cache size in /proc/cpuinfo") return -1 @@ -676,10 +672,11 @@ rffi.INT, sandboxsafe=True) - def estimate_best_nursery_size(debugprint=False): + def estimate_best_nursery_size(): """Try to estimate the best nursery size at run-time, depending on the machine we are running on. """ + debug_start("gc-L2cache") L2cache = 0 l2cache_p = lltype.malloc(rffi.LONGLONGP.TO, 1, flavor='raw') try: @@ -703,14 +700,16 @@ lltype.free(len_p, flavor='raw') finally: lltype.free(l2cache_p, flavor='raw') + debug_print("L2cache =", L2cache) + debug_stop("gc-L2cache") if L2cache > 0: - return best_nursery_size_for_L2cache(L2cache, debugprint) + return best_nursery_size_for_L2cache(L2cache) else: - # Print a warning even in non-debug builds + # Print a top-level warning even in non-debug builds llop.debug_print(lltype.Void, "Warning: cannot find your CPU L2 cache size with sysctl()") return -1 else: - def estimate_best_nursery_size(debugprint=False): + def estimate_best_nursery_size(): return -1 # XXX implement me for other platforms Modified: pypy/branch/logging2/pypy/rpython/memory/gc/hybrid.py ============================================================================== --- pypy/branch/logging2/pypy/rpython/memory/gc/hybrid.py (original) +++ pypy/branch/logging2/pypy/rpython/memory/gc/hybrid.py Fri Oct 30 18:28:17 2009 @@ -8,7 +8,8 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rlib.debug import ll_assert +from pypy.rlib.debug import ll_assert, have_debug_prints +from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rlib.rarithmetic import ovfcheck from pypy.rpython.lltypesystem import rffi @@ -117,8 +118,7 @@ def setup(self): self.large_objects_collect_trigger = self.param_space_size - if self.config.gcconfig.debugprint: - self._initial_trigger = self.large_objects_collect_trigger + self._initial_trigger = self.large_objects_collect_trigger self.rawmalloced_objects_to_trace = self.AddressStack() self.count_semispaceonly_collects = 0 @@ -271,12 +271,12 @@ def _check_rawsize_alloced(self, size_estimate, can_collect=True): self.large_objects_collect_trigger -= size_estimate if can_collect and self.large_objects_collect_trigger < 0: - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, "allocated", - self._initial_trigger - - self.large_objects_collect_trigger, - "bytes, triggering full collection") + debug_start("gc-rawsize-collect") + debug_print("allocated", (self._initial_trigger - + self.large_objects_collect_trigger), + "bytes, triggering full collection") self.semispace_collect() + debug_stop("gc-rawsize-collect") def malloc_varsize_marknsweep(self, totalsize, resizable=False): # In order to free the large objects from time to time, we @@ -341,9 +341,8 @@ None) ll_assert(not self.rawmalloced_objects_to_trace.non_empty(), "rawmalloced_objects_to_trace should be empty at start") - if self.config.gcconfig.debugprint: - self._nonmoving_copy_count = 0 - self._nonmoving_copy_size = 0 + self._nonmoving_copy_count = 0 + self._nonmoving_copy_size = 0 def _set_gcflag_unvisited(self, obj, ignored): ll_assert(not (self.header(obj).tid & GCFLAG_UNVISITED), @@ -419,9 +418,8 @@ newaddr = self.allocate_external_object(totalsize_incl_hash) if not newaddr: return llmemory.NULL # can't raise MemoryError during a collect() - if self.config.gcconfig.debugprint: - self._nonmoving_copy_count += 1 - self._nonmoving_copy_size += raw_malloc_usage(totalsize) + self._nonmoving_copy_count += 1 + self._nonmoving_copy_size += raw_malloc_usage(totalsize) llmemory.raw_memcopy(obj - self.size_gc_header(), newaddr, totalsize) # check if we need to write a hash value at the end of the new obj @@ -464,11 +462,9 @@ def finished_full_collect(self): ll_assert(not self.rawmalloced_objects_to_trace.non_empty(), "rawmalloced_objects_to_trace should be empty at end") - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, - "| [hybrid] made nonmoving: ", - self._nonmoving_copy_size, "bytes in", - self._nonmoving_copy_count, "objs") + debug_print("| [hybrid] made nonmoving: ", + self._nonmoving_copy_size, "bytes in", + self._nonmoving_copy_count, "objs") # sweep the nonmarked rawmalloced objects if self.is_collecting_gen3(): self.sweep_rawmalloced_objects(generation=3) @@ -479,8 +475,7 @@ self.large_objects_collect_trigger = self.space_size if self.is_collecting_gen3(): self.count_semispaceonly_collects = 0 - if self.config.gcconfig.debugprint: - self._initial_trigger = self.large_objects_collect_trigger + self._initial_trigger = self.large_objects_collect_trigger def sweep_rawmalloced_objects(self, generation): # free all the rawmalloced objects of the specified generation @@ -513,17 +508,18 @@ surviving_objects = self.AddressStack() # Help the flow space alive_count = alive_size = dead_count = dead_size = 0 + debug = have_debug_prints() while objects.non_empty(): obj = objects.pop() tid = self.header(obj).tid if tid & GCFLAG_UNVISITED: - if self.config.gcconfig.debugprint: + if debug: dead_count+=1 dead_size+=raw_malloc_usage(self.get_size_incl_hash(obj)) addr = obj - self.gcheaderbuilder.size_gc_header llmemory.raw_free(addr) else: - if self.config.gcconfig.debugprint: + if debug: alive_count+=1 alive_size+=raw_malloc_usage(self.get_size_incl_hash(obj)) if generation == 3: @@ -554,17 +550,14 @@ self.gen3_rawmalloced_objects = surviving_objects elif generation == -2: self.gen2_resizable_objects = surviving_objects - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, - "| [hyb] gen", generation, - "nonmoving now alive: ", - alive_size, "bytes in", - alive_count, "objs") - llop.debug_print(lltype.Void, - "| [hyb] gen", generation, - "nonmoving freed: ", - dead_size, "bytes in", - dead_count, "objs") + debug_print("| [hyb] gen", generation, + "nonmoving now alive: ", + alive_size, "bytes in", + alive_count, "objs") + debug_print("| [hyb] gen", generation, + "nonmoving freed: ", + dead_size, "bytes in", + dead_count, "objs") def id(self, ptr): obj = llmemory.cast_ptr_to_adr(ptr) Modified: pypy/branch/logging2/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/branch/logging2/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/branch/logging2/pypy/rpython/memory/gc/marksweep.py Fri Oct 30 18:28:17 2009 @@ -8,6 +8,7 @@ from pypy.rlib.objectmodel import free_non_gc_object from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rpython.memory.gc.base import GCBase @@ -242,8 +243,7 @@ # call __del__, move the object to the list of object-without-del import time from pypy.rpython.lltypesystem.lloperation import llop - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, 'collecting...') + debug_start("gc-collect") start_time = time.time() self.collect_in_progress = True size_gc_header = self.gcheaderbuilder.size_gc_header @@ -406,31 +406,22 @@ 256 * 1024 * 1024) self.total_collection_time += collect_time self.prev_collect_end_time = end_time - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, - " malloced since previous collection:", - old_malloced, "bytes") - llop.debug_print(lltype.Void, - " heap usage at start of collection: ", - self.heap_usage + old_malloced, "bytes") - llop.debug_print(lltype.Void, - " freed: ", - freed_size, "bytes") - llop.debug_print(lltype.Void, - " new heap usage: ", - curr_heap_size, "bytes") - llop.debug_print(lltype.Void, - " total time spent collecting: ", - self.total_collection_time, "seconds") - llop.debug_print(lltype.Void, - " collecting time: ", - collect_time) - llop.debug_print(lltype.Void, - " computing time: ", - collect_time) - llop.debug_print(lltype.Void, - " new threshold: ", - self.bytes_malloced_threshold) + debug_print(" malloced since previous collection:", + old_malloced, "bytes") + debug_print(" heap usage at start of collection: ", + self.heap_usage + old_malloced, "bytes") + debug_print(" freed: ", + freed_size, "bytes") + debug_print(" new heap usage: ", + curr_heap_size, "bytes") + debug_print(" total time spent collecting: ", + self.total_collection_time, "seconds") + debug_print(" collecting time: ", + collect_time) + debug_print(" computing time: ", + collect_time) + debug_print(" new threshold: ", + self.bytes_malloced_threshold) ## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, ## size_gc_header) assert self.heap_usage + old_malloced == curr_heap_size + freed_size @@ -463,8 +454,9 @@ #llop.debug_view(lltype.Void, self.malloced_objects, self.malloced_objects_with_finalizer, size_gc_header) finalizer(obj) if not self.collect_in_progress: # another collection was caused? - llop.debug_print(lltype.Void, "outer collect interrupted " - "by recursive collect") + debug_print("outer collect interrupted " + "by recursive collect") + debug_stop("gc-collect") return if not last: if self.malloced_objects_with_finalizer == next: @@ -480,6 +472,7 @@ last.next = lltype.nullptr(self.HDR) hdr = next self.collect_in_progress = False + debug_stop("gc-collect") def _mark_root(self, root): # 'root' is the address of the GCPTR gcobjectaddr = root.address[0] Modified: pypy/branch/logging2/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/logging2/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/logging2/pypy/rpython/memory/gc/semispace.py Fri Oct 30 18:28:17 2009 @@ -6,7 +6,8 @@ from pypy.rpython.memory.support import AddressDict from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi from pypy.rlib.objectmodel import free_non_gc_object -from pypy.rlib.debug import ll_assert +from pypy.rlib.debug import ll_assert, have_debug_prints +from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck from pypy.rpython.memory.gc.base import MovingGCBase @@ -61,8 +62,7 @@ self.max_space_size = self.param_max_space_size self.red_zone = 0 - if self.config.gcconfig.debugprint: - self.program_start_time = time.time() + self.program_start_time = time.time() self.tospace = llarena.arena_malloc(self.space_size, True) ll_assert(bool(self.tospace), "couldn't allocate tospace") self.top_of_space = self.tospace + self.space_size @@ -74,7 +74,7 @@ self.objects_with_weakrefs = self.AddressStack() def _teardown(self): - llop.debug_print(lltype.Void, "Teardown") + debug_print("Teardown") llarena.arena_free(self.fromspace) llarena.arena_free(self.tospace) @@ -213,18 +213,13 @@ # (this is also a hook for the HybridGC) def semispace_collect(self, size_changing=False): - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void) - llop.debug_print(lltype.Void, - ".----------- Full collection ------------------") - start_usage = self.free - self.tospace - llop.debug_print(lltype.Void, - "| used before collection: ", - start_usage, "bytes") - start_time = time.time() - else: - start_time = 0 # Help the flow space - start_usage = 0 # Help the flow space + debug_start("gc-collect") + debug_print() + debug_print(".----------- Full collection ------------------") + start_usage = self.free - self.tospace + debug_print("| used before collection: ", + start_usage, "bytes") + start_time = time.time() #llop.debug_print(lltype.Void, 'semispace_collect', int(size_changing)) # Switch the spaces. We copy everything over to the empty space @@ -254,41 +249,33 @@ self.record_red_zone() self.execute_finalizers() #llop.debug_print(lltype.Void, 'collected', self.space_size, size_changing, self.top_of_space - self.free) - if self.config.gcconfig.debugprint: + if have_debug_prints(): end_time = time.time() elapsed_time = end_time - start_time self.total_collection_time += elapsed_time self.total_collection_count += 1 total_program_time = end_time - self.program_start_time end_usage = self.free - self.tospace - llop.debug_print(lltype.Void, - "| used after collection: ", - end_usage, "bytes") - llop.debug_print(lltype.Void, - "| freed: ", - start_usage - end_usage, "bytes") - llop.debug_print(lltype.Void, - "| size of each semispace: ", - self.space_size, "bytes") - llop.debug_print(lltype.Void, - "| fraction of semispace now used: ", - end_usage * 100.0 / self.space_size, "%") + debug_print("| used after collection: ", + end_usage, "bytes") + debug_print("| freed: ", + start_usage - end_usage, "bytes") + debug_print("| size of each semispace: ", + self.space_size, "bytes") + debug_print("| fraction of semispace now used: ", + end_usage * 100.0 / self.space_size, "%") ct = self.total_collection_time cc = self.total_collection_count - llop.debug_print(lltype.Void, - "| number of semispace_collects: ", - cc) - llop.debug_print(lltype.Void, - "| i.e.: ", - cc / total_program_time, "per second") - llop.debug_print(lltype.Void, - "| total time in semispace_collect: ", - ct, "seconds") - llop.debug_print(lltype.Void, - "| i.e.: ", - ct * 100.0 / total_program_time, "%") - llop.debug_print(lltype.Void, - "`----------------------------------------------") + debug_print("| number of semispace_collects: ", + cc) + debug_print("| i.e.: ", + cc / total_program_time, "per second") + debug_print("| total time in semispace_collect: ", + ct, "seconds") + debug_print("| i.e.: ", + ct * 100.0 / total_program_time, "%") + debug_print("`----------------------------------------------") + debug_stop("gc-collect") def starting_full_collect(self): pass # hook for the HybridGC Modified: pypy/branch/logging2/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/logging2/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/logging2/pypy/rpython/memory/gctransform/framework.py Fri Oct 30 18:28:17 2009 @@ -130,7 +130,7 @@ if hasattr(translator, '_jit2gc'): self.layoutbuilder = translator._jit2gc['layoutbuilder'] else: - if translator.config.translation.gcconfig.removetypeptr: + if translator.config.translation.gcremovetypeptr: lltype2vtable = translator.rtyper.lltype2vtable else: lltype2vtable = {} @@ -888,7 +888,7 @@ def gct_getfield(self, hop): if (hop.spaceop.args[1].value == 'typeptr' and hop.spaceop.args[0].concretetype.TO._hints.get('typeptr') and - self.translator.config.translation.gcconfig.removetypeptr): + self.translator.config.translation.gcremovetypeptr): self.transform_getfield_typeptr(hop) else: GCTransformer.gct_getfield(self, hop) @@ -896,7 +896,7 @@ def gct_setfield(self, hop): if (hop.spaceop.args[1].value == 'typeptr' and hop.spaceop.args[0].concretetype.TO._hints.get('typeptr') and - self.translator.config.translation.gcconfig.removetypeptr): + self.translator.config.translation.gcremovetypeptr): self.transform_setfield_typeptr(hop) else: GCTransformer.gct_setfield(self, hop) @@ -995,7 +995,7 @@ def __init__(self, config): from pypy.rpython.memory.gc.base import choose_gc_from_config try: - assert not config.translation.gcconfig.removetypeptr + assert not config.translation.gcremovetypeptr except AttributeError: # for some tests pass GCClass, _ = choose_gc_from_config(config) Modified: pypy/branch/logging2/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/logging2/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/logging2/pypy/rpython/memory/gctypelayout.py Fri Oct 30 18:28:17 2009 @@ -219,8 +219,7 @@ type_id = self.type_info_group.add_member(fullinfo) self.id_of_type[TYPE] = type_id # store the vtable of the type (if any) immediately thereafter - # (note that if gcconfig.removetypeptr is False, lltype2vtable - # is empty) + # (note that if gcremovetypeptr is False, lltype2vtable is empty) vtable = self.lltype2vtable.get(TYPE, None) if vtable is not None: # check that if we have a vtable, we are not varsize Modified: pypy/branch/logging2/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/logging2/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/logging2/pypy/rpython/memory/test/test_transformed_gc.py Fri Oct 30 18:28:17 2009 @@ -23,7 +23,7 @@ t = TranslationContext() # XXX XXX XXX mess t.config.translation.gc = gcname - t.config.translation.gcconfig.removetypeptr = True + t.config.translation.gcremovetypeptr = True if stacklessgc: t.config.translation.gcrootfinder = "stackless" t.config.set(**extraconfigopts) Modified: pypy/branch/logging2/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/logging2/pypy/translator/c/gc.py (original) +++ pypy/branch/logging2/pypy/translator/c/gc.py Fri Oct 30 18:28:17 2009 @@ -352,7 +352,7 @@ def need_no_typeptr(self): config = self.db.translator.config - return config.translation.gcconfig.removetypeptr + return config.translation.gcremovetypeptr def OP_GC_GETTYPEPTR_GROUP(self, funcgen, op): # expands to a number of steps, as per rpython/lltypesystem/opimpl.py, From arigo at codespeak.net Fri Oct 30 18:44:55 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Oct 2009 18:44:55 +0100 (CET) Subject: [pypy-svn] r68871 - pypy/branch/logging2/pypy/translator/c/src Message-ID: <20091030174455.E1300168483@codespeak.net> Author: arigo Date: Fri Oct 30 18:44:55 2009 New Revision: 68871 Modified: pypy/branch/logging2/pypy/translator/c/src/debug.h Log: Tweat the output and add some light coloring if dumping to stderr. Modified: pypy/branch/logging2/pypy/translator/c/src/debug.h ============================================================================== --- pypy/branch/logging2/pypy/translator/c/src/debug.h (original) +++ pypy/branch/logging2/pypy/translator/c/src/debug.h Fri Oct 30 18:44:55 2009 @@ -45,11 +45,15 @@ #ifndef PYPY_NOT_MAIN_FILE #include #include +#include int pypy_ignoring_nested_prints = 0; FILE *pypy_debug_file = NULL; static bool_t debug_ready = 0; static bool_t debug_profile = 0; +static char *debug_start_colors_1 = ""; +static char *debug_start_colors_2 = ""; +static char *debug_stop_colors = ""; static char *debug_prefix = NULL; static void pypy_debug_open(void) @@ -76,7 +80,15 @@ pypy_debug_file = fopen(filename, "w"); } if (!pypy_debug_file) - pypy_debug_file = stderr; + { + pypy_debug_file = stderr; + if (isatty(2)) + { + debug_start_colors_1 = "\033[1m\033[31m"; + debug_start_colors_2 = "\033[31m"; + debug_stop_colors = "\033[0m"; + } + } debug_ready = 1; } @@ -100,11 +112,15 @@ return 1; } -static void display_startstop(const char *start, const char *category) +static void display_startstop(const char *prefix, const char *postfix, + const char *category, const char *colors) { long long timestamp; READ_TIMESTAMP(timestamp); - fprintf(pypy_debug_file, "{%llx} -%s- %s\n", timestamp, start, category); + fprintf(pypy_debug_file, "%s[%llx] %s%s%s\n%s", + colors, + timestamp, prefix, category, postfix, + debug_stop_colors); } void pypy_debug_start(const char *category) @@ -132,7 +148,7 @@ return; } } - display_startstop("start", category); + display_startstop("{", "", category, debug_start_colors_1); } void pypy_debug_stop(const char *category) @@ -143,7 +159,7 @@ if (!debug_profile) return; } - display_startstop("stop", category); + display_startstop("", "}", category, debug_start_colors_2); } #endif /* PYPY_NOT_MAIN_FILE */ From arigo at codespeak.net Fri Oct 30 18:55:02 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 30 Oct 2009 18:55:02 +0100 (CET) Subject: [pypy-svn] r68873 - in pypy/branch/logging2/pypy/translator/c: src test Message-ID: <20091030175502.DF32C168483@codespeak.net> Author: arigo Date: Fri Oct 30 18:55:02 2009 New Revision: 68873 Modified: pypy/branch/logging2/pypy/translator/c/src/debug.h pypy/branch/logging2/pypy/translator/c/test/test_standalone.py Log: Test and fix. Modified: pypy/branch/logging2/pypy/translator/c/src/debug.h ============================================================================== --- pypy/branch/logging2/pypy/translator/c/src/debug.h (original) +++ pypy/branch/logging2/pypy/translator/c/src/debug.h Fri Oct 30 18:55:02 2009 @@ -125,10 +125,10 @@ void pypy_debug_start(const char *category) { + pypy_debug_ensure_opened(); if (debug_profile) { /* profiling version */ - pypy_debug_ensure_opened(); pypy_ignoring_nested_prints++; /* disable nested debug_print */ } else @@ -140,7 +140,6 @@ pypy_ignoring_nested_prints++; return; } - pypy_debug_ensure_opened(); if (!debug_prefix || !startswith(category, debug_prefix)) { /* wrong section name, or no PYPYLOG at all, skip it */ Modified: pypy/branch/logging2/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/branch/logging2/pypy/translator/c/test/test_standalone.py (original) +++ pypy/branch/logging2/pypy/translator/c/test/test_standalone.py Fri Oct 30 18:55:02 2009 @@ -259,8 +259,6 @@ def test_debug_print_start_stop(self): def entry_point(argv): x = "got:" - if have_debug_prints(): x += "a" - debug_print("toplevel") debug_start ("mycat") if have_debug_prints(): x += "b" debug_print ("foo", 2, "bar", 3) @@ -271,6 +269,8 @@ if have_debug_prints(): x += "d" debug_print ("bok") debug_stop ("mycat") + if have_debug_prints(): x += "a" + debug_print("toplevel") os.write(1, x + '.\n') return 0 t, cbuilder = self.compile(entry_point) @@ -294,25 +294,29 @@ assert 'bok' not in err # check with PYPYLOG=:- (means print to stderr) out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': ':-'}) - assert out.strip() == 'got:abcd.' + assert out.strip() == 'got:bcda.' assert 'toplevel' in err - assert 'mycat' in err + assert '{mycat' in err + assert 'mycat}' in err assert 'foo 2 bar 3' in err - assert 'cat2' in err + assert '{cat2' in err + assert 'cat2}' in err assert 'baz' in err assert 'bok' in err # check with PYPYLOG=:somefilename path = udir.join('test_debug_xxx.log') out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': ':%s' % path}) - assert out.strip() == 'got:abcd.' + assert out.strip() == 'got:bcda.' assert not err assert path.check(file=1) data = path.read() assert 'toplevel' in data - assert 'mycat' in data + assert '{mycat' in data + assert 'mycat}' in data assert 'foo 2 bar 3' in data - assert 'cat2' in data + assert '{cat2' in data + assert 'cat2}' in data assert 'baz' in data assert 'bok' in data # check with PYPYLOG=somefilename @@ -323,22 +327,25 @@ assert path.check(file=1) data = path.read() assert 'toplevel' in data - assert 'mycat' in data + assert '{mycat' in data + assert 'mycat}' in data assert 'foo 2 bar 3' not in data - assert 'cat2' in data + assert '{cat2' in data + assert 'cat2}' in data assert 'baz' not in data assert 'bok' not in data # check with PYPYLOG=myc:somefilename (includes mycat but not cat2) path = udir.join('test_debug_xxx_myc.log') out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': 'myc:%s' % path}) - assert out.strip() == 'got:abd.' + assert out.strip() == 'got:bda.' assert not err assert path.check(file=1) data = path.read() - assert 'toplevel' in path.read() - assert 'mycat' in path.read() - assert 'foo 2 bar 3' in path.read() + assert 'toplevel' in data + assert '{mycat' in data + assert 'mycat}' in data + assert 'foo 2 bar 3' in data assert 'cat2' not in data assert 'baz' not in data assert 'bok' in data From arigo at codespeak.net Sat Oct 31 10:51:58 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 31 Oct 2009 10:51:58 +0100 (CET) Subject: [pypy-svn] r68875 - in pypy/branch/logging2/pypy: jit/metainterp rlib Message-ID: <20091031095158.A093F1680B8@codespeak.net> Author: arigo Date: Sat Oct 31 10:51:57 2009 New Revision: 68875 Modified: pypy/branch/logging2/pypy/jit/metainterp/logger.py pypy/branch/logging2/pypy/jit/metainterp/pyjitpl.py pypy/branch/logging2/pypy/rlib/debug.py Log: Use debug_start/stop/print in logger.py. Print start/stop in red also when not translated. Modified: pypy/branch/logging2/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/logger.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/logger.py Sat Oct 31 10:51:57 2009 @@ -1,48 +1,43 @@ import os -from pypy.rlib.objectmodel import compute_unique_id +from pypy.rlib.debug import have_debug_prints +from pypy.rlib.debug import debug_start, debug_stop, debug_print from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.history import Const, ConstInt, Box, \ BoxInt, ConstAddr, ConstFloat, BoxFloat, AbstractFailDescr -from pypy.rlib.streamio import open_file_as_stream class Logger(object): def __init__(self, ts, guard_number=False): - self.log_stream = None self.ts = ts self.guard_number=guard_number - def create_log(self, extension='.ops'): - if self.log_stream is not None: - return self.log_stream - s = os.environ.get('PYPYJITLOG') - if not s: - return None - s += extension - try: - self.log_stream = open_file_as_stream(s, 'w') - except OSError: - os.write(2, "could not create log file\n") - return None - return self.log_stream - def log_loop(self, inputargs, operations, number=0, type=None): - if self.log_stream is None: + if not have_debug_prints(): return - if type is not None: - self.log_stream.write("# Loop%d (%s), %d ops\n" % (number, - type, - len(operations))) - self._log_operations(inputargs, operations, {}) + if type is None: + debug_start("jit-log-noopt-loop") + self._log_operations(inputargs, operations) + debug_stop("jit-log-noopt-loop") + else: + debug_start("jit-log-opt-loop") + debug_print("# Loop", number, ":", type, + "with", len(operations), "ops") + self._log_operations(inputargs, operations) + debug_stop("jit-log-opt-loop") def log_bridge(self, inputargs, operations, number=-1): - if self.log_stream is None: + if not have_debug_prints(): return - if number != -1: - self.log_stream.write("# bridge out of Guard%d, %d ops\n" % (number, - len(operations))) - self._log_operations(inputargs, operations, {}) - + if number == -1: + debug_start("jit-log-noopt-bridge") + self._log_operations(inputargs, operations) + debug_stop("jit-log-noopt-bridge") + else: + debug_start("jit-log-opt-bridge") + debug_print("# bridge out of Guard", number, + "with", len(operations), "ops") + self._log_operations(inputargs, operations) + debug_stop("jit-log-opt-bridge") def repr_of_descr(self, descr): return descr.repr_of_descr() @@ -70,15 +65,18 @@ else: return '?' - def _log_operations(self, inputargs, operations, memo): + def _log_operations(self, inputargs, operations): + if not have_debug_prints(): + return + memo = {} if inputargs is not None: args = ", ".join([self.repr_of_arg(memo, arg) for arg in inputargs]) - self.log_stream.write('[' + args + ']\n') + debug_print('[' + args + ']') for i in range(len(operations)): op = operations[i] if op.opnum == rop.DEBUG_MERGE_POINT: loc = op.args[0]._get_str() - self.log_stream.write("debug_merge_point('%s')\n" % (loc,)) + debug_print("debug_merge_point('%s')" % (loc,)) continue args = ", ".join([self.repr_of_arg(memo, arg) for arg in op.args]) if op.result is not None: @@ -99,6 +97,5 @@ for arg in op.fail_args]) + ']' else: fail_args = '' - self.log_stream.write(res + op.getopname() + - '(' + args + ')' + fail_args + '\n') - self.log_stream.flush() + debug_print(res + op.getopname() + + '(' + args + ')' + fail_args) Modified: pypy/branch/logging2/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/pyjitpl.py Sat Oct 31 10:51:57 2009 @@ -1027,8 +1027,6 @@ self.profiler.start() self.profiler.initialized = True self.globaldata.initialized = True - self.logger_noopt.create_log('.noopt') - self.logger_ops.create_log('.ops') self.globaldata.storedebug = os.environ.get('PYPYJITRESUMELOG') def _setup_class_sizes(self): Modified: pypy/branch/logging2/pypy/rlib/debug.py ============================================================================== --- pypy/branch/logging2/pypy/rlib/debug.py (original) +++ pypy/branch/logging2/pypy/rlib/debug.py Sat Oct 31 10:51:57 2009 @@ -39,11 +39,22 @@ hop.genop('debug_print', vlist) +if sys.stderr.isatty(): + _start_colors_1 = "\033[1m\033[31m" + _start_colors_2 = "\033[31m" + _stop_colors = "\033[0m" +else: + _start_colors_1 = "" + _start_colors_2 = "" + _stop_colors = "" + def debug_start(category): - print >> sys.stderr, '[%s] debug_start %s' % (time.clock(), category) + print >> sys.stderr, '%s[%s] {%s%s' % (_start_colors_1, time.clock(), + category, _stop_colors) def debug_stop(category): - print >> sys.stderr, '[%s] debug_stop %s' % (time.clock(), category) + print >> sys.stderr, '%s[%s] %s}%s' % (_start_colors_2, time.clock(), + category, _stop_colors) class Entry(ExtRegistryEntry): _about_ = debug_start, debug_stop From arigo at codespeak.net Sat Oct 31 11:12:56 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 31 Oct 2009 11:12:56 +0100 (CET) Subject: [pypy-svn] r68876 - in pypy/branch/logging2/pypy/jit/metainterp: . test Message-ID: <20091031101256.9B9CE16807A@codespeak.net> Author: arigo Date: Sat Oct 31 11:12:56 2009 New Revision: 68876 Modified: pypy/branch/logging2/pypy/jit/metainterp/optimizeopt.py pypy/branch/logging2/pypy/jit/metainterp/pyjitpl.py pypy/branch/logging2/pypy/jit/metainterp/resume.py pypy/branch/logging2/pypy/jit/metainterp/simple_optimize.py pypy/branch/logging2/pypy/jit/metainterp/test/test_resume.py Log: Convert dump_storage() to debug_start/stop/prints. Modified: pypy/branch/logging2/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/optimizeopt.py Sat Oct 31 11:12:56 2009 @@ -518,8 +518,7 @@ def store_final_boxes_in_guard(self, op): descr = op.descr assert isinstance(descr, compile.ResumeGuardDescr) - modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo, - self.metainterp_sd.globaldata.storedebug) + modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo) newboxes = modifier.finish(self.values) if len(newboxes) > self.metainterp_sd.options.failargs_limit: raise compile.GiveUp Modified: pypy/branch/logging2/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/pyjitpl.py Sat Oct 31 11:12:56 2009 @@ -1027,7 +1027,6 @@ self.profiler.start() self.profiler.initialized = True self.globaldata.initialized = True - self.globaldata.storedebug = os.environ.get('PYPYJITRESUMELOG') def _setup_class_sizes(self): class_sizes = {} Modified: pypy/branch/logging2/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/resume.py Sat Oct 31 11:12:56 2009 @@ -4,6 +4,8 @@ from pypy.rpython.lltypesystem import rffi from pypy.rlib import rarithmetic from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.debug import have_debug_prints +from pypy.rlib.debug import debug_start, debug_stop, debug_print # Logic to encode the chain of frames and the state of the boxes at a # guard operation, and to decode it again. This is a bit advanced, @@ -177,10 +179,9 @@ class ResumeDataVirtualAdder(object): - def __init__(self, storage, memo, debug_storage=None): + def __init__(self, storage, memo): self.storage = storage self.memo = memo - self.debug_storage = debug_storage #self.virtuals = [] #self.vfieldboxes = [] @@ -256,8 +257,8 @@ self._number_virtuals(liveboxes) storage.rd_consts = self.memo.consts - if self.debug_storage: - dump_storage(self.debug_storage, storage, liveboxes) + if have_debug_prints(): + dump_storage(storage, liveboxes) return liveboxes[:] def _number_virtuals(self, liveboxes): @@ -307,6 +308,13 @@ self.fielddescrs[i], box, fieldbox) + def debug_prints(self): + assert len(self.fielddescrs) == len(self.fieldnums) + for i in range(len(self.fielddescrs)): + debug_print("\t\t", + str(self.fielddescrs[i]), + str(untag(self.fieldnums[i]))) + class VirtualInfo(AbstractVirtualStructInfo): def __init__(self, known_class, fielddescrs): AbstractVirtualStructInfo.__init__(self, fielddescrs) @@ -316,11 +324,9 @@ return metainterp.execute_and_record(rop.NEW_WITH_VTABLE, None, self.known_class) - def repr_rpython(self): - return 'VirtualInfo("%s", %s, %s)' % ( - self.known_class, - ['"%s"' % (fd,) for fd in self.fielddescrs], - [untag(i) for i in self.fieldnums]) + def debug_prints(self): + debug_print("\tvirtualinfo", self.known_class.repr_rpython()) + AbstractVirtualStructInfo.debug_prints(self) class VStructInfo(AbstractVirtualStructInfo): def __init__(self, typedescr, fielddescrs): @@ -330,11 +336,9 @@ def allocate(self, metainterp): return metainterp.execute_and_record(rop.NEW, self.typedescr) - def repr_rpython(self): - return 'VStructInfo("%s", %s, %s)' % ( - self.typedescr, - ['"%s"' % (fd,) for fd in self.fielddescrs], - [untag(i) for i in self.fieldnums]) + def debug_prints(self): + debug_print("\tvstructinfo", self.typedescr.repr_rpython()) + AbstractVirtualStructInfo.debug_prints(self) class VArrayInfo(AbstractVirtualInfo): def __init__(self, arraydescr): @@ -354,10 +358,10 @@ self.arraydescr, box, ConstInt(i), itembox) - def repr_rpython(self): - return 'VArrayInfo("%s", %s)' % ( - self.arraydescr, - [untag(i) for i in self.fieldnums]) + def debug_prints(self): + debug_print("\tvarrayinfo", self.arraydescr) + for i in self.fieldnums: + debug_print("\t\t", str(untag(i))) def rebuild_from_resumedata(metainterp, newboxes, storage, expects_virtualizables): @@ -423,38 +427,31 @@ # ____________________________________________________________ -def dump_storage(logname, storage, liveboxes): +def dump_storage(storage, liveboxes): "For profiling only." - import os - from pypy.rlib import objectmodel - assert logname is not None # annotator hack - fd = os.open(logname, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0666) - os.write(fd, 'Log(%d, [\n' % objectmodel.compute_unique_id(storage)) + from pypy.rlib.objectmodel import compute_unique_id + debug_start("jit-resume") + debug_print('Log storage', compute_unique_id(storage)) frameinfo = storage.rd_frame_info_list - while True: - os.write(fd, '\t("%s", %d, %d) at %xd,\n' % ( - frameinfo.jitcode, frameinfo.pc, frameinfo.exception_target, - objectmodel.compute_unique_id(frameinfo))) + while frameinfo is not None: + try: + jitcodename = frameinfo.jitcode.name + except AttributeError: + jitcodename = str(compute_unique_id(frameinfo.jitcode)) + debug_print('\tjitcode/pc', jitcodename, + frameinfo.pc, frameinfo.exception_target, + 'at', compute_unique_id(frameinfo)) frameinfo = frameinfo.prev - if frameinfo is None: - break - os.write(fd, '\t],\n\t[\n') numb = storage.rd_numb - while True: - os.write(fd, '\t\t%s at %xd,\n' % ([untag(i) for i in numb.nums], - objectmodel.compute_unique_id(numb))) + while numb is not None: + debug_print('\tnumb', str([untag(i) for i in numb.nums]), + 'at', compute_unique_id(numb)) numb = numb.prev - if numb is None: - break - os.write(fd, '\t], [\n') for const in storage.rd_consts: - os.write(fd, '\t"%s",\n' % (const.repr_rpython(),)) - os.write(fd, '\t], [\n') + debug_print('\tconst', const.repr_rpython()) for box in liveboxes: - os.write(fd, '\t"%s",\n' % (box.repr_rpython(),)) - os.write(fd, '\t], [\n') + debug_print('\tbox', box.repr_rpython()) if storage.rd_virtuals is not None: for virtual in storage.rd_virtuals: - os.write(fd, '\t%s,\n' % (virtual.repr_rpython(),)) - os.write(fd, '\t])\n') - os.close(fd) + virtual.debug_prints() + debug_stop("jit-resume") Modified: pypy/branch/logging2/pypy/jit/metainterp/simple_optimize.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/simple_optimize.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/simple_optimize.py Sat Oct 31 11:12:56 2009 @@ -22,8 +22,7 @@ if op.is_guard(): descr = op.descr assert isinstance(descr, compile.ResumeGuardDescr) - modifier = resume.ResumeDataVirtualAdder(descr, memo, - metainterp_sd.globaldata.storedebug) + modifier = resume.ResumeDataVirtualAdder(descr, memo) newboxes = modifier.finish(EMPTY_VALUES) descr.store_final_boxes(op, newboxes) newoperations.append(op) Modified: pypy/branch/logging2/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/test/test_resume.py Sat Oct 31 11:12:56 2009 @@ -8,7 +8,9 @@ from pypy.jit.metainterp import executor class Storage: - pass + rd_frame_info_list = None + rd_numb = None + rd_consts = [] def test_tag(): assert tag(3, 1) == rffi.r_short(3<<2|1) @@ -591,6 +593,7 @@ snapshot = Snapshot(snapshot, [ConstInt(2), ConstInt(3)]) snapshot = Snapshot(snapshot, [b1, b2, b3]) storage.rd_snapshot = snapshot + storage.rd_frame_info_list = None return storage def test_virtual_adder_int_constants(): @@ -761,6 +764,7 @@ [b4s, c1s]) # new fields liveboxes = [] modifier._number_virtuals(liveboxes) + dump_storage(storage, liveboxes) storage.rd_consts = memo.consts[:] storage.rd_numb = None # resume @@ -807,6 +811,7 @@ [c1s, b4s]) # new fields liveboxes = [] modifier._number_virtuals(liveboxes) + dump_storage(storage, liveboxes) storage.rd_consts = memo.consts[:] storage.rd_numb = None b4t = BoxPtr() From arigo at codespeak.net Sat Oct 31 11:53:55 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 31 Oct 2009 11:53:55 +0100 (CET) Subject: [pypy-svn] r68877 - pypy/branch/logging2/pypy/jit/metainterp Message-ID: <20091031105355.F32F7168006@codespeak.net> Author: arigo Date: Sat Oct 31 11:53:55 2009 New Revision: 68877 Modified: pypy/branch/logging2/pypy/jit/metainterp/compile.py pypy/branch/logging2/pypy/jit/metainterp/history.py pypy/branch/logging2/pypy/jit/metainterp/policy.py pypy/branch/logging2/pypy/jit/metainterp/pyjitpl.py Log: Add debug_start/stop/print in pyjitpl. The next step is to try to remove jitprof.py. Modified: pypy/branch/logging2/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/compile.py Sat Oct 31 11:53:55 2009 @@ -5,7 +5,7 @@ from pypy.conftest import option from pypy.jit.metainterp.resoperation import ResOperation, rop -from pypy.jit.metainterp.history import TreeLoop, log, Box, History, LoopToken +from pypy.jit.metainterp.history import TreeLoop, Box, History, LoopToken from pypy.jit.metainterp.history import AbstractFailDescr, BoxInt from pypy.jit.metainterp.history import BoxPtr, BoxObj, BoxFloat, Const from pypy.jit.metainterp import history Modified: pypy/branch/logging2/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/history.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/history.py Sat Oct 31 11:53:55 2009 @@ -3,18 +3,13 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.ootypesystem import ootype from pypy.rlib.objectmodel import we_are_translated, r_dict, Symbolic -from pypy.rlib.objectmodel import compute_hash +from pypy.rlib.objectmodel import compute_hash, compute_unique_id from pypy.rlib.rarithmetic import intmask from pypy.tool.uid import uid from pypy.conftest import option from pypy.jit.metainterp.resoperation import ResOperation, rop -import py -from pypy.tool.ansi_print import ansi_log -log = py.log.Producer('compiler') -py.log.setconsumer('compiler', ansi_log) - # ____________________________________________________________ INT = 'i' @@ -69,14 +64,9 @@ except AttributeError: return box.value -class ReprRPython: - def __init__(self): - self.seen = {} - def repr_rpython(self, box, typechars): - n = self.seen.setdefault(box, len(self.seen)) - return '%s/%s%d' % (box._get_hash_(), typechars, n) - -repr_rpython = ReprRPython().repr_rpython +def repr_rpython(box, typechars): + return '%s/%s%d' % (box._get_hash_(), typechars, + compute_unique_id(box)) class AbstractValue(object): Modified: pypy/branch/logging2/pypy/jit/metainterp/policy.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/policy.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/policy.py Sat Oct 31 11:53:55 2009 @@ -77,8 +77,9 @@ getkind(v.concretetype, supports_floats) getkind(op.result.concretetype, supports_floats) except NotImplementedError, e: - history.log.WARNING('%s, ignoring graph' % (e,)) - history.log.WARNING(' %s' % (graph,)) + from pypy.jit.metainterp.codewriter import log + log.WARNING('%s, ignoring graph' % (e,)) + log.WARNING(' %s' % (graph,)) return True return False Modified: pypy/branch/logging2/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/pyjitpl.py Sat Oct 31 11:53:55 2009 @@ -3,7 +3,7 @@ from pypy.rpython.ootypesystem import ootype from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib.debug import debug_print +from pypy.rlib.debug import debug_start, debug_stop, debug_print from pypy.jit.metainterp import history, compile, resume from pypy.jit.metainterp.history import Const, ConstInt, Box @@ -875,12 +875,10 @@ self.pc = pc self.exception_target = exception_target self.env = env - if self.metainterp.staticdata.state.debug_level >= DEBUG_DETAILED: - values = ' '.join([box.repr_rpython() for box in self.env]) - log = self.metainterp.staticdata.log - log('setup_resume_at_op %s:%d [%s] %d' % (self.jitcode.name, - self.pc, values, - self.exception_target)) + ## values = ' '.join([box.repr_rpython() for box in self.env]) + ## log('setup_resume_at_op %s:%d [%s] %d' % (self.jitcode.name, + ## self.pc, values, + ## self.exception_target)) def run_one_step(self): # Execute the frame forward. This method contains a loop that leaves @@ -1022,7 +1020,7 @@ if not self.globaldata.initialized: self._setup_class_sizes() self.cpu.setup_once() - self.log(self.jit_starting_line) + debug_print(self.jit_starting_line) if not self.profiler.initialized: self.profiler.start() self.profiler.initialized = True @@ -1078,12 +1076,8 @@ # ---------------- logging ------------------------ - def log(self, msg, event_kind='info'): - if self.state.debug_level > DEBUG_PROFILE: - if not we_are_translated(): - getattr(history.log, event_kind)(msg) - else: - debug_print(msg) + def log(self, msg): + debug_print(msg) # ____________________________________________________________ @@ -1136,12 +1130,6 @@ def is_blackholing(self): return self.history is None - def blackholing_text(self): - if self.history is None: - return " (BlackHole)" - else: - return "" - def newframe(self, jitcode): if jitcode is self.staticdata.portal_code: self.in_recursion += 1 @@ -1333,9 +1321,11 @@ op.name = self.framestack[-1].jitcode.name def switch_to_blackhole(self): + debug_print('~~~ ABORTING TRACING') + debug_stop('jit-tracing') + debug_start('jit-blackhole') self.history = None # start blackholing self.staticdata.stats.aborted() - self.staticdata.log('~~~ ABORTING TRACING', event_kind='event') self.staticdata.profiler.end_tracing() self.staticdata.profiler.start_blackhole() @@ -1350,8 +1340,6 @@ # Execute the frames forward until we raise a DoneWithThisFrame, # a ContinueRunningNormally, or a GenerateMergePoint exception. self.staticdata.stats.entered() - self.staticdata.log('~~~ ENTER' + self.blackholing_text(), - event_kind='event') try: while True: self.framestack[-1].run_one_step() @@ -1363,8 +1351,6 @@ self.staticdata.profiler.end_blackhole() else: self.staticdata.profiler.end_tracing() - self.staticdata.log('~~~ LEAVE' + self.blackholing_text(), - event_kind='event') def interpret(self): if we_are_translated(): @@ -1375,11 +1361,22 @@ except: import sys if sys.exc_info()[0] is not None: - history.log.info(sys.exc_info()[0].__name__) + codewriter.log.info(sys.exc_info()[0].__name__) raise def compile_and_run_once(self, *args): - self.staticdata.log('Switching from interpreter to compiler') + self.staticdata._setup_once() + debug_start('jit-tracing') + self.create_empty_history() + try: + return self._compile_and_run_once(*args) + finally: + if self.history is None: + debug_stop('jit-blackhole') + else: + debug_stop('jit-tracing') + + def _compile_and_run_once(self, *args): original_boxes = self.initialize_state_from_start(*args) self.current_merge_points = [(original_boxes, 0)] num_green_args = self.staticdata.num_green_args @@ -1395,9 +1392,24 @@ return self.designate_target_loop(gmp) def handle_guard_failure(self, key): - from pypy.jit.metainterp.warmspot import ContinueRunningNormallyBase assert isinstance(key, compile.ResumeGuardDescr) - resumedescr = self.initialize_state_from_guard_failure(key) + warmrunnerstate = self.staticdata.state + must_compile = warmrunnerstate.must_compile_from_failure(key) + if must_compile: + debug_start('jit-tracing') + else: + debug_start('jit-blackhole') + self.initialize_state_from_guard_failure(key, must_compile) + try: + return self._handle_guard_failure(key) + finally: + if self.history is None: + debug_stop('jit-blackhole') + else: + debug_stop('jit-tracing') + + def _handle_guard_failure(self, key): + from pypy.jit.metainterp.warmspot import ContinueRunningNormallyBase original_greenkey = key.original_greenkey # notice that here we just put the greenkey # use -1 to mark that we will have to give up @@ -1415,7 +1427,7 @@ except ContinueRunningNormallyBase: if not started_as_blackhole: warmrunnerstate = self.staticdata.state - warmrunnerstate.reset_counter_from_failure(resumedescr) + warmrunnerstate.reset_counter_from_failure(key) raise def forget_consts(self, boxes, startindex=0): @@ -1577,9 +1589,7 @@ def initialize_state_from_start(self, *args): self.in_recursion = -1 # always one portal around - self.staticdata._setup_once() self.staticdata.profiler.start_tracing() - self.create_empty_history() num_green_args = self.staticdata.num_green_args original_boxes = [] self._initialize_from_start(original_boxes, num_green_args, *args) @@ -1591,12 +1601,11 @@ self.initialize_virtualizable(original_boxes) return original_boxes - def initialize_state_from_guard_failure(self, resumedescr): + def initialize_state_from_guard_failure(self, resumedescr, must_compile): # guard failure: rebuild a complete MIFrame stack self.in_recursion = -1 # always one portal around inputargs = self.load_values_from_failure(resumedescr) warmrunnerstate = self.staticdata.state - must_compile = warmrunnerstate.must_compile_from_failure(resumedescr) if must_compile: self.history = history.History(self.cpu) self.history.inputargs = inputargs @@ -1605,7 +1614,6 @@ self.staticdata.profiler.start_blackhole() self.history = None # this means that is_blackholing() is true self.rebuild_state_after_failure(resumedescr, inputargs) - return resumedescr def load_values_from_failure(self, resumedescr): cpu = self.cpu From arigo at codespeak.net Sat Oct 31 12:04:20 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 31 Oct 2009 12:04:20 +0100 (CET) Subject: [pypy-svn] r68878 - in pypy/branch/logging2/pypy/rlib: . test Message-ID: <20091031110420.E39441680AF@codespeak.net> Author: arigo Date: Sat Oct 31 12:04:19 2009 New Revision: 68878 Modified: pypy/branch/logging2/pypy/rlib/debug.py pypy/branch/logging2/pypy/rlib/test/test_debug.py Log: A better way for tests to check that some debug_start/stop/prints really occurred. Modified: pypy/branch/logging2/pypy/rlib/debug.py ============================================================================== --- pypy/branch/logging2/pypy/rlib/debug.py (original) +++ pypy/branch/logging2/pypy/rlib/debug.py Sat Oct 31 12:04:19 2009 @@ -20,10 +20,23 @@ hop.genop('debug_assert', vlist) +class DebugLog(list): + def debug_print(self, *args): + self.append(('debug_print',) + args) + def debug_start(self, category): + self.append(('debug_start', category)) + def debug_stop(self, category): + self.append(('debug_stop', category)) + +_log = None # patched from tests to be an object of class DebugLog + # or compatible + def debug_print(*args): for arg in args: print >> sys.stderr, arg, print >> sys.stderr + if _log is not None: + _log.debug_print(*args) class Entry(ExtRegistryEntry): _about_ = debug_print @@ -51,10 +64,14 @@ def debug_start(category): print >> sys.stderr, '%s[%s] {%s%s' % (_start_colors_1, time.clock(), category, _stop_colors) + if _log is not None: + _log.debug_start(category) def debug_stop(category): print >> sys.stderr, '%s[%s] %s}%s' % (_start_colors_2, time.clock(), category, _stop_colors) + if _log is not None: + _log.debug_stop(category) class Entry(ExtRegistryEntry): _about_ = debug_start, debug_stop Modified: pypy/branch/logging2/pypy/rlib/test/test_debug.py ============================================================================== --- pypy/branch/logging2/pypy/rlib/test/test_debug.py (original) +++ pypy/branch/logging2/pypy/rlib/test/test_debug.py Sat Oct 31 12:04:19 2009 @@ -3,6 +3,7 @@ from pypy.rlib.debug import check_annotation, make_sure_not_resized from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rlib.debug import have_debug_prints +from pypy.rlib import debug from pypy.rpython.test.test_llinterp import interpret def test_check_annotation(): @@ -44,33 +45,31 @@ class DebugTests: def test_debug_print_start_stop(self): - import sys - from cStringIO import StringIO - def f(x): debug_start("mycat") debug_print("foo", 2, "bar", x) debug_stop("mycat") return have_debug_prints() - olderr = sys.stderr try: - sys.stderr = c = StringIO() + debug._log = dlog = debug.DebugLog() res = f(3) assert res == True finally: - sys.stderr = olderr - assert 'mycat' in c.getvalue() - assert 'foo 2 bar 3' in c.getvalue() + debug._log = None + assert ('debug_start', 'mycat') in dlog + assert ('debug_print', 'foo', 2, 'bar', 3) in dlog + assert ('debug_stop', 'mycat') in dlog try: - sys.stderr = c = StringIO() + debug._log = dlog = debug.DebugLog() res = self.interpret(f, [3]) assert res == True finally: - sys.stderr = olderr - assert 'mycat' in c.getvalue() - assert 'foo 2 bar 3' in c.getvalue() + debug._log = None + assert ('debug_start', 'mycat') in dlog + assert ('debug_print', 'foo', 2, 'bar', 3) in dlog + assert ('debug_stop', 'mycat') in dlog class TestLLType(DebugTests): From arigo at codespeak.net Sat Oct 31 17:42:51 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 31 Oct 2009 17:42:51 +0100 (CET) Subject: [pypy-svn] r68879 - in pypy/branch/logging2/pypy: rlib rlib/test tool Message-ID: <20091031164251.C218C1680AB@codespeak.net> Author: arigo Date: Sat Oct 31 17:42:50 2009 New Revision: 68879 Added: pypy/branch/logging2/pypy/tool/logparser.py (contents, props changed) Modified: pypy/branch/logging2/pypy/rlib/debug.py pypy/branch/logging2/pypy/rlib/test/test_debug.py Log: Start pypy.tool.logparser, a tool to load and format the debugging logs. So far, can be formatted as a PNG image giving the timeline (with PIL). Modified: pypy/branch/logging2/pypy/rlib/debug.py ============================================================================== --- pypy/branch/logging2/pypy/rlib/debug.py (original) +++ pypy/branch/logging2/pypy/rlib/debug.py Sat Oct 31 17:42:50 2009 @@ -23,10 +23,25 @@ class DebugLog(list): def debug_print(self, *args): self.append(('debug_print',) + args) - def debug_start(self, category): - self.append(('debug_start', category)) - def debug_stop(self, category): - self.append(('debug_stop', category)) + def debug_start(self, category, time=None): + self.append(('debug_start', category, time)) + def debug_stop(self, category, time=None): + for i in xrange(len(self)-1, -1, -1): + if self[i][0] == 'debug_start': + assert self[i][1] == category, ( + "nesting error: starts with %r but stops with %r" % + (self[i][1], category)) + starttime = self[i][2] + if starttime is not None or time is not None: + self[i:] = [(category, starttime, time, self[i+1:])] + else: + self[i:] = [(category, self[i+1:])] + return + assert False, ("nesting error: no start corresponding to stop %r" % + (category,)) + def __repr__(self): + import pprint + return pprint.pformat(list(self)) _log = None # patched from tests to be an object of class DebugLog # or compatible Modified: pypy/branch/logging2/pypy/rlib/test/test_debug.py ============================================================================== --- pypy/branch/logging2/pypy/rlib/test/test_debug.py (original) +++ pypy/branch/logging2/pypy/rlib/test/test_debug.py Sat Oct 31 17:42:50 2009 @@ -57,9 +57,11 @@ assert res == True finally: debug._log = None - assert ('debug_start', 'mycat') in dlog - assert ('debug_print', 'foo', 2, 'bar', 3) in dlog - assert ('debug_stop', 'mycat') in dlog + assert dlog == [ + ("mycat", [ + ('debug_print', 'foo', 2, 'bar', 3), + ]), + ] try: debug._log = dlog = debug.DebugLog() @@ -67,9 +69,11 @@ assert res == True finally: debug._log = None - assert ('debug_start', 'mycat') in dlog - assert ('debug_print', 'foo', 2, 'bar', 3) in dlog - assert ('debug_stop', 'mycat') in dlog + assert dlog == [ + ("mycat", [ + ('debug_print', 'foo', 2, 'bar', 3), + ]), + ] class TestLLType(DebugTests): Added: pypy/branch/logging2/pypy/tool/logparser.py ============================================================================== --- (empty file) +++ pypy/branch/logging2/pypy/tool/logparser.py Sat Oct 31 17:42:50 2009 @@ -0,0 +1,154 @@ +#! /usr/bin/env python +""" +Syntax: + python logparser.py + +Actions: + draw-time draw a timeline image of the log (format PPM) +""" +import autopath +import sys, re +from pypy.rlib.debug import DebugLog + + +def parse_log_file(filename): + r_start = re.compile(r"\[([0-9a-f]+)\] \{([\w-]+)$") + r_stop = re.compile(r"\[([0-9a-f]+)\] ([\w-]+)\}$") + log = DebugLog() + f = open(filename, 'r') + for line in f: + line = line.rstrip() + match = r_start.match(line) + if match: + log.debug_start(match.group(2), time=int(match.group(1), 16)) + continue + match = r_stop.match(line) + if match: + log.debug_stop(match.group(2), time=int(match.group(1), 16)) + continue + log.debug_print(line) + f.close() + return log + +def getsubcategories(log): + return [entry for entry in log if entry[0] != 'debug_print'] + +# ____________________________________________________________ + + +COLORS = { + '': (160, 160, 160), + 'gc-': (192, 0, 64), + 'gc-minor': (192, 0, 16), + 'gc-collect': (255, 0, 0), + } + +def getcolor(category): + while category not in COLORS: + category = category[:-1] + return COLORS[category] + +def getlightercolor((r, g, b)): + return ((r*2+255)//3, (g*2+255)//3, (b*2+255)//3) + +def getdarkercolor((r, g, b)): + return (r*2//3, g*2//3, b*2//3) + +def getlabel(text, _cache={}): + try: + return _cache[text] + except KeyError: + pass + from PIL import Image, ImageDraw + if None not in _cache: + image = Image.new("RGBA", (1, 1), (0, 0, 0, 0)) + draw = ImageDraw.Draw(image) + _cache[None] = draw + else: + draw = _cache[None] + sx, sy = draw.textsize(text) + texthoriz = Image.new("RGBA", (sx, sy), (0, 0, 0, 0)) + ImageDraw.Draw(texthoriz).text((0, 0), text, fill=(0, 0, 0)) + textvert = texthoriz.rotate(90) + _cache[text] = sx, sy, texthoriz, textvert + return _cache[text] + + +def get_timeline_image(log, width=900, height=150): + from PIL import Image, ImageDraw + width = int(width) + height = int(height) + maincats = getsubcategories(log) + timestart0 = maincats[0][1] + timestop0 = maincats[-1][2] + assert timestop0 > timestart0 + timefactor = float(width) / (timestop0 - timestart0) + # + def recdraw(sublist, subheight): + firstx1 = None + for category1, timestart1, timestop1, subcats in sublist: + x1 = int((timestart1 - timestart0) * timefactor) + x2 = int((timestop1 - timestart0) * timefactor) + y1 = (height - subheight) / 2 + y2 = y1 + subheight + color = getcolor(category1) + if firstx1 is None: + firstx1 = x1 + if x2 <= x1 + 1: + x2 = x1 + 1 # not wide enough: don't show start and end lines + else: + draw.line([x1, y1+1, x1, y2-1], fill=getlightercolor(color)) + x1 += 1 + x2 -= 1 + draw.line([x2, y1+1, x2, y2-1], fill=getdarkercolor(color)) + draw.line([x1, y1, x2-1, y1], fill=getlightercolor(color)) + y1 += 1 + y2 -= 1 + draw.line([x1, y2, x2-1, y2], fill=getdarkercolor(color)) + draw.rectangle([x1, y1, x2-1, y2-1], fill=color) + if subcats: + x2 = recdraw(subcats, subheight * 0.94) - 1 + sx, sy, texthoriz, textvert = getlabel(category1) + if sx <= x2-x1-8: + image.paste(texthoriz, (x1+5, y1+5), texthoriz) + elif sy <= x2-x1-2: + image.paste(textvert, (x1+1, y1+5), textvert) + return firstx1 + # + image = Image.new("RGBA", (width, height), (0, 0, 0, 0)) + draw = ImageDraw.Draw(image) + recdraw(maincats, height) + return image + +def draw_timeline_image(log, output=None, **kwds): + image = get_timeline_image(log, **kwds) + if output is None: + image.save(sys.stdout, 'png') + else: + image.save(output) + +# ____________________________________________________________ + + +ACTIONS = { + 'draw-time': (draw_timeline_image, ['width=', 'height=', 'output=']), + } + +if __name__ == '__main__': + import getopt + if len(sys.argv) < 2: + print __doc__ + sys.exit(2) + action = sys.argv[1] + func, longopts = ACTIONS[action] + options, args = getopt.gnu_getopt(sys.argv[2:], '', longopts) + if len(args) != 1: + print __doc__ + sys.exit(2) + + kwds = {} + for name, value in options: + assert name.startswith('--') + kwds[name[2:]] = value + log = parse_log_file(args[0]) + func(log, **kwds) From arigo at codespeak.net Sat Oct 31 18:24:12 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 31 Oct 2009 18:24:12 +0100 (CET) Subject: [pypy-svn] r68880 - pypy/branch/logging2/pypy/tool Message-ID: <20091031172412.E62151680A3@codespeak.net> Author: arigo Date: Sat Oct 31 18:24:12 2009 New Revision: 68880 Modified: pypy/branch/logging2/pypy/tool/logparser.py Log: Fix the docstring. Modified: pypy/branch/logging2/pypy/tool/logparser.py ============================================================================== --- pypy/branch/logging2/pypy/tool/logparser.py (original) +++ pypy/branch/logging2/pypy/tool/logparser.py Sat Oct 31 18:24:12 2009 @@ -4,7 +4,7 @@ python logparser.py Actions: - draw-time draw a timeline image of the log (format PPM) + draw-time draw a timeline image of the log (format PNG by default) """ import autopath import sys, re @@ -123,7 +123,7 @@ def draw_timeline_image(log, output=None, **kwds): image = get_timeline_image(log, **kwds) if output is None: - image.save(sys.stdout, 'png') + image.save(sys.stdout, format='png') else: image.save(output) From arigo at codespeak.net Sat Oct 31 19:24:49 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 31 Oct 2009 19:24:49 +0100 (CET) Subject: [pypy-svn] r68881 - pypy/branch/logging2/pypy/tool Message-ID: <20091031182449.DEE5A1680A3@codespeak.net> Author: arigo Date: Sat Oct 31 19:24:48 2009 New Revision: 68881 Modified: pypy/branch/logging2/pypy/tool/logparser.py Log: Don't drawn the vertical light/dark lines in case of width 3 either. Modified: pypy/branch/logging2/pypy/tool/logparser.py ============================================================================== --- pypy/branch/logging2/pypy/tool/logparser.py (original) +++ pypy/branch/logging2/pypy/tool/logparser.py Sat Oct 31 19:24:48 2009 @@ -94,9 +94,9 @@ color = getcolor(category1) if firstx1 is None: firstx1 = x1 - if x2 <= x1 + 1: - x2 = x1 + 1 # not wide enough: don't show start and end lines - else: + if x2 <= x1: + x2 = x1 + 1 # minimal width + elif x2 >= x1 + 4: draw.line([x1, y1+1, x1, y2-1], fill=getlightercolor(color)) x1 += 1 x2 -= 1